From 9b51d5254d9e7e6a4749ab6e1dfc8aad14c1448d Mon Sep 17 00:00:00 2001 From: Akinori Hattori Date: Sat, 15 Oct 2022 10:18:20 +0900 Subject: [PATCH] gh-71936: Fix race condition in multiprocessing.Pool Proxes of shared objects register a Finalizer in BaseProxy._incref(), and it will call BaseProxy._decref() when it is GCed. This may cause a race condition with Pool(maxtasksperchild=None) on Windows. A connection will be closed and raised TypeError when a GC occurs between _ConnectionBase._check_writable() and _ConnectionBase._send_bytes() in _ConnectionBase.send() in the second or later task. BaseProxy does not count references well. --- Lib/multiprocessing/managers.py | 9 +++++---- Misc/ACKS | 1 + .../2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index b6534939b4d98b..85b26517593f87 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -850,20 +850,21 @@ def _incref(self): dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) - self._idset.add(self._id) + refid = f'{self._id}.{id(self)}' + self._idset.add(refid) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, - args=(self._token, self._authkey, state, + args=(self._token, refid, self._authkey, state, self._tls, self._idset, self._Client), exitpriority=10 ) @staticmethod - def _decref(token, authkey, state, tls, idset, _Client): - idset.discard(token.id) + def _decref(token, refid, authkey, state, tls, idset, _Client): + idset.discard(refid) # check whether manager is still alive if state is None or state.value == State.STARTED: diff --git a/Misc/ACKS b/Misc/ACKS index ec5e326847b8ca..4c7f880354b2c0 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -706,6 +706,7 @@ Larry Hastings Tim Hatch Zac Hatfield-Dodds Shane Hathaway +Akinori Hattori Michael Haubenwallner Janko Hauser Flavian Hautbois diff --git a/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst b/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst new file mode 100644 index 00000000000000..ac1081e060ba2d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-15-10-18-20.gh-issue-71936.MzJjc_.rst @@ -0,0 +1 @@ +Fix a race condition in :class:`multiprocessing.Pool`.