Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a7d3581

Browse files
pitroupitrou
committed
GH-109978: Allow multiprocessing finalizers to run on a separate thread
1 parent de2a403 commit a7d3581

File tree

15 files changed

+264
-73
lines changed

15 files changed

+264
-73
lines changed

‎Lib/concurrent/futures/process.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -561,9 +561,9 @@ def shutdown_workers(self):
561561
except queue.Full:
562562
break
563563

564-
def join_executor_internals(self):
564+
def join_executor_internals(self, broken=False):
565565
with self.shutdown_lock:
566-
self._join_executor_internals()
566+
self._join_executor_internals(broken)
567567

568568
def _join_executor_internals(self, broken=False):
569569
# If broken, call_queue was closed and so can no longer be used.
@@ -759,7 +759,11 @@ def _start_executor_manager_thread(self):
759759
if not self._safe_to_dynamically_spawn_children: # ie, using fork.
760760
self._launch_processes()
761761
self._executor_manager_thread = _ExecutorManagerThread(self)
762-
self._executor_manager_thread.start()
762+
try:
763+
self._executor_manager_thread.start()
764+
except RuntimeError:
765+
self._broken = "Executor manager thread could not be started"
766+
raise BrokenProcessPool(self._broken)
763767
_threads_wakeups[self._executor_manager_thread] = \
764768
self._executor_manager_thread_wakeup
765769

@@ -860,7 +864,10 @@ def shutdown(self, wait=True, *, cancel_futures=False):
860864
self._executor_manager_thread_wakeup.wakeup()
861865

862866
if self._executor_manager_thread is not None and wait:
863-
self._executor_manager_thread.join()
867+
try:
868+
self._executor_manager_thread.join()
869+
except RuntimeError:
870+
self._executor_manager_thread.join_executor_internals(broken=True)
864871
# To reduce the risk of opening too many files, remove references to
865872
# objects that use file descriptors.
866873
self._executor_manager_thread = None

‎Lib/multiprocessing/connection.py

+4
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,10 @@ def close(self):
491491
self._listener = None
492492
listener.close()
493493

494+
@property
495+
def closed(self):
496+
return self._listener is None
497+
494498
@property
495499
def address(self):
496500
return self._listener._address

‎Lib/multiprocessing/heap.py

+6-37
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,6 @@ def __init__(self, size=mmap.PAGESIZE):
142142
self._allocated_blocks = defaultdict(set)
143143
self._arenas = []
144144

145-
# List of pending blocks to free - see comment in free() below
146-
self._pending_free_blocks = []
147-
148145
# Statistics
149146
self._n_mallocs = 0
150147
self._n_frees = 0
@@ -255,43 +252,16 @@ def _remove_allocated_block(self, block):
255252
# Arena is entirely free, discard it from this process
256253
self._discard_arena(arena)
257254

258-
def _free_pending_blocks(self):
259-
# Free all the blocks in the pending list - called with the lock held.
260-
while True:
261-
try:
262-
block = self._pending_free_blocks.pop()
263-
except IndexError:
264-
break
265-
self._add_free_block(block)
266-
self._remove_allocated_block(block)
267-
268255
def free(self, block):
269256
# free a block returned by malloc()
270-
# Since free() can be called asynchronously by the GC, it could happen
271-
# that it's called while self._lock is held: in that case,
272-
# self._lock.acquire() would deadlock (issue #12352). To avoid that, a
273-
# trylock is used instead, and if the lock can't be acquired
274-
# immediately, the block is added to a list of blocks to be freed
275-
# synchronously sometimes later from malloc() or free(), by calling
276-
# _free_pending_blocks() (appending and retrieving from a list is not
277-
# strictly thread-safe but under CPython it's atomic thanks to the GIL).
278257
if os.getpid() != self._lastpid:
279258
raise ValueError(
280259
"My pid ({0:n}) is not last pid {1:n}".format(
281260
os.getpid(),self._lastpid))
282-
if not self._lock.acquire(False):
283-
# can't acquire the lock right now, add the block to the list of
284-
# pending blocks to free
285-
self._pending_free_blocks.append(block)
286-
else:
287-
# we hold the lock
288-
try:
289-
self._n_frees += 1
290-
self._free_pending_blocks()
291-
self._add_free_block(block)
292-
self._remove_allocated_block(block)
293-
finally:
294-
self._lock.release()
261+
with self._lock:
262+
self._n_frees += 1
263+
self._add_free_block(block)
264+
self._remove_allocated_block(block)
295265

296266
def malloc(self, size):
297267
# return a block of right size (possibly rounded up)
@@ -303,8 +273,6 @@ def malloc(self, size):
303273
self.__init__() # reinitialize after fork
304274
with self._lock:
305275
self._n_mallocs += 1
306-
# allow pending blocks to be marked available
307-
self._free_pending_blocks()
308276
size = self._roundup(max(size, 1), self._alignment)
309277
(arena, start, stop) = self._malloc(size)
310278
real_stop = start + size
@@ -330,7 +298,8 @@ def __init__(self, size):
330298
raise OverflowError("Size {0:n} too large".format(size))
331299
block = BufferWrapper._heap.malloc(size)
332300
self._state = (block, size)
333-
util.Finalize(self, BufferWrapper._heap.free, args=(block,))
301+
util.Finalize(self, BufferWrapper._heap.free, args=(block,),
302+
reentrant=False)
334303

335304
def create_memoryview(self):
336305
(arena, start, stop), size = self._state

‎Lib/multiprocessing/managers.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -180,14 +180,15 @@ def serve_forever(self):
180180
except (KeyboardInterrupt, SystemExit):
181181
pass
182182
finally:
183+
self.listener.close()
183184
if sys.stdout != sys.__stdout__: # what about stderr?
184185
util.debug('resetting stdout, stderr')
185186
sys.stdout = sys.__stdout__
186187
sys.stderr = sys.__stderr__
187188
sys.exit(0)
188189

189190
def accepter(self):
190-
while True:
191+
while not self.listener.closed:
191192
try:
192193
c = self.listener.accept()
193194
except OSError:
@@ -575,7 +576,7 @@ def start(self, initializer=None, initargs=()):
575576
self, type(self)._finalize_manager,
576577
args=(self._process, self._address, self._authkey, self._state,
577578
self._Client, self._shutdown_timeout),
578-
exitpriority=0
579+
exitpriority=0, reentrant=False
579580
)
580581

581582
@classmethod
@@ -859,12 +860,11 @@ def _incref(self):
859860
self._idset.add(self._id)
860861

861862
state = self._manager and self._manager._state
862-
863863
self._close = util.Finalize(
864864
self, BaseProxy._decref,
865865
args=(self._token, self._authkey, state,
866866
self._tls, self._idset, self._Client),
867-
exitpriority=10
867+
exitpriority=10, reentrant=False
868868
)
869869

870870
@staticmethod

‎Lib/multiprocessing/pool.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
219219
p.terminate()
220220
for p in self._pool:
221221
p.join()
222+
self._pool.clear()
222223
raise
223224

224225
sentinels = self._get_sentinels()
@@ -257,7 +258,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
257258
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
258259
self._change_notifier, self._worker_handler, self._task_handler,
259260
self._result_handler, self._cache),
260-
exitpriority=15
261+
exitpriority=15, reentrant=False
261262
)
262263
self._state = RUN
263264

@@ -665,8 +666,8 @@ def join(self):
665666
self._worker_handler.join()
666667
self._task_handler.join()
667668
self._result_handler.join()
668-
for p in self._pool:
669-
p.join()
669+
while self._pool:
670+
self._pool.pop().join()
670671

671672
@staticmethod
672673
def _help_stuff_finish(inqueue, task_handler, size):

‎Lib/multiprocessing/queues.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -201,14 +201,14 @@ def _start_thread(self):
201201
self._jointhread = Finalize(
202202
self._thread, Queue._finalize_join,
203203
[weakref.ref(self._thread)],
204-
exitpriority=-5
204+
exitpriority=-5, reentrant=False
205205
)
206206

207207
# Send sentinel to the thread queue object when garbage collected
208208
self._close = Finalize(
209209
self, Queue._finalize_close,
210210
[self._buffer, self._notempty],
211-
exitpriority=10
211+
exitpriority=10, reentrant=False
212212
)
213213

214214
@staticmethod

‎Lib/multiprocessing/synchronize.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def _after_fork(obj):
7979
from .resource_tracker import register
8080
register(self._semlock.name, "semaphore")
8181
util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
82-
exitpriority=0)
82+
exitpriority=0, reentrant=False)
8383

8484
@staticmethod
8585
def _cleanup(name):

0 commit comments

Comments
 (0)