From fc7126b885ac10de9958402c37eb11d55d729731 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 16 May 2019 17:46:03 +0200 Subject: [PATCH 1/5] FIX shut down manager if parent process dies --- Lib/multiprocessing/managers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 514152298b097c..5e49d92b3241d7 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -159,6 +159,10 @@ def __init__(self, registry, address, authkey, serializer): self.id_to_local_proxy_obj = {} self.mutex = threading.Lock() + def _track_parent(self): + process.parent_process().join() + self.stop_event.set() + def serve_forever(self): ''' Run the server forever @@ -167,8 +171,10 @@ def serve_forever(self): process.current_process()._manager_server = self try: accepter = threading.Thread(target=self.accepter) - accepter.daemon = True + watcher = threading.Thread(target=self._track_parent) + accepter.daemon = watcher.daemon = True accepter.start() + watcher.start() try: while not self.stop_event.is_set(): self.stop_event.wait(1) From 87c889fa5973408978d1c78da64d49877379fb30 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 16 May 2019 17:50:33 +0200 Subject: [PATCH 2/5] TST test manager shutdown on parent termination --- Lib/test/_test_multiprocessing.py | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 071b54a713e2d8..7e4f739ac63c66 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3824,6 +3824,44 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() + def test_shared_memory_SharedMemoryManager_cleanup_if_parent_dies(self): + # If a process that spawned a SharedMemoryManager was terminated, its + # manager process should notice it, unlink the resources the manager + # delivered, and shut down. + cmd = '''if 1: + import sys, time + + from multiprocessing.managers import SharedMemoryManager + + + smm = SharedMemoryManager() + smm.start() + sl = smm.ShareableList([1, 2, 3]) + + sys.stdout.write(f'{sl.shm.name}\\n') + sys.stdout.flush() + time.sleep(100) + ''' + + p = subprocess.Popen([sys.executable, '-E', '-c', cmd], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + shm_name = p.stdout.readline().decode().strip() + p.terminate() + start_time = time.monotonic() + deadline = start_time + 60 + t = 0.1 + while time.monotonic() < deadline: + time.sleep(t) + t = min(t*2, 5) + try: + sl = shared_memory.SharedMemory(shm_name, create=False) + except FileNotFoundError: + break + else: + raise AssertionError( + "A SharedMemoryManager prevented some resources to be cleaned " + "up after its parent process was terminated") + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the From 1799897b4b87a8685c3ba0b33fd1d1897d7a9b69 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" Date: Mon, 20 May 2019 20:56:21 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2019-05-20-20-56-20.bpo-36977.0Yf6-z.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2019-05-20-20-56-20.bpo-36977.0Yf6-z.rst diff --git a/Misc/NEWS.d/next/Library/2019-05-20-20-56-20.bpo-36977.0Yf6-z.rst b/Misc/NEWS.d/next/Library/2019-05-20-20-56-20.bpo-36977.0Yf6-z.rst new file mode 100644 index 00000000000000..25c1be27ab7982 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-05-20-20-56-20.bpo-36977.0Yf6-z.rst @@ -0,0 +1 @@ +SharedMemoryManager server processes do not block anymore the release of shared memory segments if their parent process was unexpectedly terminated. \ No newline at end of file From 6e5d7edd075dab6573cfb1a2337c091ce3e94b96 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 27 May 2019 22:58:44 +0200 Subject: [PATCH 4/5] address review comments --- Lib/multiprocessing/managers.py | 5 +++-- Lib/test/_test_multiprocessing.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 5e49d92b3241d7..14c2ab0270bacc 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -160,8 +160,9 @@ def __init__(self, registry, address, authkey, serializer): self.mutex = threading.Lock() def _track_parent(self): - process.parent_process().join() - self.stop_event.set() + if parent_process := process.parent_process(): + parent_process.join() + self.stop_event.set() def serve_forever(self): ''' diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 7e4f739ac63c66..39d220458215b7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3852,7 +3852,8 @@ def test_shared_memory_SharedMemoryManager_cleanup_if_parent_dies(self): t = 0.1 while time.monotonic() < deadline: time.sleep(t) - t = min(t*2, 5) + # The shared_memory segment should be released by the + # SharedMemoryManager server process before it shuts down. try: sl = shared_memory.SharedMemory(shm_name, create=False) except FileNotFoundError: From dd6e55838743ac712abf62d38c0b05826f452658 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 28 May 2019 00:08:36 +0200 Subject: [PATCH 5/5] syntax, guards --- Lib/multiprocessing/managers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 14c2ab0270bacc..10eb1f3f164bc2 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -160,8 +160,8 @@ def __init__(self, registry, address, authkey, serializer): self.mutex = threading.Lock() def _track_parent(self): - if parent_process := process.parent_process(): - parent_process.join() + if process.parent_process() is not None: + process.parent_process().join() self.stop_event.set() def serve_forever(self): @@ -172,10 +172,14 @@ def serve_forever(self): process.current_process()._manager_server = self try: accepter = threading.Thread(target=self.accepter) - watcher = threading.Thread(target=self._track_parent) - accepter.daemon = watcher.daemon = True + accepter.daemon = True accepter.start() - watcher.start() + + if process.parent_process() is not None: + watcher = threading.Thread(target=self._track_parent) + watcher.daemon = True + watcher.start() + try: while not self.stop_event.is_set(): self.stop_event.wait(1)