From d9aeedf7500b60c82397e914f9a524b1af192846 Mon Sep 17 00:00:00 2001 From: albanD Date: Tue, 25 Jul 2023 18:11:46 -0400 Subject: [PATCH 1/5] Ensure multiprocessing lock is valid for spawn Process before serializing it --- Lib/multiprocessing/synchronize.py | 9 +++++++-- Lib/test/_test_multiprocessing.py | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 42624b543601a1..2328d332123082 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -50,8 +50,8 @@ class SemLock(object): def __init__(self, kind, value, maxvalue, *, ctx): if ctx is None: ctx = context._default_context.get_context() - name = ctx.get_start_method() - unlink_now = sys.platform == 'win32' or name == 'fork' + self.is_fork_ctx = ctx.get_start_method() == 'fork' + unlink_now = sys.platform == 'win32' or self.is_fork_ctx for i in range(100): try: sl = self._semlock = _multiprocessing.SemLock( @@ -103,6 +103,11 @@ def __getstate__(self): if sys.platform == 'win32': h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: + if self.is_fork_ctx: + raise RuntimeError('A SemLock created in a fork context is being ' + 'shared with a process in a spawn context. This is ' + 'not supported. Please use the same context to create ' + 'multiprocessing objects and Process.') h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 10754964e73bc5..6823b1691448b1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6201,3 +6201,9 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) + + def test_semlock_mixed_context(self): + queue = multiprocessing.get_context("fork").Queue() + p = multiprocessing.get_context("spawn").Process(target=close_queue, args=(queue,)) + with self.assertRaisesRegex(RuntimeError, "A SemLock created in a fork"): + p.start() From df6934792172acf9b241fdaadee07d8df222925c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:35:36 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst new file mode 100644 index 00000000000000..ebf16b9b679f90 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst @@ -0,0 +1 @@ +Ensure that multiprocessing SemLock object created in a fork context are not send to a different process created in a spawn context. This changes changes a segfault into an actionable RuntimeError in the parent process. From 152800c6e357da7f66353f91dae1f1aa672e43c5 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 23 Aug 2023 14:47:52 -0400 Subject: [PATCH 3/5] Improve test to run on macos and cover forkserver --- Lib/test/_test_multiprocessing.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6823b1691448b1..1caec4314235c8 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5421,6 +5421,24 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") + @unittest.skipIf(sys.platform == "win32", "Only Spawn on windows so no risk of mixing") + @only_run_in_spawn_testsuite("avoids redundant testing since we ignore the global context.") + def test_mixed_startmethod(self): + # Fork-based locks cannot be used with spawned process + for process_method in ["spawn", "forkserver"]: + queue = multiprocessing.get_context("fork").Queue() + p = multiprocessing.get_context(process_method).Process(target=close_queue, args=(queue,)) + with self.assertRaisesRegex(RuntimeError, "A SemLock created in a fork"): + p.start() + + # non-fork-based locks can be used with all other start methods + for queue_method in ["spawn", "forkserver"]: + for process_method in multiprocessing.get_all_start_methods(): + queue = multiprocessing.get_context(queue_method).Queue() + p = multiprocessing.get_context(process_method).Process(target=close_queue, args=(queue,)) + p.start() + p.join() + @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") @@ -6201,9 +6219,3 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) - - def test_semlock_mixed_context(self): - queue = multiprocessing.get_context("fork").Queue() - p = multiprocessing.get_context("spawn").Process(target=close_queue, args=(queue,)) - with self.assertRaisesRegex(RuntimeError, "A SemLock created in a fork"): - p.start() From 29d37e176ac4e54718d0a5d59edf0bc55a93558d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 23 Aug 2023 21:30:43 +0200 Subject: [PATCH 4/5] Fix typos and wording --- .../2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst index ebf16b9b679f90..194851dea13352 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-07-25-22-35-35.gh-issue-77377.EHAbXx.rst @@ -1 +1 @@ -Ensure that multiprocessing SemLock object created in a fork context are not send to a different process created in a spawn context. This changes changes a segfault into an actionable RuntimeError in the parent process. +Ensure that multiprocessing synchronization objects created in a fork context are not sent to a different process created in a spawn context. This changes a segfault into an actionable RuntimeError in the parent process. From e5f4e32479289998db3ae6831b08e826318e2da8 Mon Sep 17 00:00:00 2001 From: albanD Date: Wed, 23 Aug 2023 15:42:09 -0400 Subject: [PATCH 5/5] lint --- Lib/test/_test_multiprocessing.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 1caec4314235c8..a826e31f5ee079 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -5421,21 +5421,25 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") - @unittest.skipIf(sys.platform == "win32", "Only Spawn on windows so no risk of mixing") - @only_run_in_spawn_testsuite("avoids redundant testing since we ignore the global context.") + @unittest.skipIf(sys.platform == "win32", + "Only Spawn on windows so no risk of mixing") + @only_run_in_spawn_testsuite("avoids redundant testing.") def test_mixed_startmethod(self): # Fork-based locks cannot be used with spawned process for process_method in ["spawn", "forkserver"]: queue = multiprocessing.get_context("fork").Queue() - p = multiprocessing.get_context(process_method).Process(target=close_queue, args=(queue,)) - with self.assertRaisesRegex(RuntimeError, "A SemLock created in a fork"): + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) + err_msg = "A SemLock created in a fork" + with self.assertRaisesRegex(RuntimeError, err_msg): p.start() # non-fork-based locks can be used with all other start methods for queue_method in ["spawn", "forkserver"]: for process_method in multiprocessing.get_all_start_methods(): queue = multiprocessing.get_context(queue_method).Queue() - p = multiprocessing.get_context(process_method).Process(target=close_queue, args=(queue,)) + process_ctx = multiprocessing.get_context(process_method) + p = process_ctx.Process(target=close_queue, args=(queue,)) p.start() p.join()