From 412614507100bd88332885e7acf8bac541714475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 13:00:59 +0100 Subject: [PATCH 01/16] Fix signature of multiprocessing.set_executable() --- Doc/library/multiprocessing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 9bb7dd3d703ab1..cbbe1840fc3515 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1047,9 +1047,9 @@ Miscellaneous .. versionadded:: 3.4 -.. function:: set_executable() +.. function:: set_executable(executable) - Sets the path of the Python interpreter to use when starting a child process. + Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to do some thing like :: From 7b2cc9f6ca7bac6de653ca9e6cec7f0adf4321ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 15:45:51 +0100 Subject: [PATCH 02/16] Add support of path-like objects to multiprocessing.set_executable for Windows to match Unix-like systems --- Lib/multiprocessing/popen_spawn_win32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 9c4098d0fa4f1e..8a45cb5b2e8745 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -71,7 +71,7 @@ def __init__(self, process_obj): # start process try: hp, ht, pid, tid = _winapi.CreateProcess( - python_exe, cmd, + os.fsdecode(python_exe), cmd, None, None, False, 0, env, None, None) _winapi.CloseHandle(ht) except: From 43d06eaab0ce06c9f5100ea97a5f6d6bef43247d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 21:53:33 +0100 Subject: [PATCH 03/16] Update popen_spawn_win32.py --- Lib/multiprocessing/popen_spawn_win32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 8a45cb5b2e8745..9c4098d0fa4f1e 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -71,7 +71,7 @@ def __init__(self, process_obj): # start process try: hp, ht, pid, tid = _winapi.CreateProcess( - os.fsdecode(python_exe), cmd, + python_exe, cmd, None, None, False, 0, env, None, None) _winapi.CloseHandle(ht) except: From ff13b14dc1c79151a206a1bbb75223ce0a0941c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 22:00:39 +0100 Subject: [PATCH 04/16] Update spawn.py --- Lib/multiprocessing/spawn.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 7cc129e2610761..5c22b6889599c8 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -43,7 +43,10 @@ def set_executable(exe): _python_exe = exe def get_executable(): - return _python_exe + if sys.platform == 'win32': + return os.fsdecode(_python_exe) + else: + return os.fsencode(_python_exe) # # @@ -86,7 +89,8 @@ def get_command_line(**kwds): prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' prog %= ', '.join('%s=%r' % item for item in kwds.items()) opts = util._args_from_interpreter_flags() - return [_python_exe] + opts + ['-c', prog, '--multiprocessing-fork'] + exe = get_executable() + return [exe] + opts + ['-c', prog, '--multiprocessing-fork'] def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): From c195657318273da6cecdecd868cd8ab61f740d85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 22:15:05 +0100 Subject: [PATCH 05/16] Update util.py --- Lib/multiprocessing/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index a4683339820f5f..abbc4c5e6088b2 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -450,7 +450,7 @@ def spawnv_passfds(path, args, passfds): errpipe_read, errpipe_write = os.pipe() try: return _posixsubprocess.fork_exec( - args, [os.fsencode(path)], True, passfds, None, None, + args, [path], True, passfds, None, None, -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, False, False, None, None, None, -1, None) finally: From 7eaad03d142924d61f12a9912bb852ef2be35dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 22:42:10 +0100 Subject: [PATCH 06/16] Update spawn.py --- Lib/multiprocessing/spawn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 5c22b6889599c8..b60785271c6ecb 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -40,13 +40,13 @@ def set_executable(exe): global _python_exe - _python_exe = exe - -def get_executable(): if sys.platform == 'win32': - return os.fsdecode(_python_exe) + _python_exe = os.fsdecode(exe) else: - return os.fsencode(_python_exe) + _python_exe = os.fsencode(exe) + +def get_executable(): + return _python_exe # # From a41e443b3138ecf464ee69a65f449566be6f641e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 22:54:21 +0100 Subject: [PATCH 07/16] Update spawn.py --- Lib/multiprocessing/spawn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index b60785271c6ecb..09f8a229d7cccb 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -33,11 +33,6 @@ WINEXE = getattr(sys, 'frozen', False) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") -if WINSERVICE: - _python_exe = os.path.join(sys.exec_prefix, 'python.exe') -else: - _python_exe = sys.executable - def set_executable(exe): global _python_exe if sys.platform == 'win32': @@ -48,6 +43,11 @@ def set_executable(exe): def get_executable(): return _python_exe +if WINSERVICE: + set_executable(os.path.join(sys.exec_prefix, 'python.exe')) +else: + set_executable(sys.executable) + # # # From 57407dac52c0f13c095092fdc7770d54e6da01e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 23:23:46 +0100 Subject: [PATCH 08/16] Create 2022-02-11-23-11-35.bpo-46720.nY8spB.rst --- .../next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst diff --git a/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst b/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst new file mode 100644 index 00000000000000..62d6515b13b947 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst @@ -0,0 +1,2 @@ +Add support for path-like objects to :func:`multiprocessing.set_executable` for +Windows to be on a par with Unix-like systems. Patch by Géry Ogam. From 07780780ed727e3c9e26d87cc609a1b0ea542fa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 23:27:28 +0100 Subject: [PATCH 09/16] Update 2022-02-11-23-11-35.bpo-46720.nY8spB.rst --- .../next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst b/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst index 62d6515b13b947..70d5e5ef343334 100644 --- a/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst +++ b/Misc/NEWS.d/next/Library/2022-02-11-23-11-35.bpo-46720.nY8spB.rst @@ -1,2 +1,2 @@ -Add support for path-like objects to :func:`multiprocessing.set_executable` for +Add support for path-like objects to :func:`multiprocessing.set_executable` for Windows to be on a par with Unix-like systems. Patch by Géry Ogam. From e2c856bd3f20b2e0e7707f78f6bb7fe7eb9c8e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 23:35:10 +0100 Subject: [PATCH 10/16] Update multiprocessing.rst --- Doc/library/multiprocessing.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index cbbe1840fc3515..32d2e88cf6536d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1060,6 +1060,9 @@ Miscellaneous .. versionchanged:: 3.4 Now supported on Unix when the ``'spawn'`` start method is used. + .. versionchanged:: 3.11 + Accepts a :term:`path-like object`. + .. function:: set_start_method(method) Set the method which should be used to start child processes. @@ -2290,7 +2293,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.3 + .. versionadded:: 3.11 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. From 13e3eb1622ef74915810be8d0f24ec1917604f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Fri, 11 Feb 2022 23:36:49 +0100 Subject: [PATCH 11/16] Update multiprocessing.rst --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 32d2e88cf6536d..70bf8bf89ea697 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2293,7 +2293,7 @@ with the :class:`Pool` class. Wait for the worker processes to exit. One must call :meth:`close` or :meth:`terminate` before using :meth:`join`. - .. versionadded:: 3.11 + .. versionadded:: 3.3 Pool objects now support the context management protocol -- see :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. From 4c941768fa97b8bf52ebc33054d3d956ed1a5e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 12 Mar 2022 14:49:21 +0100 Subject: [PATCH 12/16] Add unit tests --- Lib/test/_test_multiprocessing.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index b2d656ab428975..6ad01bdebdbc6e 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -247,6 +247,24 @@ def test_current(self): self.assertEqual(current.ident, os.getpid()) self.assertEqual(current.exitcode, None) + def test_set_executable(self): + paths = [ + sys.executable, # str + sys.executable.encode(), # bytes + pathlib.Path(sys.executable) # os.PathLike + ] + for path in paths: + multiprocessing.set_executable(path) + p = multiprocessing.Process() + p.start() + p.join() + self.assertEqual(p.exitcode, 0) + multiprocessing.set_executable('') + p = multiprocessing.Process() + p.start() + p.join() + self.assertNotEqual(p.exitcode, 0) + def test_daemon_argument(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) From 100c9948ef7874c5ee8870de512f0a962500dbc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 12 Mar 2022 16:46:18 +0100 Subject: [PATCH 13/16] Update _test_multiprocessing.py --- Lib/test/_test_multiprocessing.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6f14651da0add5..2e8bcc1917954b 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -20,6 +20,7 @@ import subprocess import struct import operator +import pathlib import pickle import weakref import warnings @@ -260,13 +261,13 @@ def test_set_executable(self): pathlib.Path(sys.executable) # os.PathLike ] for path in paths: - multiprocessing.set_executable(path) - p = multiprocessing.Process() + self.set_executable(path) + p = self.Process() p.start() p.join() self.assertEqual(p.exitcode, 0) - multiprocessing.set_executable('') - p = multiprocessing.Process() + self.set_executable('') + p = self.Process() p.start() p.join() self.assertNotEqual(p.exitcode, 0) From ee6ab812561984e71bb9c5534cf63a47a0e33427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 12 Mar 2022 18:23:03 +0100 Subject: [PATCH 14/16] Update _test_multiprocessing.py --- Lib/test/_test_multiprocessing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 2e8bcc1917954b..9301fb10039f79 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -255,6 +255,8 @@ def test_current(self): self.assertEqual(current.exitcode, None) def test_set_executable(self): + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) paths = [ sys.executable, # str sys.executable.encode(), # bytes @@ -5798,6 +5800,7 @@ class ProcessesMixin(BaseMixin): current_process = staticmethod(multiprocessing.current_process) parent_process = staticmethod(multiprocessing.parent_process) active_children = staticmethod(multiprocessing.active_children) + set_executable = staticmethod(multiprocessing.set_executable) Pool = staticmethod(multiprocessing.Pool) Pipe = staticmethod(multiprocessing.Pipe) Queue = staticmethod(multiprocessing.Queue) From a6b16b7ec779e79abffdbe7e075c3111063fbda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9ry=20Ogam?= Date: Sat, 12 Mar 2022 21:29:03 +0100 Subject: [PATCH 15/16] Update _test_multiprocessing.py --- Lib/test/_test_multiprocessing.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9301fb10039f79..ed92da574f31fb 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -268,11 +268,6 @@ def test_set_executable(self): p.start() p.join() self.assertEqual(p.exitcode, 0) - self.set_executable('') - p = self.Process() - p.start() - p.join() - self.assertNotEqual(p.exitcode, 0) def test_args_argument(self): # bpo-45735: Using list or tuple as *args* in constructor could From 02752bd42ee3d4fda2047522ba7258653a51980f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Apr 2022 15:05:29 -0700 Subject: [PATCH 16/16] Update Lib/test/_test_multiprocessing.py --- Lib/test/_test_multiprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index ed92da574f31fb..1708d653fe8afa 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -256,7 +256,7 @@ def test_current(self): def test_set_executable(self): if self.TYPE == 'threads': - self.skipTest('test not appropriate for {}'.format(self.TYPE)) + self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str sys.executable.encode(), # bytes