From 7b907281ea1a90391e52b391f336fac33fd042cf Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Thu, 11 Jan 2018 08:21:06 -0600 Subject: [PATCH 01/12] bpo-32146: freeze_support can apply to non-Windows platforms also --- Doc/library/multiprocessing.rst | 11 +++++------ Lib/multiprocessing/context.py | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 8b3081cb80c972..701f75279d838d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -926,8 +926,8 @@ Miscellaneous .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, - **PyInstaller** and **cx_Freeze**.) + frozen to produce an application executable. (Has been tested with + **py2exe**, **PyInstaller** and **cx_Freeze**.) One needs to call this function straight after the ``if __name__ == '__main__'`` line of the main module. For example:: @@ -944,10 +944,9 @@ Miscellaneous If the ``freeze_support()`` line is omitted then trying to run the frozen executable will raise :exc:`RuntimeError`. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. + ``freeze_support()`` has no effect when invoked in a module that is being + run normally by the Python interpreter (i.e., instead of by a frozen + executable). .. function:: get_all_start_methods() diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index c98ee434249037..909f3d529d2143 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -144,7 +144,7 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + if getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() From b0d30bc48c1447356031742b44d95eaa1db17af3 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Fri, 12 Jan 2018 21:05:09 -0600 Subject: [PATCH 02/12] bpo-32146: Provisional fix for non-Windows executables (spawn) --- Lib/multiprocessing/spawn.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 1f4f3f496f51a2..c1c63ca90bd17b 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -73,6 +73,17 @@ def freeze_support(): kwds[name] = int(value) spawn_main(**kwds) sys.exit() + # fix the command line for the semaphore tracker (unix) + elif ( + (len(sys.argv) >= 3) and + (sys.argv[-2] == '-c') and + ('semaphore_tracker import main;' in sys.argv[-1]) + ): + r = int(sys.argv[-1].rsplit('(')[1].split(')')[0]) + from multiprocessing.semaphore_tracker import main; + main(r) + sys.exit() + # TODO: fix the command line for the fork server (unix) def get_command_line(**kwds): From 8baf4e4fc686c8e35d4e702bb78812453fe35077 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 14 Jan 2018 08:19:54 -0600 Subject: [PATCH 03/12] bpo-32146: Provisional fix for non-Windows executables (forkserver) --- Lib/multiprocessing/spawn.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index c1c63ca90bd17b..11b1651b9dc862 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -8,6 +8,7 @@ # Licensed to PSF under a Contributor Agreement. # +import ast import os import sys import runpy @@ -83,7 +84,22 @@ def freeze_support(): from multiprocessing.semaphore_tracker import main; main(r) sys.exit() - # TODO: fix the command line for the fork server (unix) + # fix the command line for the fork server (unix) + elif ( + (len(sys.argv) >= 3) and + (sys.argv[-2] == '-c') and + ('forkserver import main;' in sys.argv[-1]) + ): + cmd = sys.argv[-1] + main_args = cmd.split('main(')[1].rsplit(')', 1)[0].split(', ', 3) + listener_fd = int(main_args[0]) + alive_r = int(main_args[1]) + preload = ast.literal_eval(main_args[2]) + kwds = ast.literal_eval(main_args[3][2:]) + + from multiprocessing.forkserver import main + main(listener_fd, alive_r, preload, **kwds) + sys.exit() def get_command_line(**kwds): From 64456d9f7be5b0824bea9d7f36fe359042c733ac Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 14 Jan 2018 08:42:07 -0600 Subject: [PATCH 04/12] bpo-32146: Move commandline checks to separate functions --- Lib/multiprocessing/spawn.py | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 11b1651b9dc862..4c41f8663d88aa 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -60,6 +60,28 @@ def is_forking(argv): return False +def is_semaphore_tracker(argv): + ''' + Return whether commandline indicates we are running the semaphore tracker + ''' + return ( + (len(argv) >= 3) and + (argv[-2] == '-c') and + ('semaphore_tracker import main;' in sys.argv[-1]) + ) + + +def is_forkserver(argv): + ''' + Return whether commandline indicates we are running the forkserver + ''' + return ( + (len(sys.argv) >= 3) and + (sys.argv[-2] == '-c') and + ('forkserver import main;' in sys.argv[-1]) + ) + + def freeze_support(): ''' Run code for process object if this in not the main process @@ -75,21 +97,15 @@ def freeze_support(): spawn_main(**kwds) sys.exit() # fix the command line for the semaphore tracker (unix) - elif ( - (len(sys.argv) >= 3) and - (sys.argv[-2] == '-c') and - ('semaphore_tracker import main;' in sys.argv[-1]) - ): + # we extract the single argument to the semaphore tracker's main() + elif is_semaphore_tracker(sys.argv): r = int(sys.argv[-1].rsplit('(')[1].split(')')[0]) from multiprocessing.semaphore_tracker import main; main(r) sys.exit() # fix the command line for the fork server (unix) - elif ( - (len(sys.argv) >= 3) and - (sys.argv[-2] == '-c') and - ('forkserver import main;' in sys.argv[-1]) - ): + # we extract the 3 args and keywords to the forkserver's main() + elif is_forkserver(sys.argv): cmd = sys.argv[-1] main_args = cmd.split('main(')[1].rsplit(')', 1)[0].split(', ', 3) listener_fd = int(main_args[0]) From 748c2f1cc56c235b2ac0ee68286b87a3aa5abd9d Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 14 Jan 2018 17:13:19 -0600 Subject: [PATCH 05/12] bpo-32146: Add Misc/NEWS entry for freeze_support enhancements --- .../next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst diff --git a/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst b/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst new file mode 100644 index 00000000000000..5c5636bf41ffe4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-14-16-34-26.bpo-32146.6pFDbn.rst @@ -0,0 +1,2 @@ +``multiprocessing.freeze_support`` now works on non-Windows platforms as +well. From 3a0964ae759f181371d4520e0984cda71beea05c Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 14 Jan 2018 17:25:41 -0600 Subject: [PATCH 06/12] bpo-32146: Use dummy --multiprocessing- command line arguments --- Lib/multiprocessing/forkserver.py | 3 ++- Lib/multiprocessing/semaphore_tracker.py | 3 ++- Lib/multiprocessing/spawn.py | 23 ++++++++++------------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 040b46e66a0330..10342533a4a8b0 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -127,7 +127,8 @@ def ensure_running(self): cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() + args = [exe, '--multiprocessing-forkserver'] + args += util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py index 3e31bf8402ee2d..0d909b3963163a 100644 --- a/Lib/multiprocessing/semaphore_tracker.py +++ b/Lib/multiprocessing/semaphore_tracker.py @@ -66,7 +66,8 @@ def ensure_running(self): fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() + args = [exe, '--multiprocessing-semaphore-tracker'] + args += util._args_from_interpreter_flags() args += ['-c', cmd % r] pid = util.spawnv_passfds(exe, args, fds_to_pass) except: diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 4c41f8663d88aa..f850dcca5d2cc6 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -64,22 +64,20 @@ def is_semaphore_tracker(argv): ''' Return whether commandline indicates we are running the semaphore tracker ''' - return ( - (len(argv) >= 3) and - (argv[-2] == '-c') and - ('semaphore_tracker import main;' in sys.argv[-1]) - ) + if len(argv) >= 2 and argv[1] == '--multiprocessing-semaphore-tracker': + return True + else: + return False def is_forkserver(argv): ''' Return whether commandline indicates we are running the forkserver ''' - return ( - (len(sys.argv) >= 3) and - (sys.argv[-2] == '-c') and - ('forkserver import main;' in sys.argv[-1]) - ) + if len(argv) >= 2 and argv[1] == '--multiprocessing-forkserver': + return True + else: + return False def freeze_support(): @@ -99,21 +97,20 @@ def freeze_support(): # fix the command line for the semaphore tracker (unix) # we extract the single argument to the semaphore tracker's main() elif is_semaphore_tracker(sys.argv): + from multiprocessing.semaphore_tracker import main r = int(sys.argv[-1].rsplit('(')[1].split(')')[0]) - from multiprocessing.semaphore_tracker import main; main(r) sys.exit() # fix the command line for the fork server (unix) # we extract the 3 args and keywords to the forkserver's main() elif is_forkserver(sys.argv): + from multiprocessing.forkserver import main cmd = sys.argv[-1] main_args = cmd.split('main(')[1].rsplit(')', 1)[0].split(', ', 3) listener_fd = int(main_args[0]) alive_r = int(main_args[1]) preload = ast.literal_eval(main_args[2]) kwds = ast.literal_eval(main_args[3][2:]) - - from multiprocessing.forkserver import main main(listener_fd, alive_r, preload, **kwds) sys.exit() From 7b49c0078945df9bc492c61792e5c7a97c42e7c3 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Sun, 14 Jan 2018 20:00:26 -0600 Subject: [PATCH 07/12] bpo-32146: Only use special command line arguments when frozen --- Lib/multiprocessing/forkserver.py | 4 +++- Lib/multiprocessing/semaphore_tracker.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 10342533a4a8b0..57ea6f584602c5 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -127,7 +127,9 @@ def ensure_running(self): cmd %= (listener.fileno(), alive_r, self._preload_modules, data) exe = spawn.get_executable() - args = [exe, '--multiprocessing-forkserver'] + args = [exe] + if getattr(sys, 'frozen', False): + args.append('--multiprocessing-forkserver') args += util._args_from_interpreter_flags() args += ['-c', cmd] pid = util.spawnv_passfds(exe, args, fds_to_pass) diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py index 0d909b3963163a..5055aa85d0fee2 100644 --- a/Lib/multiprocessing/semaphore_tracker.py +++ b/Lib/multiprocessing/semaphore_tracker.py @@ -66,7 +66,9 @@ def ensure_running(self): fds_to_pass.append(r) # process will out live us, so no need to wait on pid exe = spawn.get_executable() - args = [exe, '--multiprocessing-semaphore-tracker'] + args = [exe] + if getattr(sys, 'frozen', False): + args.append('--multiprocessing-semaphore-tracker') args += util._args_from_interpreter_flags() args += ['-c', cmd % r] pid = util.spawnv_passfds(exe, args, fds_to_pass) From 26a63138cd40d5d447f57fdadbf4927b532f0e12 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 15 Jan 2018 13:13:01 -0600 Subject: [PATCH 08/12] bpo-32146: Factor out command line manipulation for testing --- Lib/multiprocessing/spawn.py | 108 ++++++++++++++++++++---------- Lib/test/_test_multiprocessing.py | 71 ++++++++++++++++++++ 2 files changed, 142 insertions(+), 37 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index f850dcca5d2cc6..d7a0facc73bd9b 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -50,6 +50,7 @@ def get_executable(): # # + def is_forking(argv): ''' Return whether commandline indicates we are forking @@ -60,58 +61,91 @@ def is_forking(argv): return False -def is_semaphore_tracker(argv): +def get_forking_args(argv): ''' - Return whether commandline indicates we are running the semaphore tracker + If the command line indicated we are forking, return (args, kwargs) + suitable for passing to spawn_main. Otherwise return None. ''' - if len(argv) >= 2 and argv[1] == '--multiprocessing-semaphore-tracker': - return True - else: - return False + if not is_forking(argv): + return None + + args = [] + kwds = {} + for arg in argv[2:]: + name, value = arg.split('=') + if value == 'None': + kwds[name] = None + else: + kwds[name] = int(value) + return args, kwds -def is_forkserver(argv): + +def get_semaphore_tracker_args(argv): ''' - Return whether commandline indicates we are running the forkserver + If the command line indicates we are running the semaphore tracker, + return (args, kwargs) suitable for passing to semaphore_tracker.main. + Otherwise return None. ''' - if len(argv) >= 2 and argv[1] == '--multiprocessing-forkserver': - return True - else: - return False + if len(argv) < 2 or argv[1] != '--multiprocessing-semaphore-tracker': + return None + + # command ends with main(fd) - extract fd + r = int(argv[-1].rsplit('(')[1].split(')')[0]) + + args = [r] + kwds = {} + return args, kwds + + +def get_forkserver_args(argv): + ''' + If the command line indicates we are running the forkserver, return + (args, kwargs) suitable for passing to forkserver.main. Otherwise return + None. + ''' + if len(argv) < 2 or argv[1] != '--multiprocessing-forkserver': + return None + + # command ends with main(listener_fd, alive_r, preload, **kwds) - extract + # the args and kwarfs + # listener_fd and alive_r are integers + # preload is a list + # kwds map strings to lists + main_args = argv[-1].split('main(')[1].rsplit(')', 1)[0].split(', ', 3) + listener_fd = int(main_args[0]) + alive_r = int(main_args[1]) + preload = ast.literal_eval(main_args[2]) + + args = [listener_fd, alive_r, preload] + kwds = ast.literal_eval(main_args[3][2:]) + return args, kwds def freeze_support(): ''' - Run code for process object if this in not the main process + Run code for process object if this in not the main process. ''' - if is_forking(sys.argv): - kwds = {} - for arg in sys.argv[2:]: - name, value = arg.split('=') - if value == 'None': - kwds[name] = None - else: - kwds[name] = int(value) - spawn_main(**kwds) + argv = sys.argv + + forking_args = get_forking_args(argv) + if forking_args is not None: + args, kwds = forking_args + spawn_main(*args, **kwds) sys.exit() - # fix the command line for the semaphore tracker (unix) - # we extract the single argument to the semaphore tracker's main() - elif is_semaphore_tracker(sys.argv): + + semaphore_tracker_args = get_semaphore_tracker_args(argv) + if semaphore_tracker_args is not None: from multiprocessing.semaphore_tracker import main - r = int(sys.argv[-1].rsplit('(')[1].split(')')[0]) - main(r) + args, kwds = semaphore_tracker_args + main(*args, **kwds) sys.exit() - # fix the command line for the fork server (unix) - # we extract the 3 args and keywords to the forkserver's main() - elif is_forkserver(sys.argv): + + forkserver_args = get_forkserver_args(argv) + if get_forkserver_args(sys.argv): from multiprocessing.forkserver import main - cmd = sys.argv[-1] - main_args = cmd.split('main(')[1].rsplit(')', 1)[0].split(', ', 3) - listener_fd = int(main_args[0]) - alive_r = int(main_args[1]) - preload = ast.literal_eval(main_args[2]) - kwds = ast.literal_eval(main_args[3][2:]) - main(listener_fd, alive_r, preload, **kwds) + args, kwds = forkserver_args + main(*args, **kwds) sys.exit() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 05166b91ba832a..abf28587480346 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3,6 +3,7 @@ # import unittest +import unittest.mock import queue as pyqueue import contextlib import time @@ -4493,6 +4494,76 @@ def test_empty(self): proc.join() + +# +# Issue 32146: freeze_support for fork, spawn, and forkserver start methods +# + +class TestFreezeSupport(unittest.TestCase): + def setUp(self): + import multiprocessing.spawn + self.module = multiprocessing.spawn + + def test_get_forking_args(self): + # Too few args + self.assertIsNone(self.module.get_forking_args(['./embed'])) + + # Wrong second argument + self.assertIsNone( + self.module.get_forking_args(['./embed', '-h']) + ) + + # All correct + args, kwds = self.module.get_forking_args( + ['./embed', '--multiprocessing-fork', 'pipe_handle=6', 'key=None'] + ) + self.assertEqual(args, []) + self.assertEqual(kwds, {'pipe_handle': 6, 'key': None}) + + def test_get_semaphore_tracker_args(self): + # Too few args + self.assertIsNone(self.module.get_semaphore_tracker_args(['.embed'])) + + # Wrong second argument + self.assertIsNone(self.module.get_semaphore_tracker_args( + ['./embed', '-h']) + ) + + # All correct + argv = [ + './embed', + '--multiprocessing-semaphore-tracker', + 'from multiprocessing.semaphore_tracker import main;main(5)' + ] + args, kwds = self.module.get_semaphore_tracker_args(argv) + self.assertEqual(args, [5]) + self.assertEqual(kwds, {}) + + def test_get_forkserver_args(self): + # Too few args + self.assertFalse(self.module.get_forkserver_args(['./python-embed'])) + + # Wrong second argument + self.assertFalse( + self.module.get_forkserver_args(['./python-embed', '-h']) + ) + + # All correct + argv = [ + './embed', + '--multiprocessing-forkserver', + ( + "from multiprocessing.forkserver import main; " + "main(8, 9, ['__main__'], " + "**{'sys_path': ['/embed/lib', '/embed/lib/library.zip']})" + ) + ] + args, kwds = self.module.get_forkserver_args(argv) + self.assertEqual(args, [8, 9, ['__main__']]) + self.assertEqual( + kwds, {'sys_path': ['/embed/lib', '/embed/lib/library.zip']} + ) + # # Mixins # From afdd5535fb92e19d3a77a8c6bc475b695aa0d4b2 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 15 Jan 2018 13:27:01 -0600 Subject: [PATCH 09/12] bpo-32146: Add note to docs about freeze_support on Unix --- Doc/library/multiprocessing.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 701f75279d838d..5aea3c8f8c49b4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -942,12 +942,18 @@ Miscellaneous Process(target=f).start() If the ``freeze_support()`` line is omitted then trying to run the frozen - executable will raise :exc:`RuntimeError`. + executable will cause errors (e.g., :exc:`RuntimeError`). It is needed + when using the ``'spawn'`` and ``'forkserver'`` start methods. + ``freeze_support()`` has no effect when invoked in a module that is being run normally by the Python interpreter (i.e., instead of by a frozen executable). +.. versionchanged:: 3.7 + Now supported on Unix (for the ``'spawn'`` and ``'forkserver'` start + methods) + .. function:: get_all_start_methods() Returns a list of the supported start methods, the first of which From d72742dea5d50032eaf636193e5956cd07873d82 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 15 Jan 2018 14:03:32 -0600 Subject: [PATCH 10/12] bpo-32146: Add missing backtick --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 5aea3c8f8c49b4..cafe0a2288b8c6 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -951,7 +951,7 @@ Miscellaneous executable). .. versionchanged:: 3.7 - Now supported on Unix (for the ``'spawn'`` and ``'forkserver'` start + Now supported on Unix (for the ``'spawn'`` and ``'forkserver'`` start methods) .. function:: get_all_start_methods() From 97e6a9d748902353cfb4e730c471ff05810ca160 Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Tue, 16 Jan 2018 06:12:14 -0600 Subject: [PATCH 11/12] bpo-32146: Fix and add test for extra preload modules --- Lib/multiprocessing/spawn.py | 10 +++------- Lib/test/_test_multiprocessing.py | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index d7a0facc73bd9b..019467ae1b9d7c 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -112,13 +112,9 @@ def get_forkserver_args(argv): # listener_fd and alive_r are integers # preload is a list # kwds map strings to lists - main_args = argv[-1].split('main(')[1].rsplit(')', 1)[0].split(', ', 3) - listener_fd = int(main_args[0]) - alive_r = int(main_args[1]) - preload = ast.literal_eval(main_args[2]) - - args = [listener_fd, alive_r, preload] - kwds = ast.literal_eval(main_args[3][2:]) + parsed = ast.parse(argv[-1]) + args = [ast.literal_eval(parsed.body[1].value.args[i]) for i in range(3)] + kwds = ast.literal_eval(parsed.body[1].value.keywords[0].value) return args, kwds diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index abf28587480346..b2bb386711c87a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4554,12 +4554,12 @@ def test_get_forkserver_args(self): '--multiprocessing-forkserver', ( "from multiprocessing.forkserver import main; " - "main(8, 9, ['__main__'], " + "main(8, 9, ['__main__', 'other'], " "**{'sys_path': ['/embed/lib', '/embed/lib/library.zip']})" ) ] args, kwds = self.module.get_forkserver_args(argv) - self.assertEqual(args, [8, 9, ['__main__']]) + self.assertEqual(args, [8, 9, ['__main__', 'other']]) self.assertEqual( kwds, {'sys_path': ['/embed/lib', '/embed/lib/library.zip']} ) From f0ca86b5fb59d3cd84dd2f612103823c90f12fca Mon Sep 17 00:00:00 2001 From: Bo Bayles Date: Mon, 29 Jan 2018 14:03:03 -0600 Subject: [PATCH 12/12] bpo-32146: Fix up comments --- Lib/multiprocessing/spawn.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 019467ae1b9d7c..04c5ff33f85efc 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -90,7 +90,7 @@ def get_semaphore_tracker_args(argv): if len(argv) < 2 or argv[1] != '--multiprocessing-semaphore-tracker': return None - # command ends with main(fd) - extract fd + # command ends with main(fd) - extract the fd. r = int(argv[-1].rsplit('(')[1].split(')')[0]) args = [r] @@ -108,10 +108,8 @@ def get_forkserver_args(argv): return None # command ends with main(listener_fd, alive_r, preload, **kwds) - extract - # the args and kwarfs - # listener_fd and alive_r are integers - # preload is a list - # kwds map strings to lists + # the args and kwargs. listener_fd and alive_r are integers. + # preload is a list. The kwds map strings to lists. parsed = ast.parse(argv[-1]) args = [ast.literal_eval(parsed.body[1].value.args[i]) for i in range(3)] kwds = ast.literal_eval(parsed.body[1].value.keywords[0].value)