From cab49f209ebede629e2d431cb65297cdee18570b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ndor=20Jenei?= Date: Fri, 2 Feb 2018 10:32:05 +0100 Subject: [PATCH 1/4] bpo-31804: multiprocessing calls flush on sys.stdout at exit even if it is None (pythonw) If you start Python by pythonw.exe on Windows platform then sys.stdout and sys.stderr are set to None. If you also use multiprocessing then when the child process finishes BaseProcess._bootstrap calls sys.stdout.flush() and sys.stderr.flush() finally. This causes the process return code to be not zero (it is 1). --- Lib/multiprocessing/process.py | 6 +++-- Lib/test/_test_multiprocessing.py | 41 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8fff3e105ead4c..fd770b372138c3 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,8 +314,10 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - sys.stdout.flush() - sys.stderr.flush() + if sys.stdout is not None and not sys.stdout.closed: + sys.stdout.flush() + if sys.stderr is not None and not sys.stderr.closed: + sys.stderr.flush() return exitcode diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 05166b91ba832a..359a5d7a9fdebb 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -653,6 +653,47 @@ def test_forkserver_sigkill(self): self.check_forkserver_death(signal.SIGKILL) +class TestStdOutAndErr(unittest.TestCase): + @staticmethod + def closeIO(stream_name): + getattr(sys, stream_name).close() + + @staticmethod + def removeIO(stream_name): + setattr(sys, stream_name, None) + + def test_closed_stdio(self): + """ + bpo-28326: multiprocessing.Process depends on sys.stdout being open + """ + self.run_process(self.closeIO) + + def test_no_stdio(self): + """ + bpo-31804: If you start Python by pythonw then sys.stdout and + sys.stderr are set to None. If you also use multiprocessing + then when the child process finishes BaseProcess._bootstrap + calls sys.stdout.flush() and sys.stderr.flush() finally. + This causes the process return code to be not zero (it is 1). + + This unit test sets sys.stdio and sys.stderr to None, instead of + changing the Python interpreter to use when starting a child process + to pythonw.exe because that is Windows specific. This method + cannot test if there is an error (because of stdout or stderr is None) + before the target function is called. However the errors have occurred + in the multiprocessing outro so this can test the previously mentioned + bug. Also this way the feature can be tested on other operating + systems too. + """ + self.run_process(self.removeIO) + + def run_process(self, target): + for stream_name in ('stdout', 'stderr'): + proc = multiprocessing.Process(target=target, args=(stream_name,)) + proc.start() + proc.join() + self.assertEqual(proc.exitcode, 0) + # # # From fe03c62093cea86d962608966450dca1e32f0e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ndor=20Jenei?= Date: Mon, 12 Feb 2018 12:06:29 +0100 Subject: [PATCH 2/4] bpo-31804: multiprocessing calls flush on sys.stdout at exit even if it is None (pythonw.exe) - Review changes --- Lib/multiprocessing/process.py | 8 ++++++-- Lib/test/_test_multiprocessing.py | 16 ++-------------- .../2018-02-12-11-04-46.bpo-31804.9mWA5i.rst | 3 +++ 3 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-02-12-11-04-46.bpo-31804.9mWA5i.rst diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index fd770b372138c3..ac2eb5676f16d7 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -314,10 +314,14 @@ def _bootstrap(self): finally: threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) - if sys.stdout is not None and not sys.stdout.closed: + try: sys.stdout.flush() - if sys.stderr is not None and not sys.stderr.closed: + except (AttributeError, ValueError): + pass + try: sys.stderr.flush() + except (AttributeError, ValueError): + pass return exitcode diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 359a5d7a9fdebb..0627bf5cad87b3 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -670,20 +670,8 @@ def test_closed_stdio(self): def test_no_stdio(self): """ - bpo-31804: If you start Python by pythonw then sys.stdout and - sys.stderr are set to None. If you also use multiprocessing - then when the child process finishes BaseProcess._bootstrap - calls sys.stdout.flush() and sys.stderr.flush() finally. - This causes the process return code to be not zero (it is 1). - - This unit test sets sys.stdio and sys.stderr to None, instead of - changing the Python interpreter to use when starting a child process - to pythonw.exe because that is Windows specific. This method - cannot test if there is an error (because of stdout or stderr is None) - before the target function is called. However the errors have occurred - in the multiprocessing outro so this can test the previously mentioned - bug. Also this way the feature can be tested on other operating - systems too. + bpo-31804: set sys.stdio and sys.stderr to None, instead of + changing the Python interpreter to pythonw.exe. (OS independence) """ self.run_process(self.removeIO) diff --git a/Misc/NEWS.d/next/Library/2018-02-12-11-04-46.bpo-31804.9mWA5i.rst b/Misc/NEWS.d/next/Library/2018-02-12-11-04-46.bpo-31804.9mWA5i.rst new file mode 100644 index 00000000000000..e72975c24ed6d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-12-11-04-46.bpo-31804.9mWA5i.rst @@ -0,0 +1,3 @@ +bugfix: Process return value is 0 when multiprocessing is used with closed +stdout or stderr streams, or if stdout or stderr is None (e.g. using +pythonw.exe on windows) From e860a4f5688380c396b62dc8181127267f2308a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ndor=20Jenei?= Date: Thu, 22 Feb 2018 14:41:30 +0100 Subject: [PATCH 3/4] bpo-31804: Review changes 2 - unit tests unified --- Lib/test/_test_multiprocessing.py | 91 ++++++++++++++----------------- 1 file changed, 42 insertions(+), 49 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 0627bf5cad87b3..087131a14d80f9 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -583,26 +583,52 @@ def test_wait_for_threads(self): proc.join() self.assertTrue(evt.is_set()) - @classmethod - def _test_error_on_stdio_flush(self, evt): + @staticmethod + def closeIO(stream_name, evt, type_): + if type_ == 'threads': + # it is safe to close std streams in another process, but if thread + # are used, we must not close the original std streams + closed_stream = io.StringIO() + closed_stream.close() + setattr(sys, stream_name, closed_stream) + else: + # using StringIO is not the same as using a real closed file, + # because a closed StringIO throws no exception when its flush + # method is called + getattr(sys, stream_name).close() + evt.set() + + @staticmethod + def removeIO(stream_name, evt, type_): + setattr(sys, stream_name, None) evt.set() - def test_error_on_stdio_flush(self): - streams = [io.StringIO(), None] - streams[0].close() + def test_closed_stdio(self): + """ + bpo-28326: multiprocessing.Process depends on sys.stdout being open + """ + self.run_process(self.closeIO) + + def test_no_stdio(self): + """ + bpo-31804: set sys.stdio and sys.stderr to None, instead of + changing the Python interpreter to pythonw.exe. (OS independence) + """ + self.run_process(self.removeIO) + + def run_process(self, target): for stream_name in ('stdout', 'stderr'): - for stream in streams: - old_stream = getattr(sys, stream_name) - setattr(sys, stream_name, stream) - try: - evt = self.Event() - proc = self.Process(target=self._test_error_on_stdio_flush, - args=(evt,)) - proc.start() - proc.join() - self.assertTrue(evt.is_set()) - finally: + old_stream = getattr(sys, stream_name) + evt = self.Event() + proc = self.Process(target=target, args=(stream_name, evt, self.TYPE)) + try: + proc.start() + proc.join() + finally: + if self.TYPE == 'threads': setattr(sys, stream_name, old_stream) + self.assertTrue(evt.is_set()) + self.assertEqual(proc.exitcode, 0) @classmethod def _sleep_and_set_event(self, evt, delay=0.0): @@ -653,39 +679,6 @@ def test_forkserver_sigkill(self): self.check_forkserver_death(signal.SIGKILL) -class TestStdOutAndErr(unittest.TestCase): - @staticmethod - def closeIO(stream_name): - getattr(sys, stream_name).close() - - @staticmethod - def removeIO(stream_name): - setattr(sys, stream_name, None) - - def test_closed_stdio(self): - """ - bpo-28326: multiprocessing.Process depends on sys.stdout being open - """ - self.run_process(self.closeIO) - - def test_no_stdio(self): - """ - bpo-31804: set sys.stdio and sys.stderr to None, instead of - changing the Python interpreter to pythonw.exe. (OS independence) - """ - self.run_process(self.removeIO) - - def run_process(self, target): - for stream_name in ('stdout', 'stderr'): - proc = multiprocessing.Process(target=target, args=(stream_name,)) - proc.start() - proc.join() - self.assertEqual(proc.exitcode, 0) - -# -# -# - class _UpperCaser(multiprocessing.Process): def __init__(self): From 5e91e604062f583b1a04b8045386cdc51b43dfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A1ndor=20Jenei?= Date: Thu, 22 Feb 2018 16:25:56 +0100 Subject: [PATCH 4/4] bpo-31804: Review changes 3 - decorate StringIO with _file_like and add close method to _file_like --- Lib/test/_test_multiprocessing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 087131a14d80f9..263fa1850fe0b6 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -588,7 +588,7 @@ def closeIO(stream_name, evt, type_): if type_ == 'threads': # it is safe to close std streams in another process, but if thread # are used, we must not close the original std streams - closed_stream = io.StringIO() + closed_stream = _file_like(io.StringIO()) closed_stream.close() setattr(sys, stream_name, closed_stream) else: @@ -3874,6 +3874,9 @@ def flush(self): self._delegate.write(''.join(self.cache)) self._cache = [] + def close(self): + self._delegate.close() + class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self):