From acd47e220c6ac36191efa337d2bf795fa663a6be Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 16:16:25 -0600 Subject: [PATCH 1/4] Add changes to subprocess.py from python/asycio #485 --- Lib/asyncio/subprocess.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index b2f5304f772121..32949dee27db4f 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -24,6 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +45,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -85,10 +89,19 @@ def pipe_connection_lost(self, fd, exc): reader.feed_eof() else: reader.set_exception(exc) + + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + self._maybe_close_transport(fd) def process_exited(self): - self._transport.close() - self._transport = None + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: + self._transport.close() + self._transport = None class Process: From d6c656797b1e5ec6cdcbda7825bcc2b3737ae172 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 16:20:42 -0600 Subject: [PATCH 2/4] Add test_read_sdout_after_process_exit from python/asyncio#485 --- Lib/test/test_asyncio/test_subprocess.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bba688bb5a53c7..407e8b6a7aa428 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -459,6 +459,28 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + + fut = asyncio.create_subprocess_exec(sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix From 0372b36c719d27b43cc0092ffd9fda10d50a36f4 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 16:36:49 -0600 Subject: [PATCH 3/4] Remove fd from _maybe_close_transport call --- Lib/asyncio/subprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 32949dee27db4f..06c7d4f7e4be3b 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -92,7 +92,7 @@ def pipe_connection_lost(self, fd, exc): if fd in self._pipe_fds: self._pipe_fds.remove(fd) - self._maybe_close_transport(fd) + self._maybe_close_transport() def process_exited(self): self._process_exited = True From d7f619b8992776c81f5efbba1c74ab53304d48b6 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 17:22:28 -0600 Subject: [PATCH 4/4] Add NEWS entry for bpo-29704 --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 9da682b38792a9..5a7e132c4e9086 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -256,6 +256,9 @@ Extension Modules Library ------- +- bpo-29704: asyncio.subprocess.SubprocessStreamProtocol no longer closes before + all pipes are closed. + - bpo-29615: SimpleXMLRPCDispatcher no longer chains KeyError (or any other exception) to exception(s) raised in the dispatched methods. Patch by Petr Motejlek.