From c73df721994e2a1f996b2d8dcd969abeec3cca2f Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Fri, 13 Jan 2017 15:33:49 -0600 Subject: [PATCH 1/5] Delay SubprocessStreamProtocol closing transport Delays the closing for `SubprocessStreamProtocol._transport` until all pipes have been closed and the process has exited rather than just the process exiting. This allows for reading data from the subprocess that is still pending in the pipe after the process has exited. --- asyncio/subprocess.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/asyncio/subprocess.py b/asyncio/subprocess.py index b2f5304f..3bda74f1 100644 --- a/asyncio/subprocess.py +++ b/asyncio/subprocess.py @@ -24,6 +24,7 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None + self._pipe_fds = [-1] def __repr__(self): info = [self.__class__.__name__] @@ -43,12 +44,14 @@ def connection_made(self, transport): self.stdout = streams.StreamReader(limit=self._limit, loop=self._loop) self.stdout.set_transport(stdout_transport) + self._pipe_fds.append(1) stderr_transport = transport.get_pipe_transport(2) if stderr_transport is not None: self.stderr = streams.StreamReader(limit=self._limit, loop=self._loop) self.stderr.set_transport(stderr_transport) + self._pipe_fds.append(2) stdin_transport = transport.get_pipe_transport(0) if stdin_transport is not None: @@ -85,10 +88,17 @@ def pipe_connection_lost(self, fd, exc): reader.feed_eof() else: reader.set_exception(exc) + self._maybe_close_transport(fd) def process_exited(self): - self._transport.close() - self._transport = None + self._maybe_close_transport(-1) + + def _maybe_close_transport(self, fd): + if fd in self._pipe_fds: + self._pipe_fds.remove(fd) + if len(self._pipe_fds) == 0: + self._transport.close() + self._transport = None class Process: From 12428a229888891c10b5441f1f28acd57cf7e0b6 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 18 Jan 2017 10:24:57 -0600 Subject: [PATCH 2/5] Add simple test for patch --- tests/test_subprocess.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index bba688bb..81a8c008 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -459,6 +459,28 @@ def test_popen_error(self): self.loop.run_until_complete(create) self.assertEqual(warns, []) + def test_read_stdout_after_process_exit(self): + @asyncio.coroutine + def execute(): + code = '\n'.join(['import sys, time', + 'for _ in range(64):', + ' sys.stdout.write("x" * 4096)', + 'sys.stdout.flush()', + 'sys.exit(1)']) + #fut = asyncio.create_subprocess_exec('timeout', '0.1', 'cat', '/dev/urandom', + fut = asyncio.create_subprocess_exec(sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE, + loop=self.loop) + process = yield from fut + while True: + data = yield from process.stdout.read(65536) + if data: + yield from asyncio.sleep(0.3, loop=self.loop) + else: + break + + self.loop.run_until_complete(execute()) + if sys.platform != 'win32': # Unix From 33cd1c55507f4e7e9b7353494b2104b7995bd674 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Tue, 28 Feb 2017 16:15:22 -0600 Subject: [PATCH 3/5] Remove comment about /dev/urandom --- tests/test_subprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index 81a8c008..407e8b6a 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -462,12 +462,12 @@ def test_popen_error(self): def test_read_stdout_after_process_exit(self): @asyncio.coroutine def execute(): - code = '\n'.join(['import sys, time', + code = '\n'.join(['import sys', 'for _ in range(64):', ' sys.stdout.write("x" * 4096)', 'sys.stdout.flush()', 'sys.exit(1)']) - #fut = asyncio.create_subprocess_exec('timeout', '0.1', 'cat', '/dev/urandom', + fut = asyncio.create_subprocess_exec(sys.executable, '-c', code, stdout=asyncio.subprocess.PIPE, loop=self.loop) From 16a04d2bdef89038fdd9b108eadf2822d731e226 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 15:57:28 -0600 Subject: [PATCH 4/5] Allow _maybe_close_transport to close even if... ...removal from fd list does not occur. --- asyncio/subprocess.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asyncio/subprocess.py b/asyncio/subprocess.py index 3bda74f1..7b574be5 100644 --- a/asyncio/subprocess.py +++ b/asyncio/subprocess.py @@ -96,9 +96,9 @@ def process_exited(self): def _maybe_close_transport(self, fd): if fd in self._pipe_fds: self._pipe_fds.remove(fd) - if len(self._pipe_fds) == 0: - self._transport.close() - self._transport = None + if len(self._pipe_fds) == 0: + self._transport.close() + self._transport = None class Process: From dc23e3fdb6e319022565902a8f1b03c283b01c36 Mon Sep 17 00:00:00 2001 From: "Seth M. Larson" Date: Thu, 2 Mar 2017 16:08:29 -0600 Subject: [PATCH 5/5] Add flag for process_exited, separate this from... ..._pipe_fds and change interface of _maybe_close_transport to not take an fd. --- asyncio/subprocess.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/asyncio/subprocess.py b/asyncio/subprocess.py index 7b574be5..32949dee 100644 --- a/asyncio/subprocess.py +++ b/asyncio/subprocess.py @@ -24,7 +24,8 @@ def __init__(self, limit, loop): self._limit = limit self.stdin = self.stdout = self.stderr = None self._transport = None - self._pipe_fds = [-1] + self._process_exited = False + self._pipe_fds = [] def __repr__(self): info = [self.__class__.__name__] @@ -88,15 +89,17 @@ def pipe_connection_lost(self, fd, exc): reader.feed_eof() else: reader.set_exception(exc) - self._maybe_close_transport(fd) - - def process_exited(self): - self._maybe_close_transport(-1) - def _maybe_close_transport(self, fd): if fd in self._pipe_fds: self._pipe_fds.remove(fd) - if len(self._pipe_fds) == 0: + self._maybe_close_transport(fd) + + def process_exited(self): + self._process_exited = True + self._maybe_close_transport() + + def _maybe_close_transport(self): + if len(self._pipe_fds) == 0 and self._process_exited: self._transport.close() self._transport = None