From f3481229c74762003f9e935e103c7112af8d63db Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 2 Aug 2016 18:00:18 +0200 Subject: [PATCH 1/7] Refuse handlers if the child watcher has no loop attached --- asyncio/unix_events.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/asyncio/unix_events.py b/asyncio/unix_events.py index d712749e..d06600d6 100644 --- a/asyncio/unix_events.py +++ b/asyncio/unix_events.py @@ -802,6 +802,11 @@ def __exit__(self, a, b, c): pass def add_child_handler(self, pid, callback, *args): + if self._loop is None: + raise RuntimeError( + "Cannot add child handler, " + "the child watcher does not have a loop attached") + self._callbacks[pid] = (callback, args) # Prevent a race condition in case the child is already terminated. @@ -898,6 +903,12 @@ def __exit__(self, a, b, c): def add_child_handler(self, pid, callback, *args): assert self._forks, "Must use the context manager" + + if self._loop is None: + raise RuntimeError( + "Cannot add child handler, " + "the child watcher does not have a loop attached") + with self._lock: try: returncode = self._zombies.pop(pid) From f7a5259b659f8578bf83d7901d0dde1aac8115c7 Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 2 Aug 2016 18:05:08 +0200 Subject: [PATCH 2/7] Warn when a child watcher with pending handlers looses its loop --- asyncio/unix_events.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/asyncio/unix_events.py b/asyncio/unix_events.py index d06600d6..54cfc75f 100644 --- a/asyncio/unix_events.py +++ b/asyncio/unix_events.py @@ -726,6 +726,7 @@ class BaseChildWatcher(AbstractChildWatcher): def __init__(self): self._loop = None + self._callbacks = {} def close(self): self.attach_loop(None) @@ -739,6 +740,12 @@ def _do_waitpid_all(self): def attach_loop(self, loop): assert loop is None or isinstance(loop, events.AbstractEventLoop) + if self._loop is not None and loop is None and self._callbacks: + warnings.warn( + 'A loop is being detached ' + 'from a child watcher with pending handlers', + RuntimeWarning) + if self._loop is not None: self._loop.remove_signal_handler(signal.SIGCHLD) @@ -787,10 +794,6 @@ class SafeChildWatcher(BaseChildWatcher): big number of children (O(n) each time SIGCHLD is raised) """ - def __init__(self): - super().__init__() - self._callbacks = {} - def close(self): self._callbacks.clear() super().close() @@ -871,7 +874,6 @@ class FastChildWatcher(BaseChildWatcher): """ def __init__(self): super().__init__() - self._callbacks = {} self._lock = threading.Lock() self._zombies = {} self._forks = 0 From 9ab616879f5e2d2548d3005460bee248289dcbf1 Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 2 Aug 2016 18:12:21 +0200 Subject: [PATCH 3/7] Ignore or catch warnings from f7a5259 in the tests --- tests/test_subprocess.py | 4 ++++ tests/test_unix_events.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index 58f7253b..6e09633d 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -433,6 +433,10 @@ def kill_running(): # the transport was not notified yet self.assertFalse(killed) + # Clear the handlers for FastChildWatcher to avoid a warning. + # Is that a bug? + asyncio.get_child_watcher()._callbacks.clear() + def test_popen_error(self): # Issue #24763: check that the subprocess transport is closed # when BaseSubprocessTransport fails diff --git a/tests/test_unix_events.py b/tests/test_unix_events.py index 22dc6880..880c80d6 100644 --- a/tests/test_unix_events.py +++ b/tests/test_unix_events.py @@ -11,6 +11,7 @@ import tempfile import threading import unittest +import warnings from unittest import mock if sys.platform == 'win32': @@ -1396,7 +1397,11 @@ def test_set_loop_race_condition(self, m): with mock.patch.object( old_loop, "remove_signal_handler") as m_remove_signal_handler: - self.watcher.attach_loop(None) + with warnings.catch_warnings(record=True) as warns: + self.watcher.attach_loop(None) + self.assertEqual(len(warns), 1) + self.assertEqual(warns[0].category, RuntimeWarning) + self.assertIn('A loop is being detached', warns[0].message.args[0]) m_remove_signal_handler.assert_called_once_with( signal.SIGCHLD) From d4b53cb916fcaf3603ceb055db490d136bf0663b Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 2 Aug 2016 18:37:54 +0200 Subject: [PATCH 4/7] Fix test on windows (broke in 9ab6168) --- tests/test_subprocess.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index 6e09633d..82e39133 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -435,7 +435,8 @@ def kill_running(): # Clear the handlers for FastChildWatcher to avoid a warning. # Is that a bug? - asyncio.get_child_watcher()._callbacks.clear() + if sys.platform != 'win32': + asyncio.get_child_watcher()._callbacks.clear() def test_popen_error(self): # Issue #24763: check that the subprocess transport is closed From 256e71341e20cc533b71e847093c4865b4f5b1f5 Mon Sep 17 00:00:00 2001 From: vinmic Date: Thu, 15 Sep 2016 22:47:51 +0200 Subject: [PATCH 5/7] Use assertWarnsRegex for test_set_loop_race_condition --- tests/test_unix_events.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_unix_events.py b/tests/test_unix_events.py index 880c80d6..0ab98ff8 100644 --- a/tests/test_unix_events.py +++ b/tests/test_unix_events.py @@ -1397,11 +1397,9 @@ def test_set_loop_race_condition(self, m): with mock.patch.object( old_loop, "remove_signal_handler") as m_remove_signal_handler: - with warnings.catch_warnings(record=True) as warns: + with self.assertWarnsRegex( + RuntimeWarning, 'A loop is being detached'): self.watcher.attach_loop(None) - self.assertEqual(len(warns), 1) - self.assertEqual(warns[0].category, RuntimeWarning) - self.assertIn('A loop is being detached', warns[0].message.args[0]) m_remove_signal_handler.assert_called_once_with( signal.SIGCHLD) From 99148c254f3d637e3c9f36d7031476c629aab4b2 Mon Sep 17 00:00:00 2001 From: vinmic Date: Thu, 15 Sep 2016 23:04:21 +0200 Subject: [PATCH 6/7] Add test_add_child_handler_with_no_loop_attached --- tests/test_unix_events.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_unix_events.py b/tests/test_unix_events.py index 0ab98ff8..6cf44174 100644 --- a/tests/test_unix_events.py +++ b/tests/test_unix_events.py @@ -1471,6 +1471,15 @@ def test_close(self, m): if isinstance(self.watcher, asyncio.FastChildWatcher): self.assertFalse(self.watcher._zombies) + @waitpid_mocks + def test_add_child_handler_with_no_loop_attached(self, m): + callback = mock.Mock() + with self.create_watcher() as watcher: + with self.assertRaisesRegex( + RuntimeError, + 'the child watcher does not have a loop attached'): + watcher.add_child_handler(100, callback) + class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase): def create_watcher(self): From 08190f560b010e834ddc216fd100eea78f251cec Mon Sep 17 00:00:00 2001 From: Vincent Michel Date: Tue, 20 Sep 2016 18:24:48 +0200 Subject: [PATCH 7/7] Update subprocess tests for FastChildWatcher --- tests/test_subprocess.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index 82e39133..15310238 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -433,9 +433,11 @@ def kill_running(): # the transport was not notified yet self.assertFalse(killed) - # Clear the handlers for FastChildWatcher to avoid a warning. - # Is that a bug? - if sys.platform != 'win32': + # Unlike SafeChildWatcher, FastChildWatcher does not pop the + # callbacks if waitpid() is called elsewhere. Let's clear them + # manually to avoid a warning when the watcher is detached. + if sys.platform != 'win32' and \ + isinstance(self, SubprocessFastWatcherTests): asyncio.get_child_watcher()._callbacks.clear() def test_popen_error(self):