Thanks to visit codestin.com
Credit goes to github.com

Skip to content
This repository was archived by the owner on Nov 23, 2017. It is now read-only.

Refuse handlers if the child watcher has no loop attached #391

Closed
wants to merge 7 commits into from
Closed

Refuse handlers if the child watcher has no loop attached #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f348122
Refuse handlers if the child watcher has no loop attached
Aug 2, 2016
f7a5259
Warn when a child watcher with pending handlers looses its loop
Aug 2, 2016
9ab6168
Ignore or catch warnings from f7a5259 in the tests
Aug 2, 2016
d4b53cb
Fix test on windows (broke in 9ab6168)
Aug 2, 2016
256e713
Use assertWarnsRegex for test_set_loop_race_condition
Sep 15, 2016
99148c2
Add test_add_child_handler_with_no_loop_attached
Sep 15, 2016
08190f5
Update subprocess tests for FastChildWatcher
Sep 20, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions asyncio/unix_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,7 @@ class BaseChildWatcher(AbstractChildWatcher):

def __init__(self):
self._loop = None
self._callbacks = {}

def close(self):
self.attach_loop(None)
Expand All @@ -739,6 +740,12 @@ def _do_waitpid_all(self):
def attach_loop(self, loop):
assert loop is None or isinstance(loop, events.AbstractEventLoop)

if self._loop is not None and loop is None and self._callbacks:
warnings.warn(
'A loop is being detached '
'from a child watcher with pending handlers',
RuntimeWarning)

if self._loop is not None:
self._loop.remove_signal_handler(signal.SIGCHLD)

Expand Down Expand Up @@ -787,10 +794,6 @@ class SafeChildWatcher(BaseChildWatcher):
big number of children (O(n) each time SIGCHLD is raised)
"""

def __init__(self):
super().__init__()
self._callbacks = {}

def close(self):
self._callbacks.clear()
super().close()
Expand All @@ -802,6 +805,11 @@ def __exit__(self, a, b, c):
pass

def add_child_handler(self, pid, callback, *args):
if self._loop is None:
raise RuntimeError(
"Cannot add child handler, "
"the child watcher does not have a loop attached")

self._callbacks[pid] = (callback, args)

# Prevent a race condition in case the child is already terminated.
Expand Down Expand Up @@ -866,7 +874,6 @@ class FastChildWatcher(BaseChildWatcher):
"""
def __init__(self):
super().__init__()
self._callbacks = {}
self._lock = threading.Lock()
self._zombies = {}
self._forks = 0
Expand Down Expand Up @@ -898,6 +905,12 @@ def __exit__(self, a, b, c):

def add_child_handler(self, pid, callback, *args):
assert self._forks, "Must use the context manager"

if self._loop is None:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a test for this?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

raise RuntimeError(
"Cannot add child handler, "
"the child watcher does not have a loop attached")

with self._lock:
try:
returncode = self._zombies.pop(pid)
Expand Down
7 changes: 7 additions & 0 deletions tests/test_subprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,13 @@ def kill_running():
# the transport was not notified yet
self.assertFalse(killed)

# Unlike SafeChildWatcher, FastChildWatcher does not pop the
# callbacks if waitpid() is called elsewhere. Let's clear them
# manually to avoid a warning when the watcher is detached.
if sys.platform != 'win32' and \
isinstance(self, SubprocessFastWatcherTests):
asyncio.get_child_watcher()._callbacks.clear()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is something new... Is there any reason you do this as part of this PR?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, otherwise a warning from f7a5259 is generated. Weirdly enough it only happens with the FastChildWatcher. There might be a better way to address this issue though.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just had a deeper look and we can get rid of this addition by fixing FastChildWatcher (using some SafeChildWatcher code):

diff --git a/asyncio/unix_events.py b/asyncio/unix_events.py
index 54cfc75..5ffa645 100644
--- a/asyncio/unix_events.py
+++ b/asyncio/unix_events.py
@@ -937,7 +937,15 @@ def _do_waitpid_all(self):
                 pid, status = os.waitpid(-1, os.WNOHANG)
             except ChildProcessError:
                 # No more child processes exist.
-                return
+                if not self._callbacks:
+                    return
+                # Some child processes are already reaped
+                # (may happen if waitpid() is called elsewhere).
+                pid = next(iter(self._callbacks))
+                returncode = 255
+                logger.warning(
+                    "Unknown child process pid %d, will report returncode 255",
+                    pid)
             else:
                 if pid == 0:
                     # A child process is still alive.

This also allows us to get rid of some complexity in test_sigchld_child_reaped_elsewhere:

diff --git a/tests/test_unix_events.py b/tests/test_unix_events.py
index 6cf4417..b4a6d80 100644
--- a/tests/test_unix_events.py
+++ b/tests/test_unix_events.py
@@ -1318,12 +1318,7 @@ def test_sigchld_child_reaped_elsewhere(self, m):
         with self.ignore_warnings:
             self.watcher._sig_chld()

-        if isinstance(self.watcher, asyncio.FastChildWatcher):
-            # here the FastChildWatche enters a deadlock
-            # (there is no way to prevent it)
-            self.assertFalse(callback.called)
-        else:
-            callback.assert_called_once_with(58, 255)
+        callback.assert_called_once_with(58, 255)

     @waitpid_mocks
     def test_sigchld_unknown_pid_during_registration(self, m):

All the tests run OK after those modifications. Let me know which solution you prefer.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not so sure about this solution. Let's keep the current hack-ish code.


def test_popen_error(self):
# Issue #24763: check that the subprocess transport is closed
# when BaseSubprocessTransport fails
Expand Down
14 changes: 13 additions & 1 deletion tests/test_unix_events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import tempfile
import threading
import unittest
import warnings
from unittest import mock

if sys.platform == 'win32':
Expand Down Expand Up @@ -1396,7 +1397,9 @@ def test_set_loop_race_condition(self, m):
with mock.patch.object(
old_loop, "remove_signal_handler") as m_remove_signal_handler:

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about using with self.assertWarnsRegexp?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, it's much cleaner now!

self.watcher.attach_loop(None)
with self.assertWarnsRegex(
RuntimeWarning, 'A loop is being detached'):
self.watcher.attach_loop(None)

m_remove_signal_handler.assert_called_once_with(
signal.SIGCHLD)
Expand Down Expand Up @@ -1468,6 +1471,15 @@ def test_close(self, m):
if isinstance(self.watcher, asyncio.FastChildWatcher):
self.assertFalse(self.watcher._zombies)

@waitpid_mocks
def test_add_child_handler_with_no_loop_attached(self, m):
callback = mock.Mock()
with self.create_watcher() as watcher:
with self.assertRaisesRegex(
RuntimeError,
'the child watcher does not have a loop attached'):
watcher.add_child_handler(100, callback)


class SafeChildWatcherTests (ChildWatcherTestsMixin, test_utils.TestCase):
def create_watcher(self):
Expand Down