Try to fix some of current failures.

Carreau · Carreau · commit 25f92b9988c8 · 2021-10-17T12:47:25.000-07:00
See error found in CI below. Hopefully this fixes it.

There are manycomplexity, we need to make sure the loop has no task
after each tests, so we need to close and restore the subprocess
watcher.

We make it a context manager and provide a decorator.

    ======================================================================
    ERROR: IPython.core.tests.test_magic.test_script_out
    ----------------------------------------------------------------------
    Traceback (most recent call last):
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/nose/case.py", line 198, in runTest
        self.test(*self.arg)
      File "/home/runner/work/ipython/ipython/IPython/testing/decorators.py", line 223, in skipper_func
        return f(*args, **kwargs)
      File "/home/runner/work/ipython/ipython/IPython/core/tests/test_magic.py", line 953, in test_script_out
        ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
      File "/home/runner/work/ipython/ipython/IPython/core/interactiveshell.py", line 2417, in run_cell_magic
        result = fn(*args, **kwargs)
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/decorator.py", line 232, in fun
        return caller(func, *(extras + args), **kw)
      File "/home/runner/work/ipython/ipython/IPython/core/magic.py", line 187, in &lt;lambda&gt;
        call = lambda f, *a, **k: f(*a, **k)
      File "/home/runner/work/ipython/ipython/IPython/core/magics/script.py", line 216, in shebang
        stdin=asyncio.subprocess.PIPE,
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/base_events.py", line 587, in run_until_complete
        return future.result()
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/subprocess.py", line 217, in create_subprocess_exec
        stderr=stderr, **kwds)
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/base_events.py", line 1544, in subprocess_exec
        bufsize, **kwargs)
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/unix_events.py", line 193, in _make_subprocess_transport
        self._child_watcher_callback, transp)
      File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/unix_events.py", line 941, in add_child_handler
        "Cannot add child handler, "
    RuntimeError: Cannot add child handler, the child watcher does not have a loop attached
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -13,6 +13,7 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       matrix:
         python-version: [3.8]
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -45,6 +45,6 @@ jobs:
         cp /tmp/.coverage ./
     - name: pytest
       run: |
-        pytest
+        pytest -v
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v1
diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py
@@ -3,24 +3,24 @@
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import asyncio
+import atexit
 import errno
+import functools
 import os
-import sys
 import signal
+import sys
 import time
-import asyncio
-import atexit
-
+from asyncio import SafeChildWatcher
+from contextlib import contextmanager
 from subprocess import CalledProcessError
 
+from traitlets import Dict, List, default
+
 from IPython.core import magic_arguments
-from IPython.core.magic import  (
-    Magics, magics_class, line_magic, cell_magic
-)
+from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
 from IPython.lib.backgroundjobs import BackgroundJobManager
 from IPython.utils.process import arg_split
-from traitlets import List, Dict, default
-
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
@@ -67,6 +67,39 @@ def script_args(f):
         f = arg(f)
     return f
 
+
+@contextmanager
+def safe_watcher():
+    if sys.platform == "win32":
+        yield
+        return
+
+    policy = asyncio.get_event_loop_policy()
+    old_watcher = policy.get_child_watcher()
+    if isinstance(old_watcher, SafeChildWatcher):
+        yield
+        return
+
+    loop = asyncio.get_event_loop()
+    try:
+        watcher = asyncio.SafeChildWatcher()
+        watcher.attach_loop(loop)
+        policy.set_child_watcher(watcher)
+        yield
+    finally:
+        watcher.close()
+        policy.set_child_watcher(old_watcher)
+
+
+def dec_safe_watcher(fun):
+    @functools.wraps(fun)
+    def _inner(*args, **kwargs):
+        with safe_watcher():
+            return fun(*args, **kwargs)
+
+    return _inner
+
+
 @magics_class
 class ScriptMagics(Magics):
     """Magics for talking to scripts
@@ -157,6 +190,7 @@ def named_script_magic(line, cell):
     @magic_arguments.magic_arguments()
     @script_args
     @cell_magic("script")
+    @dec_safe_watcher
     def shebang(self, line, cell):
         """Run a cell via a shell command
         
@@ -204,7 +238,6 @@ async def _stream_communicate(process, cell):
         if sys.platform.startswith("win"):
             asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
         loop = asyncio.get_event_loop()
-
         argv = arg_split(line, posix=not sys.platform.startswith("win"))
         args, cmd = self.shebang.parser.parse_known_args(argv)
         try:
diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py
@@ -4,38 +4,40 @@
 Needs to be run by nose (to make ipython session available).
 """
 
+import asyncio
 import io
 import os
 import re
+import shlex
 import sys
 import warnings
-from textwrap import dedent
-from unittest import TestCase
-from unittest import mock
 from importlib import invalidate_caches
 from io import StringIO
 from pathlib import Path
+from textwrap import dedent
+from unittest import TestCase, mock
 
 import nose.tools as nt
 
-import shlex
-
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
-from IPython.core.magic import (Magics, magics_class, line_magic,
-                                cell_magic,
-                                register_line_magic, register_cell_magic)
-from IPython.core.magics import execution, script, code, logging, osm
+from IPython.core.magic import (
+    Magics,
+    cell_magic,
+    line_magic,
+    magics_class,
+    register_cell_magic,
+    register_line_magic,
+)
+from IPython.core.magics import code, execution, logging, osm, script
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
-from IPython.utils.tempdir import (TemporaryDirectory,
-                                    TemporaryWorkingDirectory)
 from IPython.utils.process import find_cmd
-from .test_debugger import PdbTestInput
+from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 
-import pytest
+from .test_debugger import PdbTestInput
 
 
 @magic.magics_class
@@ -947,32 +949,48 @@ def test_script_config():
     sm = script.ScriptMagics(shell=ip)
     nt.assert_in('whoda', sm.magics['cell'])
 
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 def test_script_out():
+    assert asyncio.get_event_loop().is_running() is False
+
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
+    assert asyncio.get_event_loop().is_running() is False
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
 
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 def test_script_err():
     ip = get_ipython()
+    assert asyncio.get_event_loop().is_running() is False
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
+    assert asyncio.get_event_loop().is_running() is False
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
+
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 def test_script_out_err():
+
     ip = get_ipython()
-    ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
-    nt.assert_equal(ip.user_ns['output'], 'hi\n')
-    nt.assert_equal(ip.user_ns['error'], 'hello\n')
+    ip.run_cell_magic(
+        "script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2"
+    )
+    nt.assert_equal(ip.user_ns["output"], "hi\n")
+    nt.assert_equal(ip.user_ns["error"], "hello\n")
 
+
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 async def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
     nt.assert_equal((await ip.user_ns["output"].read()), b"hi\n")
-    ip.user_ns['output'].close()
+    ip.user_ns["output"].close()
+    asyncio.get_event_loop().stop()
 
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 async def test_script_bg_err():
     ip = get_ipython()
@@ -981,6 +999,7 @@ async def test_script_bg_err():
     ip.user_ns["error"].close()
 
 
+@dec.skip_iptest_but_not_pytest
 @dec.skip_win32
 async def test_script_bg_out_err():
     ip = get_ipython()
