Merge pull request #13436 from Carreau/fix-async-with

Carreau · web-flow · commit 6edc05a6e52e · 2022-01-17T18:01:42.000+01:00
Fix and test for "async with does not allow new lines".
diff --git a/IPython/core/inputtransformer2.py b/IPython/core/inputtransformer2.py
@@ -507,9 +507,12 @@ def make_tokens_by_line(lines:List[str]):
 
     #   reexported from token on 3.7+
     NEWLINE, NL = tokenize.NEWLINE, tokenize.NL  # type: ignore
-    tokens_by_line:List[List[Any]] = [[]]
-    if len(lines) > 1 and not lines[0].endswith(('\n', '\r', '\r\n', '\x0b', '\x0c')):
-        warnings.warn("`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified")
+    tokens_by_line: List[List[Any]] = [[]]
+    if len(lines) > 1 and not lines[0].endswith(("\n", "\r", "\r\n", "\x0b", "\x0c")):
+        warnings.warn(
+            "`make_tokens_by_line` received a list of lines which do not have lineending markers ('\\n', '\\r', '\\r\\n', '\\x0b', '\\x0c'), behavior will be unspecified",
+            stacklevel=2,
+        )
     parenlev = 0
     try:
         for token in tokenize.generate_tokens(iter(lines).__next__):
@@ -782,9 +785,6 @@ def __init__(self, extra_flags=0):
         super().__init__()
         self.flags |= extra_flags
 
-    def __call__(self, *args, **kwds):
-        return compile(*args, **kwds)
-
 
 class MaybeAsyncCommandCompiler(CommandCompiler):
     def __init__(self, extra_flags=0):
diff --git a/IPython/core/tests/test_inputtransformer2.py b/IPython/core/tests/test_inputtransformer2.py
@@ -4,6 +4,7 @@
 more complex. See test_inputtransformer2_line for tests for line-based
 transformations.
 """
+import platform
 import string
 import sys
 from textwrap import dedent
@@ -291,6 +292,7 @@ def test_check_complete_param(code, expected, number):
     assert cc(code) == (expected, number)
 
 
+@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
 @pytest.mark.xfail(
     reason="Bug in python 3.9.8 – bpo 45738",
     condition=sys.version_info in [(3, 9, 8, "final", 0), (3, 11, 0, "alpha", 2)],
@@ -319,33 +321,48 @@ def test_check_complete():
     assert cc("def f():\n  x=0\n  \\\n  ") == ("incomplete", 2)
 
 
-def test_check_complete_II():
+@pytest.mark.xfail(platform.python_implementation() == "PyPy", reason="fail on pypy")
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        ('''def foo():\n    """''', ("incomplete", 4)),
+        ("""async with example:\n    pass""", ("incomplete", 4)),
+        ("""async with example:\n    pass\n    """, ("complete", None)),
+    ],
+)
+def test_check_complete_II(value, expected):
     """
     Test that multiple line strings are properly handled.
 
     Separate test function for convenience
 
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc('''def foo():\n    """''') == ("incomplete", 4)
-
-
-def test_check_complete_invalidates_sunken_brackets():
+    assert cc(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        (")", ("invalid", None)),
+        ("]", ("invalid", None)),
+        ("}", ("invalid", None)),
+        (")(", ("invalid", None)),
+        ("][", ("invalid", None)),
+        ("}{", ("invalid", None)),
+        ("]()(", ("invalid", None)),
+        ("())(", ("invalid", None)),
+        (")[](", ("invalid", None)),
+        ("()](", ("invalid", None)),
+    ],
+)
+def test_check_complete_invalidates_sunken_brackets(value, expected):
     """
     Test that a single line with more closing brackets than the opening ones is
     interpreted as invalid
     """
     cc = ipt2.TransformerManager().check_complete
-    assert cc(")") == ("invalid", None)
-    assert cc("]") == ("invalid", None)
-    assert cc("}") == ("invalid", None)
-    assert cc(")(") == ("invalid", None)
-    assert cc("][") == ("invalid", None)
-    assert cc("}{") == ("invalid", None)
-    assert cc("]()(") == ("invalid", None)
-    assert cc("())(") == ("invalid", None)
-    assert cc(")[](") == ("invalid", None)
-    assert cc("()](") == ("invalid", None)
+    assert cc(value) == expected
 
 
 def test_null_cleanup_transformer():
