Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 1098b2c

Browse files
lysnikolaoulysnikolaou
committed
Fix issues due to breaking tokenize changes in 3.12
1 parent cf7c266 commit 1098b2c

6 files changed

Lines changed: 83 additions & 19 deletions

File tree

‎IPython/core/inputsplitter.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
assign_from_system,
4545
assemble_python_lines,
4646
)
47+
from IPython.utils import tokenutil
4748

4849
# These are available in this module for backwards compatibility.
4950
from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
@@ -128,7 +129,7 @@ def partial_tokens(s):
128129
readline = io.StringIO(s).readline
129130
token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
130131
try:
131-
for token in tokenize.generate_tokens(readline):
132+
for token in tokenutil.generate_tokens_catch_errors(readline):
132133
yield token
133134
except tokenize.TokenError as e:
134135
# catch EOF error
@@ -150,9 +151,17 @@ def find_next_indent(code):
150151
tokens.pop()
151152
if not tokens:
152153
return 0
153-
while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT}):
154+
155+
while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT, tokenize.ERRORTOKEN}):
154156
tokens.pop()
155157

158+
# Starting in Python 3.12, the tokenize module adds implicit newlines at the end
159+
# of input. We need to remove those if we're in a multiline statement
160+
if tokens[-1].type == IN_MULTILINE_STATEMENT:
161+
while tokens[-2].type in {tokenize.NL}:
162+
tokens.pop(-2)
163+
164+
156165
if tokens[-1].type == INCOMPLETE_STRING:
157166
# Inside a multiline string
158167
return 0

‎IPython/core/inputtransformer.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,11 @@
99
import functools
1010
import re
1111
import tokenize
12-
from tokenize import generate_tokens, untokenize, TokenError
12+
from tokenize import untokenize, TokenError
1313
from io import StringIO
1414

1515
from IPython.core.splitinput import LineInfo
16+
from IPython.utils import tokenutil
1617

1718
#-----------------------------------------------------------------------------
1819
# Globals
@@ -127,7 +128,7 @@ def __init__(self, func):
127128

128129
def reset_tokenizer(self):
129130
it = iter(self.buf)
130-
self.tokenizer = generate_tokens(it.__next__)
131+
self.tokenizer = tokenutil.generate_tokens_catch_errors(it.__next__)
131132

132133
def push(self, line):
133134
self.buf.append(line + '\n')
@@ -295,7 +296,7 @@ def _line_tokens(line):
295296
readline = StringIO(line).readline
296297
toktypes = set()
297298
try:
298-
for t in generate_tokens(readline):
299+
for t in tokenutil.generate_tokens_catch_errors(readline):
299300
toktypes.add(t[0])
300301
except TokenError as e:
301302
# There are only two cases where a TokenError is raised.

‎IPython/core/inputtransformer2.py‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@
1313
import ast
1414
from codeop import CommandCompiler, Compile
1515
import re
16+
import sys
1617
import tokenize
1718
from typing import List, Tuple, Optional, Any
1819
import warnings
1920

21+
from IPython.utils import tokenutil
22+
2023
_indent_re = re.compile(r'^[ \t]+')
2124

2225
def leading_empty_lines(lines):
@@ -269,9 +272,7 @@ def transform(self, lines: List[str]):
269272
class SystemAssign(TokenTransformBase):
270273
"""Transformer for assignments from system commands (a = !foo)"""
271274
@classmethod
272-
def find(cls, tokens_by_line):
273-
"""Find the first system assignment (a = !foo) in the cell.
274-
"""
275+
def find_pre_312(cls, tokens_by_line):
275276
for line in tokens_by_line:
276277
assign_ix = _find_assign_op(line)
277278
if (assign_ix is not None) \
@@ -287,6 +288,25 @@ def find(cls, tokens_by_line):
287288
break
288289
ix += 1
289290

291+
@classmethod
292+
def find_post_312(cls, tokens_by_line):
293+
for line in tokens_by_line:
294+
assign_ix = _find_assign_op(line)
295+
if (assign_ix is not None) \
296+
and not line[assign_ix].line.strip().startswith('=') \
297+
and (len(line) >= assign_ix + 2) \
298+
and (line[assign_ix + 1].type == tokenize.OP) \
299+
and (line[assign_ix + 1].string == '!'):
300+
return cls(line[assign_ix + 1].start)
301+
302+
@classmethod
303+
def find(cls, tokens_by_line):
304+
"""Find the first system assignment (a = !foo) in the cell.
305+
"""
306+
if sys.version_info < (3, 12):
307+
return cls.find_pre_312(tokens_by_line)
308+
return cls.find_post_312(tokens_by_line)
309+
290310
def transform(self, lines: List[str]):
291311
"""Transform a system assignment found by the ``find()`` classmethod.
292312
"""
@@ -511,7 +531,8 @@ def make_tokens_by_line(lines:List[str]):
511531
)
512532
parenlev = 0
513533
try:
514-
for token in tokenize.generate_tokens(iter(lines).__next__):
534+
for token in tokenutil.generate_tokens_catch_errors(iter(lines).__next__,
535+
extra_errors_to_catch=['expected EOF']):
515536
tokens_by_line[-1].append(token)
516537
if (token.type == NEWLINE) \
517538
or ((token.type == NL) and (parenlev <= 0)):
@@ -677,9 +698,13 @@ def check_complete(self, cell: str):
677698
if not lines:
678699
return 'complete', None
679700

680-
if lines[-1].endswith('\\'):
681-
# Explicit backslash continuation
682-
return 'incomplete', find_last_indent(lines)
701+
for line in reversed(lines):
702+
if not line.strip():
703+
continue
704+
elif line.strip('\n').endswith('\\'):
705+
return 'incomplete', find_last_indent(lines)
706+
else:
707+
break
683708

684709
try:
685710
for transform in self.cleanup_transforms:
@@ -717,7 +742,8 @@ def check_complete(self, cell: str):
717742
if not tokens_by_line:
718743
return 'incomplete', find_last_indent(lines)
719744

720-
if tokens_by_line[-1][-1].type != tokenize.ENDMARKER:
745+
if (tokens_by_line[-1][-1].type != tokenize.ENDMARKER
746+
and tokens_by_line[-1][-1].type != tokenize.ERRORTOKEN):
721747
# We're in a multiline string or expression
722748
return 'incomplete', find_last_indent(lines)
723749

‎IPython/core/tests/test_inputtransformer2.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -297,14 +297,18 @@ def __init__(self, s):
297297
_find_assign_op([Tk(s) for s in ("", "(", "a", "=", "b", ")", "=", "5")]) == 6
298298
)
299299

300-
300+
extra_closing_paren_param = (
301+
pytest.param("(\n))", "invalid", None)
302+
if sys.version_info >= (3, 12)
303+
else pytest.param("(\n))", "incomplete", 0)
304+
)
301305
examples = [
302306
pytest.param("a = 1", "complete", None),
303307
pytest.param("for a in range(5):", "incomplete", 4),
304308
pytest.param("for a in range(5):\n if a > 0:", "incomplete", 8),
305309
pytest.param("raise = 2", "invalid", None),
306310
pytest.param("a = [1,\n2,", "incomplete", 0),
307-
pytest.param("(\n))", "incomplete", 0),
311+
extra_closing_paren_param,
308312
pytest.param("\\\r\n", "incomplete", 0),
309313
pytest.param("a = '''\n hi", "incomplete", 3),
310314
pytest.param("def a():\n x=1\n global x", "invalid", None),

‎IPython/utils/tests/test_pycolorize.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#-----------------------------------------------------------------------------
1919

2020
# our own
21+
import sys
2122
from IPython.utils.PyColorize import Parser
2223
import io
2324
import pytest
@@ -40,7 +41,7 @@ def function(arg, *args, kwarg=True, **kwargs):
4041
False == None
4142
4243
with io.open(ru'unicode', encoding='utf-8'):
43-
raise ValueError("\n escape \r sequence")
44+
raise ValueError("escape \r sequence")
4445
4546
print("wěird ünicoðe")
4647
@@ -64,6 +65,6 @@ def test_parse_sample(style):
6465

6566
def test_parse_error(style):
6667
p = Parser(style=style)
67-
f1 = p.format(")", "str")
68+
f1 = p.format(r"\ " if sys.version_info >= (3, 12) else ")", "str")
6869
if style != "NoColor":
6970
assert "ERROR" in f1

‎IPython/utils/tokenutil.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,31 @@ def generate_tokens(readline):
2121
# catch EOF error
2222
return
2323

24+
def generate_tokens_catch_errors(readline, extra_errors_to_catch=None):
25+
default_errors_to_catch = ['unterminated string literal', 'invalid non-printable character',
26+
'after line continuation character']
27+
assert extra_errors_to_catch is None or isinstance(extra_errors_to_catch, list)
28+
errors_to_catch = default_errors_to_catch + (extra_errors_to_catch or [])
29+
30+
tokens = []
31+
try:
32+
for token in tokenize.generate_tokens(readline):
33+
tokens.append(token)
34+
yield token
35+
except tokenize.TokenError as exc:
36+
if any(error in exc.args[0] for error in errors_to_catch):
37+
if tokens:
38+
start = tokens[-1].start[0], tokens[-1].end[0]
39+
end = start
40+
line = tokens[-1].line
41+
else:
42+
start = end = (1, 0)
43+
line = ''
44+
yield tokenize.TokenInfo(tokenize.ERRORTOKEN, '', start, end, line)
45+
else:
46+
# Catch EOF
47+
raise
48+
2449
def line_at_cursor(cell, cursor_pos=0):
2550
"""Return the line in a cell at a given cursor position
2651
@@ -123,5 +148,3 @@ def token_at_cursor(cell, cursor_pos=0):
123148
return names[-1]
124149
else:
125150
return ''
126-
127-

0 commit comments

Comments
 (0)