Thanks to visit codestin.com
Credit goes to github.com

Skip to content

gh-107015: Remove async_hacks from the tokenizer #107018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 26, 2023
Merged

gh-107015: Remove async_hacks from the tokenizer #107018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e0733cf
gh-107015: Remove async_hacks from the tokenizer
pablogsal Jul 22, 2023
a4830c5
fix tests
pablogsal Jul 25, 2023
a68132f
fix docs
pablogsal Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Doc/library/ast.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2146,7 +2146,7 @@ and classes for traversing abstract syntax trees:
Currently ``major`` must equal to ``3``. For example, setting
``feature_version=(3, 4)`` will allow the use of ``async`` and
``await`` as variable names. The lowest supported version is
``(3, 4)``; the highest is ``sys.version_info[0:2]``.
``(3, 7)``; the highest is ``sys.version_info[0:2]``.

If source contains a null character ('\0'), :exc:`ValueError` is raised.

Expand All @@ -2169,6 +2169,9 @@ and classes for traversing abstract syntax trees:
.. versionchanged:: 3.8
Added ``type_comments``, ``mode='func_type'`` and ``feature_version``.

.. versionchanged:: 3.13
The minimum supported version for feature_version is now (3,7)


.. function:: unparse(ast_obj)

Expand Down
4 changes: 0 additions & 4 deletions Doc/library/token-list.inc
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 7 additions & 3 deletions Doc/library/token.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,21 @@ the :mod:`tokenize` module.


.. versionchanged:: 3.5
Added :data:`AWAIT` and :data:`ASYNC` tokens.
Added :data:`!AWAIT` and :data:`!ASYNC` tokens.

.. versionchanged:: 3.7
Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens.

.. versionchanged:: 3.7
Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are
Removed :data:`!AWAIT` and :data:`!ASYNC` tokens. "async" and "await" are
now tokenized as :data:`NAME` tokens.

.. versionchanged:: 3.8
Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`.
Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed
Added :data:`!AWAIT` and :data:`!ASYNC` tokens back (they're needed
to support parsing older Python versions for :func:`ast.parse` with
``feature_version`` set to 6 or lower).

.. versionchanged:: 3.13
Removed :data:`!AWAIT` and :data:`!ASYNC` tokens again.

2 changes: 0 additions & 2 deletions Grammar/Tokens
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,6 @@ COLONEQUAL ':='
EXCLAMATION '!'

OP
AWAIT
ASYNC
TYPE_IGNORE
TYPE_COMMENT
SOFT_KEYWORD
Expand Down
34 changes: 17 additions & 17 deletions Grammar/python.gram
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,11 @@ simple_stmt[stmt_ty] (memo):
| &'nonlocal' nonlocal_stmt

compound_stmt[stmt_ty]:
| &('def' | '@' | ASYNC) function_def
| &('def' | '@' | 'async') function_def
| &'if' if_stmt
| &('class' | '@') class_def
| &('with' | ASYNC) with_stmt
| &('for' | ASYNC) for_stmt
| &('with' | 'async') with_stmt
| &('for' | 'async') for_stmt
| &'try' try_stmt
| &'while' while_stmt
| match_stmt
Expand Down Expand Up @@ -272,7 +272,7 @@ function_def_raw[stmt_ty]:
_PyAST_FunctionDef(n->v.Name.id,
(params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) }
| ASYNC 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
| 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
CHECK_VERSION(
stmt_ty,
5,
Expand Down Expand Up @@ -385,7 +385,7 @@ for_stmt[stmt_ty]:
| invalid_for_stmt
| 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
_PyAST_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
| 'async' 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
CHECK_VERSION(stmt_ty, 5, "Async for loops are", _PyAST_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
| invalid_for_target

Expand All @@ -398,9 +398,9 @@ with_stmt[stmt_ty]:
CHECK_VERSION(stmt_ty, 9, "Parenthesized context managers are", _PyAST_With(a, b, NULL, EXTRA)) }
| 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
_PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| ASYNC 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block {
| 'async' 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block {
CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NULL, EXTRA)) }
| ASYNC 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
| 'async' 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
| invalid_with_stmt

Expand Down Expand Up @@ -814,7 +814,7 @@ power[expr_ty]:
# Primary elements are things like "obj.something.something", "obj[something]", "obj(something)", "obj" ...

await_primary[expr_ty] (memo):
| AWAIT a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) }
| 'await' a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) }
| primary

primary[expr_ty]:
Expand Down Expand Up @@ -966,7 +966,7 @@ for_if_clauses[asdl_comprehension_seq*]:
| a[asdl_comprehension_seq*]=for_if_clause+ { a }

for_if_clause[comprehension_ty]:
| ASYNC 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
| 'async' 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) }
| 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* {
_PyAST_comprehension(a, b, c, 0, p->arena) }
Expand Down Expand Up @@ -1284,7 +1284,7 @@ invalid_with_item:
RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) }

invalid_for_target:
| ASYNC? 'for' a=star_expressions {
| 'async'? 'for' a=star_expressions {
RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) }

invalid_group:
Expand All @@ -1301,12 +1301,12 @@ invalid_import_from_targets:
RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") }

invalid_with_stmt:
| [ASYNC] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
| [ASYNC] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
| ['async'] 'with' ','.(expression ['as' star_target])+ NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
| ['async'] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
invalid_with_stmt_indent:
| [ASYNC] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
| ['async'] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT {
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }
| [ASYNC] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
| ['async'] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT {
RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) }

invalid_try_stmt:
Expand Down Expand Up @@ -1367,11 +1367,11 @@ invalid_while_stmt:
| a='while' named_expression ':' NEWLINE !INDENT {
RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) }
invalid_for_stmt:
| [ASYNC] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
| [ASYNC] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
| ['async'] 'for' star_targets 'in' star_expressions NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
| ['async'] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT {
RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) }
invalid_def_raw:
| [ASYNC] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
| ['async'] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT {
RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) }
invalid_class_def_raw:
| 'class' NAME ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") }
Expand Down
22 changes: 10 additions & 12 deletions Include/internal/pycore_token.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,16 @@ extern "C" {
#define COLONEQUAL 53
#define EXCLAMATION 54
#define OP 55
#define AWAIT 56
#define ASYNC 57
#define TYPE_IGNORE 58
#define TYPE_COMMENT 59
#define SOFT_KEYWORD 60
#define FSTRING_START 61
#define FSTRING_MIDDLE 62
#define FSTRING_END 63
#define COMMENT 64
#define NL 65
#define ERRORTOKEN 66
#define N_TOKENS 68
#define TYPE_IGNORE 56
#define TYPE_COMMENT 57
#define SOFT_KEYWORD 58
#define FSTRING_START 59
#define FSTRING_MIDDLE 60
#define FSTRING_END 61
#define COMMENT 62
#define NL 63
#define ERRORTOKEN 64
#define N_TOKENS 66
#define NT_OFFSET 256

/* Special definitions for cooperation with parser */
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/test_peg_generator/test_c_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ def test_ternary_operator(self) -> None:
a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) }
)
for_if_clauses[asdl_comprehension_seq*]: (
a[asdl_comprehension_seq*]=(y=[ASYNC] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
{ _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a }
)
"""
Expand Down
Loading