From 717ce38f9f4fd43d7d5dda2e5f7d8cc98628acef Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 13 Sep 2026 11:47:32 +0100 Subject: [PATCH 1/2] gh-153568: Parse arithmetic with operator loops Turn rules marked with operator_loop into a simple loop over operators and operands. Keep the cached result and the existing error rules. --- Grammar/python.gram | 12 +- InternalDocs/parser.md | 51 ++ ...10-44-21.gh-issue-153568.operatorloops.rst | 2 + Parser/parser.c | 690 ++++++++++++++++++ Tools/peg_generator/pegen/c_generator.py | 219 +++++- 5 files changed, 967 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 2713ba9466a0b1d..caa56675b8418f5 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -817,21 +817,21 @@ is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, I # Bitwise operators # ----------------- -bitwise_or[expr_ty]: +bitwise_or[expr_ty] (operator_loop): | a=bitwise_or '|' b=bitwise_xor { _PyAST_BinOp(a, BitOr, b, EXTRA) } | invalid_bitwise_or | bitwise_xor -bitwise_xor[expr_ty]: +bitwise_xor[expr_ty] (operator_loop): | a=bitwise_xor '^' b=bitwise_and { _PyAST_BinOp(a, BitXor, b, EXTRA) } | bitwise_and -bitwise_and[expr_ty]: +bitwise_and[expr_ty] (operator_loop): | a=bitwise_and '&' b=shift_expr { _PyAST_BinOp(a, BitAnd, b, EXTRA) } | invalid_bitwise_and | shift_expr -shift_expr[expr_ty]: +shift_expr[expr_ty] (operator_loop): | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } | sum @@ -839,13 +839,13 @@ shift_expr[expr_ty]: # Arithmetic operators # -------------------- -sum[expr_ty]: +sum[expr_ty] (operator_loop): | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) } | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) } | invalid_arithmetic | term -term[expr_ty]: +term[expr_ty] (operator_loop): | a=term '*' b=factor { _PyAST_BinOp(a, Mult, b, EXTRA) } | a=term '/' b=factor { _PyAST_BinOp(a, Div, b, EXTRA) } | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index a6de8d456b6f71c..3ed74349ae3996a 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -209,6 +209,57 @@ and "hidden left-recursion" like: rule: 'optional'? rule '@' some_other_rule ``` +Explicit operator loops +---------------------- + +A C grammar can request an operator loop for a particular left-recursive rule +with the `(operator_loop)` flag: + +``` + sum[expr_ty] (operator_loop): + | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) } + | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) } + | term +``` + +Unmarked rules keep the general left-recursion algorithm. The generator checks +each marked rule and raises a rule-specific error if its shape or operand +dependency is unsupported. The flag takes no value and cannot be combined with +other rule flags. In `skip_actions` mode the generator still validates marked +rules, but emits the general algorithm instead of AST-folding loops. + +The flag also asserts an action contract; the generator cannot prove properties +of arbitrary C code. Actions reachable from a marked rule must preserve the +grammar-derived token position and parser-control state when they succeed. +They must not make hidden calls to generated parser rules, inspect or modify +provisional left-recursion memo entries, or mutate the buffered input. The +existing generated `without_invalid` scopes remain supported. AST constructors, +feature checks, warnings and normal error reporting remain permitted. Existing +restrictions on modifying shared AST nodes also apply. Changes to relevant +actions and their C helpers must preserve this contract. + +After excluding alternatives guarded by `call_invalid_rules`, each marked rule +must consist of `left=self exact-token right=operand` alternatives followed by +one plain operand alternative. Operator tokens must be distinct. The result and +operand types must be `expr_ty`. Recursive actions must construct `_PyAST_BinOp` +using the bound operands and a constant operator kind, optionally wrapped in +`CHECK_VERSION`. Cuts, predicates, other recursive actions and other alternative +shapes are unsupported. + +The operand must consume input and cannot call the marked rule again at the same +position through the grammar. It must use ordinary `(memo)`, or also be marked +`(operator_loop)` and pass validation. This preserves the +original algorithm's final base retry as memo reuse, rather than executing an +operand action again. The loop retains result memoization but does not publish +intermediate seeds. Normal right-operand failure restores the position before +the operator; fatal errors propagate. + +Loops run when `call_invalid_rules` is false. The original leader and raw rule +remain available while it is true. This includes diagnostic parsing: an +`expression_without_invalid` scope temporarily disables invalid rules and may +therefore use loops. The transformation changes stack usage and allocation +order; the flag does not promise identical resource-limit boundaries. + Variables in the grammar ------------------------ diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst b/Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst new file mode 100644 index 000000000000000..26f2177eab9210f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst @@ -0,0 +1,2 @@ +Speed up arithmetic parsing by generating operator loops for explicitly +marked grammar rules. diff --git a/Parser/parser.c b/Parser/parser.c index 800db5b490fea98..ba8b5266e0b5691 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -13503,10 +13503,96 @@ is_bitwise_or_rule(Parser *p) // Left-recursive // bitwise_or: bitwise_or '|' bitwise_xor | invalid_bitwise_or | bitwise_xor +static expr_ty +bitwise_or_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, bitwise_or_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = bitwise_xor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 18: { + expr_ty a = _res; + p->mark++; + expr_ty b = bitwise_xor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , BitOr , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, bitwise_or_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty bitwise_or_raw(Parser *); static expr_ty bitwise_or_rule(Parser *p) { + if (!p->call_invalid_rules) { + return bitwise_or_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } @@ -13644,10 +13730,96 @@ bitwise_or_raw(Parser *p) // Left-recursive // bitwise_xor: bitwise_xor '^' bitwise_and | bitwise_and +static expr_ty +bitwise_xor_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, bitwise_xor_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = bitwise_and_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 32: { + expr_ty a = _res; + p->mark++; + expr_ty b = bitwise_and_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , BitXor , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, bitwise_xor_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty bitwise_xor_raw(Parser *); static expr_ty bitwise_xor_rule(Parser *p) { + if (!p->call_invalid_rules) { + return bitwise_xor_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } @@ -13766,10 +13938,96 @@ bitwise_xor_raw(Parser *p) // Left-recursive // bitwise_and: bitwise_and '&' shift_expr | invalid_bitwise_and | shift_expr +static expr_ty +bitwise_and_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, bitwise_and_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = shift_expr_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 19: { + expr_ty a = _res; + p->mark++; + expr_ty b = shift_expr_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , BitAnd , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, bitwise_and_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty bitwise_and_raw(Parser *); static expr_ty bitwise_and_rule(Parser *p) { + if (!p->call_invalid_rules) { + return bitwise_and_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } @@ -13907,10 +14165,125 @@ bitwise_and_raw(Parser *p) // Left-recursive // shift_expr: shift_expr '<<' sum | shift_expr '>>' sum | sum +static expr_ty +shift_expr_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, shift_expr_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = sum_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 33: { + expr_ty a = _res; + p->mark++; + expr_ty b = sum_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , LShift , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 34: { + expr_ty a = _res; + p->mark++; + expr_ty b = sum_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , RShift , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, shift_expr_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty shift_expr_raw(Parser *); static expr_ty shift_expr_rule(Parser *p) { + if (!p->call_invalid_rules) { + return shift_expr_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } @@ -14068,10 +14441,125 @@ shift_expr_raw(Parser *p) // Left-recursive // sum: sum '+' term | sum '-' term | invalid_arithmetic | term +static expr_ty +sum_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, sum_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = term_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 14: { + expr_ty a = _res; + p->mark++; + expr_ty b = term_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , Add , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 15: { + expr_ty a = _res; + p->mark++; + expr_ty b = term_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , Sub , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, sum_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty sum_raw(Parser *); static expr_ty sum_rule(Parser *p) { + if (!p->call_invalid_rules) { + return sum_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } @@ -14254,10 +14742,212 @@ sum_raw(Parser *p) // | term '%' factor // | term '@' factor // | factor +static expr_ty +term_operator_loop(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + if (_PyPegen_is_memoized(p, term_type, &_res)) { + p->level--; + return _res; + } + int _mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + int _start_lineno = p->tokens[_mark]->lineno; + UNUSED(_start_lineno); // Only used by EXTRA macro + int _start_col_offset = p->tokens[_mark]->col_offset; + UNUSED(_start_col_offset); // Only used by EXTRA macro + _res = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (_res == NULL) { + p->mark = _mark; + goto done; + } + while (1) { + int _operator_mark = p->mark; + if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + switch (p->tokens[p->mark]->type) { + case 16: { + expr_ty a = _res; + p->mark++; + expr_ty b = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , Mult , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 17: { + expr_ty a = _res; + p->mark++; + expr_ty b = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , Div , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 47: { + expr_ty a = _res; + p->mark++; + expr_ty b = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , FloorDiv , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 24: { + expr_ty a = _res; + p->mark++; + expr_ty b = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyAST_BinOp ( a , Mod , b , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + case 49: { + expr_ty a = _res; + p->mark++; + expr_ty b = factor_rule(p); + if (p->error_indicator) { + p->level--; + return NULL; + } + if (b == NULL) { + p->mark = _operator_mark; + goto done; + } + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = CHECK_VERSION ( expr_ty , 5 , "The '@' operator is" , _PyAST_BinOp ( a , MatMult , b , EXTRA ) ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + break; + } + default: goto done; + } + } + done: + if (_PyPegen_insert_memo(p, _mark, term_type, _res) < 0) { + p->error_indicator = 1; + p->level--; + return NULL; + } + p->level--; + return _res; +} static expr_ty term_raw(Parser *); static expr_ty term_rule(Parser *p) { + if (!p->call_invalid_rules) { + return term_operator_loop(p); + } if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); } diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index d9236dfb22835bd..0e100e6bb3b1314 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -4,7 +4,7 @@ from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any +from typing import IO, Any, NoReturn from pegen import grammar from pegen.grammar import ( @@ -390,6 +390,8 @@ def __init__( self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor( self, exact_tokens, non_exact_tokens ) + self.exact_tokens = exact_tokens + self.operator_loops: dict[str, Rhs] = {} self._varname_counter = 0 self.debug = debug self.skip_actions = skip_actions @@ -454,6 +456,7 @@ def out_of_memory_goto(self, expr: str, goto_target: str) -> None: def generate(self, filename: str) -> None: self.collect_rules() + self.operator_loops = self._find_operator_loops() basename = os.path.basename(filename) self.print(f"// @generated by pegen from {basename}") header = self.grammar.metas.get("header", EXTENSION_PREFIX) @@ -564,6 +567,11 @@ def _check_for_errors(self) -> None: def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: self.print("{") with self.indent(): + if node.name in self.operator_loops: + self.print("if (!p->call_invalid_rules) {") + with self.indent(): + self.print(f"return {node.name}_operator_loop(p);") + self.print("}") self.add_level() self.print(f"{result_type} _res = NULL;") self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") @@ -670,6 +678,213 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") self.add_return("_seq") + def _find_operator_loops(self) -> dict[str, Rhs]: + """Validate marked rules under their explicit C-action contract. + + Compute a conservative fixed point of rules that consume on success. + Unknown nodes, lookahead and optional items do not prove consumption. + C actions can hide parser calls and cursor changes from this analysis; + the opt-in contract is documented in InternalDocs/parser.md. + """ + def reject(name: str, reason: str) -> NoReturn: + raise ValueError(f"rule {name!r}: operator_loop {reason}") + + marked = {} + for name, rule in self.all_rules.items(): + if "operator_loop" in rule.flags: + marked[name] = rule + if not marked: + return {} + consuming: set[str] = set() + + def consumes(node: Any) -> bool: + if isinstance(node, NameLeaf): + return node.value not in self.all_rules or node.value in consuming + if isinstance(node, StringLeaf): + return True + if isinstance(node, (NamedItem, Repeat1, Gather, Forced)): + return consumes(node.item if isinstance(node, NamedItem) else node.node) + if isinstance(node, Group): + return consumes(node.rhs) + if isinstance(node, Alt): + return any(consumes(item) for item in node.items) + if isinstance(node, Rhs): + return bool(node.alts) and all(consumes(alt) for alt in node.alts) + return False + + while True: + added = {name for name, rule in self.all_rules.items() + if consumes(rule.rhs)} - consuming + if not added: + break + consuming.update(added) + + # Include calls inside predicates and nullable prefixes, which must not + # re-enter an operator rule before its final memo has been installed. + def initial_calls(node: Any) -> set[str]: + if isinstance(node, NameLeaf): + return {node.value} if node.value in self.all_rules else set() + if isinstance(node, NamedItem): + return initial_calls(node.item) + if isinstance(node, (Lookahead, Opt, Repeat0, Repeat1, Gather, Forced)): + return initial_calls(node.node) + if isinstance(node, Group): + return initial_calls(node.rhs) + if isinstance(node, Rhs): + return set().union(*(initial_calls(alt) for alt in node.alts)) + if isinstance(node, Alt): + result: set[str] = set() + for item in node.items: + result.update(initial_calls(item)) + if consumes(item): + break + return result + return set() + + graph = {name: initial_calls(rule.rhs) + for name, rule in self.all_rules.items()} + operator_kinds = {kind.__name__ for kind in ast.operator.__subclasses__()} + operator_kind = "(?:" + "|".join(sorted(operator_kinds)) + ")" + candidates: dict[str, Rhs] = {} + for name, rule in marked.items(): + if rule.type != "expr_ty": + reject(name, "requires result type expr_ty") + if not rule.left_recursive or not rule.leader: + reject(name, "requires a direct left-recursion leader") + if rule.flags != {"operator_loop"}: + reject(name, "cannot be combined with other rule flags") + if name.endswith("without_invalid"): + reject(name, "does not support without_invalid rules") + # These are precisely the alternatives visit_Alt disables on pass 1. + alts = [alt for alt in rule.flatten().alts + if not (len(alt.items) == 1 + and isinstance(alt.items[0].item, NameLeaf) + and str(alt.items[0]).startswith("invalid_") + and not alt.action)] + if len(alts) < 2: + reject(name, "requires recursive alternatives followed by one operand alternative") + base = alts[-1] + if (base.action or len(base.items) != 1 + or not isinstance(base.items[0].item, NameLeaf) + or base.items[0].type): + reject(name, "requires a final plain operand rule without an action or type cast") + operand = base.items[0].item.value + if operand not in self.all_rules or self.all_rules[operand].type != "expr_ty": + reject(name, f"requires operand {operand!r} to be an expr_ty rule") + if operand not in consuming: + reject(name, f"cannot prove that operand {operand!r} consumes input") + reached: set[str] = set() + pending = [operand] + while pending: + current = pending.pop() + if current not in reached: + reached.add(current) + pending.extend(graph.get(current, ())) + if name in reached: + reject(name, f"operand {operand!r} can re-enter the rule without consuming input") + tokens: set[int] = set() + for alt in alts[:-1]: + if len(alt.items) != 3: + reject(name, "requires each recursive alternative to be left=self token right=operand") + left, operator, right = alt.items + if (not isinstance(left.item, NameLeaf) or left.item.value != name + or not isinstance(right.item, NameLeaf) or right.item.value != operand + or not isinstance(operator.item, StringLeaf) + or not left.name or not right.name or left.name == right.name + or left.name in operator_kinds or right.name in operator_kinds + or left.type or right.type or operator.name or operator.type): + reject(name, "requires distinct operand bindings, an unnamed exact token, and no casts") + token = self.exact_tokens.get(ast.literal_eval(operator.item.value)) + if token is None: + reject(name, f"operator {operator.item.value} is not an exact token") + if token in tokens: + reject(name, f"operator {operator.item.value} occurs in more than one alternative") + tokens.add(token) + # Only immutable AST construction, optionally with the existing + # feature-version check. Custom actions need a separate proof. + action = (re.escape(left.name) + r"\s*,\s*" + operator_kind + + r"\s*,\s*" + re.escape(right.name)) + action = r"_PyAST_BinOp\s*\(\s*" + action + r"\s*,\s*EXTRA\s*\)" + versioned = (r'CHECK_VERSION\s*\(\s*expr_ty\s*,\s*[0-9]+\s*,\s*' + r'"(?:[^"\\]|\\.)*"\s*,\s*' + action + r"\s*\)") + if not re.fullmatch(r"\s*(?:" + action + "|" + versioned + r")\s*", alt.action or ""): + reject(name, "requires a BinOp action on the bound operands, optionally CHECK_VERSION") + candidates[name] = Rhs(alts) + + # The seed algorithm retries the base after its last unsuccessful grow. + # Require that retry to reuse a result, not run an operand action again. + for name, rhs in candidates.items(): + operand = rhs.alts[-1].items[0].item.value + rule = self.all_rules[operand] + if operand not in candidates and not self._should_memoize(rule): + reject(name, f"requires operand {operand!r} to use (operator_loop), " + "or ordinary (memo)") + # Validate the annotations even when the existing action-free debugging + # mode requests ordinary PEG emission instead of AST folding. + return {} if self.skip_actions else candidates + + def _handle_operator_loop(self, node: Rule, rhs: Rhs) -> None: + self.print(f"static expr_ty\n{node.name}_operator_loop(Parser *p)") + self.print("{") + with self.indent(): + self.add_level() + self._check_for_errors() + self.print("expr_ty _res = NULL;") + self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") + with self.indent(): + self.add_return("_res") + self.print("}") + self.print("int _mark = p->mark;") + self._set_up_token_start_metadata_extraction() + operand = rhs.alts[-1].items[0].item.value + self.print(f"_res = {operand}_rule(p);") + self._check_for_errors() + self.print("if (_res == NULL) {") + with self.indent(): + self.print("p->mark = _mark;") + self.print("goto done;") + self.print("}") + self.print("while (1) {") + with self.indent(): + self.print("int _operator_mark = p->mark;") + self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") + with self.indent(): + self.print("p->error_indicator = 1;") + self.add_return("NULL") + self.print("}") + self.print("switch (p->tokens[p->mark]->type) {") + with self.indent(): + for alt in rhs.alts[:-1]: + left, operator, right = alt.items + token = self.exact_tokens[ast.literal_eval(operator.item.value)] + self.print(f"case {token}: {{") + with self.indent(): + self.print(f"expr_ty {left.name} = _res;") + self.print("p->mark++;") + self.print(f"expr_ty {right.name} = {operand}_rule(p);") + self._check_for_errors() + self.print(f"if ({right.name} == NULL) {{") + with self.indent(): + self.print("p->mark = _operator_mark;") + self.print("goto done;") + self.print("}") + self._set_up_token_end_metadata_extraction() + self.emit_action(alt) + self.print("break;") + self.print("}") + self.print("default: goto done;") + self.print("}") + self.print("}") + self.print(" done:") + with self.indent(): + self.print(f"if (_PyPegen_insert_memo(p, _mark, {node.name}_type, _res) < 0) {{") + with self.indent(): + self.print("p->error_indicator = 1;") + self.add_return("NULL") + self.print("}") + self.add_return("_res") + self.print("}") + def visit_Rule(self, node: Rule) -> None: is_loop = node.is_loop() is_gather = node.is_gather() @@ -683,6 +898,8 @@ def visit_Rule(self, node: Rule) -> None: for line in str(node).splitlines(): self.print(f"// {line}") + if node.name in self.operator_loops: + self._handle_operator_loop(node, self.operator_loops[node.name]) if node.left_recursive and node.leader: self.print(f"static {result_type} {node.name}_raw(Parser *);") From 1c7ff6757ee1abcf34a6186beb2ce8a6ed34eebc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 13 Sep 2026 14:04:34 +0100 Subject: [PATCH 2/2] gh-153568: Fix generator typing and the news path --- ...2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst | 0 Tools/peg_generator/pegen/c_generator.py | 9 +++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) rename Misc/NEWS.d/next/{Core and Builtins => Core_and_Builtins}/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst (100%) diff --git a/Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst similarity index 100% rename from Misc/NEWS.d/next/Core and Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst rename to Misc/NEWS.d/next/Core_and_Builtins/2026-09-13-10-44-21.gh-issue-153568.operatorloops.rst diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 0e100e6bb3b1314..613f11c991e8d25 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -814,7 +814,9 @@ def initial_calls(node: Any) -> set[str]: # The seed algorithm retries the base after its last unsuccessful grow. # Require that retry to reuse a result, not run an operand action again. for name, rhs in candidates.items(): - operand = rhs.alts[-1].items[0].item.value + operand_item = rhs.alts[-1].items[0].item + assert isinstance(operand_item, NameLeaf) + operand = operand_item.value rule = self.all_rules[operand] if operand not in candidates and not self._should_memoize(rule): reject(name, f"requires operand {operand!r} to use (operator_loop), " @@ -836,7 +838,9 @@ def _handle_operator_loop(self, node: Rule, rhs: Rhs) -> None: self.print("}") self.print("int _mark = p->mark;") self._set_up_token_start_metadata_extraction() - operand = rhs.alts[-1].items[0].item.value + operand_item = rhs.alts[-1].items[0].item + assert isinstance(operand_item, NameLeaf) + operand = operand_item.value self.print(f"_res = {operand}_rule(p);") self._check_for_errors() self.print("if (_res == NULL) {") @@ -856,6 +860,7 @@ def _handle_operator_loop(self, node: Rule, rhs: Rhs) -> None: with self.indent(): for alt in rhs.alts[:-1]: left, operator, right = alt.items + assert isinstance(operator.item, StringLeaf) token = self.exact_tokens[ast.literal_eval(operator.item.value)] self.print(f"case {token}: {{") with self.indent():