From 366f205d9c690f46877552235a3a47cd3c7b0644 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 14 Oct 2024 21:24:02 +0100 Subject: [PATCH 01/10] gh-125331: Allow the parser to activate future imports on the fly --- Grammar/python.gram | 4 +++- Parser/action_helpers.c | 21 +++++++++++++++++++++ Parser/parser.c | 2 +- Parser/pegen.h | 2 ++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index b47028460b94f4..0c8011e5de64b3 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -207,7 +207,9 @@ import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyAST_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } + _PyPegen_check_future_import(p, + _PyAST_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) + ) } | 'from' a=('.' | '...')+ 'import' b=import_from_targets { _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } import_from_targets[asdl_alias_seq*]: diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 2fe8d11badcbac..852a967fbefbc5 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1694,3 +1694,24 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, assert(current_pos == n_elements); return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena); } + +stmt_ty +_PyPegen_check_future_import(Parser *p, stmt_ty importfrom) { + if (importfrom->kind != ImportFrom_kind) { + goto exit; + } + asdl_alias_seq *names = importfrom->v.ImportFrom.names; + if (asdl_seq_LEN(names) != 1) { + goto exit; + } + identifier mod = importfrom->v.ImportFrom.module; + if (PyUnicode_CompareWithASCIIString(mod, "__future__") != 0) { + goto exit; + } + alias_ty alias = asdl_seq_GET(names, 0); + if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { + p->flags |= PyPARSE_BARRY_AS_BDFL; + } +exit: + return importfrom; +} \ No newline at end of file diff --git a/Parser/parser.c b/Parser/parser.c index 9ff58ab7e7bfd5..1e1deb082ff27f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -3356,7 +3356,7 @@ import_from_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_ImportFrom ( b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ); + _res = _PyPegen_check_future_import ( p , _PyAST_ImportFrom ( b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 32c64e7774b878..5c7752287ba6c1 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -361,6 +361,8 @@ asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // TODO: move to the correct place in this file expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b); +stmt_ty _PyPegen_check_future_import(Parser *p, stmt_ty importfrom); + // Generated function in parse.c - function definition in python.gram void *_PyPegen_parse(Parser *); From 4ef188ce8274eaa9e28cb9cec3a50fbf6d1145ec Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 15 Oct 2024 15:26:44 +0100 Subject: [PATCH 02/10] Apply suggestions from code review Co-authored-by: Lysandros Nikolaou Co-authored-by: Nice Zombies Signed-off-by: Pablo Galindo --- Grammar/python.gram | 4 +-- Lib/test/test_flufl.py | 15 ++++++++++ Lib/test/test_grammar.py | 1 - ...-10-29-23-30-35.gh-issue-125331.quKQ7V.rst | 4 +++ Parser/action_helpers.c | 28 ++++++++----------- Parser/parser.c | 2 +- Parser/pegen.h | 4 +-- 7 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 0c8011e5de64b3..c86892d8967248 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -207,9 +207,7 @@ import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_check_future_import(p, - _PyAST_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) - ) } + _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } | 'from' a=('.' | '...')+ 'import' b=import_from_targets { _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } import_from_targets[asdl_alias_seq*]: diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index fd264c926bd575..f9bc460af2c60e 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -34,6 +34,21 @@ def test_guido_as_bdfl(self): # parser reports the start of the token self.assertEqual(cm.exception.offset, 3) + def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): + # Check that the future import is handled by the parser + # even if the compiler flags are not passed. + code = "from __future__ import barry_as_FLUFL;2 {0} 3" + compile(code.format('<>'), '', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile(code.format('!='), '', 'exec') + self.assertRegex(str(cm.exception), "with Barry as BDFL, use '<>' instead of '!='") + self.assertIn('2 != 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + self.assertEqual(cm.exception.lineno, 1) + self.assertEqual(cm.exception.offset, len(code) - 4) + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 6a841587f49166..ee15c5bf2b030f 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1972,6 +1972,5 @@ async def foo(): with self.assertRaises(Done): foo().send(None) - if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst new file mode 100644 index 00000000000000..c4698f90a70aa3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst @@ -0,0 +1,4 @@ +Using ``from __future__ import barry_as_FLUFL`` works now when using +multiple statements on the same line. Additionally the effect is respected +even if the import is used even if the flags are not passed to +:func:`compile`. Patch by Pablo Galindo diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 852a967fbefbc5..a5deceb55eabc5 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1696,22 +1696,16 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } stmt_ty -_PyPegen_check_future_import(Parser *p, stmt_ty importfrom) { - if (importfrom->kind != ImportFrom_kind) { - goto exit; - } - asdl_alias_seq *names = importfrom->v.ImportFrom.names; - if (asdl_seq_LEN(names) != 1) { - goto exit; - } - identifier mod = importfrom->v.ImportFrom.module; - if (PyUnicode_CompareWithASCIIString(mod, "__future__") != 0) { - goto exit; - } - alias_ty alias = asdl_seq_GET(names, 0); - if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { - p->flags |= PyPARSE_BARRY_AS_BDFL; +_PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * names, int level, + int lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { + if (PyUnicode_CompareWithASCIIString(module, "__future__") == 0) { + for (Py_ssize_t i = 0; i < asdl_seq_LEN(names); i++) { + alias_ty alias = asdl_seq_GET(names, i); + if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { + p->flags |= PyPARSE_BARRY_AS_BDFL; + } + } } -exit: - return importfrom; + return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena); } \ No newline at end of file diff --git a/Parser/parser.c b/Parser/parser.c index 1e1deb082ff27f..9b02d1340163a2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -3356,7 +3356,7 @@ import_from_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyPegen_check_future_import ( p , _PyAST_ImportFrom ( b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ) ); + _res = _PyPegen_checked_future_import ( p , b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index 5c7752287ba6c1..651659ac6c926b 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -346,6 +346,8 @@ mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *); void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); +stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq *, + int , int, int , int , int , PyArena *); // Parser API @@ -361,8 +363,6 @@ asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // TODO: move to the correct place in this file expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b); -stmt_ty _PyPegen_check_future_import(Parser *p, stmt_ty importfrom); - // Generated function in parse.c - function definition in python.gram void *_PyPegen_parse(Parser *); From c3165412940fd39db0dc372727a1311770ca3cfd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 30 Oct 2024 11:41:51 +0000 Subject: [PATCH 03/10] Fix whitespace --- Lib/test/test_grammar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ee15c5bf2b030f..6a841587f49166 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1972,5 +1972,6 @@ async def foo(): with self.assertRaises(Done): foo().send(None) + if __name__ == '__main__': unittest.main() From d12081aecb013aca732bcb5e29daa2211d82bc06 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 13:25:13 +0000 Subject: [PATCH 04/10] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lysandros Nikolaou Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst | 4 ++-- Parser/action_helpers.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst index c4698f90a70aa3..9e48ecb4c93297 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst @@ -1,4 +1,4 @@ Using ``from __future__ import barry_as_FLUFL`` works now when using -multiple statements on the same line. Additionally the effect is respected -even if the import is used even if the flags are not passed to +multiple statements on the same line. Additionally, the effect is respected +even if the import is used but flags are not passed to :func:`compile`. Patch by Pablo Galindo diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index a5deceb55eabc5..f47a7aa26cc4d1 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1697,8 +1697,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena) { + int lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { if (PyUnicode_CompareWithASCIIString(module, "__future__") == 0) { for (Py_ssize_t i = 0; i < asdl_seq_LEN(names); i++) { alias_ty alias = asdl_seq_GET(names, i); @@ -1708,4 +1708,4 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na } } return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena); -} \ No newline at end of file +} From 68b29e4099b19ad46efe17f75e4a8422a54a9268 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 13:25:36 +0000 Subject: [PATCH 05/10] Update Lib/test/test_flufl.py --- Lib/test/test_flufl.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index f9bc460af2c60e..164aba54f2859a 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -47,8 +47,5 @@ def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): self.assertEqual(cm.exception.lineno, 1) self.assertEqual(cm.exception.offset, len(code) - 4) - - - if __name__ == '__main__': unittest.main() From 53a237ee9947952f9c2140b9e817c7a3d68c4b56 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 13:25:58 +0000 Subject: [PATCH 06/10] Update Lib/test/test_flufl.py --- Lib/test/test_flufl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index 164aba54f2859a..667bf977314917 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -47,5 +47,6 @@ def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): self.assertEqual(cm.exception.lineno, 1) self.assertEqual(cm.exception.offset, len(code) - 4) + if __name__ == '__main__': unittest.main() From 941d2ed7deeb98ee7e49255baabd6cef7e469adc Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 16:47:50 +0000 Subject: [PATCH 07/10] Update Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst index 9e48ecb4c93297..00172fa8187d93 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst @@ -1,4 +1,4 @@ Using ``from __future__ import barry_as_FLUFL`` works now when using multiple statements on the same line. Additionally, the effect is respected even if the import is used but flags are not passed to -:func:`compile`. Patch by Pablo Galindo +:func:`compile`. Patch by Pablo Galindo. From f3772917d5b2b9fad77ff51d3e9cb1c1d9a47e91 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 30 Oct 2024 22:32:36 +0000 Subject: [PATCH 08/10] Update action_helpers.c Co-authored-by: Nice Zombies --- Parser/action_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index f47a7aa26cc4d1..52d63401a6e6b5 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1699,7 +1699,7 @@ stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * names, int level, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { - if (PyUnicode_CompareWithASCIIString(module, "__future__") == 0) { + if (level == 0 && PyUnicode_CompareWithASCIIString(module, "__future__") == 0) { for (Py_ssize_t i = 0; i < asdl_seq_LEN(names); i++) { alias_ty alias = asdl_seq_GET(names, i); if (PyUnicode_CompareWithASCIIString(alias->name, "barry_as_FLUFL") == 0) { From babe18bf5eb3cd92a0726893e2bf109495a049df Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 11 Dec 2024 10:11:22 +0100 Subject: [PATCH 09/10] Update Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst Co-authored-by: Jelle Zijlstra --- .../2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst index 00172fa8187d93..a87467a5ba554b 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst @@ -1,4 +1,5 @@ -Using ``from __future__ import barry_as_FLUFL`` works now when using -multiple statements on the same line. Additionally, the effect is respected -even if the import is used but flags are not passed to -:func:`compile`. Patch by Pablo Galindo. +``from __future__ import barry_as_FLUFL`` now works in more contexts, +including when it is used in files, with the ``-c`` flag, and in the REPL +when there are multiple statements on the same line. Previously, it worked +only on subsequent lines in the REPL, and when the appropriate flags were +passed directly to :func:`compile`. Patch by Pablo Galindo. From 76d8c9f56382f13b13bce59b2c0c5416bef2bac0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 13 Feb 2025 00:50:30 +0000 Subject: [PATCH 10/10] fixup! Update Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-23-30-35.gh-issue-125331.quKQ7V.rst --- Lib/test/test_flufl.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py index 667bf977314917..d77e481c81d598 100644 --- a/Lib/test/test_flufl.py +++ b/Lib/test/test_flufl.py @@ -47,6 +47,19 @@ def test_barry_as_bdfl_look_ma_with_no_compiler_flags(self): self.assertEqual(cm.exception.lineno, 1) self.assertEqual(cm.exception.offset, len(code) - 4) + def test_barry_as_bdfl_relative_import(self): + code = "from .__future__ import barry_as_FLUFL;2 {0} 3" + compile(code.format('!='), '', 'exec') + with self.assertRaises(SyntaxError) as cm: + compile(code.format('<>'), '', 'exec') + self.assertRegex(str(cm.exception), "") + self.assertIn('2 <> 3', cm.exception.text) + self.assertEqual(cm.exception.filename, '') + self.assertEqual(cm.exception.lineno, 1) + self.assertEqual(cm.exception.offset, len(code) - 4) + + + if __name__ == '__main__': unittest.main()