From 16989daa2ab502017bb4f89efb1441b720868ec8 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Mon, 14 Dec 2020 07:58:36 +0000 Subject: [PATCH 1/4] Add positional only args support to lib2to3 pgen2. This adds 3.8's PEP-570 support to lib2to3's pgen2. lib2to3, while being deprecated is still used by things to parse all versions of Python code today. We need it to support parsing modern 3.8 and 3.9 constructs. --- Lib/lib2to3/Grammar.txt | 52 ++++++++++++++++++++++++++++---- Lib/lib2to3/tests/test_parser.py | 22 ++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index e007dc188af503..fa7b15061d941c 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -18,15 +18,55 @@ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' -typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tname] (',' tname ['=' test])* [',' ['**' tname [',']]] | '**' tname [',']) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + +# The following definition for typedarglist is equivalent to this set of rules: +# +# arguments = argument (',' argument)* +# argument = tfpdef ['=' test] +# kwargs = '**' tname [','] +# args = '*' [tname] +# kwonly_kwargs = (',' argument)* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# typedargslist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# typedarglist = arguments ',' '/' [',' [typedargslist_no_posonly]])|(typedargslist_no_posonly)" +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +typedargslist: tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [ + ',' [((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [','])] + ] | ((tfpdef ['=' test] ',')* ('*' [tname] (',' tname ['=' test])* + [',' ['**' tname [',']]] | '**' tname [',']) + | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + tname: NAME [':' test] tfpdef: tname | '(' tfplist ')' tfplist: tfpdef (',' tfpdef)* [','] -varargslist: ((vfpdef ['=' test] ',')* - ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]] | '**' vname [',']) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + +# The following definition for varargslist is equivalent to this set of rules: +# +# arguments = argument (',' argument )* +# argument = vfpdef ['=' test] +# kwargs = '**' vname [','] +# args = '*' [vname] +# kwonly_kwargs = (',' argument )* [',' [kwargs]] +# args_kwonly_kwargs = args kwonly_kwargs | kwargs +# poskeyword_args_kwonly_kwargs = arguments [',' [args_kwonly_kwargs]] +# vararglist_no_posonly = poskeyword_args_kwonly_kwargs | args_kwonly_kwargs +# varargslist = arguments ',' '/' [','[(vararglist_no_posonly)]] | (vararglist_no_posonly) +# +# It needs to be fully expanded to allow our LL(1) parser to work on it. + +varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ + ((vfpdef ['=' test] ',')* ('*' [vname] (',' vname ['=' test])* + [',' ['**' vname [',']]] | '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + ]] | ((vfpdef ['=' test] ',')* + ('*' [vname] (',' vname ['=' test])* [',' ['**' vname [',']]]| '**' vname [',']) + | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + vname: NAME vfpdef: vname | '(' vfplist ')' vfplist: vfpdef (',' vfpdef)* [','] diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index ba2bb787332edd..13f05b4491b9cf 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -630,6 +630,7 @@ def test_multiline_str_literals(self): class TestNamedAssignments(GrammarTest): + """Also known as the walrus operator.""" def test_named_assignment_if(self): driver.parse_string("if f := x(): pass\n") @@ -644,6 +645,27 @@ def test_named_assignment_listcomp(self): driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n") +class TestPositionalOnlyArgs(GrammarTest): + + def test_one_pos_only_arg(self): + driver.parse_string("def one_pos_only_arg(a, /): pass\n") + + def test_all_markers(self): + driver.parse_string( + "def all_markers(a, b=2, /, c, d=4, *, e=5, f): pass\n") + + def test_all_with_args_and_kwargs(self): + driver.parse_string( + """def all_markers_with_args_and_kwargs( + aa, b, /, _cc, d, *args, e, f_f, **kwargs, + ): + pass\n""") + + def test_lambda_soup(self): + driver.parse_string( + "lambda a, b, /, c, d, *args, e, f, **kw: kw\n") + + class TestPickleableException(unittest.TestCase): def test_ParseError(self): err = ParseError('msg', 2, None, (1, 'context')) From 093b468f953b1830eabca5541304a35ad7681268 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Mon, 14 Dec 2020 08:14:45 +0000 Subject: [PATCH 2/4] Add tests for complex *expr and **expr's. --- Lib/lib2to3/tests/test_parser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 13f05b4491b9cf..5e270ce643ea6f 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -272,6 +272,12 @@ def test_dict_display_1(self): def test_dict_display_2(self): self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") + def test_complex_star_expression(self): + self.validate("func(* [] or [1])") + + def test_complex_double_star_expression(self): + self.validate("func(**{1: 3} if False else {x: x for x in range(3)})") + def test_argument_unpacking_1(self): self.validate("""f(a, *b, *c, d)""") From 6e728b057a989ddedaaa60810bc47128c00510e8 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Mon, 14 Dec 2020 08:24:00 +0000 Subject: [PATCH 3/4] NEWS via blurb --- .../next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst diff --git a/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst b/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst new file mode 100644 index 00000000000000..5678d8c595ca08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-12-14-08-23-57.bpo-36541.qdEtZv.rst @@ -0,0 +1,2 @@ +Fixed lib2to3.pgen2 to be able to parse PEP-570 positional only argument +syntax. From 5d7321ce0a3aac20d5c437c3d182b074ee919cb2 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Mon, 14 Dec 2020 16:39:33 +0000 Subject: [PATCH 4/4] Add another test case. --- Lib/lib2to3/tests/test_parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 5e270ce643ea6f..d5db66b9b1e7b9 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -671,6 +671,9 @@ def test_lambda_soup(self): driver.parse_string( "lambda a, b, /, c, d, *args, e, f, **kw: kw\n") + def test_only_positional_or_keyword(self): + driver.parse_string("def func(a,b,/,*,g,e=3): pass\n") + class TestPickleableException(unittest.TestCase): def test_ParseError(self):