Fix 'gather' rules in the python parser generator (GH-22021)

pablogsal · web-flow · commit e55a0e971b4b · 2020-09-03T15:29:55.000+01:00
Currently, empty sequences in gather rules make the conditional for
gather rules fail as empty sequences evaluate as "False". We need to
explicitly check for "None" (the failure condition) to avoid false
negatives.
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
@@ -74,7 +74,7 @@ def test_typed_rules(self) -> None:
             "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))"
         )
 
-    def test_repeat_with_separator_rules(self) -> None:
+    def test_gather(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
@@ -85,6 +85,20 @@ def test_repeat_with_separator_rules(self) -> None:
             "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
         ))
         self.assertEqual(str(rules["thing"]), "thing: NUMBER")
+        parser_class = make_parser(grammar)
+        node = parse_string("42\n", parser_class)
+        assert node == [
+            [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
+            TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
+        ]
+        node = parse_string("1, 2\n", parser_class)
+        assert node == [
+            [
+                [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")],
+                [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")],
+            ],
+            TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"),
+        ]
 
     def test_expr_grammar(self) -> None:
         grammar = """
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
@@ -217,6 +217,9 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None:
                     else:
                         self.print("and")
                     self.visit(item)
+                    if is_gather:
+                        self.print("is not None")
+
             self.print("):")
             with self.indent():
                 action = node.action
