Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4ab362c

Browse files
isidenticalisidentical
authored
bpo-39638: Keep ASDL signatures in the AST nodes (GH-18515)
1 parent 5b66ec1 commit 4ab362c

7 files changed

Lines changed: 343 additions & 132 deletions

File tree

‎Doc/whatsnew/3.9.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,9 @@ be used to unparse an :class:`ast.AST` object and produce a string with code
164164
that would produce an equivalent :class:`ast.AST` object when parsed.
165165
(Contributed by Pablo Galindo and Batuhan Taskaya in :issue:`38870`.)
166166

167+
Added docstrings to AST nodes that contains the ASDL signature used to
168+
construct that node. (Contributed by Batuhan Taskaya in :issue:`39638`.)
169+
167170
asyncio
168171
-------
169172

‎Lib/test/test_asdl_parser.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,10 @@ def test_product(self):
6363
def test_attributes(self):
6464
stmt = self.types['stmt']
6565
self.assertEqual(len(stmt.attributes), 4)
66-
self.assertEqual(str(stmt.attributes[0]), 'Field(int, lineno)')
67-
self.assertEqual(str(stmt.attributes[1]), 'Field(int, col_offset)')
68-
self.assertEqual(str(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
69-
self.assertEqual(str(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
66+
self.assertEqual(repr(stmt.attributes[0]), 'Field(int, lineno)')
67+
self.assertEqual(repr(stmt.attributes[1]), 'Field(int, col_offset)')
68+
self.assertEqual(repr(stmt.attributes[2]), 'Field(int, end_lineno, opt=True)')
69+
self.assertEqual(repr(stmt.attributes[3]), 'Field(int, end_col_offset, opt=True)')
7070

7171
def test_constructor_fields(self):
7272
ehandler = self.types['excepthandler']

‎Lib/test/test_ast.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -636,6 +636,16 @@ def test_issue39579_dotted_name_end_col_offset(self):
636636
attr_b = tree.body[0].decorator_list[0].value
637637
self.assertEqual(attr_b.end_col_offset, 4)
638638

639+
def test_ast_asdl_signature(self):
640+
self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)")
641+
self.assertEqual(ast.GtE.__doc__, "GtE")
642+
self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)")
643+
self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn")
644+
expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()]
645+
expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}"
646+
self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions)
647+
648+
639649
class ASTHelpers_Test(unittest.TestCase):
640650
maxDiff = None
641651

‎Misc/NEWS.d/next/Library/2020-02-15-20-38-11.bpo-39638.wm_RS3.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Keep ASDL signatures in the docstrings for ``AST`` nodes. Patch by Batuhan
2+
Taskaya

‎Parser/asdl.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ def __init__(self, type, name=None, seq=False, opt=False):
7272
self.seq = seq
7373
self.opt = opt
7474

75+
def __str__(self):
76+
if self.seq:
77+
extra = "*"
78+
elif self.opt:
79+
extra = "?"
80+
else:
81+
extra = ""
82+
83+
return "{}{} {}".format(self.type, extra, self.name)
84+
7585
def __repr__(self):
7686
if self.seq:
7787
extra = ", seq=True"

‎Parser/asdl_c.py‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ def reflow_lines(s, depth):
6060
lines.append(padding + cur)
6161
return lines
6262

63+
def reflow_c_string(s, depth):
64+
return '"%s"' % s.replace('\n', '\\n"\n%s"' % (' ' * depth * TABSIZE))
65+
6366
def is_simple(sum):
6467
"""Return True if a sum is a simple.
6568
@@ -71,6 +74,21 @@ def is_simple(sum):
7174
return False
7275
return True
7376

77+
def asdl_of(name, obj):
78+
if isinstance(obj, asdl.Product) or isinstance(obj, asdl.Constructor):
79+
fields = ", ".join(map(str, obj.fields))
80+
if fields:
81+
fields = "({})".format(fields)
82+
return "{}{}".format(name, fields)
83+
else:
84+
if is_simple(obj):
85+
types = " | ".join(type.name for type in obj.types)
86+
else:
87+
sep = "\n{}| ".format(" " * (len(name) + 1))
88+
types = sep.join(
89+
asdl_of(type.name, type) for type in obj.types
90+
)
91+
return "{} = {}".format(name, types)
7492

7593
class EmitVisitor(asdl.VisitorBase):
7694
"""Visit that emits lines"""
@@ -764,7 +782,7 @@ def visitModule(self, mod):
764782
};
765783
766784
static PyObject *
767-
make_type(const char *type, PyObject* base, const char* const* fields, int num_fields)
785+
make_type(const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc)
768786
{
769787
PyObject *fnames, *result;
770788
int i;
@@ -778,11 +796,12 @@ def visitModule(self, mod):
778796
}
779797
PyTuple_SET_ITEM(fnames, i, field);
780798
}
781-
result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
799+
result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOOOs}",
782800
type, base,
783801
astmodulestate_global->_fields, fnames,
784802
astmodulestate_global->__module__,
785-
astmodulestate_global->_ast);
803+
astmodulestate_global->_ast,
804+
astmodulestate_global->__doc__, doc);
786805
Py_DECREF(fnames);
787806
return result;
788807
}
@@ -947,8 +966,9 @@ def visitProduct(self, prod, name):
947966
fields = name+"_fields"
948967
else:
949968
fields = "NULL"
950-
self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d);' %
969+
self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d,' %
951970
(name, name, fields, len(prod.fields)), 1)
971+
self.emit('%s);' % reflow_c_string(asdl_of(name, prod), 2), 2, reflow=False)
952972
self.emit("if (!state->%s_type) return 0;" % name, 1)
953973
self.emit_type("AST_type")
954974
self.emit_type("%s_type" % name)
@@ -961,8 +981,9 @@ def visitProduct(self, prod, name):
961981
self.emit_defaults(name, prod.attributes, 1)
962982

963983
def visitSum(self, sum, name):
964-
self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' %
984+
self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0,' %
965985
(name, name), 1)
986+
self.emit('%s);' % reflow_c_string(asdl_of(name, sum), 2), 2, reflow=False)
966987
self.emit_type("%s_type" % name)
967988
self.emit("if (!state->%s_type) return 0;" % name, 1)
968989
if sum.attributes:
@@ -980,8 +1001,9 @@ def visitConstructor(self, cons, name, simple):
9801001
fields = cons.name+"_fields"
9811002
else:
9821003
fields = "NULL"
983-
self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d);' %
1004+
self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d,' %
9841005
(cons.name, cons.name, name, fields, len(cons.fields)), 1)
1006+
self.emit('%s);' % reflow_c_string(asdl_of(cons.name, cons), 2), 2, reflow=False)
9851007
self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
9861008
self.emit_type("%s_type" % cons.name)
9871009
self.emit_defaults(cons.name, cons.fields, 1)
@@ -1279,8 +1301,15 @@ def generate_module_def(f, mod):
12791301
visitor.visit(mod)
12801302
visitor_list.add(visitor)
12811303

1282-
state_strings = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
1283-
module_state = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
1304+
state_strings = {
1305+
"_ast",
1306+
"_fields",
1307+
"__doc__",
1308+
"__dict__",
1309+
"__module__",
1310+
"_attributes",
1311+
}
1312+
module_state = state_strings.copy()
12841313
for visitor in visitor_list:
12851314
for identifier in visitor.identifiers:
12861315
module_state.add(identifier)

0 commit comments

Comments
 (0)