Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node when

vstinner · vstinner · commit f9827ea61859 · 2015-11-06T17:01:48.000+01:00
compiling AST from Python objects.
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
@@ -585,8 +585,9 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq *
 arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq *
                            kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg,
                            asdl_seq * defaults, PyArena *arena);
-#define arg(a0, a1, a2) _Py_arg(a0, a1, a2)
-arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena);
+#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4)
+arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset,
+               PyArena *arena);
 #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2)
 keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena);
 #define alias(a0, a1, a2) _Py_alias(a0, a1, a2)
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -11,6 +11,9 @@ Release date: TBA
 Core and Builtins
 -----------------
 
+- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node
+  when compiling AST from Python objects.
+
 - Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()
   and eval() are passed bytes-like objects.  These objects are not
   necessarily terminated by a null byte, but the functions assumed they were.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
@@ -275,7 +275,9 @@ def emit_function(self, name, ctype, args, attrs, union=True):
 
     def visitProduct(self, prod, name):
         self.emit_function(name, get_c_type(name),
-                           self.get_args(prod.fields), [], union=False)
+                           self.get_args(prod.fields),
+                           self.get_args(prod.attributes),
+                           union=False)
 
 
 class FunctionVisitor(PrototypeVisitor):
@@ -329,7 +331,8 @@ def emit(s, depth=0, reflow=True):
             self.emit(s, depth, reflow)
         for argtype, argname, opt in args:
             emit("p->%s = %s;" % (argname, argname), 1)
-        assert not attrs
+        for argtype, argname, opt in attrs:
+            emit("p->%s = %s;" % (argname, argname), 1)
 
 
 class PickleVisitor(EmitVisitor):
@@ -452,10 +455,15 @@ def visitProduct(self, prod, name):
         self.emit("PyObject* tmp = NULL;", 1)
         for f in prod.fields:
             self.visitFieldDeclaration(f, name, prod=prod, depth=1)
+        for a in prod.attributes:
+            self.visitFieldDeclaration(a, name, prod=prod, depth=1)
         self.emit("", 0)
         for f in prod.fields:
             self.visitField(f, name, prod=prod, depth=1)
+        for a in prod.attributes:
+            self.visitField(a, name, prod=prod, depth=1)
         args = [f.name for f in prod.fields]
+        args.extend([a.name for a in prod.attributes])
         self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
         self.emit("return 0;", 1)
         self.emit("failed:", 0)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
@@ -2372,7 +2372,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq *
 }
 
 arg_ty
-arg(identifier arg, expr_ty annotation, PyArena *arena)
+arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena
+    *arena)
 {
     arg_ty p;
     if (!arg) {
@@ -2385,6 +2386,8 @@ arg(identifier arg, expr_ty annotation, PyArena *arena)
         return NULL;
     p->arg = arg;
     p->annotation = annotation;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
     return p;
 }
 
@@ -7086,6 +7089,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     PyObject* tmp = NULL;
     identifier arg;
     expr_ty annotation;
+    int lineno;
+    int col_offset;
 
     if (_PyObject_HasAttrId(obj, &PyId_arg)) {
         int res;
@@ -7108,7 +7113,29 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena)
     } else {
         annotation = NULL;
     }
-    *out = arg(arg, annotation, arena);
+    if (_PyObject_HasAttrId(obj, &PyId_lineno)) {
+        int res;
+        tmp = _PyObject_GetAttrId(obj, &PyId_lineno);
+        if (tmp == NULL) goto failed;
+        res = obj2ast_int(tmp, &lineno, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg");
+        return 1;
+    }
+    if (_PyObject_HasAttrId(obj, &PyId_col_offset)) {
+        int res;
+        tmp = _PyObject_GetAttrId(obj, &PyId_col_offset);
+        if (tmp == NULL) goto failed;
+        res = obj2ast_int(tmp, &col_offset, arena);
+        if (res != 0) goto failed;
+        Py_CLEAR(tmp);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg");
+        return 1;
+    }
+    *out = arg(arg, annotation, lineno, col_offset, arena);
     return 0;
 failed:
     Py_XDECREF(tmp);
diff --git a/Python/ast.c b/Python/ast.c
@@ -1173,11 +1173,9 @@ ast_for_arg(struct compiling *c, const node *n)
             return NULL;
     }
 
-    ret = arg(name, annotation, c->c_arena);
+    ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena);
     if (!ret)
         return NULL;
-    ret->lineno = LINENO(n);
-    ret->col_offset = n->n_col_offset;
     return ret;
 }
 
@@ -1233,11 +1231,10 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start,
                     goto error;
                 if (forbidden_name(c, argname, ch, 0))
                     goto error;
-                arg = arg(argname, annotation, c->c_arena);
+                arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset,
+                          c->c_arena);
                 if (!arg)
                     goto error;
-                arg->lineno = LINENO(ch);
-                arg->col_offset = ch->n_col_offset;
                 asdl_seq_SET(kwonlyargs, j++, arg);
                 i += 2; /* the name and the comma */
                 break;
