#! /usr/bin/env python

"""Generate C code from an ASDL description."""

import sys

import textwrap

import types

from argparse import ArgumentParser

from contextlib import contextmanager

from pathlib import Path

import asdl

TABSIZE = 4

MAX_COL = 80

AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n"

builtin_type_to_c_type = {

"identifier": "PyUnicode_Type",

"string": "PyUnicode_Type",

"int": "PyLong_Type",

"constant": "PyBaseObject_Type",

}

def get_c_type(name):

"""Return a string for the C name of the type.

This function special cases the default types provided by asdl.

"""

if name in asdl.builtin_types:

return name

else:

return "%s_ty" % name

def reflow_lines(s, depth):

"""Reflow the line s indented depth tabs.

Return a sequence of lines where no line extends beyond MAX_COL

when properly indented. The first line is properly indented based

exclusively on depth * TABSIZE. All following lines -- these are

the reflowed lines generated by this function -- start at the same

column as the first character beyond the opening { in the first

line.

"""

size = MAX_COL - depth * TABSIZE

if len(s) < size:

return [s]

lines = []

cur = s

padding = ""

while len(cur) > size:

i = cur.rfind(' ', 0, size)

# XXX this should be fixed for real

if i == -1 and 'GeneratorExp' in cur:

i = size + 3

assert i != -1, "Impossible line %d to reflow: %r" % (size, s)

lines.append(padding + cur[:i])

if len(lines) == 1:

# find new size based on brace

j = cur.find('{', 0, i)

if j >= 0:

j += 2 # account for the brace and the space after it

size -= j

padding = " " * j

else:

j = cur.find('(', 0, i)

if j >= 0:

j += 1 # account for the paren (no space after it)

size -= j

padding = " " * j

cur = cur[i+1:]

else:

lines.append(padding + cur)

return lines

def reflow_c_string(s, depth):

return '"%s"' % s.replace('\n', '\\n"\n%s"' % (' ' * depth * TABSIZE))

def is_simple(sum_type):

"""Return True if a sum is a simple.

A sum is simple if its types have no fields and itself

doesn't have any attributes. Instances of these types are

cached at C level, and they act like singletons when propagating

parser generated nodes into Python level, e.g.

unaryop = Invert | Not | UAdd | USub

"""

return not (

sum_type.attributes or

any(constructor.fields for constructor in sum_type.types)

)

def asdl_of(name, obj):

if isinstance(obj, asdl.Product) or isinstance(obj, asdl.Constructor):

fields = ", ".join(map(str, obj.fields))

if fields:

fields = "({})".format(fields)

return "{}{}".format(name, fields)

else:

if is_simple(obj):

types = " | ".join(type.name for type in obj.types)

else:

sep = "\n{}| ".format(" " * (len(name) + 1))

types = sep.join(

asdl_of(type.name, type) for type in obj.types

)

return "{} = {}".format(name, types)

class EmitVisitor(asdl.VisitorBase):

"""Visit that emits lines"""

def __init__(self, file, metadata = None):

self.file = file

self._metadata = metadata

super(EmitVisitor, self).__init__()

def emit(self, s, depth, reflow=True):

# XXX reflow long lines?

if reflow:

lines = reflow_lines(s, depth)

else:

lines = [s]

for line in lines:

if line:

line = (" " * TABSIZE * depth) + line

self.file.write(line + "\n")

@property

def metadata(self):

if self._metadata is None:

raise ValueError(

"%s was expecting to be annnotated with metadata"

% type(self).__name__

)

return self._metadata

@metadata.setter

def metadata(self, value):

self._metadata = value

class MetadataVisitor(asdl.VisitorBase):

ROOT_TYPE = "AST"

def __init__(self, *args, **kwargs):

super().__init__(*args, **kwargs)

# Metadata:

# - simple_sums: Tracks the list of compound type

# names where all the constructors

# belonging to that type lack of any

# fields.

# - identifiers: All identifiers used in the AST declarations

# - singletons: List of all constructors that originates from

# simple sums.

# - types: List of all top level type names

#

self.metadata = types.SimpleNamespace(

simple_sums=set(),

identifiers=set(),

singletons=set(),

types={self.ROOT_TYPE},

)

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type):

self.visit(type.value, type.name)

def visitSum(self, sum, name):

self.metadata.types.add(name)

simple_sum = is_simple(sum)

if simple_sum:

self.metadata.simple_sums.add(name)

for constructor in sum.types:

if simple_sum:

self.metadata.singletons.add(constructor.name)

self.visitConstructor(constructor)

self.visitFields(sum.attributes)

def visitConstructor(self, constructor):

self.metadata.types.add(constructor.name)

self.visitFields(constructor.fields)

def visitProduct(self, product, name):

self.metadata.types.add(name)

self.visitFields(product.attributes)

self.visitFields(product.fields)

def visitFields(self, fields):

for field in fields:

self.visitField(field)

def visitField(self, field):

self.metadata.identifiers.add(field.name)

class TypeDefVisitor(EmitVisitor):

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type, depth=0):

self.visit(type.value, type.name, depth)

def visitSum(self, sum, name, depth):

if is_simple(sum):

self.simple_sum(sum, name, depth)

else:

self.sum_with_constructors(sum, name, depth)

def simple_sum(self, sum, name, depth):

enum = []

for i in range(len(sum.types)):

type = sum.types[i]

enum.append("%s=%d" % (type.name, i + 1))

enums = ", ".join(enum)

ctype = get_c_type(name)

s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)

self.emit(s, depth)

self.emit("", depth)

def sum_with_constructors(self, sum, name, depth):

ctype = get_c_type(name)

s = "typedef struct _%(name)s *%(ctype)s;" % locals()

self.emit(s, depth)

self.emit("", depth)

def visitProduct(self, product, name, depth):

ctype = get_c_type(name)

s = "typedef struct _%(name)s *%(ctype)s;" % locals()

self.emit(s, depth)

self.emit("", depth)

class SequenceDefVisitor(EmitVisitor):

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type, depth=0):

self.visit(type.value, type.name, depth)

def visitSum(self, sum, name, depth):

if is_simple(sum):

return

self.emit_sequence_constructor(name, depth)

def emit_sequence_constructor(self, name,depth):

ctype = get_c_type(name)

self.emit("""\

typedef struct {

_ASDL_SEQ_HEAD

%(ctype)s typed_elements[1];

} asdl_%(name)s_seq;""" % locals(), reflow=False, depth=depth)

self.emit("", depth)

self.emit("asdl_%(name)s_seq *_Py_asdl_%(name)s_seq_new(Py_ssize_t size, PyArena *arena);" % locals(), depth)

self.emit("", depth)

def visitProduct(self, product, name, depth):

self.emit_sequence_constructor(name, depth)

class StructVisitor(EmitVisitor):

"""Visitor to generate typedefs for AST."""

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type, depth=0):

self.visit(type.value, type.name, depth)

def visitSum(self, sum, name, depth):

if not is_simple(sum):

self.sum_with_constructors(sum, name, depth)

def sum_with_constructors(self, sum, name, depth):

def emit(s, depth=depth):

self.emit(s % sys._getframe(1).f_locals, depth)

enum = []

for i in range(len(sum.types)):

type = sum.types[i]

enum.append("%s_kind=%d" % (type.name, i + 1))

emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")

emit("struct _%(name)s {")

emit("enum _%(name)s_kind kind;", depth + 1)

emit("union {", depth + 1)

for t in sum.types:

self.visit(t, depth + 2)

emit("} v;", depth + 1)

for field in sum.attributes:

# rudimentary attribute handling

type = str(field.type)

assert type in asdl.builtin_types, type

emit("%s %s;" % (type, field.name), depth + 1);

emit("};")

emit("")

def visitConstructor(self, cons, depth):

if cons.fields:

self.emit("struct {", depth)

for f in cons.fields:

self.visit(f, depth + 1)

self.emit("} %s;" % cons.name, depth)

self.emit("", depth)

def visitField(self, field, depth):

# XXX need to lookup field.type, because it might be something

# like a builtin...

ctype = get_c_type(field.type)

name = field.name

if field.seq:

if field.type in self.metadata.simple_sums:

self.emit("asdl_int_seq *%(name)s;" % locals(), depth)

else:

_type = field.type

self.emit("asdl_%(_type)s_seq *%(name)s;" % locals(), depth)

else:

self.emit("%(ctype)s %(name)s;" % locals(), depth)

def visitProduct(self, product, name, depth):

self.emit("struct _%(name)s {" % locals(), depth)

for f in product.fields:

self.visit(f, depth + 1)

for field in product.attributes:

# rudimentary attribute handling

type = str(field.type)

assert type in asdl.builtin_types, type

self.emit("%s %s;" % (type, field.name), depth + 1);

self.emit("};", depth)

self.emit("", depth)

def ast_func_name(name):

return f"_PyAST_{name}"

class PrototypeVisitor(EmitVisitor):

"""Generate function prototypes for the .h file"""

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type):

self.visit(type.value, type.name)

def visitSum(self, sum, name):

if is_simple(sum):

pass # XXX

else:

for t in sum.types:

self.visit(t, name, sum.attributes)

def get_args(self, fields):

"""Return list of C argument info, one for each field.

Argument info is 3-tuple of a C type, variable name, and flag

that is true if type can be NULL.

"""

args = []

unnamed = {}

for f in fields:

if f.name is None:

name = f.type

c = unnamed[name] = unnamed.get(name, 0) + 1

if c > 1:

name = "name%d" % (c - 1)

else:

name = f.name

# XXX should extend get_c_type() to handle this

if f.seq:

if f.type in self.metadata.simple_sums:

ctype = "asdl_int_seq *"

else:

ctype = f"asdl_{f.type}_seq *"

else:

ctype = get_c_type(f.type)

args.append((ctype, name, f.opt or f.seq))

return args

def visitConstructor(self, cons, type, attrs):

args = self.get_args(cons.fields)

attrs = self.get_args(attrs)

ctype = get_c_type(type)

self.emit_function(cons.name, ctype, args, attrs)

def emit_function(self, name, ctype, args, attrs, union=True):

args = args + attrs

if args:

argstr = ", ".join(["%s %s" % (atype, aname)

for atype, aname, opt in args])

argstr += ", PyArena *arena"

else:

argstr = "PyArena *arena"

self.emit("%s %s(%s);" % (ctype, ast_func_name(name), argstr), False)

def visitProduct(self, prod, name):

self.emit_function(name, get_c_type(name),

self.get_args(prod.fields),

self.get_args(prod.attributes),

union=False)

class FunctionVisitor(PrototypeVisitor):

"""Visitor to generate constructor functions for AST."""

def emit_function(self, name, ctype, args, attrs, union=True):

def emit(s, depth=0, reflow=True):

self.emit(s, depth, reflow)

argstr = ", ".join(["%s %s" % (atype, aname)

for atype, aname, opt in args + attrs])

if argstr:

argstr += ", PyArena *arena"

else:

argstr = "PyArena *arena"

self.emit("%s" % ctype, 0)

emit("%s(%s)" % (ast_func_name(name), argstr))

emit("{")

emit("%s p;" % ctype, 1)

for argtype, argname, opt in args:

if not opt and argtype != "int":

emit("if (!%s) {" % argname, 1)

emit("PyErr_SetString(PyExc_ValueError,", 2)

msg = "field '%s' is required for %s" % (argname, name)

emit(' "%s");' % msg,

2, reflow=False)

emit('return NULL;', 2)

emit('}', 1)

emit("p = (%s)_PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);

emit("if (!p)", 1)

emit("return NULL;", 2)

if union:

self.emit_body_union(name, args, attrs)

else:

self.emit_body_struct(name, args, attrs)

emit("return p;", 1)

emit("}")

emit("")

def emit_body_union(self, name, args, attrs):

def emit(s, depth=0, reflow=True):

self.emit(s, depth, reflow)

emit("p->kind = %s_kind;" % name, 1)

for argtype, argname, opt in args:

emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)

for argtype, argname, opt in attrs:

emit("p->%s = %s;" % (argname, argname), 1)

def emit_body_struct(self, name, args, attrs):

def emit(s, depth=0, reflow=True):

self.emit(s, depth, reflow)

for argtype, argname, opt in args:

emit("p->%s = %s;" % (argname, argname), 1)

for argtype, argname, opt in attrs:

emit("p->%s = %s;" % (argname, argname), 1)

class PickleVisitor(EmitVisitor):

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type):

self.visit(type.value, type.name)

def visitSum(self, sum, name):

pass

def visitProduct(self, sum, name):

pass

def visitConstructor(self, cons, name):

pass

def visitField(self, sum):

pass

class Obj2ModPrototypeVisitor(PickleVisitor):

def visitProduct(self, prod, name):

code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);"

self.emit(code % (name, get_c_type(name)), 0)

visitSum = visitProduct

class Obj2ModVisitor(PickleVisitor):

attribute_special_defaults = {

"end_lineno": "lineno",

"end_col_offset": "col_offset",

}

@contextmanager

def recursive_call(self, node, level):

self.emit('if (_Py_EnterRecursiveCall(" while traversing \'%s\' node")) {' % node, level, reflow=False)

self.emit('goto failed;', level + 1)

self.emit('}', level)

yield

self.emit('_Py_LeaveRecursiveCall();', level)

def funcHeader(self, name):

ctype = get_c_type(name)

self.emit("int", 0)

self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)

self.emit("{", 0)

self.emit("int isinstance;", 1)

self.emit("", 0)

def sumTrailer(self, name, add_label=False):

self.emit("", 0)

# there's really nothing more we can do if this fails ...

error = "expected some sort of %s, but got %%R" % name

format = "PyErr_Format(PyExc_TypeError, \"%s\", obj);"

self.emit(format % error, 1, reflow=False)

if add_label:

self.emit("failed:", 1)

self.emit("Py_XDECREF(tmp);", 1)

self.emit("return -1;", 1)

self.emit("}", 0)

self.emit("", 0)

def simpleSum(self, sum, name):

self.funcHeader(name)

for t in sum.types:

line = ("isinstance = PyObject_IsInstance(obj, "

"state->%s_type);")

self.emit(line % (t.name,), 1)

self.emit("if (isinstance == -1) {", 1)

self.emit("return -1;", 2)

self.emit("}", 1)

self.emit("if (isinstance) {", 1)

self.emit("*out = %s;" % t.name, 2)

self.emit("return 0;", 2)

self.emit("}", 1)

self.sumTrailer(name)

def buildArgs(self, fields):

return ", ".join(fields + ["arena"])

def complexSum(self, sum, name):

self.funcHeader(name)

self.emit("PyObject *tmp = NULL;", 1)

self.emit("PyObject *tp;", 1)

for a in sum.attributes:

self.visitAttributeDeclaration(a, name, sum=sum)

self.emit("", 0)

# XXX: should we only do this for 'expr'?

self.emit("if (obj == Py_None) {", 1)

self.emit("*out = NULL;", 2)

self.emit("return 0;", 2)

self.emit("}", 1)

for a in sum.attributes:

self.visitField(a, name, sum=sum, depth=1)

for t in sum.types:

self.emit("tp = state->%s_type;" % (t.name,), 1)

self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1)

self.emit("if (isinstance == -1) {", 1)

self.emit("return -1;", 2)

self.emit("}", 1)

self.emit("if (isinstance) {", 1)

for f in t.fields:

self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)

self.emit("", 0)

for f in t.fields:

self.visitField(f, t.name, sum=sum, depth=2)

args = [f.name for f in t.fields] + [a.name for a in sum.attributes]

self.emit("*out = %s(%s);" % (ast_func_name(t.name), self.buildArgs(args)), 2)

self.emit("if (*out == NULL) goto failed;", 2)

self.emit("return 0;", 2)

self.emit("}", 1)

self.sumTrailer(name, True)

def visitAttributeDeclaration(self, a, name, sum=sum):

ctype = get_c_type(a.type)

self.emit("%s %s;" % (ctype, a.name), 1)

def visitSum(self, sum, name):

if is_simple(sum):

self.simpleSum(sum, name)

else:

self.complexSum(sum, name)

def visitProduct(self, prod, name):

ctype = get_c_type(name)

self.emit("int", 0)

self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)

self.emit("{", 0)

self.emit("PyObject* tmp = NULL;", 1)

for f in prod.fields:

self.visitFieldDeclaration(f, name, prod=prod, depth=1)

for a in prod.attributes:

self.visitFieldDeclaration(a, name, prod=prod, depth=1)

self.emit("", 0)

for f in prod.fields:

self.visitField(f, name, prod=prod, depth=1)

for a in prod.attributes:

self.visitField(a, name, prod=prod, depth=1)

args = [f.name for f in prod.fields]

args.extend([a.name for a in prod.attributes])

self.emit("*out = %s(%s);" % (ast_func_name(name), self.buildArgs(args)), 1)

self.emit("if (*out == NULL) goto failed;", 1)

self.emit("return 0;", 1)

self.emit("failed:", 0)

self.emit("Py_XDECREF(tmp);", 1)

self.emit("return -1;", 1)

self.emit("}", 0)

self.emit("", 0)

def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):

ctype = get_c_type(field.type)

if field.seq:

if self.isSimpleType(field):

self.emit("asdl_int_seq* %s;" % field.name, depth)

else:

_type = field.type

self.emit(f"asdl_{field.type}_seq* {field.name};", depth)

else:

ctype = get_c_type(field.type)

self.emit("%s %s;" % (ctype, field.name), depth)

def isNumeric(self, field):

return get_c_type(field.type) in ("int", "bool")

def isSimpleType(self, field):

return field.type in self.metadata.simple_sums or self.isNumeric(field)

def visitField(self, field, name, sum=None, prod=None, depth=0):

ctype = get_c_type(field.type)

line = "if (PyObject_GetOptionalAttr(obj, state->%s, &tmp) < 0) {"

self.emit(line % field.name, depth)

self.emit("return -1;", depth+1)

self.emit("}", depth)

if field.seq:

self.emit("if (tmp == NULL) {", depth)

self.emit("tmp = PyList_New(0);", depth+1)

self.emit("if (tmp == NULL) {", depth+1)

self.emit("return -1;", depth+2)

self.emit("}", depth+1)

self.emit("}", depth)

self.emit("{", depth)

else:

if not field.opt:

self.emit("if (tmp == NULL) {", depth)

message = "required field \\\"%s\\\" missing from %s" % (field.name, name)

format = "PyErr_SetString(PyExc_TypeError, \"%s\");"

self.emit(format % message, depth+1, reflow=False)

self.emit("return -1;", depth+1)

else:

self.emit("if (tmp == NULL || tmp == Py_None) {", depth)

self.emit("Py_CLEAR(tmp);", depth+1)

if self.isNumeric(field):

if field.name in self.attribute_special_defaults:

self.emit(

"%s = %s;" % (field.name, self.attribute_special_defaults[field.name]),

depth+1,

)

else:

self.emit("%s = 0;" % field.name, depth+1)

elif not self.isSimpleType(field):

self.emit("%s = NULL;" % field.name, depth+1)

else:

raise TypeError("could not determine the default value for %s" % field.name)

self.emit("}", depth)

self.emit("else {", depth)

self.emit("int res;", depth+1)

if field.seq:

self.emit("Py_ssize_t len;", depth+1)

self.emit("Py_ssize_t i;", depth+1)

self.emit("if (!PyList_Check(tmp)) {", depth+1)

self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "

"be a list, not a %%.200s\", _PyType_Name(Py_TYPE(tmp)));" %

(name, field.name),

depth+2, reflow=False)

self.emit("goto failed;", depth+2)

self.emit("}", depth+1)

self.emit("len = PyList_GET_SIZE(tmp);", depth+1)

if self.isSimpleType(field):

self.emit("%s = _Py_asdl_int_seq_new(len, arena);" % field.name, depth+1)

else:

self.emit("%s = _Py_asdl_%s_seq_new(len, arena);" % (field.name, field.type), depth+1)

self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)

self.emit("for (i = 0; i < len; i++) {", depth+1)

self.emit("%s val;" % ctype, depth+2)

self.emit("PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i));", depth+2)

with self.recursive_call(name, depth+2):

self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" %

field.type, depth+2, reflow=False)

self.emit("Py_DECREF(tmp2);", depth+2)

self.emit("if (res != 0) goto failed;", depth+2)

self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2)

self.emit("PyErr_SetString(PyExc_RuntimeError, \"%s field \\\"%s\\\" "

"changed size during iteration\");" %

(name, field.name),

depth+3, reflow=False)

self.emit("goto failed;", depth+3)

self.emit("}", depth+2)

self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2)

self.emit("}", depth+1)

else:

with self.recursive_call(name, depth+1):

self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" %

(field.type, field.name), depth+1)

self.emit("if (res != 0) goto failed;", depth+1)

self.emit("Py_CLEAR(tmp);", depth+1)

self.emit("}", depth)

class SequenceConstructorVisitor(EmitVisitor):

def visitModule(self, mod):

for dfn in mod.dfns:

self.visit(dfn)

def visitType(self, type):

self.visit(type.value, type.name)

def visitProduct(self, prod, name):

self.emit_sequence_constructor(name, get_c_type(name))

def visitSum(self, sum, name):

if not is_simple(sum):

self.emit_sequence_constructor(name, get_c_type(name))

def emit_sequence_constructor(self, name, type):

self.emit(f"GENERATE_ASDL_SEQ_CONSTRUCTOR({name}, {type})", depth=0)

class PyTypesDeclareVisitor(PickleVisitor):

def visitProduct(self, prod, name):

self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, void*);" % name, 0)

if prod.attributes:

self.emit("static const char * const %s_attributes[] = {" % name, 0)

for a in prod.attributes:

self.emit('"%s",' % a.name, 1)

self.emit("};", 0)

if prod.fields:

self.emit("static const char * const %s_fields[]={" % name,0)

for f in prod.fields:

self.emit('"%s",' % f.name, 1)

self.emit("};", 0)

def visitSum(self, sum, name):

if sum.attributes:

self.emit("static const char * const %s_attributes[] = {" % name, 0)

for a in sum.attributes:

self.emit('"%s",' % a.name, 1)

self.emit("};", 0)

ptype = "void*"

if is_simple(sum):

ptype = get_c_type(name)

self.emit("static PyObject* ast2obj_%s(struct ast_state *state, struct validator *vstate, %s);" % (name, ptype), 0)

for t in sum.types:

self.visitConstructor(t, name)

def visitConstructor(self, cons, name):

if cons.fields:

self.emit("static const char * const %s_fields[]={" % cons.name, 0)

for t in cons.fields:

self.emit('"%s",' % t.name, 1)

self.emit("};",0)

class AnnotationsVisitor(PickleVisitor):

def visitModule(self, mod):

self.file.write(textwrap.dedent('''

static int

add_ast_annotations(struct ast_state *state)

{

bool cond;

'''))

for dfn in mod.dfns:

self.visit(dfn)

self.file.write(textwrap.dedent('''

return 1;

}

'''))

def visitProduct(self, prod, name):

self.emit_annotations(name, prod.fields)

def visitSum(self, sum, name):

for t in sum.types:

self.visitConstructor(t, name)

def visitConstructor(self, cons, name):

self.emit_annotations(cons.name, cons.fields)

def emit_annotations(self, name, fields):

self.emit(f"PyObject *{name}_annotations = PyDict_New();", 1)

self.emit(f"if (!{name}_annotations) return 0;", 1)

for field in fields:

self.emit("{", 1)

if field.type in builtin_type_to_c_type:

self.emit(f"PyObject *type = (PyObject *)&{builtin_type_to_c_type[field.type]};", 2)

else:

self.emit(f"PyObject *type = state->{field.type}_type;", 2)

if field.opt:

self.emit("type = _Py_union_type_or(type, Py_None);", 2)

self.emit("cond = type != NULL;", 2)

self.emit_annotations_error(name, 2)

elif field.seq:

self.emit("type = Py_GenericAlias((PyObject *)&PyList_Type, type);", 2)

self.emit("cond = type != NULL;", 2)

self.emit_annotations_error(name, 2)

else:

self.emit("Py_INCREF(type);", 2)

self.emit(f"cond = PyDict_SetItemString({name}_annotations, \"{field.name}\", type) == 0;", 2)

self.emit("Py_DECREF(type);", 2)

self.emit_annotations_error(name, 2)

self.emit("}", 1)

self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "_field_types", {name}_annotations) == 0;', 1)

self.emit_annotations_error(name, 1)

self.emit(f'cond = PyObject_SetAttrString(state->{name}_type, "__annotations__", {name}_annotations) == 0;', 1)

self.emit_annotations_error(name, 1)

self.emit(f"Py_DECREF({name}_annotations);", 1)

def emit_annotations_error(self, name, depth):

self.emit("if (!cond) {", depth)

self.emit(f"Py_DECREF({name}_annotations);", depth + 1)

self.emit("return 0;", depth + 1)

self.emit("}", depth)

class PyTypesVisitor(PickleVisitor):

def visitModule(self, mod):

self.emit("""

typedef struct {

PyObject_HEAD

PyObject *dict;

} AST_object;

static void

ast_dealloc(AST_object *self)

{

/* bpo-31095: UnTrack is needed before calling any callbacks */

PyTypeObject *tp = Py_TYPE(self);

PyObject_GC_UnTrack(self);

Py_CLEAR(self->dict);

freefunc free_func = PyType_GetSlot(tp, Py_tp_free);

assert(free_func != NULL);

free_func(self);

Py_DECREF(tp);

}

static int

ast_traverse(AST_object *self, visitproc visit, void *arg)

{

Py_VISIT(Py_TYPE(self));

Py_VISIT(self->dict);

return 0;

}

static int

ast_clear(AST_object *self)

{

Py_CLEAR(self->dict);

return 0;

}

static int

ast_type_init(PyObject *self, PyObject *args, PyObject *kw)

{

struct ast_state *state = get_ast_state();

if (state == NULL) {

return -1;

}

Py_ssize_t i, numfields = 0;

int res = -1;

PyObject *key, *value, *fields, *remaining_fields = NULL;

if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), state->_fields, &fields) < 0) {

goto cleanup;

}

if (fields) {

numfields = PySequence_Size(fields);

if (numfields == -1) {

goto cleanup;

}

remaining_fields = PySet_New(fields);

}

else {

remaining_fields = PySet_New(NULL);

}

if (remaining_fields == NULL) {

goto cleanup;

}

res = 0; /* if no error occurs, this stays 0 to the end */

if (numfields < PyTuple_GET_SIZE(args)) {

PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "

"%zd positional argument%s",

_PyType_Name(Py_TYPE(self)),

numfields, numfields == 1 ? "" : "s");

res = -1;

goto cleanup;

}

for (i = 0; i < PyTuple_GET_SIZE(args); i++) {

/* cannot be reached when fields is NULL */

PyObject *name = PySequence_GetItem(fields, i);

if (!name) {

res = -1;

goto cleanup;

}

res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));

if (PySet_Discard(remaining_fields, name) < 0) {

res = -1;

Py_DECREF(name);

goto cleanup;

}

Py_DECREF(name);

if (res < 0) {

goto cleanup;

}

}

if (kw) {

i = 0; /* needed by PyDict_Next */

while (PyDict_Next(kw, &i, &key, &value)) {

int contains = PySequence_Contains(fields, key);

if (contains == -1) {

res = -1;

goto cleanup;

}

else if (contains == 1) {

int p = PySet_Discard(remaining_fields, key);

if (p == -1) {

res = -1;

goto cleanup;

}

if (p == 0) {

PyErr_Format(PyExc_TypeError,

"%.400s got multiple values for argument '%U'",

Py_TYPE(self)->tp_name, key);

res = -1;

goto cleanup;

}

}

else if (

PyUnicode_CompareWithASCIIString(key, "lineno") != 0 &&

PyUnicode_CompareWithASCIIString(key, "col_offset") != 0 &&

PyUnicode_CompareWithASCIIString(key, "end_lineno") != 0 &&

PyUnicode_CompareWithASCIIString(key, "end_col_offset") != 0

) {

if (PyErr_WarnFormat(

PyExc_DeprecationWarning, 1,

"%.400s.__init__ got an unexpected keyword argument '%U'. "

"Support for arbitrary keyword arguments is deprecated "

"and will be removed in Python 3.15.",

Py_TYPE(self)->tp_name, key

) < 0) {

res = -1;

goto cleanup;

}

}

res = PyObject_SetAttr(self, key, value);

if (res < 0) {

goto cleanup;

}

}

}

Py_ssize_t size = PySet_Size(remaining_fields);

PyObject *field_types = NULL, *remaining_list = NULL;

if (size > 0) {

if (!PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), &_Py_ID(_field_types),

&field_types)) {

res = -1;

goto cleanup;

}

remaining_list = PySequence_List(remaining_fields);

if (!remaining_list) {

goto set_remaining_cleanup;

}

for (Py_ssize_t i = 0; i < size; i++) {

PyObject *name = PyList_GET_ITEM(remaining_list, i);

PyObject *type = PyDict_GetItemWithError(field_types, name);

if (!type) {

if (!PyErr_Occurred()) {

PyErr_SetObject(PyExc_KeyError, name);

}

goto set_remaining_cleanup;

}

if (_PyUnion_Check(type)) {

// optional field

// do nothing, we'll have set a None default on the class

}

else if (Py_IS_TYPE(type, &Py_GenericAliasType)) {

// list field

PyObject *empty = PyList_New(0);