Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 49dadba

Browse files
msullivanmsullivan
authored
Rework vtable handling, emit glue methods when override changes method type (mypyc/mypyc#154)
When we have: ``` class A: def f(self, x: int) -> object: ... class B(A): def f(self, x: object) -> int: ... ``` then `A.f` and `B.f` have different function signatures in the generated C, because `int` and `object` have different runtime representations. So `B` then needs to have two versions of `f` in its symbol table: one for when it is called as an `A`, and one for as a `B`. This means we need to generate a glue function to convert from `A.f` to `B.f`. To support this, we restructure a lot of how vtables are represented in the compiler.
1 parent b913793 commit 49dadba

10 files changed

Lines changed: 448 additions & 101 deletions

File tree

‎mypyc/emitclass.py‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
from mypyc.common import PREFIX, NATIVE_PREFIX, REG_PREFIX
88
from mypyc.emit import Emitter
99
from mypyc.emitfunc import native_function_header
10-
from mypyc.ops import ClassIR, FuncIR, RType, Environment, object_rprimitive
10+
from mypyc.ops import (
11+
ClassIR, FuncIR, RType, Environment, object_rprimitive, FuncSignature,
12+
VTableMethod, VTableAttr,
13+
)
1114
from mypyc.sametype import is_same_type
1215
from mypyc.namegen import NameGenerator
1316

@@ -53,7 +56,8 @@ def emit_line() -> None:
5356

5457
emitter.emit_line('static PyObject *{}(void);'.format(setup_name))
5558
# TODO: Use RInstance
56-
ctor = FuncIR(cl.name, None, module, init_args, object_rprimitive, [], Environment())
59+
ctor = FuncIR(cl.name, None, module, FuncSignature(init_args, object_rprimitive),
60+
[], Environment())
5761
emitter.emit_line(native_function_header(ctor, emitter.names) + ';')
5862

5963
emit_line()
@@ -204,22 +208,14 @@ def generate_vtable(base: ClassIR,
204208
vtable_name: str,
205209
emitter: Emitter) -> None:
206210
emitter.emit_line('static CPyVTableItem {}[] = {{'.format(vtable_name))
207-
for cl in reversed(base.mro):
208-
for attr in cl.attributes:
209-
emitter.emit_line(
210-
'(CPyVTableItem){},'.format(native_getter_name(cl, attr, emitter.names)))
211-
emitter.emit_line(
212-
'(CPyVTableItem){},'.format(native_setter_name(cl, attr, emitter.names)))
213-
for fn in cl.methods:
214-
# TODO: This is gross, and inefficient, and wrong if the type changes.
215-
# This logic should all live on the genops side, I think
216-
search = base.mro if fn.name != '__init__' else [cl]
217-
for cl2 in search:
218-
m = cl2.get_method(fn.name)
219-
if m:
220-
emitter.emit_line('(CPyVTableItem){}{},'.format(NATIVE_PREFIX,
221-
m.cname(emitter.names)))
222-
break
211+
for entry in base.vtable_entries:
212+
if isinstance(entry, VTableMethod):
213+
emitter.emit_line('(CPyVTableItem){}{},'.format(NATIVE_PREFIX,
214+
entry.method.cname(emitter.names)))
215+
else:
216+
cl, attr, is_getter = entry
217+
namer = native_getter_name if is_getter else native_setter_name
218+
emitter.emit_line('(CPyVTableItem){},'.format(namer(cl, attr, emitter.names)))
223219
emitter.emit_line('};')
224220

225221

@@ -340,7 +336,7 @@ def generate_methods_table(cl: ClassIR,
340336
name: str,
341337
emitter: Emitter) -> None:
342338
emitter.emit_line('static PyMethodDef {}[] = {{'.format(name))
343-
for fn in cl.methods:
339+
for fn in cl.methods.values():
344340
emitter.emit_line('{{"{}",'.format(fn.name))
345341
emitter.emit_line(' (PyCFunction){}{},'.format(PREFIX, fn.cname(emitter.names)))
346342
emitter.emit_line(' METH_VARARGS | METH_KEYWORDS, NULL},')

‎mypyc/emitwrapper.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def generate_wrapper_function(fn: FuncIR, emitter: Emitter) -> None:
2121
emitter.emit_line('{} {{'.format(wrapper_function_header(fn, emitter.names)))
2222

2323
# If fn is a method, then the first argument is a self param
24-
real_args = fn.args[:]
24+
real_args = list(fn.args)
2525
if fn.class_name:
2626
arg = real_args.pop(0)
2727
emitter.emit_line('PyObject *obj_{} = self;'.format(arg.name))

‎mypyc/genops.py‎

Lines changed: 128 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ def f(x: int) -> int:
1414
return r3
1515
"""
1616

17-
from typing import Dict, List, Tuple, Optional, Union
17+
from typing import Dict, List, Tuple, Optional, Union, Sequence
1818

1919
from mypy.nodes import (
2020
Node, MypyFile, SymbolNode, FuncDef, ReturnStmt, AssignmentStmt, OpExpr, IntExpr, NameExpr,
@@ -47,7 +47,8 @@ def f(x: int) -> int:
4747
is_int_rprimitive, float_rprimitive, is_float_rprimitive, bool_rprimitive, list_rprimitive,
4848
is_list_rprimitive, dict_rprimitive, is_dict_rprimitive, str_rprimitive, is_tuple_rprimitive,
4949
tuple_rprimitive, none_rprimitive, is_none_rprimitive, object_rprimitive, PrimitiveOp,
50-
ERR_FALSE, OpDescription, RegisterOp, is_object_rprimitive, LiteralsMap,
50+
ERR_FALSE, OpDescription, RegisterOp, is_object_rprimitive, LiteralsMap, FuncSignature,
51+
VTableAttr, VTableMethod,
5152
)
5253
from mypyc.ops_primitive import binary_ops, unary_ops, func_ops, method_ops, name_ref_ops
5354
from mypyc.ops_list import list_len_op, list_get_item_op, list_set_item_op, new_list_op
@@ -56,7 +57,7 @@ def f(x: int) -> int:
5657
none_op, iter_op, next_op, no_err_occurred_op, py_getattr_op, py_setattr_op,
5758
)
5859
from mypyc.subtype import is_subtype
59-
from mypyc.sametype import is_same_type
60+
from mypyc.sametype import is_same_type, is_same_method_signature
6061

6162

6263
def build_ir(modules: List[MypyFile],
@@ -81,8 +82,8 @@ def build_ir(modules: List[MypyFile],
8182
prepare_class_def(cdef, mapper)
8283

8384
# Generate IR for all modules.
85+
module_names = [mod.fullname() for mod in modules]
8486
for module in modules:
85-
module_names = [mod.fullname() for mod in modules]
8687
builder = IRBuilder(types, mapper, module_names)
8788
module.accept(builder)
8889
ir = ModuleIR(
@@ -96,11 +97,44 @@ def build_ir(modules: List[MypyFile],
9697

9798
# Compute vtables.
9899
for _, cdef in classes:
99-
mapper.type_to_ir[cdef.info].compute_vtable()
100+
compute_vtable(mapper.type_to_ir[cdef.info])
100101

101102
return result
102103

103104

105+
def compute_vtable(cls: ClassIR) -> None:
106+
"""Compute the vtable structure for a class."""
107+
if cls.vtable is not None: return
108+
cls.vtable = {}
109+
entries = cls.vtable_entries
110+
if cls.base:
111+
compute_vtable(cls.base)
112+
assert cls.base.vtable is not None
113+
cls.vtable.update(cls.base.vtable)
114+
prefix = cls.base.vtable_entries
115+
else:
116+
prefix = []
117+
118+
# Include the vtable from the parent classes, but handle method overrides.
119+
for entry in prefix:
120+
if isinstance(entry, VTableMethod):
121+
method = entry.method
122+
if method.name in cls.methods:
123+
if is_same_method_signature(method.sig, cls.methods[method.name].sig):
124+
entry = VTableMethod(cls, cls.methods[method.name])
125+
else:
126+
entry = VTableMethod(cls, cls.glue_methods[(entry.cls, method.name)])
127+
entries.append(entry)
128+
129+
for attr in cls.attributes:
130+
cls.vtable[attr] = len(entries)
131+
entries.append(VTableAttr(cls, attr, is_getter=True))
132+
entries.append(VTableAttr(cls, attr, is_getter=False))
133+
for fn in cls.methods.values():
134+
cls.vtable[fn.name] = len(entries)
135+
entries.append(VTableMethod(cls, fn))
136+
137+
104138
class Mapper:
105139
"""Keep track of mappings from mypy concepts to IR concepts.
106140
@@ -155,6 +189,13 @@ def type_to_rtype(self, typ: Type) -> RType:
155189
return self.type_to_rtype(typ.upper_bound)
156190
assert False, '%s unsupported' % type(typ)
157191

192+
def fdef_to_sig(self, fdef: FuncDef) -> FuncSignature:
193+
assert isinstance(fdef.type, CallableType)
194+
args = [RuntimeArg(arg.variable.name(), self.type_to_rtype(fdef.type.arg_types[i]))
195+
for i, arg in enumerate(fdef.arguments)]
196+
ret = self.type_to_rtype(fdef.type.ret_type)
197+
return FuncSignature(args, ret)
198+
158199
def c_name_for_literal(self, value: Union[int, float, str]) -> str:
159200
# Include type to distinguish between 1 and 1.0, and so on.
160201
key = (type(value), value)
@@ -177,6 +218,8 @@ def prepare_class_def(cdef: ClassDef, mapper: Mapper) -> None:
177218
if isinstance(node.node, Var):
178219
assert node.node.type, "Class member missing type"
179220
ir.attributes[name] = mapper.type_to_rtype(node.node.type)
221+
elif isinstance(node.node, FuncDef):
222+
ir.method_types[name] = mapper.fdef_to_sig(node.node)
180223

181224
# Set up the parent class
182225
assert len(info.bases) == 1, "Only single inheritance is supported"
@@ -282,9 +325,21 @@ def visit_class_def(self, cdef: ClassDef) -> Value:
282325
ir = self.mapper.type_to_ir[cdef.info]
283326
for name, node in sorted(cdef.info.names.items(), key=lambda x: x[0]):
284327
if isinstance(node.node, FuncDef):
285-
func = self.gen_func_def(node.node, cdef.name)
328+
func = self.gen_func_def(node.node, ir.method_sig(node.node.name()), cdef.name)
286329
self.functions.append(func)
287-
ir.methods.append(func)
330+
ir.methods[func.name] = func
331+
332+
# If this overrides a parent class method with a different type, we need
333+
# to generate a glue method to mediate between them.
334+
for cls in ir.mro[1:]:
335+
if (name in cls.method_types
336+
and not is_same_method_signature(ir.method_types[name],
337+
cls.method_types[name])):
338+
f = self.gen_glue_method(cls.method_types[name], func, ir, cls,
339+
node.node.line)
340+
ir.glue_methods[(cls, name)] = f
341+
self.functions.append(f)
342+
288343
return INVALID_VALUE
289344

290345
def visit_import(self, node: Import) -> Value:
@@ -324,7 +379,57 @@ def visit_import_all(self, node: ImportAll) -> Value:
324379

325380
return INVALID_VALUE
326381

327-
def gen_func_def(self, fdef: FuncDef, class_name: Optional[str] = None) -> FuncIR:
382+
def gen_glue_method(self, sig: FuncSignature, target: FuncIR,
383+
cls: ClassIR, base: ClassIR, line: int) -> FuncIR:
384+
"""Generate glue methods that mediate between different method types in subclasses.
385+
386+
For example, if we have:
387+
388+
class A:
389+
def f(self, x: int) -> object: ...
390+
391+
then it is totally permissable to have a subclass
392+
393+
class B(A):
394+
def f(self, x: object) -> int: ...
395+
396+
since '(object) -> int' is a subtype of '(int) -> object' by the usual
397+
contra/co-variant function subtyping rules.
398+
399+
The trickiness here is that int and object have different
400+
runtime representations in mypyc, so A.f and B.f have
401+
different signatures at the native C level. To deal with this,
402+
we need to generate glue methods that mediate between the
403+
different versions by coercing the arguments and return
404+
values.
405+
"""
406+
self.enter()
407+
408+
rt_args = (RuntimeArg(sig.args[0].name, RInstance(cls)),) + sig.args[1:]
409+
410+
# The environment operates on Vars, so we make some up
411+
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
412+
args = [self.environment.add_local(var, type, is_arg=True)
413+
for var, type in fake_vars] # type: List[Value]
414+
self.ret_types[-1] = sig.ret_type
415+
416+
arg_types = [arg.type for arg in target.sig.args]
417+
args = self.coerce_native_call_args(args, arg_types, line)
418+
retval = self.add(MethodCall(target.ret_type,
419+
args[0],
420+
target.name,
421+
args[1:],
422+
line))
423+
retval = self.coerce(retval, sig.ret_type, line)
424+
self.add(Return(retval))
425+
426+
blocks, env, ret_type = self.leave()
427+
return FuncIR(target.name + '__' + base.name + '_glue',
428+
cls.name, self.module_name,
429+
FuncSignature(rt_args, ret_type), blocks, env)
430+
431+
def gen_func_def(self, fdef: FuncDef, sig: FuncSignature,
432+
class_name: Optional[str] = None) -> FuncIR:
328433
# If there is more than one environment in the environment stack, then we are visiting a
329434
# non-global function.
330435
is_nested = len(self.environments) > 1
@@ -340,7 +445,7 @@ def gen_func_def(self, fdef: FuncDef, class_name: Optional[str] = None) -> FuncI
340445
assert arg.variable.type, "Function argument missing type"
341446
self.environment.add_local(arg.variable, self.type_to_rtype(arg.variable.type),
342447
is_arg=True)
343-
self.ret_types[-1] = self.convert_return_type(fdef)
448+
self.ret_types[-1] = sig.ret_type
344449

345450
fdef.body.accept(self)
346451

@@ -351,34 +456,23 @@ def gen_func_def(self, fdef: FuncDef, class_name: Optional[str] = None) -> FuncI
351456
self.add_implicit_unreachable()
352457

353458
blocks, env, ret_type = self.leave()
354-
args = self.convert_args(fdef)
355459

356460
if is_nested:
357461
namespace = self.generate_function_namespace()
358-
func_ir = self.generate_function_class(fdef, namespace, blocks, env, ret_type)
462+
func_ir = self.generate_function_class(fdef, namespace, blocks, sig, env)
359463

360464
# Instantiate the callable class and load it into a register in the current environment
361465
# immediately so that it does not have to be loaded every time the function is called.
362466
self.instantiate_function_class(fdef, namespace)
363467
else:
364-
func_ir = FuncIR(fdef.name(), class_name, self.module_name, args, ret_type, blocks,
468+
func_ir = FuncIR(fdef.name(), class_name, self.module_name, sig, blocks,
365469
env)
366470
return func_ir
367471

368472
def visit_func_def(self, fdef: FuncDef) -> Value:
369-
self.functions.append(self.gen_func_def(fdef))
473+
self.functions.append(self.gen_func_def(fdef, self.mapper.fdef_to_sig(fdef)))
370474
return INVALID_VALUE
371475

372-
def convert_args(self, fdef: FuncDef) -> List[RuntimeArg]:
373-
assert isinstance(fdef.type, CallableType)
374-
ann = fdef.type
375-
return [RuntimeArg(arg.variable.name(), self.type_to_rtype(ann.arg_types[i]))
376-
for i, arg in enumerate(fdef.arguments)]
377-
378-
def convert_return_type(self, fdef: FuncDef) -> RType:
379-
assert isinstance(fdef.type, CallableType)
380-
return self.type_to_rtype(fdef.type.ret_type)
381-
382476
def add_implicit_return(self) -> None:
383477
block = self.blocks[-1][-1]
384478
if not block.ops or not isinstance(block.ops[-1], Return):
@@ -885,8 +979,8 @@ def py_method_call(self,
885979
return self.add(PyMethodCall(obj, method, arg_boxes))
886980

887981
def coerce_native_call_args(self,
888-
args: List[Value],
889-
arg_types: List[RType],
982+
args: Sequence[Value],
983+
arg_types: Sequence[RType],
890984
line: int) -> List[Value]:
891985
coerced_arg_regs = []
892986
for reg, arg_type in zip(args, arg_types):
@@ -1377,8 +1471,8 @@ def generate_function_class(self,
13771471
fdef: FuncDef,
13781472
namespace: str,
13791473
blocks: List[BasicBlock],
1380-
env: Environment,
1381-
ret_type: RType) -> FuncIR:
1474+
sig: FuncSignature,
1475+
env: Environment) -> FuncIR:
13821476
"""Generates a callable class representing a nested function.
13831477
13841478
This takes a FuncDef and its associated namespace, blocks, environment, and return type and
@@ -1390,27 +1484,22 @@ class is generated using the names of the functions that enclose the given neste
13901484
Returns a newly constructed FuncIR associated with the given FuncDef.
13911485
"""
13921486
class_name = '{}_{}_obj'.format(fdef.name(), namespace)
1393-
args = self.convert_args(fdef)
1394-
args.insert(0, RuntimeArg('self', object_rprimitive))
1395-
func_ir = FuncIR('__call__', class_name, self.module_name, args, ret_type, blocks, env)
1487+
sig = FuncSignature((RuntimeArg('self', object_rprimitive),) + sig.args, sig.ret_type)
1488+
func_ir = FuncIR('__call__', class_name, self.module_name, sig, blocks, env)
13961489
class_ir = ClassIR(class_name, self.module_name)
1397-
class_ir.methods.append(func_ir)
1490+
class_ir.methods['__call__'] = func_ir
13981491
self.classes.append(class_ir)
13991492
return func_ir
14001493

14011494
def instantiate_function_class(self, fdef: FuncDef, namespace: str) -> Value:
14021495
"""Assigns a callable class to a register named after the given function definition."""
1403-
temp_reg = self.load_function_class(fdef, namespace)
1496+
temp_reg = self.add(Call(self.mapper.fdef_to_sig(fdef).ret_type,
1497+
'{}.{}_{}_obj'.format(self.module_name, fdef.name(), namespace),
1498+
[],
1499+
fdef.line))
14041500
func_reg = self.environment.add_local(fdef, object_rprimitive)
14051501
return self.add(Assign(func_reg, temp_reg))
14061502

1407-
def load_function_class(self, fdef: FuncDef, namespace: str) -> Value:
1408-
"""Loads a callable class representing a nested function into a register."""
1409-
return self.add(Call(self.convert_return_type(fdef),
1410-
'{}.{}_{}_obj'.format(self.module_name, fdef.name(), namespace),
1411-
[],
1412-
fdef.line))
1413-
14141503
def load_global(self, expr: NameExpr) -> Value:
14151504
"""Loads a Python-level global.
14161505

0 commit comments

Comments
 (0)