Support import cycles (mypyc/mypyc#195)

JukkaL · msullivan · commit 633972739137 · 2018-06-28T10:12:26.000-07:00
This approach is pretty janky -- the module init function can be called multiple times if there are import cycles, and we just return the previously created module object (which may be only partially initialized) if it's available. This Seems To Work but might break in some unforeseen ways. However, I propose that we keep this for now, at least until we support module top levels -- as that's where the potential problems with this approach seem likely to materialize. I manually verified that references to other modules within an import cycle seem to generate efficient code (i.e. not go through Python semantics) but there are no tests for that yet. I'll create a follow-up issue about this. Fixes mypyc/mypyc#164.
diff --git a/mypyc/emitmodule.py b/mypyc/emitmodule.py
@@ -82,6 +82,7 @@ def generate_c_for_modules(self) -> str:
         module_irs = [module_ir for _, module_ir in self.modules]
 
         for module_name, module in self.modules:
+            self.declare_module(module_name, emitter)
             self.declare_internal_globals(module_name, emitter)
             self.declare_imports(module.imports, emitter)
 
@@ -162,8 +163,12 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module
         else:
             declaration = 'PyObject *CPyInit_{}(void)'
         emitter.emit_lines(declaration.format(module_name),
-                           '{',
-                           'PyObject *m;')
+                           '{')
+        module_static = self.module_static_name(module_name, emitter)
+        emitter.emit_lines('if ({} != NULL) {{'.format(module_static),
+                           'Py_INCREF({});'.format(module_static),
+                           'return {};'.format(module_static),
+                           '}')
         for cl in module.classes:
             type_struct = emitter.type_struct_name(cl)
             if cl.traits:
@@ -175,15 +180,16 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module
 
             emitter.emit_lines('if (PyType_Ready(&{}) < 0)'.format(type_struct),
                                '    return NULL;')
-        emitter.emit_lines('m = PyModule_Create(&{}module);'.format(module_prefix),
-                           'if (m == NULL)',
+        emitter.emit_lines('{} = PyModule_Create(&{}module);'.format(module_static, module_prefix),
+                           'if ({} == NULL)'.format(module_static),
                            '    return NULL;')
         module_globals = emitter.static_name('globals', module_name)
-        emitter.emit_lines('{} = PyModule_GetDict(m);'.format(module_globals),
+        emitter.emit_lines('{} = PyModule_GetDict({});'.format(module_globals, module_static),
                            'if ({} == NULL)'.format(module_globals),
                            '    return NULL;')
         self.generate_imports_init_section(module.imports, emitter)
         self.generate_from_imports_init_section(
+            module_static,
             module.imports,
             module.from_imports,
             emitter,
@@ -216,8 +222,9 @@ def generate_module_def(self, emitter: Emitter, module_name: str, module: Module
             type_struct = emitter.type_struct_name(cl)
             emitter.emit_lines(
                 'Py_INCREF(&{});'.format(type_struct),
-                'PyModule_AddObject(m, "{}", (PyObject *)&{});'.format(name, type_struct))
-        emitter.emit_line('return m;')
+                'PyModule_AddObject({}, "{}", (PyObject *)&{});'.format(module_static, name,
+                                                                        type_struct))
+        emitter.emit_line('return {};'.format(module_static))
         emitter.emit_line('}')
 
     def toposort_declarations(self) -> List[HeaderDeclaration]:
@@ -262,13 +269,16 @@ def declare_internal_globals(self, module_name: str, emitter: Emitter) -> None:
         static_name = emitter.static_name('globals', module_name)
         self.declare_global('PyObject *', static_name)
 
-    def declare_import(self, imp: str, emitter: Emitter) -> None:
-        static_name = emitter.static_name('module', imp)
+    def module_static_name(self, module_name: str, emitter: Emitter) -> str:
+        return emitter.static_name('module', module_name)
+
+    def declare_module(self, module_name: str, emitter: Emitter) -> None:
+        static_name = self.module_static_name(module_name, emitter)
         self.declare_global('CPyModule *', static_name)
 
     def declare_imports(self, imps: Iterable[str], emitter: Emitter) -> None:
         for imp in imps:
-            self.declare_import(imp, emitter)
+            self.declare_module(imp, emitter)
 
     def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None:
         symbol = emitter.static_name(identifier, None)
@@ -280,7 +290,7 @@ def generate_imports_init_section(self, imps: List[str], emitter: Emitter) -> No
             self.generate_import(imp, emitter, check_for_null=True)
 
     def generate_import(self, imp: str, emitter: Emitter, check_for_null: bool) -> None:
-        c_name = emitter.static_name('module', imp)
+        c_name = self.module_static_name(imp, emitter)
         if check_for_null:
             emitter.emit_line('if ({} == NULL) {{'.format(c_name))
         emitter.emit_line('{} = PyImport_ImportModule("{}");'.format(c_name, imp))
@@ -290,21 +300,22 @@ def generate_import(self, imp: str, emitter: Emitter, check_for_null: bool) -> N
             emitter.emit_line('}')
 
     def generate_from_imports_init_section(self,
+            module_static: str,
             imps: List[str],
             from_imps: Dict[str, List[Tuple[str, str]]],
             emitter: Emitter) -> None:
         for imp, import_names in from_imps.items():
             # Only import it again if we haven't imported it from the main
             # imports section
             if imp not in imps:
-                c_name = emitter.static_name('module', imp)
+                c_name = self.module_static_name(imp, emitter)
                 emitter.emit_line('CPyModule *{};'.format(c_name))
                 self.generate_import(imp, emitter, check_for_null=False)
 
             for original_name, as_name in import_names:
                 # Obtain a reference to the original object
                 object_temp_name = emitter.temp_name()
-                c_name = emitter.static_name('module', imp)
+                c_name = self.module_static_name(imp, emitter)
                 emitter.emit_line('PyObject *{} = PyObject_GetAttrString({}, "{}");'.format(
                     object_temp_name,
                     c_name,
@@ -315,7 +326,8 @@ def generate_from_imports_init_section(self,
                     '    return NULL;',
                 )
                 # and add it to the namespace of the current module, which eats the ref
-                emitter.emit_line('if (PyModule_AddObject(m, "{}", {}) < 0)'.format(
+                emitter.emit_line('if (PyModule_AddObject({}, "{}", {}) < 0)'.format(
+                    module_static,
                     as_name,
                     object_temp_name,
                 ))
@@ -324,7 +336,7 @@ def generate_from_imports_init_section(self,
             # This particular import isn't saved as a global so we should decref it
             # and not keep it around
             if imp not in imps:
-                c_name = emitter.static_name('module', imp)
+                c_name = self.module_static_name(imp, emitter)
                 emitter.emit_line('Py_DECREF({});'.format(c_name))
 
 
diff --git a/mypyc/genops.py b/mypyc/genops.py
@@ -981,16 +981,16 @@ def visit_int_expr(self, expr: IntExpr) -> Value:
     def visit_float_expr(self, expr: FloatExpr) -> Value:
         return self.load_static_float(expr.value)
 
-    def is_native_name_expr(self, expr: NameExpr) -> bool:
-        assert expr.node, "RefExpr not resolved"
+    def is_native_ref_expr(self, expr: RefExpr) -> bool:
+        if expr.node is None:
+            return False
         if '.' in expr.node.fullname():
             module_name = '.'.join(expr.node.fullname().split('.')[:-1])
             return module_name in self.modules
-
         return True
 
-    def is_native_module_name_expr(self, expr: NameExpr) -> bool:
-        return self.is_native_name_expr(expr) and expr.kind == GDEF
+    def is_native_module_ref_expr(self, expr: RefExpr) -> bool:
+        return self.is_native_ref_expr(expr) and expr.kind == GDEF
 
     def visit_name_expr(self, expr: NameExpr) -> Value:
         assert expr.node, "RefExpr not resolved"
@@ -1059,14 +1059,17 @@ def visit_call_expr(self, expr: CallExpr) -> Value:
             callee = callee.analyzed.expr  # Unwrap type application
 
         if isinstance(callee, MemberExpr):
-            # TODO: Could be call to module-level function
-            return self.translate_method_call(expr, callee)
+            if self.is_native_ref_expr(callee):
+                # Call to module-level function or such
+                return self.translate_call(expr, callee)
+            else:
+                return self.translate_method_call(expr, callee)
         else:
             return self.translate_call(expr, callee)
 
     def translate_call(self, expr: CallExpr, callee: Expression) -> Value:
         """Translate a non-method call."""
-        assert isinstance(callee, NameExpr)  # TODO: Allow arbitrary callees
+        assert isinstance(callee, RefExpr)  # TODO: Allow arbitrary callees
 
         # Gen the args
         fullname = callee.fullname
@@ -1080,9 +1083,9 @@ def translate_call(self, expr: CallExpr, callee: Expression) -> Value:
         if (fullname == 'builtins.isinstance'
                 and len(expr.args) == 2
                 and expr.arg_kinds == [ARG_POS, ARG_POS]
-                and isinstance(expr.args[1], NameExpr)
+                and isinstance(expr.args[1], RefExpr)
                 and isinstance(expr.args[1].node, TypeInfo)
-                and self.is_native_module_name_expr(expr.args[1])):
+                and self.is_native_module_ref_expr(expr.args[1])):
             # Special case native isinstance() checks as this makes them much faster.
             return self.primitive_op(fast_isinstance_op, args, expr.line)
 
@@ -1111,13 +1114,13 @@ def translate_call(self, expr: CallExpr, callee: Expression) -> Value:
             function = self.accept(callee)
             return self.py_call(function, args, target_type, expr.line)
 
-    def get_native_signature(self, callee: NameExpr) -> Optional[CallableType]:
+    def get_native_signature(self, callee: RefExpr) -> Optional[CallableType]:
         """Get the signature of a native function, or return None if not available.
 
         This only works for normal functions, not methods.
         """
         signature = None
-        if self.is_native_module_name_expr(callee):
+        if self.is_native_module_ref_expr(callee):
             node = callee.node
             if isinstance(node, TypeInfo):
                 node = node['__init__'].node
@@ -1599,7 +1602,7 @@ def instantiate_function_class(self, fdef: FuncDef, namespace: str) -> Value:
         func_reg = self.environment.add_local(fdef, object_rprimitive)
         return self.add(Assign(func_reg, temp_reg))
 
-    def is_builtin_name_expr(self, expr: NameExpr) -> bool:
+    def is_builtin_ref_expr(self, expr: RefExpr) -> bool:
         assert expr.node, "RefExpr not resolved"
         return '.' in expr.node.fullname() and expr.node.fullname().split('.')[0] == 'builtins'
 
@@ -1610,9 +1613,9 @@ def load_global(self, expr: NameExpr) -> Value:
         from the _globals dictionary in the C-generated code.
         """
         # If the global is from 'builtins', turn it into a module attr load instead
-        if self.is_builtin_name_expr(expr):
+        if self.is_builtin_ref_expr(expr):
             return self.load_static_module_attr(expr)
-        if self.is_native_module_name_expr(expr) and isinstance(expr.node, TypeInfo):
+        if self.is_native_module_ref_expr(expr) and isinstance(expr.node, TypeInfo):
             assert expr.fullname is not None
             return self.load_native_type_object(expr.fullname)
         _globals = self.add(LoadStatic(object_rprimitive, 'globals', self.module_name))
diff --git a/mypyc/ops.py b/mypyc/ops.py
@@ -13,7 +13,7 @@
 from abc import abstractmethod, abstractproperty
 import re
 from typing import (
-    List, Sequence, Dict, Generic, TypeVar, Optional, Any, NamedTuple, Tuple, NewType, Callable,
+    List, Sequence, Dict, Generic, TypeVar, Optional, Any, NamedTuple, Tuple, Callable,
     Union, Iterable, Type,
 )
 from collections import OrderedDict
diff --git a/test-data/module-output.test b/test-data/module-output.test
@@ -8,6 +8,7 @@ def f(x: int) -> int:
 #include <Python.h>
 #include <CPy.h>
 
+static CPyModule *CPyStatic_module;
 static PyObject *CPyStatic_globals;
 static CPyModule *CPyStatic_builtins_module;
 static CPyTagged CPyDef_f(CPyTagged cpy_r_x);
@@ -29,19 +30,22 @@ static struct PyModuleDef module = {
 
 PyMODINIT_FUNC PyInit_prog(void)
 {
-    PyObject *m;
-    m = PyModule_Create(&module);
-    if (m == NULL)
+    if (CPyStatic_module != NULL) {
+        Py_INCREF(CPyStatic_module);
+        return CPyStatic_module;
+    }
+    CPyStatic_module = PyModule_Create(&module);
+    if (CPyStatic_module == NULL)
         return NULL;
-    CPyStatic_globals = PyModule_GetDict(m);
+    CPyStatic_globals = PyModule_GetDict(CPyStatic_module);
     if (CPyStatic_globals == NULL)
         return NULL;
     if (CPyStatic_builtins_module == NULL) {
         CPyStatic_builtins_module = PyImport_ImportModule("builtins");
         if (CPyStatic_builtins_module == NULL)
             return NULL;
     }
-    return m;
+    return CPyStatic_module;
 }
 
 static CPyTagged CPyDef_f(CPyTagged cpy_r_x) {
diff --git a/test-data/run-multimodule.test b/test-data/run-multimodule.test
@@ -212,3 +212,121 @@ Traceback (most recent call last):
     fail2()
   File "tmp/other.py", line 3, in fail2
 IndexError: list assignment index out of range
+
+[case testMultiModuleCycle]
+import other
+
+def f1() -> int:
+    return other.f2()
+
+def f3() -> int:
+    return 5
+[file other.py]
+import native
+
+def f2() -> int:
+    return native.f3()
+[file driver.py]
+from native import f1
+assert f1() == 5
+
+[case testMultiModuleCycleWithClasses]
+import other
+
+class D: pass
+
+def f() -> other.C:
+    return other.C()
+
+def g(c: other.C) -> D:
+    return c.d
+
+[file other.py]
+import native
+
+class C:
+    def __init__(self) -> None:
+        self.d = native.D()
+
+def h(d: native.D) -> None:
+    pass
+
+[file driver.py]
+from native import f, g
+from other import C, h
+
+c = f()
+assert isinstance(c, C)
+assert g(c) is c.d
+h(c.d)
+
+try:
+    g(1)
+except TypeError:
+    pass
+else:
+    assert False
+
+try:
+    h(1)
+except TypeError:
+    pass
+else:
+    assert False
+
+[case testMultiModuleCycleWithInheritance]
+import other
+
+class Deriv1(other.Base1):
+    pass
+
+class Base2:
+    y: int
+    def __init__(self) -> None:
+        self.y = 2
+
+[file other.py]
+import native
+
+class Base1:
+    x: int
+    def __init__(self) -> None:
+        self.x = 1
+
+class Deriv2(native.Base2):
+    pass
+
+[file driver.py]
+from native import Deriv1
+from other import Deriv2
+a = Deriv1()
+assert a.x == 1
+b = Deriv2()
+assert b.y == 2
+
+[case testImportCycleWithNonCompiledModule]
+import m
+
+class C: pass
+
+def f1() -> int:
+    m.D()
+    return m.f2()
+
+def f3() -> int:
+    return 2
+
+[file m.py]
+# This module is NOT compiled
+import native
+
+class D: pass
+
+def f2() -> int:
+    native.C()
+    return native.f3()
+
+[file driver.py]
+from native import f1
+
+assert f1() == 2

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@`
`13`	`13`	`from abc import abstractmethod, abstractproperty`
`14`	`14`	`import re`
`15`	`15`	`from typing import (`
`16`		`- List, Sequence, Dict, Generic, TypeVar, Optional, Any, NamedTuple, Tuple, NewType, Callable,`
	`16`	`+ List, Sequence, Dict, Generic, TypeVar, Optional, Any, NamedTuple, Tuple, Callable,`
`17`	`17`	`Union, Iterable, Type,`
`18`	`18`	`)`
`19`	`19`	`from collections import OrderedDict`