From 5a4f0f6d3ac3bc4cb232257af881f1254ce01ca7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 6 Dec 2020 16:48:25 +0000 Subject: [PATCH 1/3] [mypyc] Fix reading package imported inside a function This fixes an issue where this code resulted in an unbound local `p` error: ``` def f() -> None: import p.submodule print(p.x) # Runtime error here ``` We now look up `p` from the global modules dictionary instead of trying to use an undefined local variable. --- mypyc/irbuild/expression.py | 21 ++++++++++--- mypyc/irbuild/statement.py | 49 ++++++++++++++++-------------- mypyc/test-data/irbuild-basic.test | 37 ++++++++++++++++++++++ mypyc/test-data/run-imports.test | 22 +++++++++----- 4 files changed, 95 insertions(+), 34 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 14c11e07090df..34212aebe0cb2 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -26,10 +26,10 @@ from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD from mypyc.primitives.registry import CFunctionDescription, builtin_names from mypyc.primitives.generic_ops import iter_op -from mypyc.primitives.misc_ops import new_slice_op, ellipsis_op, type_op +from mypyc.primitives.misc_ops import new_slice_op, ellipsis_op, type_op, get_module_dict_op from mypyc.primitives.list_ops import list_append_op, list_extend_op, list_slice_op from mypyc.primitives.tuple_ops import list_tuple_op, tuple_slice_op -from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op +from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op, dict_get_item_op from mypyc.primitives.set_ops import new_set_op, set_add_op, set_update_op from mypyc.primitives.str_ops import str_slice_op from mypyc.primitives.int_ops import int_comparison_op_mapping @@ -85,8 +85,21 @@ def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value: expr.node.name), expr.node.line) - # TODO: Behavior currently only defined for Var and FuncDef node types. - return builder.read(builder.get_assignment_target(expr), expr.line) + # TODO: Behavior currently only defined for Var, FuncDef and MypyFile node types. + if isinstance(expr.node, MypyFile): + # Load reference to a module imported inside function from + # the modules dictionary. It would be closer to Python + # semantics to access modules imported inside functions + # via local variables, but this is tricky since the mypy + # AST doesn't include a Var node for the module. We + # instead load the module separately on each access. + mod_dict = builder.call_c(get_module_dict_op, [], expr.line) + obj = builder.call_c(dict_get_item_op, + [mod_dict, builder.load_static_unicode(expr.node.fullname)], + expr.line) + return obj + else: + return builder.read(builder.get_assignment_target(expr), expr.line) return builder.load_global(expr) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index b83bc4beafe9e..d97f3d57fce8e 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -119,32 +119,35 @@ def transform_operator_assignment_stmt(builder: IRBuilder, stmt: OperatorAssignm def transform_import(builder: IRBuilder, node: Import) -> None: if node.is_mypy_only: return - globals = builder.load_globals_dict() for node_id, as_name in node.ids: builder.gen_import(node_id, node.line) - # Update the globals dict with the appropriate module: - # * For 'import foo.bar as baz' we add 'foo.bar' with the name 'baz' - # * For 'import foo.bar' we add 'foo' with the name 'foo' - # Typically we then ignore these entries and access things directly - # via the module static, but we will use the globals version for modules - # that mypy couldn't find, since it doesn't analyze module references - # from those properly. - - # Miscompiling imports inside of functions, like below in import from. - if as_name: - name = as_name - base = node_id - else: - base = name = node_id.split('.')[0] - - # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( - mod_dict = builder.call_c(get_module_dict_op, [], node.line) - obj = builder.call_c(dict_get_item_op, - [mod_dict, builder.load_static_unicode(base)], node.line) - builder.gen_method_call( - globals, '__setitem__', [builder.load_static_unicode(name), obj], - result_type=None, line=node.line) + if builder.non_function_scope(): + # Update the globals dict with the appropriate module: + # * For 'import foo.bar as baz' we add 'foo.bar' with the name 'baz' + # * For 'import foo.bar' we add 'foo' with the name 'foo' + # Typically we then ignore these entries and access things directly + # via the module static, but we will use the globals version for modules + # that mypy couldn't find, since it doesn't analyze module references + # from those properly. + + # Miscompiling imports inside of functions, like below in import from. + if as_name: + name = as_name + base = node_id + else: + base = name = node_id.split('.')[0] + + globals = builder.load_globals_dict() + # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( + mod_dict = builder.call_c(get_module_dict_op, [], node.line) + # Get top-level module/package object. + obj = builder.call_c(dict_get_item_op, + [mod_dict, builder.load_static_unicode(base)], node.line) + + builder.gen_method_call( + globals, '__setitem__', [builder.load_static_unicode(name), obj], + result_type=None, line=node.line) def transform_import_from(builder: IRBuilder, node: ImportFrom) -> None: diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index fc60b99ea29b7..6019227dd82bd 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -3637,3 +3637,40 @@ L0: c = r2 r3 = (c, b, a) return r3 + +[case testLocalImportSubmodule] +def f() -> int: + import p.m + return p.x +[file p/__init__.py] +x = 1 +[file p/m.py] +[out] +def f(): + r0, r1 :: object + r2 :: bit + r3 :: str + r4 :: object + r5 :: dict + r6 :: str + r7 :: object + r8 :: str + r9 :: object + r10 :: int +L0: + r0 = p.m :: module + r1 = load_address _Py_NoneStruct + r2 = r0 != r1 + if r2 goto L2 else goto L1 :: bool +L1: + r3 = load_global CPyStatic_unicode_1 :: static ('p.m') + r4 = PyImport_Import(r3) + p.m = r4 :: module +L2: + r5 = PyImport_GetModuleDict() + r6 = load_global CPyStatic_unicode_2 :: static ('p') + r7 = CPyDict_GetItem(r5, r6) + r8 = load_global CPyStatic_unicode_3 :: static ('x') + r9 = CPyObject_GetAttr(r7, r8) + r10 = unbox(int, r9) + return r10 diff --git a/mypyc/test-data/run-imports.test b/mypyc/test-data/run-imports.test index 6b5a70cf6ced3..0190545bfe8d9 100644 --- a/mypyc/test-data/run-imports.test +++ b/mypyc/test-data/run-imports.test @@ -5,9 +5,20 @@ import testmodule def f(x: int) -> int: return testmodule.factorial(5) + def g(x: int) -> int: from welp import foo return foo(x) + +def test_import() -> None: + assert f(5) == 120 + assert g(5) == 5 + +def test_import_submodule_within_function() -> None: + import pkg.mod + assert pkg.x == 1 + assert pkg.mod.y == 2 + [file testmodule.py] def factorial(x: int) -> int: if x == 0: @@ -17,13 +28,10 @@ def factorial(x: int) -> int: [file welp.py] def foo(x: int) -> int: return x -[file driver.py] -from native import f, g -print(f(5)) -print(g(5)) -[out] -120 -5 +[file pkg/__init__.py] +x = 1 +[file pkg/mod.py] +y = 2 [case testImportMissing] # The unchecked module is configured by the test harness to not be From 7341cafdbaa8bdb06a8e7ec1d6faee79686895c1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 6 Dec 2020 20:28:31 +0000 Subject: [PATCH 2/3] Add local imports back to globals() since some imports break otherwise --- mypyc/irbuild/statement.py | 53 ++++++++++++------------- mypyc/test-data/irbuild-basic.test | 62 ++++++++++++++++++------------ mypyc/test-data/run-imports.test | 17 +++++++- 3 files changed, 81 insertions(+), 51 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index d97f3d57fce8e..b56175ae3e2f6 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -119,35 +119,36 @@ def transform_operator_assignment_stmt(builder: IRBuilder, stmt: OperatorAssignm def transform_import(builder: IRBuilder, node: Import) -> None: if node.is_mypy_only: return + globals = builder.load_globals_dict() for node_id, as_name in node.ids: builder.gen_import(node_id, node.line) - if builder.non_function_scope(): - # Update the globals dict with the appropriate module: - # * For 'import foo.bar as baz' we add 'foo.bar' with the name 'baz' - # * For 'import foo.bar' we add 'foo' with the name 'foo' - # Typically we then ignore these entries and access things directly - # via the module static, but we will use the globals version for modules - # that mypy couldn't find, since it doesn't analyze module references - # from those properly. - - # Miscompiling imports inside of functions, like below in import from. - if as_name: - name = as_name - base = node_id - else: - base = name = node_id.split('.')[0] - - globals = builder.load_globals_dict() - # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( - mod_dict = builder.call_c(get_module_dict_op, [], node.line) - # Get top-level module/package object. - obj = builder.call_c(dict_get_item_op, - [mod_dict, builder.load_static_unicode(base)], node.line) - - builder.gen_method_call( - globals, '__setitem__', [builder.load_static_unicode(name), obj], - result_type=None, line=node.line) + # Update the globals dict with the appropriate module: + # * For 'import foo.bar as baz' we add 'foo.bar' with the name 'baz' + # * For 'import foo.bar' we add 'foo' with the name 'foo' + # Typically we then ignore these entries and access things directly + # via the module static, but we will use the globals version for modules + # that mypy couldn't find, since it doesn't analyze module references + # from those properly. + + # TODO: Don't add local imports to the global namespace + + # Miscompiling imports inside of functions, like below in import from. + if as_name: + name = as_name + base = node_id + else: + base = name = node_id.split('.')[0] + + # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( + mod_dict = builder.call_c(get_module_dict_op, [], node.line) + # Get top-level module/package object. + obj = builder.call_c(dict_get_item_op, + [mod_dict, builder.load_static_unicode(base)], node.line) + + builder.gen_method_call( + globals, '__setitem__', [builder.load_static_unicode(name), obj], + result_type=None, line=node.line) def transform_import_from(builder: IRBuilder, node: ImportFrom) -> None: diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index 6019227dd82bd..5917ef529413b 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -3647,30 +3647,44 @@ x = 1 [file p/m.py] [out] def f(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4 :: object - r5 :: dict - r6 :: str - r7 :: object - r8 :: str - r9 :: object - r10 :: int + r0 :: dict + r1, r2 :: object + r3 :: bit + r4 :: str + r5 :: object + r6 :: dict + r7 :: str + r8 :: object + r9 :: str + r10 :: int32 + r11 :: bit + r12 :: dict + r13 :: str + r14 :: object + r15 :: str + r16 :: object + r17 :: int L0: - r0 = p.m :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 = __main__.globals :: static + r1 = p.m :: module + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = load_global CPyStatic_unicode_1 :: static ('p.m') - r4 = PyImport_Import(r3) - p.m = r4 :: module + r4 = load_global CPyStatic_unicode_1 :: static ('p.m') + r5 = PyImport_Import(r4) + p.m = r5 :: module L2: - r5 = PyImport_GetModuleDict() - r6 = load_global CPyStatic_unicode_2 :: static ('p') - r7 = CPyDict_GetItem(r5, r6) - r8 = load_global CPyStatic_unicode_3 :: static ('x') - r9 = CPyObject_GetAttr(r7, r8) - r10 = unbox(int, r9) - return r10 + r6 = PyImport_GetModuleDict() + r7 = load_global CPyStatic_unicode_2 :: static ('p') + r8 = CPyDict_GetItem(r6, r7) + r9 = load_global CPyStatic_unicode_2 :: static ('p') + r10 = CPyDict_SetItem(r0, r9, r8) + r11 = r10 >= 0 :: signed + r12 = PyImport_GetModuleDict() + r13 = load_global CPyStatic_unicode_2 :: static ('p') + r14 = CPyDict_GetItem(r12, r13) + r15 = load_global CPyStatic_unicode_3 :: static ('x') + r16 = CPyObject_GetAttr(r14, r15) + r17 = unbox(int, r16) + return r17 diff --git a/mypyc/test-data/run-imports.test b/mypyc/test-data/run-imports.test index 0190545bfe8d9..3a87c92c3617e 100644 --- a/mypyc/test-data/run-imports.test +++ b/mypyc/test-data/run-imports.test @@ -10,7 +10,7 @@ def g(x: int) -> int: from welp import foo return foo(x) -def test_import() -> None: +def test_import_basics() -> None: assert f(5) == 120 assert g(5) == 5 @@ -19,6 +19,19 @@ def test_import_submodule_within_function() -> None: assert pkg.x == 1 assert pkg.mod.y == 2 +# TODO: Don't add local imports to globals() +# +# def test_local_import_not_in_globals() -> None: +# import nob +# assert 'nob' not in globals() + +def test_import_module_without_stub_in_function() -> None: + # 'virtualenv' must not have a stub in typeshed for this test case + import virtualenv # type: ignore + # TODO: We shouldn't add local imports to globals() + # assert 'virtualenv' not in globals() + assert isinstance(virtualenv.__name__, str) + [file testmodule.py] def factorial(x: int) -> int: if x == 0: @@ -32,6 +45,8 @@ def foo(x: int) -> int: x = 1 [file pkg/mod.py] y = 2 +[file nob.py] +z = 3 [case testImportMissing] # The unchecked module is configured by the test harness to not be From fc26c1bbbc58d55580949f06a93c564b118c2530 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 6 Dec 2020 20:30:53 +0000 Subject: [PATCH 3/3] Add test cases --- mypyc/test-data/run-imports.test | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mypyc/test-data/run-imports.test b/mypyc/test-data/run-imports.test index 3a87c92c3617e..78b167861ae82 100644 --- a/mypyc/test-data/run-imports.test +++ b/mypyc/test-data/run-imports.test @@ -19,6 +19,10 @@ def test_import_submodule_within_function() -> None: assert pkg.x == 1 assert pkg.mod.y == 2 +def test_import_as_submodule_within_function() -> None: + import pkg.mod as mm + assert mm.y == 2 + # TODO: Don't add local imports to globals() # # def test_local_import_not_in_globals() -> None: @@ -32,6 +36,14 @@ def test_import_module_without_stub_in_function() -> None: # assert 'virtualenv' not in globals() assert isinstance(virtualenv.__name__, str) +def test_import_as_module_without_stub_in_function() -> None: + # 'virtualenv' must not have a stub in typeshed for this test case + import virtualenv as vv # type: ignore + assert 'virtualenv' not in globals() + # TODO: We shouldn't add local imports to globals() + # assert 'vv' not in globals() + assert isinstance(vv.__name__, str) + [file testmodule.py] def factorial(x: int) -> int: if x == 0: