New semantic analyzer: fix crash caused by leaking placeholders (#6845)

JukkaL · web-flow · commit 924de06a4715 · 2019-05-16T13:02:28.000-04:00
This fixes a crash when processing the latest unittest stubs.

Previously we didn't always defer a target when placeholder nodes
were added to a symbol table. This could result in crashes.
diff --git a/mypy/newsemanal/semanal.py b/mypy/newsemanal/semanal.py
@@ -313,7 +313,6 @@ def add_implicit_module_attrs(self, file_node: MypyFile) -> None:
                 self.add_symbol(name, var, None)
             else:
                 self.add_symbol(name, PlaceholderNode(self.qualified_name(name), file_node), None)
-                self.defer()
 
     def prepare_file(self, file_node: MypyFile) -> None:
         """Prepare a freshly parsed file for semantic analysis."""
@@ -848,7 +847,12 @@ def visit_class_def(self, defn: ClassDef) -> None:
     def analyze_class(self, defn: ClassDef) -> None:
         fullname = self.qualified_name(defn.name)
         if not defn.info and not self.is_core_builtin_class(defn):
-            self.add_symbol(defn.name, PlaceholderNode(fullname, defn, True), defn)
+            # Add placeholder so that self-references in base classes can be
+            # resolved.  We don't want this to cause a deferral, since if there
+            # are no incomplete references, we'll replace this with a TypeInfo
+            # before returning.
+            self.add_symbol(defn.name, PlaceholderNode(fullname, defn, True), defn,
+                            can_defer=False)
 
         tag = self.track_incomplete_refs()
 
@@ -3900,7 +3904,11 @@ def lookup_qualified(self, name: str, ctx: Context,
             return None
 
     def defer(self) -> None:
-        """Defer current analysis target to be analyzed again."""
+        """Defer current analysis target to be analyzed again.
+
+        This must be called if something in the current target is
+        incomplete or has a placeholder node.
+        """
         self.deferred = True
 
     def track_incomplete_refs(self) -> Tag:
@@ -4115,7 +4123,8 @@ def current_symbol_kind(self) -> int:
         return kind
 
     def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
-                   module_public: bool = True, module_hidden: bool = False) -> bool:
+                   module_public: bool = True, module_hidden: bool = False,
+                   can_defer: bool = True) -> bool:
         """Add symbol to the currently active symbol table.
 
         Generally additions to symbol table should go through this method or
@@ -4124,6 +4133,8 @@ def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
 
         Return True if we actually added the symbol, or False if we refused to do so
         (because something is not ready).
+
+        If can_defer is True, defer current target if adding a placeholder.
         """
         if self.is_func_scope():
             kind = LDEF
@@ -4135,7 +4146,7 @@ def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
                                  node,
                                  module_public=module_public,
                                  module_hidden=module_hidden)
-        return self.add_symbol_table_node(name, symbol, context)
+        return self.add_symbol_table_node(name, symbol, context, can_defer)
 
     def add_symbol_skip_local(self, name: str, node: SymbolNode) -> None:
         """Same as above, but skipping the local namespace.
@@ -4175,21 +4186,26 @@ def is_global_or_nonlocal(self, name: str) -> bool:
                      or name in self.nonlocal_decls[-1]))
 
     def add_symbol_table_node(self, name: str, symbol: SymbolTableNode,
-                              context: Optional[Context] = None) -> bool:
+                              context: Optional[Context] = None,
+                              can_defer: bool = True) -> bool:
         """Add symbol table node to the currently active symbol table.
 
         Return True if we actually added the symbol, or False if we refused to do so
         (because something is not ready).
+
+        If can_defer is True, defer current target if adding a placeholder.
         """
         names = self.current_symbol_table()
         existing = names.get(name)
+        if isinstance(symbol.node, PlaceholderNode) and can_defer:
+            self.defer()
         if (existing is not None
                 and context is not None
                 and (not isinstance(existing.node, PlaceholderNode)
-                     or isinstance(symbol.node, PlaceholderNode) and
-                     # Allow replacing becomes_typeinfo=False with becomes_typeinfo=True.
-                     # This can happen for type aliases and NewTypes.
-                     not symbol.node.becomes_typeinfo)):
+                     or (isinstance(symbol.node, PlaceholderNode) and
+                         # Allow replacing becomes_typeinfo=False with becomes_typeinfo=True.
+                         # This can happen for type aliases and NewTypes.
+                         not symbol.node.becomes_typeinfo))):
             # There is an existing node, so this may be a redefinition.
             # If the new node points to the same node as the old one,
             # or if both old and new nodes are placeholders, we don't
diff --git a/mypy/newsemanal/semanal_newtype.py b/mypy/newsemanal/semanal_newtype.py
@@ -50,7 +50,8 @@ def process_newtype_declaration(self, s: AssignmentStmt) -> bool:
         if (not call.analyzed or
                 isinstance(call.analyzed, NewTypeExpr) and not call.analyzed.info):
             # Start from labeling this as a future class, as we do for normal ClassDefs.
-            self.api.add_symbol(name, PlaceholderNode(fullname, s, becomes_typeinfo=True), s)
+            self.api.add_symbol(name, PlaceholderNode(fullname, s, becomes_typeinfo=True), s,
+                                can_defer=False)
 
         old_type, should_defer = self.check_newtype_args(name, call, s)
         if not call.analyzed:
diff --git a/mypy/newsemanal/semanal_shared.py b/mypy/newsemanal/semanal_shared.py
@@ -133,7 +133,8 @@ def current_symbol_table(self) -> SymbolTable:
 
     @abstractmethod
     def add_symbol(self, name: str, node: SymbolNode, context: Optional[Context],
-                   module_public: bool = True, module_hidden: bool = False) -> bool:
+                   module_public: bool = True, module_hidden: bool = False,
+                   can_defer: bool = True) -> bool:
         """Add symbol to the current symbol table."""
         raise NotImplementedError
 
diff --git a/test-data/unit/check-newsemanal.test b/test-data/unit/check-newsemanal.test
@@ -1009,7 +1009,8 @@ class B: ...
 import a
 [file a.py]
 C = 1
-from b import *  # E: Name 'C' already defined on line 1
+from b import *  # E: Name 'C' already defined on line 1 \
+                 # E: Incompatible import of "C" (imported name has type "Type[C]", local name has type "int")
 
 [file b.py]
 import a
@@ -2200,3 +2201,40 @@ x = 1
 # We want to check that the first definition creates the variable.
 def x() -> None: ...  # E: Name 'x' already defined on line 1
 y = 2
+
+[case testNewAnalyzerImportStarSpecialCase]
+import unittest
+[file unittest/__init__.pyi]
+from unittest.suite import *
+
+def load_tests() -> TestSuite: ...
+[file unittest/suite.pyi]
+from typing import Union # Iterable not imported
+import unittest.case
+
+_TestType = Union[unittest.case.TestCase]
+
+class BaseTestSuite(Iterable[_TestType]):
+    ...
+
+class TestSuite(BaseTestSuite):
+    ...
+
+[file unittest/case.pyi]
+class TestCase: ...
+
+[out]
+tmp/unittest/suite.pyi:6: error: Name 'Iterable' is not defined
+tmp/unittest/suite.pyi:6: note: Did you forget to import it from "typing"? (Suggestion: "from typing import Iterable")
+
+[case testNewAnalyzerNewTypeSpecialCase]
+from typing import NewType
+from typing_extensions import Final, Literal
+
+X = NewType('X', int)
+
+var1: Final = 1
+
+def force1(x: Literal[1]) -> None: pass
+
+force1(reveal_type(var1))            # E: Revealed type is 'Literal[1]'
