Add machinery for marking places we subvert the type system (#5467)

msullivan · web-flow · commit 57004569b62a · 2018-08-15T11:58:58.000-07:00
We need this for compiling with mypyc, which inserts runtime
typechecks that cause problems when we subvert the type system. So
when compiling with mypyc, we turn those places into Any, while
keeping the types around for normal typechecks.

Since this causes the runtype types to be Any, this is best used
in places where efficient access to properties is not important.
For those cases some other technique should be used.
diff --git a/mypy/bogus_type.py b/mypy/bogus_type.py
@@ -0,0 +1,22 @@
+"""A Bogus[T] type alias for marking when we subvert the type system
+
+We need this for compiling with mypyc, which inserts runtime
+typechecks that cause problems when we subvert the type system. So
+when compiling with mypyc, we turn those places into Any, while
+keeping the types around for normal typechecks.
+
+Since this causes the runtime types to be Any, this is best used
+in places where efficient access to properties is not important.
+For those cases some other technique should be used.
+"""
+
+from mypy_extensions import FlexibleAlias
+from typing import TypeVar, Any
+
+T = TypeVar('T')
+
+SUPPRESS_BOGUS_TYPES = False
+if SUPPRESS_BOGUS_TYPES:
+    Bogus = FlexibleAlias[T, Any]
+else:
+    Bogus = FlexibleAlias[T, T]
diff --git a/mypy/nodes.py b/mypy/nodes.py
@@ -16,6 +16,8 @@
 from mypy.visitor import NodeVisitor, StatementVisitor, ExpressionVisitor
 from mypy.mypyc_hacks import SymbolTable
 
+from mypy.bogus_type import Bogus
+
 
 class Context:
     """Base type for objects that are valid as error message locations."""
@@ -177,8 +179,11 @@ class SymbolNode(Node):
     @abstractmethod
     def name(self) -> str: pass
 
+    # fullname can often be None even though the type system
+    # disagrees. We mark this with Bogus to let mypyc know not to
+    # worry about it.
     @abstractmethod
-    def fullname(self) -> str: pass
+    def fullname(self) -> Bogus[str]: pass
 
     @abstractmethod
     def serialize(self) -> JsonDict: pass
@@ -195,10 +200,8 @@ def deserialize(cls, data: JsonDict) -> 'SymbolNode':
 class MypyFile(SymbolNode):
     """The abstract syntax tree of a single source file."""
 
-    # Module name ('__main__' for initial file)
-    _name = None      # type: str
     # Fully qualified module name
-    _fullname = None  # type: str
+    _fullname = None  # type: Bogus[str]
     # Path to the file (None if not known)
     path = ''
     # Top-level definitions and statements
@@ -238,9 +241,9 @@ def __init__(self,
             self.ignored_lines = set()
 
     def name(self) -> str:
-        return self._name
+        return '' if not self._fullname else self._fullname.split('.')[-1]
 
-    def fullname(self) -> str:
+    def fullname(self) -> Bogus[str]:
         return self._fullname
 
     def accept(self, visitor: NodeVisitor[T]) -> T:
@@ -252,7 +255,6 @@ def is_package_init_file(self) -> bool:
 
     def serialize(self) -> JsonDict:
         return {'.class': 'MypyFile',
-                '_name': self._name,
                 '_fullname': self._fullname,
                 'names': self.names.serialize(self._fullname),
                 'is_stub': self.is_stub,
@@ -264,7 +266,6 @@ def serialize(self) -> JsonDict:
     def deserialize(cls, data: JsonDict) -> 'MypyFile':
         assert data['.class'] == 'MypyFile', data
         tree = MypyFile([], [])
-        tree._name = data['_name']
         tree._fullname = data['_fullname']
         tree.names = SymbolTable.deserialize(data['names'])
         tree.is_stub = data['is_stub']
@@ -405,12 +406,12 @@ def __init__(self) -> None:
         self.is_static = False
         # Name with module prefix
         # TODO: Type should be Optional[str]
-        self._fullname = cast(str, None)
+        self._fullname = cast(Bogus[str], None)
 
     @abstractmethod
     def name(self) -> str: pass
 
-    def fullname(self) -> str:
+    def fullname(self) -> Bogus[str]:
         return self._fullname
 
 
@@ -660,7 +661,7 @@ def __init__(self, func: FuncDef, decorators: List[Expression],
     def name(self) -> str:
         return self.func.name()
 
-    def fullname(self) -> str:
+    def fullname(self) -> Bogus[str]:
         return self.func.fullname()
 
     @property
@@ -725,7 +726,7 @@ def __init__(self, name: str, type: 'Optional[mypy.types.Type]' = None) -> None:
         super().__init__()
         self._name = name   # Name without module prefix
         # TODO: Should be Optional[str]
-        self._fullname = cast(str, None)  # Name with module prefix
+        self._fullname = cast(Bogus[str], None)  # Name with module prefix
         # TODO: Should be Optional[TypeInfo]
         self.info = VAR_NO_INFO
         self.type = type  # type: Optional[mypy.types.Type] # Declared or inferred type, or None
@@ -748,7 +749,7 @@ def __init__(self, name: str, type: 'Optional[mypy.types.Type]' = None) -> None:
     def name(self) -> str:
         return self._name
 
-    def fullname(self) -> str:
+    def fullname(self) -> Bogus[str]:
         return self._fullname
 
     def accept(self, visitor: StatementVisitor[T]) -> T:
@@ -780,7 +781,7 @@ class ClassDef(Statement):
     """Class definition"""
 
     name = None  # type: str       # Name of the class without module prefix
-    fullname = None  # type: str   # Fully qualified name of the class
+    fullname = None  # type: Bogus[str]   # Fully qualified name of the class
     defs = None  # type: Block
     type_vars = None  # type: List[mypy.types.TypeVarDef]
     # Base class expressions (not semantically analyzed -- can be arbitrary expressions)
@@ -2053,7 +2054,7 @@ class is generic then it will be a type constructor of higher kind.
     the appropriate number of arguments.
     """
 
-    _fullname = None  # type: str          # Fully qualified name
+    _fullname = None  # type: Bogus[str]          # Fully qualified name
     # Fully qualified name for the module this type was defined in. This
     # information is also in the fullname, but is harder to extract in the
     # case of nested class definitions.
@@ -2193,7 +2194,7 @@ def name(self) -> str:
         """Short name."""
         return self.defn.name
 
-    def fullname(self) -> str:
+    def fullname(self) -> Bogus[str]:
         return self._fullname
 
     def is_generic(self) -> bool:
diff --git a/mypy/treetransform.py b/mypy/treetransform.py
@@ -61,7 +61,6 @@ def visit_mypy_file(self, node: MypyFile) -> MypyFile:
         # NOTE: The 'names' and 'imports' instance variables will be empty!
         new = MypyFile(self.statements(node.defs), [], node.is_bom,
                        ignored_lines=set(node.ignored_lines))
-        new._name = node._name
         new._fullname = node._fullname
         new.path = node.path
         new.names = SymbolTable()
diff --git a/mypy/types.py b/mypy/types.py
@@ -21,6 +21,7 @@
 )
 from mypy.sharedparse import argument_elide_name
 from mypy.util import IdMapper
+from mypy.bogus_type import Bogus
 
 from mypy.mypyc_hacks import TypeOfAny
 
@@ -320,8 +321,9 @@ def accept(self, visitor: 'TypeVisitor[T]') -> T:
         return visitor.visit_any(self)
 
     def copy_modified(self,
-                      type_of_any: TypeOfAny = _dummy,
-                      original_any: Optional['AnyType'] = _dummy,
+                      # Mark with Bogus because _dummy is just an object (with type Any)
+                      type_of_any: Bogus[TypeOfAny] = _dummy,
+                      original_any: Bogus[Optional['AnyType']] = _dummy,
                       ) -> 'AnyType':
         if type_of_any is _dummy:
             type_of_any = self.type_of_any
@@ -720,22 +722,22 @@ def __init__(self,
             self.def_extras = {}
 
     def copy_modified(self,
-                      arg_types: List[Type] = _dummy,
-                      arg_kinds: List[int] = _dummy,
-                      arg_names: List[Optional[str]] = _dummy,
-                      ret_type: Type = _dummy,
-                      fallback: Instance = _dummy,
-                      name: Optional[str] = _dummy,
-                      definition: SymbolNode = _dummy,
-                      variables: List[TypeVarDef] = _dummy,
-                      line: int = _dummy,
-                      column: int = _dummy,
-                      is_ellipsis_args: bool = _dummy,
-                      implicit: bool = _dummy,
-                      special_sig: Optional[str] = _dummy,
-                      from_type_type: bool = _dummy,
-                      bound_args: List[Optional[Type]] = _dummy,
-                      def_extras: Dict[str, Any] = _dummy) -> 'CallableType':
+                      arg_types: Bogus[List[Type]] = _dummy,
+                      arg_kinds: Bogus[List[int]] = _dummy,
+                      arg_names: Bogus[List[Optional[str]]] = _dummy,
+                      ret_type: Bogus[Type] = _dummy,
+                      fallback: Bogus[Instance] = _dummy,
+                      name: Bogus[Optional[str]] = _dummy,
+                      definition: Bogus[SymbolNode] = _dummy,
+                      variables: Bogus[List[TypeVarDef]] = _dummy,
+                      line: Bogus[int] = _dummy,
+                      column: Bogus[int] = _dummy,
+                      is_ellipsis_args: Bogus[bool] = _dummy,
+                      implicit: Bogus[bool] = _dummy,
+                      special_sig: Bogus[Optional[str]] = _dummy,
+                      from_type_type: Bogus[bool] = _dummy,
+                      bound_args: Bogus[List[Optional[Type]]] = _dummy,
+                      def_extras: Bogus[Dict[str, Any]] = _dummy) -> 'CallableType':
         return CallableType(
             arg_types=arg_types if arg_types is not _dummy else self.arg_types,
             arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds,
@@ -1434,8 +1436,9 @@ class TypeType(Type):
     # a generic class instance, a union, Any, a type variable...
     item = None  # type: Type
 
-    def __init__(self, item: Union[Instance, AnyType, TypeVarType, TupleType, NoneTyp,
-                                   CallableType], *, line: int = -1, column: int = -1) -> None:
+    def __init__(self, item: Bogus[Union[Instance, AnyType, TypeVarType, TupleType, NoneTyp,
+                                         CallableType]], *,
+                 line: int = -1, column: int = -1) -> None:
         """To ensure Type[Union[A, B]] is always represented as Union[Type[A], Type[B]], item of
         type UnionType must be handled through make_normalized static method.
         """
diff --git a/mypy_bootstrap.ini b/mypy_bootstrap.ini
@@ -0,0 +1,15 @@
+[mypy]
+disallow_untyped_defs = True
+disallow_subclassing_any = True
+warn_no_return = True
+strict_optional = True
+no_implicit_optional = True
+disallow_any_generics = True
+disallow_any_unimported = True
+warn_redundant_casts = True
+warn_unused_configs = True
+always_true = SUPPRESS_BOGUS_TYPES
+
+# needs py2 compatibility
+[mypy-mypy.test.testextensions]
+disallow_untyped_defs = False
diff --git a/mypy_self_check.ini b/mypy_self_check.ini
@@ -8,6 +8,7 @@ disallow_any_generics = True
 disallow_any_unimported = True
 warn_redundant_casts = True
 warn_unused_configs = True
+always_false = SUPPRESS_BOGUS_TYPES
 
 # needs py2 compatibility
 [mypy-mypy.test.testextensions]
diff --git a/setup.py b/setup.py
@@ -104,6 +104,7 @@ def run(self):
       classifiers=classifiers,
       cmdclass={'build_py': CustomPythonBuild},
       install_requires = ['typed-ast >= 1.1.0, < 1.2.0',
+                          'mypy_extensions >= 0.4.0, < 0.5.0',
                           ],
       extras_require = {
           ':python_version < "3.5"': 'typing >= 3.5.3',
diff --git a/test-requirements.txt b/test-requirements.txt
@@ -3,6 +3,7 @@ flake8
 flake8-bugbear; python_version >= '3.5'
 flake8-pyi; python_version >= '3.6'
 lxml==4.2.4
+mypy_extensions==0.4.0
 psutil==5.4.0
 pytest>=3.4
 pytest-xdist>=1.22
