Enable Unpack/TypeVarTuple support (#16354)

ilevkivskyi · JelleZijlstra · web-flow · commit ad0e183b0df7 · 2023-10-30T12:14:00.000Z
Fixes #12280 Fixes #14697 In this PR: * Enable `TypeVarTuple` and `Unpack` features. * Delete the old blanket `--enable-incomplete-features` flag that was deprecated a year ago. * Switch couple corner cases to `PreciseTupleTypes` feature. * Add the draft docs about the new feature. * Handle a previously unhandled case where variadic tuple appears in string formatting (discovered on mypy self-check, where `PreciseTupleTypes` is already enabled). --------- Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst
@@ -991,6 +991,58 @@ format into the specified directory.
     library or specify mypy installation with the setuptools extra
     ``mypy[reports]``.
 
+
+Enabling incomplete/experimental features
+*****************************************
+
+.. option:: --enable-incomplete-feature FEATURE
+
+    Some features may require several mypy releases to implement, for example
+    due to their complexity, potential for backwards incompatibility, or
+    ambiguous semantics that would benefit from feedback from the community.
+    You can enable such features for early preview using this flag. Note that
+    it is not guaranteed that all features will be ultimately enabled by
+    default. In *rare cases* we may decide to not go ahead with certain
+    features.
+
+List of currently incomplete/experimental features:
+
+* ``PreciseTupleTypes``: this feature will infer more precise tuple types in
+  various scenarios. Before variadic types were added to the Python type system
+  by :pep:`646`, it was impossible to express a type like "a tuple with
+  at least two integers". The best type available was ``tuple[int, ...]``.
+  Therefore, mypy applied very lenient checking for variable-length tuples.
+  Now this type can be expressed as ``tuple[int, int, *tuple[int, ...]]``.
+  For such more precise types (when explicitly *defined* by a user) mypy,
+  for example, warns about unsafe index access, and generally handles them
+  in a type-safe manner. However, to avoid problems in existing code, mypy
+  does not *infer* these precise types when it technically can. Here are
+  notable examples where ``PreciseTupleTypes`` infers more precise types:
+
+  .. code-block:: python
+
+     numbers: tuple[int, ...]
+
+     more_numbers = (1, *numbers, 1)
+     reveal_type(more_numbers)
+     # Without PreciseTupleTypes: tuple[int, ...]
+     # With PreciseTupleTypes: tuple[int, *tuple[int, ...], int]
+
+     other_numbers = (1, 1) + numbers
+     reveal_type(other_numbers)
+     # Without PreciseTupleTypes: tuple[int, ...]
+     # With PreciseTupleTypes: tuple[int, int, *tuple[int, ...]]
+
+     if len(numbers) > 2:
+         reveal_type(numbers)
+         # Without PreciseTupleTypes: tuple[int, ...]
+         # With PreciseTupleTypes: tuple[int, int, int, *tuple[int, ...]]
+     else:
+         reveal_type(numbers)
+         # Without PreciseTupleTypes: tuple[int, ...]
+         # With PreciseTupleTypes: tuple[()] | tuple[int] | tuple[int, int]
+
+
 Miscellaneous
 *************
 
diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py
@@ -97,7 +97,7 @@
     YieldExpr,
     YieldFromExpr,
 )
-from mypy.options import TYPE_VAR_TUPLE
+from mypy.options import PRECISE_TUPLE_TYPES
 from mypy.plugin import (
     FunctionContext,
     FunctionSigContext,
@@ -3377,7 +3377,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
                         ):
                             return self.concat_tuples(proper_left_type, proper_right_type)
                 elif (
-                    TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature
+                    PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
                     and isinstance(proper_right_type, Instance)
                     and self.chk.type_is_iterable(proper_right_type)
                 ):
@@ -3411,7 +3411,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
                 if is_named_instance(proper_right_type, "builtins.dict"):
                     use_reverse = USE_REVERSE_NEVER
 
-        if TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature:
+        if PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature:
             # Handle tuple[X, ...] + tuple[Y, Z] = tuple[*tuple[X, ...], Y, Z].
             if (
                 e.op == "+"
@@ -4988,7 +4988,7 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type:
                         j += len(tt.items)
                 else:
                     if (
-                        TYPE_VAR_TUPLE in self.chk.options.enable_incomplete_feature
+                        PRECISE_TUPLE_TYPES in self.chk.options.enable_incomplete_feature
                         and not seen_unpack_in_items
                     ):
                         # Handle (x, *y, z), where y is e.g. tuple[Y, ...].
diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py
@@ -47,8 +47,11 @@
     TupleType,
     Type,
     TypeOfAny,
+    TypeVarTupleType,
     TypeVarType,
     UnionType,
+    UnpackType,
+    find_unpack_in_list,
     get_proper_type,
     get_proper_types,
 )
@@ -728,6 +731,22 @@ def check_simple_str_interpolation(
         rep_types: list[Type] = []
         if isinstance(rhs_type, TupleType):
             rep_types = rhs_type.items
+            unpack_index = find_unpack_in_list(rep_types)
+            if unpack_index is not None:
+                # TODO: we should probably warn about potentially short tuple.
+                # However, without special-casing for tuple(f(i) for in other_tuple)
+                # this causes false positive on mypy self-check in report.py.
+                extras = max(0, len(checkers) - len(rep_types) + 1)
+                unpacked = rep_types[unpack_index]
+                assert isinstance(unpacked, UnpackType)
+                unpacked = get_proper_type(unpacked.type)
+                if isinstance(unpacked, TypeVarTupleType):
+                    unpacked = get_proper_type(unpacked.upper_bound)
+                assert (
+                    isinstance(unpacked, Instance) and unpacked.type.fullname == "builtins.tuple"
+                )
+                unpack_items = [unpacked.args[0]] * extras
+                rep_types = rep_types[:unpack_index] + unpack_items + rep_types[unpack_index + 1 :]
         elif isinstance(rhs_type, AnyType):
             return
         elif isinstance(rhs_type, Instance) and rhs_type.type.fullname == "builtins.tuple":
diff --git a/mypy/main.py b/mypy/main.py
@@ -22,7 +22,7 @@
 from mypy.find_sources import InvalidSourceList, create_source_list
 from mypy.fscache import FileSystemCache
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths, get_search_dirs, mypy_path
-from mypy.options import INCOMPLETE_FEATURES, BuildType, Options
+from mypy.options import COMPLETE_FEATURES, INCOMPLETE_FEATURES, BuildType, Options
 from mypy.split_namespace import SplitNamespace
 from mypy.version import __version__
 
@@ -1151,10 +1151,7 @@ def add_invertible_flag(
     # --debug-serialize will run tree.serialize() even if cache generation is disabled.
     # Useful for mypy_primer to detect serialize errors earlier.
     parser.add_argument("--debug-serialize", action="store_true", help=argparse.SUPPRESS)
-    # This one is deprecated, but we will keep it for few releases.
-    parser.add_argument(
-        "--enable-incomplete-features", action="store_true", help=argparse.SUPPRESS
-    )
+
     parser.add_argument(
         "--disable-bytearray-promotion", action="store_true", help=argparse.SUPPRESS
     )
@@ -1334,14 +1331,10 @@ def set_strict_flags() -> None:
 
     # Validate incomplete features.
     for feature in options.enable_incomplete_feature:
-        if feature not in INCOMPLETE_FEATURES:
+        if feature not in INCOMPLETE_FEATURES | COMPLETE_FEATURES:
             parser.error(f"Unknown incomplete feature: {feature}")
-    if options.enable_incomplete_features:
-        print(
-            "Warning: --enable-incomplete-features is deprecated, use"
-            " --enable-incomplete-feature=FEATURE instead"
-        )
-        options.enable_incomplete_feature = list(INCOMPLETE_FEATURES)
+        if feature in COMPLETE_FEATURES:
+            print(f"Warning: {feature} is already enabled by default")
 
     # Compute absolute path for custom typeshed (if present).
     if options.custom_typeshed_dir is not None:
diff --git a/mypy/options.py b/mypy/options.py
@@ -69,11 +69,12 @@ class BuildType:
     }
 ) - {"debug_cache"}
 
-# Features that are currently incomplete/experimental
+# Features that are currently (or were recently) incomplete/experimental
 TYPE_VAR_TUPLE: Final = "TypeVarTuple"
 UNPACK: Final = "Unpack"
 PRECISE_TUPLE_TYPES: Final = "PreciseTupleTypes"
-INCOMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK, PRECISE_TUPLE_TYPES))
+INCOMPLETE_FEATURES: Final = frozenset((PRECISE_TUPLE_TYPES,))
+COMPLETE_FEATURES: Final = frozenset((TYPE_VAR_TUPLE, UNPACK))
 
 
 class Options:
@@ -307,7 +308,6 @@ def __init__(self) -> None:
         self.dump_type_stats = False
         self.dump_inference_stats = False
         self.dump_build_stats = False
-        self.enable_incomplete_features = False  # deprecated
         self.enable_incomplete_feature: list[str] = []
         self.timing_stats: str | None = None
         self.line_checking_stats: str | None = None
diff --git a/mypy/semanal.py b/mypy/semanal.py
@@ -179,7 +179,7 @@
     type_aliases_source_versions,
     typing_extensions_aliases,
 )
-from mypy.options import TYPE_VAR_TUPLE, Options
+from mypy.options import Options
 from mypy.patterns import (
     AsPattern,
     ClassPattern,
@@ -4417,9 +4417,6 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool:
             else:
                 self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s)
 
-        if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s):
-            return False
-
         name = self.extract_typevarlike_name(s, call)
         if name is None:
             return False
diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py
@@ -10,7 +10,6 @@
 from mypy.build import Graph
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource, FindModuleCache, SearchPaths
-from mypy.options import TYPE_VAR_TUPLE, UNPACK
 from mypy.test.config import test_data_prefix, test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, module_from_path
 from mypy.test.helpers import (
@@ -125,8 +124,6 @@ def run_case_once(
         # Parse options after moving files (in case mypy.ini is being moved).
         options = parse_options(original_program_text, testcase, incremental_step)
         options.use_builtins_fixtures = True
-        if not testcase.name.endswith("_no_incomplete"):
-            options.enable_incomplete_feature += [TYPE_VAR_TUPLE, UNPACK]
         options.show_traceback = True
 
         # Enable some options automatically based on test file name.
diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py
@@ -28,7 +28,7 @@
 from mypy.errors import CompileError
 from mypy.find_sources import create_source_list
 from mypy.modulefinder import BuildSource
-from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
+from mypy.options import Options
 from mypy.server.mergecheck import check_consistency
 from mypy.server.update import sort_messages_preserving_file_order
 from mypy.test.config import test_temp_dir
@@ -149,7 +149,6 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         options.use_fine_grained_cache = self.use_cache and not build_cache
         options.cache_fine_grained = self.use_cache
         options.local_partial_types = True
-        options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         # Treat empty bodies safely for these test cases.
         options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
         if re.search("flags:.*--follow-imports", source) is None:
diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py
@@ -10,7 +10,7 @@
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource
 from mypy.nodes import TypeInfo
-from mypy.options import TYPE_VAR_TUPLE, UNPACK, Options
+from mypy.options import Options
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import (
@@ -45,7 +45,6 @@ def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Opti
     options.semantic_analysis_only = True
     options.show_traceback = True
     options.python_version = PYTHON3_VERSION
-    options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
     options.force_uppercase_builtins = True
     return options
 
diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py
@@ -5,7 +5,6 @@
 from mypy import build
 from mypy.errors import CompileError
 from mypy.modulefinder import BuildSource
-from mypy.options import TYPE_VAR_TUPLE, UNPACK
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, normalize_error_messages, parse_options
@@ -38,7 +37,6 @@ def test_transform(testcase: DataDrivenTestCase) -> None:
         options = parse_options(src, testcase, 1)
         options.use_builtins_fixtures = True
         options.semantic_analysis_only = True
-        options.enable_incomplete_feature = [TYPE_VAR_TUPLE, UNPACK]
         options.show_traceback = True
         options.force_uppercase_builtins = True
         result = build.build(
diff --git a/mypy/typeanal.py b/mypy/typeanal.py
@@ -35,7 +35,7 @@
     check_arg_names,
     get_nongen_builtins,
 )
-from mypy.options import UNPACK, Options
+from mypy.options import Options
 from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
 from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs
 from mypy.tvar_scope import TypeVarLikeScope
@@ -664,8 +664,6 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ
             # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args)
             return self.named_type("builtins.bool")
         elif fullname in ("typing.Unpack", "typing_extensions.Unpack"):
-            if not self.api.incomplete_feature_enabled(UNPACK, t):
-                return AnyType(TypeOfAny.from_error)
             if len(t.args) != 1:
                 self.fail("Unpack[...] requires exactly one type argument", t)
                 return AnyType(TypeOfAny.from_error)
diff --git a/test-data/unit/check-flags.test b/test-data/unit/check-flags.test
@@ -2190,18 +2190,6 @@ x: int = ""  # E: Incompatible types in assignment (expression has type "str", v
 # flags: --hide-error-codes
 x: int = ""  # E: Incompatible types in assignment (expression has type "str", variable has type "int")
 
-[case testTypeVarTupleDisabled_no_incomplete]
-from typing_extensions import TypeVarTuple
-Ts = TypeVarTuple("Ts")  # E: "TypeVarTuple" support is experimental, use --enable-incomplete-feature=TypeVarTuple to enable
-[builtins fixtures/tuple.pyi]
-
-[case testTypeVarTupleEnabled_no_incomplete]
-# flags: --enable-incomplete-feature=TypeVarTuple
-from typing_extensions import TypeVarTuple
-Ts = TypeVarTuple("Ts")  # OK
-[builtins fixtures/tuple.pyi]
-
-
 [case testDisableBytearrayPromotion]
 # flags: --disable-bytearray-promotion
 def f(x: bytes) -> None: ...
diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test
@@ -1100,12 +1100,28 @@ reveal_type(b)  # N: Revealed type is "Tuple[builtins.int, builtins.int, builtin
 [case testTupleWithStarExpr2]
 a = [1]
 b = (0, *a)
+reveal_type(b)  # N: Revealed type is "builtins.tuple[builtins.int, ...]"
+[builtins fixtures/tuple.pyi]
+
+[case testTupleWithStarExpr2Precise]
+# flags: --enable-incomplete-feature=PreciseTupleTypes
+a = [1]
+b = (0, *a)
 reveal_type(b)  # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]]]"
 [builtins fixtures/tuple.pyi]
 
 [case testTupleWithStarExpr3]
 a = ['']
 b = (0, *a)
+reveal_type(b)  # N: Revealed type is "builtins.tuple[builtins.object, ...]"
+c = (*a, '')
+reveal_type(c)  # N: Revealed type is "builtins.tuple[builtins.str, ...]"
+[builtins fixtures/tuple.pyi]
+
+[case testTupleWithStarExpr3Precise]
+# flags: --enable-incomplete-feature=PreciseTupleTypes
+a = ['']
+b = (0, *a)
 reveal_type(b)  # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.str, ...]]]"
 c = (*a, '')
 reveal_type(c)  # N: Revealed type is "Tuple[Unpack[builtins.tuple[builtins.str, ...]], builtins.str]"
diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test
@@ -1653,6 +1653,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None:
 [builtins fixtures/tuple.pyi]
 
 [case testPackingVariadicTuplesHomogeneous]
+# flags: --enable-incomplete-feature=PreciseTupleTypes
 from typing import Tuple
 from typing_extensions import Unpack
 
@@ -1689,6 +1690,7 @@ def foo(arg: Tuple[int, Unpack[Ts], str]) -> None:
 [builtins fixtures/isinstancelist.pyi]
 
 [case testVariadicTupleInTupleContext]
+# flags: --enable-incomplete-feature=PreciseTupleTypes
 from typing import Tuple, Optional
 from typing_extensions import TypeVarTuple, Unpack
 
@@ -1701,6 +1703,7 @@ vt2 = 1, *test(), 2  # E: Need type annotation for "vt2"
 [builtins fixtures/tuple.pyi]
 
 [case testVariadicTupleConcatenation]
+# flags: --enable-incomplete-feature=PreciseTupleTypes
 from typing import Tuple
 from typing_extensions import TypeVarTuple, Unpack
 
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
@@ -1421,14 +1421,6 @@ b \d+
 b\.c \d+
 .*
 
-[case testCmdlineEnableIncompleteFeatures]
-# cmd: mypy --enable-incomplete-features a.py
-[file a.py]
-pass
-[out]
-Warning: --enable-incomplete-features is deprecated, use --enable-incomplete-feature=FEATURE instead
-== Return code: 0
-
 [case testShadowTypingModuleEarlyLoad]
 # cmd: mypy dir
 [file dir/__init__.py]
@@ -1585,3 +1577,13 @@ disable_error_code =
 always_true =
   MY_VAR,
 [out]
+
+[case testTypeVarTupleUnpackEnabled]
+# cmd: mypy --enable-incomplete-feature=TypeVarTuple --enable-incomplete-feature=Unpack a.py
+[file a.py]
+from typing_extensions import TypeVarTuple
+Ts = TypeVarTuple("Ts")
+[out]
+Warning: TypeVarTuple is already enabled by default
+Warning: Unpack is already enabled by default
+== Return code: 0
