From eb1a7ab74c24829131bb177f0bc24aa6bba82799 Mon Sep 17 00:00:00 2001 From: Matthew Rahtz Date: Sun, 13 Mar 2022 11:38:32 +0000 Subject: [PATCH 1/3] Refactor typevar substitution into separate helper for easier testing --- Lib/typing.py | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index dd68e71db1558c..58de28710e5b12 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1269,11 +1269,6 @@ def __getitem__(self, args): if (self._paramspec_tvars and any(isinstance(t, ParamSpec) for t in self.__parameters__)): args = _prepare_paramspec_params(self, args) - elif not any(isinstance(p, TypeVarTuple) for p in self.__parameters__): - # We only run this if there are no TypeVarTuples, because we - # don't check variadic generic arity at runtime (to reduce - # complexity of typing.py). - _check_generic(self, args, len(self.__parameters__)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) @@ -1296,18 +1291,7 @@ def _determine_new_args(self, args): params = self.__parameters__ # In the example above, this would be {T3: str} - new_arg_by_param = {} - for i, param in enumerate(params): - if isinstance(param, TypeVarTuple): - j = len(args) - (len(params) - i - 1) - if j < i: - raise TypeError(f"Too few arguments for {self}") - new_arg_by_param.update(zip(params[:i], args[:i])) - new_arg_by_param[param] = args[i: j] - new_arg_by_param.update(zip(params[i + 1:], args[j:])) - break - else: - new_arg_by_param.update(zip(params, args)) + new_arg_by_param = _determine_typevar_substitution(self, params, args) new_args = [] for old_arg in self.__args__: @@ -1401,6 +1385,28 @@ def __iter__(self): yield Unpack[self] +def _determine_typevar_substitution(cls, params, args): + new_arg_by_param = {} + for i, param in enumerate(params): + if isinstance(param, TypeVarTuple): + j = len(args) - (len(params) - i - 1) + if j < i: + raise TypeError("Too few type arguments") + new_arg_by_param.update(zip(params[:i], args[:i])) + new_arg_by_param[param] = args[i: j] + new_arg_by_param.update(zip(params[i + 1:], args[j:])) + if any(isinstance(param, TypeVarTuple) for param in params[i + 1:]): + raise TypeError("Only one TypeVarTuple may be used in type parameters") + break + else: + # We only run this if there are no TypeVarTuples, because we + # don't check variadic generic arity at runtime (to reduce + # complexity of typing.py). + _check_generic(cls, args, len(params)) + new_arg_by_param.update(zip(params, args)) + return new_arg_by_param + + # _nparams is the number of accepted parameters, e.g. 0 for Hashable, # 1 for List and 2 for Dict. It may be -1 if variable number of # parameters are accepted (needs custom __getitem__). From f500d5909fcca23392a361271957a3e55f86f9d1 Mon Sep 17 00:00:00 2001 From: Matthew Rahtz Date: Sun, 13 Mar 2022 11:38:59 +0000 Subject: [PATCH 2/3] Deal properly with e.g. *tuple[int] --- Lib/typing.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Lib/typing.py b/Lib/typing.py index 58de28710e5b12..bf427e23c5a02f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1385,7 +1385,23 @@ def __iter__(self): yield Unpack[self] +def _replace_degenerate_unpacked_tuples( + args: tuple[type, ...] +) -> tuple[type, ...]: + """Replaces e.g. `*tuple[int]` with just `int` in `args`.""" + new_args = [] + for arg in args: + if (_is_unpacked_tuple(arg) + and not _is_unpacked_arbitrary_length_tuple(arg)): + arg_tuple = arg.__args__[0] # The actual tuple[int] + new_args.extend(arg_tuple.__args__) + else: + new_args.append(arg) + return tuple(new_args) + + def _determine_typevar_substitution(cls, params, args): + args = _replace_degenerate_unpacked_tuples(args) new_arg_by_param = {} for i, param in enumerate(params): if isinstance(param, TypeVarTuple): From 8611053d00b2cc95035031cf435c2cf9b1577c44 Mon Sep 17 00:00:00 2001 From: Matthew Rahtz Date: Sun, 13 Mar 2022 11:39:16 +0000 Subject: [PATCH 3/3] Add more tests for typevar substitution --- Lib/test/test_typing.py | 120 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b212b523048809..cdcebc23919cce 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -31,6 +31,7 @@ from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs from typing import TypeGuard +from typing import _determine_typevar_substitution import abc import textwrap import typing @@ -843,6 +844,125 @@ class C(Generic[Unpack[Ts]]): pass self.assertNotEqual(C[Unpack[Ts1]], C[Unpack[Ts2]]) +class TypeVarSubstitutionTests(BaseTestCase): + + def test_valid_substitution(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + Ts = TypeVarTuple('Ts') + + # These are tuples of (typevars, args, expected_result). + test_cases = [ + # TypeVars only + ((T1,), (int,), {T1: int}), + ((T1,), (tuple[int, ...],), {T1: tuple[int, ...]}), + ((T1,), (Unpack[tuple[int]],), {T1: int}), + ((T1, T2), (int, str), {T1: int, T2: str}), + ((T1, T2), (tuple[int, ...], tuple[str, ...]), {T1: tuple[int, ...], T2: tuple[str, ...]}), + ((T1, T2), (Unpack[tuple[int, str]],), {T1: int, T2: str}), + # TypeVarTuple only + ((Ts,), (), {Ts: ()}), + ((Ts,), (int,), {Ts: (int,)}), + ((Ts,), (tuple[int, ...],), {Ts: (tuple[int, ...],)}), + ((Ts,), (Unpack[tuple[int]],), {Ts: (int,)}), + ((Ts,), (int, str), {Ts: (int, str)}), + ((Ts,), (tuple[int, ...], tuple[str, ...]), {Ts: (tuple[int, ...], tuple[str, ...])}), + ((Ts,), (Unpack[tuple[int, ...]],), {Ts: (Unpack[tuple[int, ...]],)}), + # TypeVarTuple at the beginning + ((Ts, T1), (int,), {Ts: (), T1: int}), + ((Ts, T1), (tuple[int, ...],), {Ts: (), T1: tuple[int, ...]}), + ((Ts, T1), (int, str), {Ts: (int,), T1: str}), + ((Ts, T1), (int, str, float), {Ts: (int, str), T1: float}), + ((Ts, T1), (Unpack[tuple[int, ...]], str), {Ts: (Unpack[tuple[int, ...]],), T1: str}), + ((Ts, T1), (Unpack[tuple[int, ...]], str, bool), {Ts: (Unpack[tuple[int, ...]], str), T1: bool}), + # TypeVarTuple at the end + ((T1, Ts), (int,), {T1: int, Ts: ()}), + ((T1, Ts), (int, str), {T1: int, Ts: (str,)}), + ((T1, Ts), (int, str, float), {T1: int, Ts: (str, float)}), + ((T1, Ts), (int, Unpack[tuple[str, ...]]), {T1: int, Ts: (Unpack[tuple[str, ...]],)}), + ((T1, Ts), (int, str, Unpack[tuple[float, ...]]), {T1: int, Ts: (str, Unpack[tuple[float, ...]],)}), + # TypeVarTuple in the middle + ((T1, Ts, T2), (int, str), {T1: int, Ts: (), T2: str}), + ((T1, Ts, T2), (int, float, str), {T1: int, Ts: (float,), T2: str}), + ((T1, Ts, T2), (int, Unpack[tuple[int, ...]], str), {T1: int, Ts: (Unpack[tuple[int, ...]],), T2: str}), + ((T1, Ts, T2), (int, float, Unpack[tuple[bool, ...]], str), {T1: int, Ts: (float, Unpack[tuple[bool, ...]],), T2: str}), + ((T1, Ts, T2), (int, Unpack[tuple[bool, ...]], float, str), {T1: int, Ts: (Unpack[tuple[bool, ...]], float), T2: str}), + ((T1, Ts, T2), (int, complex, Unpack[tuple[bool, ...]], float, str), {T1: int, Ts: (complex, Unpack[tuple[bool, ...]], float), T2: str}) + ] + for typevars, args, expected_result in test_cases: + with self.subTest(f'typevars={typevars}, args={args}'): + self.assertEqual( + expected_result, + _determine_typevar_substitution( + cls=None, params=typevars, args=args + ) + ) + + def test_too_few_args_raises_exception(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + # We don't include test cases including TypeVarTuples because we + # decided not to implement arity checking of variadic generics + # in order to reduce complexity. + test_cases = [ + # One TypeVar: invalid if 0 args + ((T1,), ()), + ((T1,), (Unpack[tuple[()]],)), + # Two TypeVars: invalid if <= 1 args + ((T1, T2), (int,)), + ((T1, T2), (Unpack[tuple[int]],)), + ] + for typevars, args in test_cases: + with self.subTest(f'typevars={typevars}, args={args}'): + with self.assertRaises(TypeError): + _determine_typevar_substitution( + cls=None, params=typevars, args=args + ) + + def test_too_many_args_raises_exception(self): + T1 = TypeVar('T1') + T2 = TypeVar('T2') + + # We don't include test cases including TypeVarTuples because we + # decided not to implement arity checking of variadic generics + # in order to reduce complexity. + test_cases = [ + # One TypeVar: invalid if >= 2 args + ((T1,), (int, int)), + ((T1,), (Unpack[tuple[int, int]],)), + ((T1,), (Unpack[tuple[int]], Unpack[tuple[int]])), + # Two TypeVars: invalid if >= 3 args + ((T1, T2), (int, int, int)), + ((T1, T2), (Unpack[tuple[int, int, int]],)), + ((T1, T2), (Unpack[tuple[int]], Unpack[tuple[int]], Unpack[tuple[int]])), + ] + + for typevars, args in test_cases: + with self.subTest(f'typevars={typevars}, args={args}'): + with self.assertRaises(TypeError): + _determine_typevar_substitution( + cls=None, params=typevars, args=args + ) + + def test_too_many_typevartuples_raises_exception(self): + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + test_cases = [ + ((T, Ts, Ts), (int, str)), + ((Ts, T, Ts), (int, str)), + ((Ts, Ts, T), (int, str)) + ] + + for typevars, args in test_cases: + with self.subTest(f'typevars={typevars}, args={args}'): + with self.assertRaises(TypeError): + _determine_typevar_substitution( + cls=None, params=typevars, args=args + ) + + class UnionTests(BaseTestCase): def test_basics(self):