From a1a016709973d8efbff55b32b2f5da6821bda5da Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 2 Sep 2023 22:45:04 +0100 Subject: [PATCH 01/14] Handle various edge cases for homogeneous tuples --- mypy/constraints.py | 64 ++++++++++++++++++-- mypy/subtypes.py | 77 ++++++++++++++++++++++++- test-data/unit/check-typevar-tuple.test | 67 +++++++++++++++++++++ 3 files changed, 200 insertions(+), 8 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 0e59b5459fd4..88a320072d0a 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1049,7 +1049,8 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: ) res.extend(unpack_constraints) else: - # Negate direction due to function argument type contravariance. + # TODO: do we need some special-casing when unpack is present in actual + # but not in template? res.extend( infer_callable_arguments_constraints(template, cactual, self.direction) ) @@ -1170,11 +1171,27 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: res: list[Constraint] = [] if unpack_index is not None: if is_varlength_tuple: + # Variadic tuple can be only a supertype of a tuple type, but even if + # direction is opposite, inferring something may give better error messages. unpack_type = template.items[unpack_index] assert isinstance(unpack_type, UnpackType) - unpacked_type = unpack_type.type - assert isinstance(unpacked_type, TypeVarTupleType) - return [Constraint(type_var=unpacked_type, op=self.direction, target=actual)] + unpacked_type = get_proper_type(unpack_type.type) + if isinstance(unpacked_type, TypeVarTupleType): + res = [ + Constraint(type_var=unpacked_type, op=self.direction, target=actual) + ] + else: + assert ( + isinstance(unpacked_type, Instance) + and unpacked_type.type.fullname == "builtins.tuple" + ) + res = infer_constraints(unpacked_type, actual, self.direction) + assert isinstance(actual, Instance) + for i, ti in enumerate(template.items): + if i == unpack_index: + continue + res.extend(infer_constraints(ti, actual.args[0], self.direction)) + return res else: assert isinstance(actual, TupleType) unpack_constraints = build_constraints_for_simple_unpack( @@ -1184,8 +1201,33 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: template_items: tuple[Type, ...] = () res.extend(unpack_constraints) elif isinstance(actual, TupleType): - actual_items = tuple(actual.items) - template_items = tuple(template.items) + a_unpack_index = find_unpack_in_list(actual.items) + if a_unpack_index is not None: + a_unpack = actual.items[a_unpack_index] + assert isinstance(a_unpack, UnpackType) + a_unpacked = get_proper_type(a_unpack.type) + if len(actual.items) + 1 <= len(template.items): + a_prefix_len = a_unpack_index + a_suffix_len = len(actual.items) - a_unpack_index - 1 + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + tuple(template.items), a_prefix_len, a_suffix_len + ) + actual_items = tuple(actual.items[:a_prefix_len]) + if a_suffix_len: + actual_items += tuple(actual.items[-a_suffix_len:]) + template_items = t_prefix + t_suffix + if isinstance(a_unpacked, Instance): + assert a_unpacked.type.fullname == "builtins.tuple" + for tm in t_middle: + res.extend( + infer_constraints(tm, a_unpacked.args[0], self.direction) + ) + else: + actual_items = () + template_items = () + else: + actual_items = tuple(actual.items) + template_items = tuple(template.items) else: return res @@ -1439,6 +1481,16 @@ def build_constraints_for_simple_unpack( res.extend(infer_constraints(tp.args[0], a_tp.args[0], direction)) elif isinstance(tp, TypeVarTupleType): res.append(Constraint(tp, direction, TupleType(list(middle), tp.tuple_fallback))) + elif actual_unpack is not None: + actual_unpack_type = actual_args[actual_unpack] + assert isinstance(actual_unpack_type, UnpackType) + a_unpacked = get_proper_type(actual_unpack_type.type) + if isinstance(a_unpacked, Instance) and a_unpacked.type.fullname == "builtins.tuple": + t_unpack = template_args[template_unpack] + assert isinstance(t_unpack, UnpackType) + tp = get_proper_type(t_unpack.type) + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + res.extend(infer_constraints(tp.args[0], a_unpacked.args[0], direction)) return res diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 58ae4efdf582..02148363d378 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -58,8 +58,10 @@ UninhabitedType, UnionType, UnpackType, + find_unpack_in_list, get_proper_type, is_named_instance, + split_with_prefix_and_suffix, ) from mypy.types_utils import flatten_types from mypy.typestate import SubtypeKind, type_state @@ -278,7 +280,13 @@ def _is_subtype( left = get_proper_type(left) right = get_proper_type(right) - if not proper_subtype and isinstance(right, (AnyType, UnboundType, ErasedType)): + # Note: Unpack type should not be a subtype of Any, since it may represent + # multiple types. This should always go through the visitor, to check arity. + if ( + not proper_subtype + and isinstance(right, (AnyType, UnboundType, ErasedType)) + and not isinstance(left, UnpackType) + ): # TODO: should we consider all types proper subtypes of UnboundType and/or # ErasedType as we do for non-proper subtyping. return True @@ -437,6 +445,14 @@ def visit_instance(self, left: Instance) -> bool: right = self.right if isinstance(right, TupleType) and right.partial_fallback.type.is_enum: return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) + if isinstance(right, TupleType) and len(right.items) == 1: + # Non-normalized Tuple type (may be left after semantic analysis). + item = right.items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + return self._is_subtype(left, unpacked) + return False if isinstance(right, Instance): if type_state.is_cached_subtype_check(self._subtype_kind, left, right): return True @@ -761,8 +777,12 @@ def visit_tuple_type(self, left: TupleType) -> bool: return True return False elif isinstance(right, TupleType): + # If right has a variadic unpack this needs special handling. If there is a TypeVarTuple + # unpack, item count must coincide. If the left has variadic unpack but right + # doesn't have one, we will fall through to False down the line. + if self.variadic_tuple_subtype(left, right): + return True if len(left.items) != len(right.items): - # TODO: handle tuple with variadic items better. return False if any(not self._is_subtype(l, r) for l, r in zip(left.items, right.items)): return False @@ -778,6 +798,59 @@ def visit_tuple_type(self, left: TupleType) -> bool: else: return False + def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: + right_unpack_index = find_unpack_in_list(right.items) + if right_unpack_index is None: + return False + right_unpack = right.items[right_unpack_index] + assert isinstance(right_unpack, UnpackType) + right_unpacked = get_proper_type(right_unpack.type) + if not isinstance(right_unpacked, Instance): + return False + assert right_unpacked.type.fullname == "builtins.tuple" + right_item = right_unpacked.args[0] + right_prefix = right_unpack_index + right_suffix = len(right.items) - right_prefix - 1 + left_unpack_index = find_unpack_in_list(left.items) + if left_unpack_index is None: + if len(left.items) < right_prefix + right_suffix: + return False + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(left.items), right_prefix, right_suffix + ) + if not all( + self._is_subtype(li, ri) for li, ri in zip(prefix, right.items[:right_prefix]) + ): + return False + if right_suffix and not all( + self._is_subtype(li, ri) for li, ri in zip(suffix, right.items[-right_suffix:]) + ): + return False + return all(self._is_subtype(li, right_item) for li in middle) + else: + if len(left.items) < len(right.items): + return False + left_unpack = left.items[left_unpack_index] + assert isinstance(left_unpack, UnpackType) + left_unpacked = get_proper_type(left_unpack.type) + if not isinstance(left_unpacked, Instance): + return False + assert left_unpacked.type.fullname == "builtins.tuple" + left_item = left_unpacked.args[0] + if not self._is_subtype(left_item, right_item): + return False + left_prefix = left_unpack_index + left_suffix = len(left.items) - left_prefix - 1 + max_overlap = max(0, right_prefix - left_prefix, right_suffix - left_suffix) + for overlap in range(max_overlap + 1): + repr_items = left.items[:left_prefix] + [left_item] * overlap + if left_suffix: + repr_items += left.items[-left_suffix:] + left_repr = left.copy_modified(items=repr_items) + if not self._is_subtype(left_repr, right): + return False + return True + def visit_typeddict_type(self, left: TypedDictType) -> bool: right = self.right if isinstance(right, Instance): diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 2b47ff30cdfb..b4a5da9f11ff 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1233,3 +1233,70 @@ def test(d: A[int, str]) -> None: else: reveal_type(d) # E: Statement is unreachable [builtins fixtures/isinstancelist.pyi] + +[case testVariadicTupleSubtyping] +from typing import Tuple +from typing_extensions import Unpack + +def f1(x: Tuple[float, ...]) -> None: ... +def f2(x: Tuple[float, Unpack[Tuple[float, ...]]]) -> None: ... +def f3(x: Tuple[Unpack[Tuple[float, ...]], float]) -> None: ... +def f4(x: Tuple[float, Unpack[Tuple[float, ...]], float]) -> None: ... + +t1: Tuple[int, int] +t2: Tuple[int, Unpack[Tuple[int, ...]]] +t3: Tuple[Unpack[Tuple[int, ...]], int] +t4: Tuple[int, Unpack[Tuple[int, ...]], int] +t5: Tuple[int, ...] + +f1(t1) +f1(t2) +f1(t3) +f1(t4) +f1(t5) + +f2(t1) +f2(t2) +f2(t3) +f2(t4) +f2(t5) # E: Argument 1 to "f2" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]]]" + +f3(t1) +f3(t2) +f3(t3) +f3(t4) +f3(t5) # E: Argument 1 to "f3" has incompatible type "Tuple[int, ...]"; expected "Tuple[Unpack[Tuple[float, ...]], float]" + +f4(t1) +f4(t2) # E: Argument 1 to "f4" has incompatible type "Tuple[int, Unpack[Tuple[int, ...]]]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" +f4(t3) # E: Argument 1 to "f4" has incompatible type "Tuple[Unpack[Tuple[int, ...]], int]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" +f4(t4) +f4(t5) # E: Argument 1 to "f4" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" + +t5_verbose: Tuple[Unpack[Tuple[int, ...]]] +t5 = t5_verbose # OK +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleInference] +from typing import List, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +T = TypeVar("T") +def f(x: Tuple[int, Unpack[Tuple[T, ...]]]) -> T: ... + +vt0: Tuple[int, ...] +f(vt0) # E: Argument 1 to "f" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]]]" + +vt1: Tuple[Unpack[Tuple[int, ...]], int] +reveal_type(f(vt1)) # N: Revealed type is "builtins.int" + +S = TypeVar("S") +Ts = TypeVarTuple("Ts") +def g(x: Tuple[T, Unpack[Ts], S]) -> Tuple[T, Unpack[Ts], S]: ... +g(vt0) # E: Argument 1 to "g" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]], int]" + +U = TypeVar("U") +def h(x: List[Tuple[T, S, U]]) -> Tuple[T, S, U]: ... +vt2: Tuple[Unpack[Tuple[int, ...]], int] +vt2 = h(reveal_type([])) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, builtins.int]]" +[builtins fixtures/tuple.pyi] From 37a28df221be7f82d386eb1ff576d0888ac229a2 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 7 Sep 2023 23:06:04 +0100 Subject: [PATCH 02/14] Start working on instances --- mypy/erasetype.py | 8 +- mypy/meet.py | 2 + mypy/subtypes.py | 133 ++++++------------------ test-data/unit/check-typevar-tuple.test | 10 +- 4 files changed, 47 insertions(+), 106 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index d1a01fb6c779..63b4099b8612 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -77,7 +77,13 @@ def visit_deleted_type(self, t: DeletedType) -> ProperType: return t def visit_instance(self, t: Instance) -> ProperType: - return Instance(t.type, [AnyType(TypeOfAny.special_form)] * len(t.args), t.line) + args = [] + for tv in t.type.defn.type_vars: + if isinstance(tv, TypeVarTupleType): + args.append(tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)])) + else: + args.append(AnyType(TypeOfAny.special_form)) + return Instance(t.type, args, t.line) def visit_type_var(self, t: TypeVarType) -> ProperType: return AnyType(TypeOfAny.special_form) diff --git a/mypy/meet.py b/mypy/meet.py index 2efde4ac7588..920f8c0ce605 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -825,6 +825,8 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: elif is_proper_subtype(t, self.s): # A named tuple that inherits from a normal class return t + elif self.s.type.has_type_var_tuple_type and is_subtype(t, self.s): + return t return self.default(self.s) def visit_typeddict_type(self, t: TypedDictType) -> ProperType: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 02148363d378..5c1055b23436 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -445,13 +445,20 @@ def visit_instance(self, left: Instance) -> bool: right = self.right if isinstance(right, TupleType) and right.partial_fallback.type.is_enum: return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) - if isinstance(right, TupleType) and len(right.items) == 1: - # Non-normalized Tuple type (may be left after semantic analysis). - item = right.items[0] - if isinstance(item, UnpackType): - unpacked = get_proper_type(item.type) + if isinstance(right, TupleType): + if len(right.items) == 1: + # Non-normalized Tuple type (may be left after semantic analysis). + item = right.items[0] + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, Instance): + return self._is_subtype(left, unpacked) + if len(left.args) == 1 and isinstance(left.args[0], UnpackType): + unpacked = get_proper_type(left.args[0].type) if isinstance(unpacked, Instance): - return self._is_subtype(left, unpacked) + assert unpacked.type.fullname == "builtins.tuple" + if isinstance(get_proper_type(unpacked.args[0]), AnyType): + return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) return False if isinstance(right, Instance): if type_state.is_cached_subtype_check(self._subtype_kind, left, right): @@ -492,106 +499,24 @@ def visit_instance(self, left: Instance) -> bool: t = erased nominal = True if right.type.has_type_var_tuple_type: - assert left.type.type_var_tuple_prefix is not None - assert left.type.type_var_tuple_suffix is not None assert right.type.type_var_tuple_prefix is not None assert right.type.type_var_tuple_suffix is not None - split_result = fully_split_with_mapped_and_template( - left.args, - left.type.type_var_tuple_prefix, - left.type.type_var_tuple_suffix, - right.args, - right.type.type_var_tuple_prefix, - right.type.type_var_tuple_suffix, - ) - if split_result is None: - return False - - ( - left_prefix, - left_mprefix, - left_middle, - left_msuffix, - left_suffix, - right_prefix, - right_mprefix, - right_middle, - right_msuffix, - right_suffix, - ) = split_result - - left_unpacked = extract_unpack(left_middle) - right_unpacked = extract_unpack(right_middle) - - # Helper for case 2 below so we can treat them the same. - def check_mixed( - unpacked_type: ProperType, compare_to: tuple[Type, ...] - ) -> bool: - if ( - isinstance(unpacked_type, Instance) - and unpacked_type.type.fullname == "builtins.tuple" - ): - return all(is_equivalent(l, unpacked_type.args[0]) for l in compare_to) - if isinstance(unpacked_type, TypeVarTupleType): - return False - if isinstance(unpacked_type, AnyType): - return True - if isinstance(unpacked_type, TupleType): - if len(unpacked_type.items) != len(compare_to): - return False - for t1, t2 in zip(unpacked_type.items, compare_to): - if not is_equivalent(t1, t2): - return False - return True - return False - - # Case 1: Both are unpacks, in this case we check what is being - # unpacked is the same. - if left_unpacked is not None and right_unpacked is not None: - if not is_equivalent(left_unpacked, right_unpacked): - return False - - # Case 2: Only one of the types is an unpack. The equivalence - # case is mostly the same but we check some additional - # things when unpacking on the right. - elif left_unpacked is not None and right_unpacked is None: - if not check_mixed(left_unpacked, right_middle): - return False - elif left_unpacked is None and right_unpacked is not None: - if not check_mixed(right_unpacked, left_middle): - return False - - # Case 3: Neither type is an unpack. In this case we just compare - # the items themselves. - else: - if len(left_middle) != len(right_middle): - return False - for left_t, right_t in zip(left_middle, right_middle): - if not is_equivalent(left_t, right_t): - return False - - assert len(left_mprefix) == len(right_mprefix) - assert len(left_msuffix) == len(right_msuffix) - - for left_item, right_item in zip( - left_mprefix + left_msuffix, right_mprefix + right_msuffix - ): - if not is_equivalent(left_item, right_item): - return False - - left_items = t.args[: right.type.type_var_tuple_prefix] - right_items = right.args[: right.type.type_var_tuple_prefix] - if right.type.type_var_tuple_suffix: - left_items += t.args[-right.type.type_var_tuple_suffix :] - right_items += right.args[-right.type.type_var_tuple_suffix :] - unpack_index = right.type.type_var_tuple_prefix - assert unpack_index is not None - type_params = zip( - left_prefix + left_suffix, - right_prefix + right_suffix, - right.type.defn.type_vars[:unpack_index] - + right.type.defn.type_vars[unpack_index + 1 :], - ) + prefix = right.type.type_var_tuple_prefix + suffix = right.type.type_var_tuple_suffix + tvt = right.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + left_prefix, left_middle, left_suffix = split_with_prefix_and_suffix(t.args, prefix, suffix) + right_prefix, right_middle, right_suffix = split_with_prefix_and_suffix(right.args, prefix, suffix) + left_args = left_prefix + (TupleType(list(left_middle), fallback),) + left_suffix + right_args = right_prefix + (TupleType(list(right_middle), fallback),) + right_suffix + if len(t.args) == 1 and isinstance(t.args[0], UnpackType): + unpacked = get_proper_type(t.args[0].type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if isinstance(get_proper_type(unpacked.args[0]), AnyType) and not self.proper_subtype: + return True + type_params = zip(left_args, right_args, right.type.defn.type_vars) else: type_params = zip(t.args, right.args, right.type.defn.type_vars) if not self.subtype_context.ignore_type_params: diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index b4a5da9f11ff..6c39b63b7cfd 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1221,7 +1221,7 @@ def foo(x: Tuple[Unpack[Ts]]) -> Tuple[Unpack[Ts]]: [case testTypeVarTupleWithIsInstance] # flags: --warn-unreachable -from typing import Tuple +from typing import Generic, Tuple from typing_extensions import TypeVarTuple, Unpack TP = TypeVarTuple("TP") @@ -1232,6 +1232,14 @@ def test(d: A[int, str]) -> None: reveal_type(d) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.A[builtins.int, builtins.str]]" else: reveal_type(d) # E: Statement is unreachable + +class B(Generic[Unpack[TP]]): ... + +def test2(d: B[int, str]) -> None: + if isinstance(d, B): + reveal_type(d) # N: Revealed type is "__main__.B[builtins.int, builtins.str]" + else: + reveal_type(d) # E: Statement is unreachable [builtins fixtures/isinstancelist.pyi] [case testVariadicTupleSubtyping] From d8ce5c580a31d354d99a72e04af0201c6f6f33cb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2023 00:11:22 +0100 Subject: [PATCH 03/14] Handle meets and joins --- mypy/erasetype.py | 6 ++- mypy/join.py | 126 +++++++++++++++++++++++++++++++++++++++++++--- mypy/meet.py | 99 +++++++++++++++++++++++++++++++++--- mypy/subtypes.py | 22 +++++--- 4 files changed, 233 insertions(+), 20 deletions(-) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 63b4099b8612..0a3b209bfc65 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -77,10 +77,12 @@ def visit_deleted_type(self, t: DeletedType) -> ProperType: return t def visit_instance(self, t: Instance) -> ProperType: - args = [] + args: list[Type] = [] for tv in t.type.defn.type_vars: if isinstance(tv, TypeVarTupleType): - args.append(tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)])) + args.append( + tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)]) + ) else: args.append(AnyType(TypeOfAny.special_form)) return Instance(t.type, args, t.line) diff --git a/mypy/join.py b/mypy/join.py index 806c644a680c..7c0be216edef 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -43,8 +43,10 @@ UninhabitedType, UnionType, UnpackType, + find_unpack_in_list, get_proper_type, get_proper_types, + split_with_prefix_and_suffix, ) @@ -67,7 +69,22 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: args: list[Type] = [] # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. - for ta, sa, type_var in zip(t.args, s.args, t.type.defn.type_vars): + if t.type.has_type_var_tuple_type: + assert s.type.type_var_tuple_prefix is not None + assert s.type.type_var_tuple_suffix is not None + prefix = s.type.type_var_tuple_prefix + suffix = s.type.type_var_tuple_suffix + tvt = s.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + s_prefix, s_middle, s_suffix = split_with_prefix_and_suffix(s.args, prefix, suffix) + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix(t.args, prefix, suffix) + s_args = s_prefix + (TupleType(list(s_middle), fallback),) + s_suffix + t_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + else: + t_args = t.args + s_args = s.args + for ta, sa, type_var in zip(t_args, s_args, t.type.defn.type_vars): ta_proper = get_proper_type(ta) sa_proper = get_proper_type(sa) new_type: Type | None = None @@ -93,6 +110,15 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: # If the types are different but equivalent, then an Any is involved # so using a join in the contravariant case is also OK. new_type = join_types(ta, sa, self) + elif isinstance(type_var, TypeVarTupleType): + new_type = get_proper_type(join_types(ta, sa, self)) + if isinstance(new_type, Instance): + assert new_type.type.fullname == "builtins.tuple" + new_type = UnpackType(new_type) + else: + assert isinstance(new_type, TupleType) + args.extend(new_type.items) + continue else: # ParamSpec type variables behave the same, independent of variance if not is_equivalent(ta, sa): @@ -440,6 +466,96 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: return join_types(t, call) return join_types(t.fallback, s) + def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + s_unpack_index = find_unpack_in_list(s.items) + t_unpack_index = find_unpack_in_list(t.items) + if s_unpack_index is None and t_unpack_index is None: + if s.length() == t.length(): + items: list[Type] = [] + for i in range(t.length()): + items.append(join_types(t.items[i], s.items[i])) + return items + return None + if s_unpack_index is not None and t_unpack_index is not None: + s_unpack = s.items[s_unpack_index] + assert isinstance(s_unpack, UnpackType) + s_unpacked = get_proper_type(s_unpack.type) + t_unpack = t.items[t_unpack_index] + assert isinstance(t_unpack, UnpackType) + t_unpacked = get_proper_type(t_unpack.type) + if s.length() == t.length() and s_unpack_index == t_unpack_index: + prefix_len = t_unpack_index + suffix_len = t.length() - t_unpack_index - 1 + items = [] + for si, ti in zip(s.items[:prefix_len], t.items[:prefix_len]): + items.append(join_types(si, ti)) + joined = join_types(s_unpacked, t_unpacked) + if isinstance(joined, TypeVarTupleType): + items.append(UnpackType(joined)) + elif isinstance(joined, Instance) and joined.type.fullname == "builtins.tuple": + items.append(UnpackType(joined)) + else: + if isinstance(t_unpacked, Instance): + assert t_unpacked.type.fullname == "builtins.tuple" + tuple_instance = t_unpacked + else: + assert isinstance(t_unpacked, TypeVarTupleType) + tuple_instance = t_unpacked.tuple_fallback + items.append( + UnpackType( + tuple_instance.copy_modified( + args=[object_from_instance(tuple_instance)] + ) + ) + ) + if suffix_len: + for si, ti in zip(s.items[-suffix_len:], t.items[-suffix_len:]): + items.append(join_types(si, ti)) + if s.length() == 1 or t.length() == 1: + if not (isinstance(s_unpacked, Instance) and isinstance(t_unpacked, Instance)): + return None + assert s_unpacked.type.fullname == "builtins.tuple" + assert t_unpacked.type.fullname == "builtins.tuple" + mid_joined = join_types(s_unpacked.args[0], t_unpacked.args[0]) + t_other = [a for i, a in enumerate(t.items) if i != t_unpack_index] + s_other = [a for i, a in enumerate(s.items) if i != s_unpack_index] + other_joined = join_type_list(s_other + t_other) + mid_joined = join_types(mid_joined, other_joined) + return [UnpackType(s_unpacked.copy_modified(args=[mid_joined]))] + # TODO: are there other case we can handle? + return None + if s_unpack_index is not None: + variadic = s + unpack_index = s_unpack_index + fixed = t + else: + assert t_unpack_index is not None + variadic = t + unpack_index = t_unpack_index + fixed = s + unpack = variadic.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if not isinstance(unpacked, Instance): + return None + if fixed.length() < variadic.length() - 1: + return None + prefix_len = unpack_index + suffix_len = variadic.length() - prefix_len - 1 + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(fixed.items), prefix_len, suffix_len + ) + items = [] + for fi, vi in zip(prefix, variadic.items[:prefix_len]): + items.append(join_types(fi, vi)) + mid_joined = join_type_list(list(middle)) + mid_joined = join_types(mid_joined, unpacked.args[0]) + items.append(UnpackType(unpacked.copy_modified(args=[mid_joined]))) + if suffix_len: + for fi, vi in zip(suffix, variadic.items[-suffix_len:]): + items.append(join_types(fi, vi)) + return items + def visit_tuple_type(self, t: TupleType) -> ProperType: # When given two fixed-length tuples: # * If they have the same length, join their subtypes item-wise: @@ -452,17 +568,15 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: # Tuple[int, bool] + Tuple[bool, ...] becomes Tuple[int, ...] # * Joining with any Sequence also returns a Sequence: # Tuple[int, bool] + List[bool] becomes Sequence[int] - if isinstance(self.s, TupleType) and self.s.length() == t.length(): + if isinstance(self.s, TupleType): if self.instance_joiner is None: self.instance_joiner = InstanceJoiner() fallback = self.instance_joiner.join_instances( mypy.typeops.tuple_fallback(self.s), mypy.typeops.tuple_fallback(t) ) assert isinstance(fallback, Instance) - if self.s.length() == t.length(): - items: list[Type] = [] - for i in range(t.length()): - items.append(join_types(t.items[i], self.s.items[i])) + items = self.join_tuples(self.s, t) + if items is not None: return TupleType(items, fallback) else: return fallback diff --git a/mypy/meet.py b/mypy/meet.py index 920f8c0ce605..8df135cac21d 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -44,8 +44,10 @@ UninhabitedType, UnionType, UnpackType, + find_unpack_in_list, get_proper_type, get_proper_types, + split_with_prefix_and_suffix, ) # TODO Describe this module. @@ -721,8 +723,36 @@ def visit_instance(self, t: Instance) -> ProperType: args: list[Type] = [] # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. - for ta, sia in zip(t.args, self.s.args): - args.append(self.meet(ta, sia)) + if t.type.has_type_var_tuple_type: + s = self.s + assert s.type.type_var_tuple_prefix is not None + assert s.type.type_var_tuple_suffix is not None + prefix = s.type.type_var_tuple_prefix + suffix = s.type.type_var_tuple_suffix + tvt = s.type.defn.type_vars[prefix] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + s_prefix, s_middle, s_suffix = split_with_prefix_and_suffix( + s.args, prefix, suffix + ) + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + t.args, prefix, suffix + ) + s_args = s_prefix + (TupleType(list(s_middle), fallback),) + s_suffix + t_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + else: + t_args = t.args + s_args = self.s.args + for ta, sa, tv in zip(t_args, s_args, t.type.defn.type_vars): + meet = self.meet(ta, sa) + if isinstance(tv, TypeVarTupleType): + if isinstance(meet, TupleType): + args.extend(meet.items) + continue + else: + assert isinstance(meet, UninhabitedType) + meet = UnpackType(tv.tuple_fallback.copy_modified(args=[meet])) + args.append(meet) return Instance(t.type, args) else: if state.strict_optional: @@ -811,11 +841,68 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: return meet_types(t, call) return meet_types(t.fallback, s) + def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + s_unpack_index = find_unpack_in_list(s.items) + t_unpack_index = find_unpack_in_list(t.items) + if s_unpack_index is None and t_unpack_index is None: + if s.length() == t.length(): + items: list[Type] = [] + for i in range(t.length()): + items.append(self.meet(t.items[i], s.items[i])) + return items + return None + if s_unpack_index is not None and t_unpack_index is not None: + # TODO: handle more cases (like strictly shorter prefix/suffix). + if s.length() == 1 and t.length() == 1: + s_unpack = s.items[0] + assert isinstance(s_unpack, UnpackType) + s_unpacked = get_proper_type(s_unpack.type) + t_unpack = t.items[0] + assert isinstance(t_unpack, UnpackType) + t_unpacked = get_proper_type(t_unpack.type) + if not (isinstance(s_unpacked, Instance) and isinstance(t_unpacked, Instance)): + return None + meet = self.meet(s_unpacked, t_unpacked) + if not isinstance(meet, Instance): + return None + return [UnpackType(meet)] + return None + if s_unpack_index is not None: + variadic = s + unpack_index = s_unpack_index + fixed = t + else: + assert t_unpack_index is not None + variadic = t + unpack_index = t_unpack_index + fixed = s + unpack = variadic.items[unpack_index] + assert isinstance(unpack, UnpackType) + unpacked = get_proper_type(unpack.type) + if not isinstance(unpacked, Instance): + return None + if fixed.length() < variadic.length() - 1: + return None + prefix_len = unpack_index + suffix_len = variadic.length() - prefix_len - 1 + prefix, middle, suffix = split_with_prefix_and_suffix( + tuple(fixed.items), prefix_len, suffix_len + ) + items = [] + for fi, vi in zip(prefix, variadic.items[:prefix_len]): + items.append(self.meet(fi, vi)) + for mi in middle: + items.append(self.meet(mi, unpacked.args[0])) + if suffix_len: + for fi, vi in zip(suffix, variadic.items[-suffix_len:]): + items.append(self.meet(fi, vi)) + return items + def visit_tuple_type(self, t: TupleType) -> ProperType: - if isinstance(self.s, TupleType) and self.s.length() == t.length(): - items: list[Type] = [] - for i in range(t.length()): - items.append(self.meet(t.items[i], self.s.items[i])) + if isinstance(self.s, TupleType): + items = self.meet_tuples(self.s, t) + if items is None: + return self.default(self.s) # TODO: What if the fallbacks are different? return TupleType(items, tuple_fallback(t)) elif isinstance(self.s, Instance): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5c1055b23436..1911f517fa6b 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -66,7 +66,6 @@ from mypy.types_utils import flatten_types from mypy.typestate import SubtypeKind, type_state from mypy.typevars import fill_typevars_with_any -from mypy.typevartuples import extract_unpack, fully_split_with_mapped_and_template # Flags for detected protocol members IS_SETTABLE: Final = 1 @@ -506,15 +505,26 @@ def visit_instance(self, left: Instance) -> bool: tvt = right.type.defn.type_vars[prefix] assert isinstance(tvt, TypeVarTupleType) fallback = tvt.tuple_fallback - left_prefix, left_middle, left_suffix = split_with_prefix_and_suffix(t.args, prefix, suffix) - right_prefix, right_middle, right_suffix = split_with_prefix_and_suffix(right.args, prefix, suffix) - left_args = left_prefix + (TupleType(list(left_middle), fallback),) + left_suffix - right_args = right_prefix + (TupleType(list(right_middle), fallback),) + right_suffix + left_prefix, left_middle, left_suffix = split_with_prefix_and_suffix( + t.args, prefix, suffix + ) + right_prefix, right_middle, right_suffix = split_with_prefix_and_suffix( + right.args, prefix, suffix + ) + left_args = ( + left_prefix + (TupleType(list(left_middle), fallback),) + left_suffix + ) + right_args = ( + right_prefix + (TupleType(list(right_middle), fallback),) + right_suffix + ) if len(t.args) == 1 and isinstance(t.args[0], UnpackType): unpacked = get_proper_type(t.args[0].type) if isinstance(unpacked, Instance): assert unpacked.type.fullname == "builtins.tuple" - if isinstance(get_proper_type(unpacked.args[0]), AnyType) and not self.proper_subtype: + if ( + isinstance(get_proper_type(unpacked.args[0]), AnyType) + and not self.proper_subtype + ): return True type_params = zip(left_args, right_args, right.type.defn.type_vars) else: From 31e82f8bc35aa502969da2250b499d88e49952df Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2023 15:33:16 +0100 Subject: [PATCH 04/14] Update constraint inference --- mypy/constraints.py | 139 ++++++++--------------------------- mypy/test/testconstraints.py | 22 +----- mypy/test/testsubtypes.py | 52 +------------ mypy/typevartuples.py | 134 --------------------------------- 4 files changed, 34 insertions(+), 313 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 88a320072d0a..ef92c777622b 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Final, Iterable, List, Sequence, cast +from typing import TYPE_CHECKING, Final, Iterable, List, Sequence import mypy.subtypes import mypy.typeops @@ -58,7 +58,6 @@ ) from mypy.types_utils import is_union_with_any from mypy.typestate import type_state -from mypy.typevartuples import extract_unpack, split_with_mapped_and_template if TYPE_CHECKING: from mypy.infer import ArgumentInferContext @@ -747,26 +746,19 @@ def visit_instance(self, template: Instance) -> list[Constraint]: if instance.type.has_type_var_tuple_type: assert instance.type.type_var_tuple_prefix is not None assert instance.type.type_var_tuple_suffix is not None - assert mapped.type.type_var_tuple_prefix is not None - assert mapped.type.type_var_tuple_suffix is not None - - unpack_constraints, instance_args, mapped_args = build_constraints_for_unpack( - instance.args, - instance.type.type_var_tuple_prefix, - instance.type.type_var_tuple_suffix, - mapped.args, - mapped.type.type_var_tuple_prefix, - mapped.type.type_var_tuple_suffix, - self.direction, + prefix_len = instance.type.type_var_tuple_prefix + suffix_len = instance.type.type_var_tuple_suffix + tvt = instance.type.defn.type_vars[prefix_len] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + i_prefix, i_middle, i_suffix = split_with_prefix_and_suffix( + instance.args, prefix_len, suffix_len ) - res.extend(unpack_constraints) - - tvars_prefix, _, tvars_suffix = split_with_prefix_and_suffix( - tuple(tvars), - instance.type.type_var_tuple_prefix, - instance.type.type_var_tuple_suffix, + m_prefix, m_middle, m_suffix = split_with_prefix_and_suffix( + mapped.args, prefix_len, suffix_len ) - tvars = cast("list[TypeVarLikeType]", list(tvars_prefix + tvars_suffix)) + instance_args = i_prefix + (TupleType(list(i_middle), fallback),) + i_suffix + mapped_args = m_prefix + (TupleType(list(m_middle), fallback),) + m_suffix else: mapped_args = mapped.args instance_args = instance.args @@ -806,44 +798,36 @@ def visit_instance(self, template: Instance) -> list[Constraint]: ) res.append(Constraint(mapped_arg, SUBTYPE_OF, suffix)) res.append(Constraint(mapped_arg, SUPERTYPE_OF, suffix)) - else: - # This case should have been handled above. - assert not isinstance(tvar, TypeVarTupleType) + elif isinstance(tvar, TypeVarTupleType): + # Handle variadic type variables covariantly for consistency. + res.extend(infer_constraints(mapped_arg, instance_arg, self.direction)) return res elif self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname): mapped = map_instance_to_supertype(instance, template.type) tvars = template.type.defn.type_vars if template.type.has_type_var_tuple_type: - assert mapped.type.type_var_tuple_prefix is not None - assert mapped.type.type_var_tuple_suffix is not None assert template.type.type_var_tuple_prefix is not None assert template.type.type_var_tuple_suffix is not None - - unpack_constraints, mapped_args, template_args = build_constraints_for_unpack( - mapped.args, - mapped.type.type_var_tuple_prefix, - mapped.type.type_var_tuple_suffix, - template.args, - template.type.type_var_tuple_prefix, - template.type.type_var_tuple_suffix, - self.direction, + prefix_len = template.type.type_var_tuple_prefix + suffix_len = template.type.type_var_tuple_suffix + tvt = template.type.defn.type_vars[prefix_len] + assert isinstance(tvt, TypeVarTupleType) + fallback = tvt.tuple_fallback + t_prefix, t_middle, t_suffix = split_with_prefix_and_suffix( + template.args, prefix_len, suffix_len ) - res.extend(unpack_constraints) - - tvars_prefix, _, tvars_suffix = split_with_prefix_and_suffix( - tuple(tvars), - template.type.type_var_tuple_prefix, - template.type.type_var_tuple_suffix, + m_prefix, m_middle, m_suffix = split_with_prefix_and_suffix( + mapped.args, prefix_len, suffix_len ) - tvars = cast("list[TypeVarLikeType]", list(tvars_prefix + tvars_suffix)) + template_args = t_prefix + (TupleType(list(t_middle), fallback),) + t_suffix + mapped_args = m_prefix + (TupleType(list(m_middle), fallback),) + m_suffix else: mapped_args = mapped.args template_args = template.args # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. for tvar, mapped_arg, template_arg in zip(tvars, mapped_args, template_args): - assert not isinstance(tvar, TypeVarTupleType) if isinstance(tvar, TypeVarType): # The constraints for generic type parameters depend on variance. # Include constraints from both directions if invariant. @@ -878,9 +862,9 @@ def visit_instance(self, template: Instance) -> list[Constraint]: ) res.append(Constraint(template_arg, SUBTYPE_OF, suffix)) res.append(Constraint(template_arg, SUPERTYPE_OF, suffix)) - else: - # This case should have been handled above. - assert not isinstance(tvar, TypeVarTupleType) + elif isinstance(tvar, TypeVarTupleType): + # Handle variadic type variables covariantly for consistency. + res.extend(infer_constraints(template_arg, mapped_arg, self.direction)) return res if ( template.type.is_protocol @@ -1494,71 +1478,6 @@ def build_constraints_for_simple_unpack( return res -def build_constraints_for_unpack( - # TODO: this naming is misleading, these should be "actual", not "mapped" - # both template and actual can be mapped before, depending on direction. - # Also the convention is to put template related args first. - mapped: tuple[Type, ...], - mapped_prefix_len: int | None, - mapped_suffix_len: int | None, - template: tuple[Type, ...], - template_prefix_len: int, - template_suffix_len: int, - direction: int, -) -> tuple[list[Constraint], tuple[Type, ...], tuple[Type, ...]]: - # TODO: this function looks broken: - # a) it should take into account variances, but it doesn't - # b) it looks like both call sites always pass identical values to args (2, 3) and (5, 6) - # because after map_instance_to_supertype() both template and actual have same TypeInfo. - if mapped_prefix_len is None: - mapped_prefix_len = template_prefix_len - if mapped_suffix_len is None: - mapped_suffix_len = template_suffix_len - - split_result = split_with_mapped_and_template( - mapped, - mapped_prefix_len, - mapped_suffix_len, - template, - template_prefix_len, - template_suffix_len, - ) - assert split_result is not None - ( - mapped_prefix, - mapped_middle, - mapped_suffix, - template_prefix, - template_middle, - template_suffix, - ) = split_result - - template_unpack = extract_unpack(template_middle) - res = [] - - if template_unpack is not None: - if isinstance(template_unpack, TypeVarTupleType): - res.append( - Constraint( - template_unpack, - direction, - TupleType(list(mapped_middle), template_unpack.tuple_fallback), - ) - ) - elif ( - isinstance(template_unpack, Instance) - and template_unpack.type.fullname == "builtins.tuple" - ): - for item in mapped_middle: - res.extend(infer_constraints(template_unpack.args[0], item, direction)) - - elif isinstance(template_unpack, TupleType): - if len(template_unpack.items) == len(mapped_middle): - for template_arg, item in zip(template_unpack.items, mapped_middle): - res.extend(infer_constraints(template_arg, item, direction)) - return res, mapped_prefix + mapped_suffix, template_prefix + template_suffix - - def infer_directed_arg_constraints(left: Type, right: Type, direction: int) -> list[Constraint]: """Infer constraints between two arguments using direction between original callables.""" if isinstance(left, (ParamSpecType, UnpackType)) or isinstance( diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py index f40996145cba..3f05723213ea 100644 --- a/mypy/test/testconstraints.py +++ b/mypy/test/testconstraints.py @@ -20,6 +20,8 @@ def test_basic_type_variable(self) -> None: Constraint(type_var=fx.t, op=direction, target=fx.a) ] + # TODO: some tests here use non-normalized variadic forms (mostly fixed tuple unpacks). + # We may want to either replace these with normalized ones, or add matching normalized tests. def test_basic_type_var_tuple_subtype(self) -> None: fx = self.fx assert infer_constraints( @@ -82,26 +84,6 @@ def test_unpack_homogenous_tuple_with_prefix_and_suffix(self) -> None: Constraint(type_var=fx.u, op=SUPERTYPE_OF, target=fx.d), } - def test_unpack_tuple(self) -> None: - fx = self.fx - assert set( - infer_constraints( - Instance( - fx.gvi, - [ - UnpackType( - TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) - ) - ], - ), - Instance(fx.gvi, [fx.a, fx.b]), - SUPERTYPE_OF, - ) - ) == { - Constraint(type_var=fx.t, op=SUPERTYPE_OF, target=fx.a), - Constraint(type_var=fx.s, op=SUPERTYPE_OF, target=fx.b), - } - def test_unpack_with_prefix_and_suffix(self) -> None: fx = self.fx assert set( diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index 464f64d2b846..541ba47ced4a 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -198,6 +198,8 @@ def test_type_callable_subtyping(self) -> None: self.fx.callable_type(self.fx.a, self.fx.b), self.fx.callable(self.fx.a, self.fx.b) ) + # TODO: some tests here use non-normalized variadic forms (mostly fixed tuple unpacks). + # We may want to either replace these with normalized ones, or add matching normalized tests. def test_type_var_tuple(self) -> None: self.assert_subtype(Instance(self.fx.gvi, []), Instance(self.fx.gvi, [])) self.assert_subtype( @@ -221,10 +223,6 @@ def test_type_var_tuple(self) -> None: Instance(self.fx.gvi, [UnpackType(self.fx.us)]), ) - self.assert_subtype( - Instance(self.fx.gvi, [UnpackType(self.fx.anyt)]), - Instance(self.fx.gvi, [self.fx.anyt]), - ) self.assert_not_subtype( Instance(self.fx.gvi, [UnpackType(self.fx.ss)]), Instance(self.fx.gvi, []) ) @@ -272,51 +270,7 @@ def test_type_var_tuple_with_prefix_suffix(self) -> None: Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss), self.fx.b, self.fx.c]), ) - def test_type_var_tuple_unpacked_varlength_tuple(self) -> None: - self.assert_subtype( - Instance( - self.fx.gvi, - [ - UnpackType( - TupleType( - [self.fx.a, self.fx.b], - fallback=Instance(self.fx.std_tuplei, [self.fx.o]), - ) - ) - ], - ), - Instance(self.fx.gvi, [self.fx.a, self.fx.b]), - ) - def test_type_var_tuple_unpacked_tuple(self) -> None: - self.assert_subtype( - Instance( - self.fx.gvi, - [ - UnpackType( - TupleType( - [self.fx.a, self.fx.b], - fallback=Instance(self.fx.std_tuplei, [self.fx.o]), - ) - ) - ], - ), - Instance(self.fx.gvi, [self.fx.a, self.fx.b]), - ) - self.assert_subtype( - Instance( - self.fx.gvi, - [ - UnpackType( - TupleType( - [self.fx.a, self.fx.b], - fallback=Instance(self.fx.std_tuplei, [self.fx.o]), - ) - ) - ], - ), - Instance(self.fx.gvi, [self.fx.anyt, self.fx.anyt]), - ) self.assert_not_subtype( Instance( self.fx.gvi, @@ -348,7 +302,7 @@ def test_type_var_tuple_unpacked_tuple(self) -> None: ) def test_type_var_tuple_unpacked_variable_length_tuple(self) -> None: - self.assert_equivalent( + self.assert_subtype( Instance(self.fx.gvi, [self.fx.a, self.fx.a]), Instance(self.fx.gvi, [UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))]), ) diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py index bcb5e96b615c..af2effbd4035 100644 --- a/mypy/typevartuples.py +++ b/mypy/typevartuples.py @@ -9,7 +9,6 @@ ProperType, Type, UnpackType, - find_unpack_in_list, get_proper_type, split_with_prefix_and_suffix, ) @@ -25,139 +24,6 @@ def split_with_instance( ) -def split_with_mapped_and_template( - mapped: tuple[Type, ...], - mapped_prefix_len: int | None, - mapped_suffix_len: int | None, - template: tuple[Type, ...], - template_prefix_len: int, - template_suffix_len: int, -) -> ( - tuple[ - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - ] - | None -): - split_result = fully_split_with_mapped_and_template( - mapped, - mapped_prefix_len, - mapped_suffix_len, - template, - template_prefix_len, - template_suffix_len, - ) - if split_result is None: - return None - - ( - mapped_prefix, - mapped_middle_prefix, - mapped_middle_middle, - mapped_middle_suffix, - mapped_suffix, - template_prefix, - template_middle_prefix, - template_middle_middle, - template_middle_suffix, - template_suffix, - ) = split_result - - return ( - mapped_prefix + mapped_middle_prefix, - mapped_middle_middle, - mapped_middle_suffix + mapped_suffix, - template_prefix + template_middle_prefix, - template_middle_middle, - template_middle_suffix + template_suffix, - ) - - -def fully_split_with_mapped_and_template( - mapped: tuple[Type, ...], - mapped_prefix_len: int | None, - mapped_suffix_len: int | None, - template: tuple[Type, ...], - template_prefix_len: int, - template_suffix_len: int, -) -> ( - tuple[ - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - tuple[Type, ...], - ] - | None -): - if mapped_prefix_len is not None: - assert mapped_suffix_len is not None - mapped_prefix, mapped_middle, mapped_suffix = split_with_prefix_and_suffix( - tuple(mapped), mapped_prefix_len, mapped_suffix_len - ) - else: - mapped_prefix = tuple() - mapped_suffix = tuple() - mapped_middle = mapped - - template_prefix, template_middle, template_suffix = split_with_prefix_and_suffix( - tuple(template), template_prefix_len, template_suffix_len - ) - - unpack_prefix = find_unpack_in_list(template_middle) - if unpack_prefix is None: - return ( - mapped_prefix, - (), - mapped_middle, - (), - mapped_suffix, - template_prefix, - (), - template_middle, - (), - template_suffix, - ) - - unpack_suffix = len(template_middle) - unpack_prefix - 1 - # mapped_middle is too short to do the unpack - if unpack_prefix + unpack_suffix > len(mapped_middle): - return None - - ( - mapped_middle_prefix, - mapped_middle_middle, - mapped_middle_suffix, - ) = split_with_prefix_and_suffix(mapped_middle, unpack_prefix, unpack_suffix) - ( - template_middle_prefix, - template_middle_middle, - template_middle_suffix, - ) = split_with_prefix_and_suffix(template_middle, unpack_prefix, unpack_suffix) - - return ( - mapped_prefix, - mapped_middle_prefix, - mapped_middle_middle, - mapped_middle_suffix, - mapped_suffix, - template_prefix, - template_middle_prefix, - template_middle_middle, - template_middle_suffix, - template_suffix, - ) - - def extract_unpack(types: Sequence[Type]) -> ProperType | None: """Given a list of types, extracts either a single type from an unpack, or returns None.""" if len(types) == 1: From 75a9024de83ab5ef05c86162058a9ef0553b04ec Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2023 17:42:21 +0100 Subject: [PATCH 05/14] Add some comments --- mypy/constraints.py | 18 +++++++++++++++--- mypy/erasetype.py | 1 + mypy/join.py | 22 ++++++++++++++++++++++ mypy/meet.py | 23 ++++++++++++++++++++++- mypy/semanal_typeargs.py | 3 ++- mypy/subtypes.py | 30 +++++++++++++++++++++++++++++- 6 files changed, 91 insertions(+), 6 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index ef92c777622b..3572997dc464 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -744,6 +744,8 @@ def visit_instance(self, template: Instance) -> list[Constraint]: tvars = mapped.type.defn.type_vars if instance.type.has_type_var_tuple_type: + # Variadic types need special handling to map each type argument to + # the correct corresponding type variable. assert instance.type.type_var_tuple_prefix is not None assert instance.type.type_var_tuple_suffix is not None prefix_len = instance.type.type_var_tuple_prefix @@ -807,6 +809,8 @@ def visit_instance(self, template: Instance) -> list[Constraint]: mapped = map_instance_to_supertype(instance, template.type) tvars = template.type.defn.type_vars if template.type.has_type_var_tuple_type: + # Variadic types need special handling to map each type argument to + # the correct corresponding type variable. assert template.type.type_var_tuple_prefix is not None assert template.type.type_var_tuple_suffix is not None prefix_len = template.type.type_var_tuple_prefix @@ -1034,7 +1038,7 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]: res.extend(unpack_constraints) else: # TODO: do we need some special-casing when unpack is present in actual - # but not in template? + # callable but not in template callable? res.extend( infer_callable_arguments_constraints(template, cactual, self.direction) ) @@ -1170,10 +1174,12 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: and unpacked_type.type.fullname == "builtins.tuple" ) res = infer_constraints(unpacked_type, actual, self.direction) - assert isinstance(actual, Instance) + assert isinstance(actual, Instance) # ensured by is_varlength_tuple == True for i, ti in enumerate(template.items): if i == unpack_index: + # This one we just handled above. continue + # For Tuple[T, *Ts, S] <: tuple[X, ...] infer also T <: X and S <: X. res.extend(infer_constraints(ti, actual.args[0], self.direction)) return res else: @@ -1187,6 +1193,9 @@ def visit_tuple_type(self, template: TupleType) -> list[Constraint]: elif isinstance(actual, TupleType): a_unpack_index = find_unpack_in_list(actual.items) if a_unpack_index is not None: + # The case where template tuple doesn't have an unpack, but actual tuple + # has an unpack. We can infer something if actual unpack is a variadic tuple. + # Tuple[T, S, U] <: tuple[X, *tuple[Y, ...], Z] => T <: X, S <: Y, U <: Z. a_unpack = actual.items[a_unpack_index] assert isinstance(a_unpack, UnpackType) a_unpacked = get_proper_type(a_unpack.type) @@ -1435,7 +1444,8 @@ def build_constraints_for_simple_unpack( common_prefix = min(template_prefix, actual_prefix) common_suffix = min(template_suffix, actual_suffix) if actual_prefix >= template_prefix and actual_suffix >= template_suffix: - # This is the only case where we can guarantee there will be no partial overlap. + # This is the only case where we can guarantee there will be no partial overlap + # (note however partial overlap is OK for variadic tuples, it is handled below). t_unpack = template_args[template_unpack] # Handle constraints from prefixes/suffixes first. @@ -1466,6 +1476,8 @@ def build_constraints_for_simple_unpack( elif isinstance(tp, TypeVarTupleType): res.append(Constraint(tp, direction, TupleType(list(middle), tp.tuple_fallback))) elif actual_unpack is not None: + # A special case for a variadic tuple unpack, we simply infer T <: X from + # Tuple[..., *tuple[T, ...], ...] <: Tuple[..., *tuple[X, ...], ...]. actual_unpack_type = actual_args[actual_unpack] assert isinstance(actual_unpack_type, UnpackType) a_unpacked = get_proper_type(actual_unpack_type.type) diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 0a3b209bfc65..24471f918319 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -79,6 +79,7 @@ def visit_deleted_type(self, t: DeletedType) -> ProperType: def visit_instance(self, t: Instance) -> ProperType: args: list[Type] = [] for tv in t.type.defn.type_vars: + # Valid erasure for *Ts is *tuple[Any, ...], not just Any. if isinstance(tv, TypeVarTupleType): args.append( tv.tuple_fallback.copy_modified(args=[AnyType(TypeOfAny.special_form)]) diff --git a/mypy/join.py b/mypy/join.py index 7c0be216edef..48d8ef3dc8dd 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -70,6 +70,9 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. if t.type.has_type_var_tuple_type: + # We handle joins of variadic instances by simply creating correct mapping + # for type arguments and compute the individual joins same as for regular + # instances. All the heavy lifting is done in the join of tuple types. assert s.type.type_var_tuple_prefix is not None assert s.type.type_var_tuple_suffix is not None prefix = s.type.type_var_tuple_prefix @@ -112,6 +115,9 @@ def join_instances(self, t: Instance, s: Instance) -> ProperType: new_type = join_types(ta, sa, self) elif isinstance(type_var, TypeVarTupleType): new_type = get_proper_type(join_types(ta, sa, self)) + # Put the joined arguments back into instance in the normal form: + # a) Tuple[X, Y, Z] -> [X, Y, Z] + # b) tuple[X, ...] -> [*tuple[X, ...]] if isinstance(new_type, Instance): assert new_type.type.fullname == "builtins.tuple" new_type = UnpackType(new_type) @@ -467,6 +473,12 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: return join_types(t.fallback, s) def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + """Join two tuple types while handling variadic entries. + + This is surprisingly tricky, and we don't handle some tricky corner cases. + Most of the trickiness comes from the variadic tuple items like *tuple[X, ...] + since they can have arbitrary partial overlaps (while *Ts can't be split). + """ s_unpack_index = find_unpack_in_list(s.items) t_unpack_index = find_unpack_in_list(t.items) if s_unpack_index is None and t_unpack_index is None: @@ -477,6 +489,7 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: return items return None if s_unpack_index is not None and t_unpack_index is not None: + # The most complex case: both tuples have an upack item. s_unpack = s.items[s_unpack_index] assert isinstance(s_unpack, UnpackType) s_unpacked = get_proper_type(s_unpack.type) @@ -484,6 +497,9 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: assert isinstance(t_unpack, UnpackType) t_unpacked = get_proper_type(t_unpack.type) if s.length() == t.length() and s_unpack_index == t_unpack_index: + # We can handle a case where arity is perfectly aligned, e.g. + # join(Tuple[X1, *tuple[Y1, ...], Z1], Tuple[X2, *tuple[Y2, ...], Z2]). + # We can essentially perform the join elementwise. prefix_len = t_unpack_index suffix_len = t.length() - t_unpack_index - 1 items = [] @@ -512,6 +528,9 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: for si, ti in zip(s.items[-suffix_len:], t.items[-suffix_len:]): items.append(join_types(si, ti)) if s.length() == 1 or t.length() == 1: + # Another case we can handle is when one of tuple is purely variadic + # (i.e. a non-normalized form of tuple[X, ...]), in this case the join + # will be again purely variadic. if not (isinstance(s_unpacked, Instance) and isinstance(t_unpacked, Instance)): return None assert s_unpacked.type.fullname == "builtins.tuple" @@ -533,12 +552,15 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: variadic = t unpack_index = t_unpack_index fixed = s + # Case where one tuple has variadic item and the other one doesn't. The join will + # be variadic, since fixed tuple is a subtype of variadic, but not vice versa. unpack = variadic.items[unpack_index] assert isinstance(unpack, UnpackType) unpacked = get_proper_type(unpack.type) if not isinstance(unpacked, Instance): return None if fixed.length() < variadic.length() - 1: + # There are no non-trivial types that are supertype of both. return None prefix_len = unpack_index suffix_len = variadic.length() - prefix_len - 1 diff --git a/mypy/meet.py b/mypy/meet.py index 8df135cac21d..cceb34354314 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -724,6 +724,9 @@ def visit_instance(self, t: Instance) -> ProperType: # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. if t.type.has_type_var_tuple_type: + # We handle meet of variadic instances by simply creating correct mapping + # for type arguments and compute the individual meets same as for regular + # instances. All the heavy lifting is done in the meet of tuple types. s = self.s assert s.type.type_var_tuple_prefix is not None assert s.type.type_var_tuple_suffix is not None @@ -746,6 +749,8 @@ def visit_instance(self, t: Instance) -> ProperType: for ta, sa, tv in zip(t_args, s_args, t.type.defn.type_vars): meet = self.meet(ta, sa) if isinstance(tv, TypeVarTupleType): + # Correctly unpack possible outcomes of meets of tuples: it can be + # either another tuple type or Never (normalized as *tuple[Never, ...]) if isinstance(meet, TupleType): args.extend(meet.items) continue @@ -842,6 +847,14 @@ def visit_overloaded(self, t: Overloaded) -> ProperType: return meet_types(t.fallback, s) def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: + """Meet two tuple types while handling variadic entries. + + This is surprisingly tricky, and we don't handle some tricky corner cases. + Most of the trickiness comes from the variadic tuple items like *tuple[X, ...] + since they can have arbitrary partial overlaps (while *Ts can't be split). This + function is roughly a mirror of join_tuples() w.r.t. to the fact that fixed + tuples are subtypes of variadic ones but not vice versa. + """ s_unpack_index = find_unpack_in_list(s.items) t_unpack_index = find_unpack_in_list(t.items) if s_unpack_index is None and t_unpack_index is None: @@ -852,7 +865,12 @@ def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: return items return None if s_unpack_index is not None and t_unpack_index is not None: - # TODO: handle more cases (like strictly shorter prefix/suffix). + # The only simple case we can handle if both tuples are variadic + # is when they are purely variadic. Other cases are tricky because + # a variadic item is effectively a union of tuples of all length, thus + # potentially causing overlap between a suffix in `s` and a prefix + # in `t` (see how this is handled in is_subtype() for details). + # TODO: handle more cases (like when both prefix/suffix are strictly shorter). if s.length() == 1 and t.length() == 1: s_unpack = s.items[0] assert isinstance(s_unpack, UnpackType) @@ -876,6 +894,7 @@ def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: variadic = t unpack_index = t_unpack_index fixed = s + # If one tuple is variadic one, and the other one is fixed, the meet will be fixed. unpack = variadic.items[unpack_index] assert isinstance(unpack, UnpackType) unpacked = get_proper_type(unpack.type) @@ -913,6 +932,8 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: # A named tuple that inherits from a normal class return t elif self.s.type.has_type_var_tuple_type and is_subtype(t, self.s): + # This is a bit ad-hoc but more principled handling is tricky, and this + # special case is important for type narrowing in binder to work. return t return self.default(self.s) diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 3e11951376c9..ea4d8e38c07c 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -207,7 +207,8 @@ def visit_unpack_type(self, typ: UnpackType) -> None: return if isinstance(proper_type, TypeVarTupleType): return - # TODO: this should probably be .has_base("builtins.tuple"), also elsewhere. + # TODO: this should probably be .has_base("builtins.tuple"), also elsewhere. This is + # tricky however, since this needs map_instance_to_supertype() available in many places. if isinstance(proper_type, Instance) and proper_type.type.fullname == "builtins.tuple": return if not isinstance(proper_type, (UnboundType, AnyType)): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 1911f517fa6b..e38d8497a09a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -446,18 +446,24 @@ def visit_instance(self, left: Instance) -> bool: return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) if isinstance(right, TupleType): if len(right.items) == 1: - # Non-normalized Tuple type (may be left after semantic analysis). + # Non-normalized Tuple type (may be left after semantic analysis + # because semanal_typearg visitor is not a type translator). item = right.items[0] if isinstance(item, UnpackType): unpacked = get_proper_type(item.type) if isinstance(unpacked, Instance): return self._is_subtype(left, unpacked) if len(left.args) == 1 and isinstance(left.args[0], UnpackType): + # Special case to consider Foo[*tuple[Any, ...]] (i.e. bare Foo) a + # subtype of Foo[], when Foo is user defined variadic tuple type. + # TODO: be sure we handle Bar <: Foo[*tuple[Any, ...]] <: Foo[]. unpacked = get_proper_type(left.args[0].type) if isinstance(unpacked, Instance): assert unpacked.type.fullname == "builtins.tuple" if isinstance(get_proper_type(unpacked.args[0]), AnyType): return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) + # TODO: we need a special case similar to above to consider (something that maps to) + # tuple[Any, ...] a subtype of Tuple[]. return False if isinstance(right, Instance): if type_state.is_cached_subtype_check(self._subtype_kind, left, right): @@ -498,6 +504,8 @@ def visit_instance(self, left: Instance) -> bool: t = erased nominal = True if right.type.has_type_var_tuple_type: + # For variadic instances we simply find the correct type argument mappings, + # all the heavy lifting is done by the tuple subtyping. assert right.type.type_var_tuple_prefix is not None assert right.type.type_var_tuple_suffix is not None prefix = right.type.type_var_tuple_prefix @@ -734,13 +742,23 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: + """Check subtyping between two potentially variadic tuples. + + Most non-trivial cases here are due to variadic unpacks like *tuple[X, ...], + we handle such unpacks as infinite unions Tuple[()] | Tuple[X] | Tuple[X, X] | ... + + Note: the cases where right is fixed or has *Ts unpack should be handled + by the caller. + """ right_unpack_index = find_unpack_in_list(right.items) if right_unpack_index is None: + # This case should be handled by the caller. return False right_unpack = right.items[right_unpack_index] assert isinstance(right_unpack, UnpackType) right_unpacked = get_proper_type(right_unpack.type) if not isinstance(right_unpacked, Instance): + # This case should be handled by the caller. return False assert right_unpacked.type.fullname == "builtins.tuple" right_item = right_unpacked.args[0] @@ -748,6 +766,8 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: right_suffix = len(right.items) - right_prefix - 1 left_unpack_index = find_unpack_in_list(left.items) if left_unpack_index is None: + # Simple case: left is fixed, simply find correct mapping to the right + # (effectively selecting item with matching length from an infinite union). if len(left.items) < right_prefix + right_suffix: return False prefix, middle, suffix = split_with_prefix_and_suffix( @@ -764,14 +784,22 @@ def variadic_tuple_subtype(self, left: TupleType, right: TupleType) -> bool: return all(self._is_subtype(li, right_item) for li in middle) else: if len(left.items) < len(right.items): + # There are some items on the left that will never have a matching length + # on the right. return False left_unpack = left.items[left_unpack_index] assert isinstance(left_unpack, UnpackType) left_unpacked = get_proper_type(left_unpack.type) if not isinstance(left_unpacked, Instance): + # *Ts unpacks can't be split. return False assert left_unpacked.type.fullname == "builtins.tuple" left_item = left_unpacked.args[0] + + # The most tricky case with two variadic unpacks we handle similar to union + # subtyping: *each* item on the left, must be a subtype of *some* item on the right. + # For this we first check the "asymptotic case", i.e. that both unpacks a subtypes, + # and then check subtyping for all finite overlaps. if not self._is_subtype(left_item, right_item): return False left_prefix = left_unpack_index From 38cd66c1f4662aba609bcd49a4c3f9f4707f4ce1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2023 17:49:43 +0100 Subject: [PATCH 06/14] Update comment --- mypy/constraints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 3572997dc464..ca43cdc49c5f 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1409,9 +1409,8 @@ def build_constraints_for_simple_unpack( templates: T1, T2, Ts, Ts, Ts, ... actuals: A1, As, As, As, ... - Note: this function can only be called for builtin variadic constructors: Tuple and Callable, - for Instances variance depends on position, and a much more complex function - build_constraints_for_unpack() should be used. + Note: this function can only be called for builtin variadic constructors: Tuple and Callable. + For instances, you should first find correct type argument mapping. """ template_unpack = find_unpack_in_list(template_args) assert template_unpack is not None From 60df996291871798146d191406113bd2f2e9a1b0 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 9 Sep 2023 22:43:39 +0100 Subject: [PATCH 07/14] Some more fixes --- mypy/semanal.py | 3 +- mypy/subtypes.py | 21 ++++++++----- mypy/test/testconstraints.py | 11 +------ test-data/unit/check-typevar-tuple.test | 42 +++++++++++++++++++++++++ test-data/unit/semanal-types.test | 7 +++-- 5 files changed, 63 insertions(+), 21 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ec4d32aefeb9..70403eed57ae 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4414,7 +4414,8 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: typevartuple_var = TypeVarTupleExpr( name, self.qualified_name(name), - self.object_type(), + # Upper bound for *Ts is *tuple[object, ...], it can never be object. + tuple_fallback.copy_modified(), tuple_fallback, default, INVARIANT, diff --git a/mypy/subtypes.py b/mypy/subtypes.py index e38d8497a09a..fdde1c24670e 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -453,18 +453,25 @@ def visit_instance(self, left: Instance) -> bool: unpacked = get_proper_type(item.type) if isinstance(unpacked, Instance): return self._is_subtype(left, unpacked) - if len(left.args) == 1 and isinstance(left.args[0], UnpackType): + if left.type.has_base(right.partial_fallback.type.fullname): # Special case to consider Foo[*tuple[Any, ...]] (i.e. bare Foo) a # subtype of Foo[], when Foo is user defined variadic tuple type. - # TODO: be sure we handle Bar <: Foo[*tuple[Any, ...]] <: Foo[]. - unpacked = get_proper_type(left.args[0].type) - if isinstance(unpacked, Instance): - assert unpacked.type.fullname == "builtins.tuple" - if isinstance(get_proper_type(unpacked.args[0]), AnyType): - return self._is_subtype(left, mypy.typeops.tuple_fallback(right)) + mapped = map_instance_to_supertype(left, right.partial_fallback.type) + if len(mapped.args) == 1 and isinstance(mapped.args[0], UnpackType): + unpacked = get_proper_type(mapped.args[0].type) + if isinstance(unpacked, Instance): + assert unpacked.type.fullname == "builtins.tuple" + if isinstance(get_proper_type(unpacked.args[0]), AnyType): + return not self.proper_subtype # TODO: we need a special case similar to above to consider (something that maps to) # tuple[Any, ...] a subtype of Tuple[]. return False + if isinstance(right, TypeVarTupleType): + # tuple[Any, ...] is like Any in the world of tuples (see special case above). + if left.type.has_base("builtins.tuple"): + mapped = map_instance_to_supertype(left, right.tuple_fallback.type) + if isinstance(get_proper_type(mapped.args[0]), AnyType): + return not self.proper_subtype if isinstance(right, Instance): if type_state.is_cached_subtype_check(self._subtype_kind, left, right): return True diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py index 3f05723213ea..43ccb370b202 100644 --- a/mypy/test/testconstraints.py +++ b/mypy/test/testconstraints.py @@ -88,16 +88,7 @@ def test_unpack_with_prefix_and_suffix(self) -> None: fx = self.fx assert set( infer_constraints( - Instance( - fx.gv2i, - [ - fx.u, - UnpackType( - TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) - ), - fx.u, - ], - ), + Instance(fx.gv2i, [fx.u, fx.t, fx.s, fx.u]), Instance(fx.gv2i, [fx.a, fx.b, fx.c, fx.d]), SUPERTYPE_OF, ) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 6c39b63b7cfd..8ecbe191378b 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1308,3 +1308,45 @@ def h(x: List[Tuple[T, S, U]]) -> Tuple[T, S, U]: ... vt2: Tuple[Unpack[Tuple[int, ...]], int] vt2 = h(reveal_type([])) # N: Revealed type is "builtins.list[Tuple[builtins.int, builtins.int, builtins.int]]" [builtins fixtures/tuple.pyi] + +[case testVariadicSelfTypeErasure] +from typing import Generic +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class Array(Generic[Unpack[Ts]]): + def _close(self) -> None: ... + + def close(self) -> None: + self._close() +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleAnySubtype] +from typing import Any, Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C1(B[Unpack[Tuple[Any, ...]]]): ... +c1 = C1() +class C2(B): ... +c2 = C2() +x: B[int, str] +x = c1 +x = c2 +[builtins fixtures/tuple.pyi] + +[case testVariadicTupleAnySubtypeTupleType] +from typing import Any, Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Tuple[Unpack[Ts]]): ... +class C1(B[Unpack[Tuple[Any, ...]]]): ... +c1 = C1() +class C2(B): ... +c2 = C2() +x: B[int, str] +x = c1 +x = c2 +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/semanal-types.test b/test-data/unit/semanal-types.test index 71a5c6dd87b5..5e05d099b958 100644 --- a/test-data/unit/semanal-types.test +++ b/test-data/unit/semanal-types.test @@ -1559,8 +1559,8 @@ MypyFile:1( ImportFrom:1(typing_extensions, [TypeVarTuple]) AssignmentStmt:2( NameExpr(TV* [__main__.TV]) - TypeVarTupleExpr:2())) - + TypeVarTupleExpr:2( + UpperBound(builtins.tuple[builtins.object, ...])))) [builtins fixtures/tuple.pyi] [case testTypeVarTupleCallable] @@ -1576,7 +1576,8 @@ MypyFile:1( ImportFrom:2(typing, [Callable]) AssignmentStmt:3( NameExpr(Ts* [__main__.Ts]) - TypeVarTupleExpr:3()) + TypeVarTupleExpr:3( + UpperBound(builtins.tuple[builtins.object, ...]))) FuncDef:5( foo Args( From 4991a9a29ff5e15ab694d839a85968b2f093e070 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2023 00:15:00 +0100 Subject: [PATCH 08/14] Update solver; update/add a TODO --- mypy/expandtype.py | 3 ++- mypy/solve.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index c29fcb167777..b233561e19c2 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -255,7 +255,8 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: variables=[*t.prefix.variables, *repl.variables], ) else: - # TODO: replace this with "assert False" + # We could encode Any as trivial parameters etc., but it would be too verbose. + # TODO: assert this is a trivial type, like Any, Never, or object. return repl def visit_type_var_tuple(self, t: TypeVarTupleType) -> Type: diff --git a/mypy/solve.py b/mypy/solve.py index 17e1ca047818..7cdf1c10c9b5 100644 --- a/mypy/solve.py +++ b/mypy/solve.py @@ -144,6 +144,8 @@ def solve_with_dependent( if all(not lowers[tv] and not uppers[tv] for tv in scc): best_free = choose_free([originals[tv] for tv in scc], original_vars) if best_free: + # TODO: failing to choose may cause leaking type variables, + # we need to fail gracefully instead. free_vars.append(best_free.id) free_solutions[best_free.id] = best_free @@ -323,13 +325,15 @@ def test(x: U) -> U: ... best = sorted(scc, key=lambda x: (x.id not in original_vars, x.id.raw_id))[0] if isinstance(best, TypeVarType): return best.copy_modified(values=values, upper_bound=common_upper_bound) - if is_trivial_bound(common_upper_bound_p): + if is_trivial_bound(common_upper_bound_p, allow_tuple=True): # TODO: support more cases for ParamSpecs/TypeVarTuples return best return None -def is_trivial_bound(tp: ProperType) -> bool: +def is_trivial_bound(tp: ProperType, allow_tuple: bool = False) -> bool: + if isinstance(tp, Instance) and tp.type.fullname == "builtins.tuple": + return allow_tuple and is_trivial_bound(get_proper_type(tp.args[0])) return isinstance(tp, Instance) and tp.type.fullname == "builtins.object" From 7fc37dae9572f215733c12422f6f8a856ccf73d8 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 10 Sep 2023 17:03:40 +0100 Subject: [PATCH 09/14] Fix overload check; update upper bound in couple more places --- mypy/test/typefixture.py | 11 ++++++----- mypy/typeops.py | 7 +++---- test-data/unit/check-typevar-tuple.test | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/mypy/test/typefixture.py b/mypy/test/typefixture.py index 81af765f8585..b7bde16e6be2 100644 --- a/mypy/test/typefixture.py +++ b/mypy/test/typefixture.py @@ -233,9 +233,10 @@ def make_type_var_tuple(name: str, id: int, upper_bound: Type) -> TypeVarTupleTy AnyType(TypeOfAny.from_omitted_generics), ) - self.ts = make_type_var_tuple("Ts", 1, self.o) # Ts`1 (type var tuple) - self.ss = make_type_var_tuple("Ss", 2, self.o) # Ss`2 (type var tuple) - self.us = make_type_var_tuple("Us", 3, self.o) # Us`3 (type var tuple) + obj_tuple = self.std_tuple.copy_modified(args=[self.o]) + self.ts = make_type_var_tuple("Ts", 1, obj_tuple) # Ts`1 (type var tuple) + self.ss = make_type_var_tuple("Ss", 2, obj_tuple) # Ss`2 (type var tuple) + self.us = make_type_var_tuple("Us", 3, obj_tuple) # Us`3 (type var tuple) self.gvi = self.make_type_info("GV", mro=[self.oi], typevars=["Ts"], typevar_tuple_index=0) self.gv2i = self.make_type_info( @@ -325,8 +326,8 @@ def make_type_info( n, n, id, - self.o, - self.std_tuple, + self.std_tuple.copy_modified(args=[self.o]), + self.std_tuple.copy_modified(args=[self.o]), AnyType(TypeOfAny.from_omitted_generics), ) ) diff --git a/mypy/typeops.py b/mypy/typeops.py index 3efa3cc3e965..3f50232f04c1 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -104,8 +104,8 @@ def tuple_fallback(typ: TupleType) -> Instance: if isinstance(item, UnpackType): unpacked_type = get_proper_type(item.type) if isinstance(unpacked_type, TypeVarTupleType): - items.append(unpacked_type.upper_bound) - elif ( + unpacked_type = get_proper_type(unpacked_type.upper_bound) + if ( isinstance(unpacked_type, Instance) and unpacked_type.type.fullname == "builtins.tuple" ): @@ -654,8 +654,7 @@ def erase_def_to_union_or_bound(tdef: TypeVarLikeType) -> Type: # TODO(PEP612): fix for ParamSpecType if isinstance(tdef, ParamSpecType): return AnyType(TypeOfAny.from_error) - assert isinstance(tdef, TypeVarType) - if tdef.values: + if isinstance(tdef, TypeVarType) and tdef.values: return make_simplified_union(tdef.values) else: return tdef.upper_bound diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 8ecbe191378b..832769317bdf 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1350,3 +1350,19 @@ x: B[int, str] x = c1 x = c2 [builtins fixtures/tuple.pyi] + +[case testTypeVarTupleAnyOverload] +from typing import Any, Generic, overload, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class Array(Generic[Unpack[Ts]]): ... + +class A: + @overload + def f(self, x: Tuple[Unpack[Ts]]) -> Array[Unpack[Ts]]: ... + @overload + def f(self, x: Any) -> Any: ... + def f(self, x: Any) -> Any: + ... +[builtins fixtures/tuple.pyi] From f4080385098975e4b66a3ef864a30057e84b4911 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 11 Sep 2023 23:44:20 +0100 Subject: [PATCH 10/14] Fix incremental mode (also don't visit void) --- mypy/fixup.py | 10 +++++++--- test-data/unit/check-incremental.test | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 2b2e1210ee4e..cb11416b78b7 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -81,11 +81,17 @@ def visit_type_info(self, info: TypeInfo) -> None: info.update_tuple_type(info.tuple_type) if info.special_alias: info.special_alias.alias_tvars = list(info.defn.type_vars) + for i, t in enumerate(info.defn.type_vars): + if isinstance(t, TypeVarTupleType): + info.special_alias.tvar_tuple_index = i if info.typeddict_type: info.typeddict_type.accept(self.type_fixer) info.update_typeddict_type(info.typeddict_type) if info.special_alias: info.special_alias.alias_tvars = list(info.defn.type_vars) + for i, t in enumerate(info.defn.type_vars): + if isinstance(t, TypeVarTupleType): + info.special_alias.tvar_tuple_index = i if info.declared_metaclass: info.declared_metaclass.accept(self.type_fixer) if info.metaclass_type: @@ -314,6 +320,7 @@ def visit_param_spec(self, p: ParamSpecType) -> None: p.default.accept(self) def visit_type_var_tuple(self, t: TypeVarTupleType) -> None: + t.tuple_fallback.accept(self) t.upper_bound.accept(self) t.default.accept(self) @@ -336,9 +343,6 @@ def visit_union_type(self, ut: UnionType) -> None: for it in ut.items: it.accept(self) - def visit_void(self, o: Any) -> None: - pass # Nothing to descend into. - def visit_type_type(self, t: TypeType) -> None: t.item.accept(self) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index fcab0545b982..f80aa298ffc5 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -6422,3 +6422,22 @@ P = ParamSpec("P") class C(Generic[P]): def __init__(self, fn: Callable[P, int]) -> None: ... [builtins fixtures/dict.pyi] + +[case testVariadicTupleIncrementalUpdateNoCrash] +import m +[file m.py] +from typing import Any +from lib import C + +x: C[Any] +[file m.py.2] +from lib import C + +x: C[int] +[file lib.py] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class C(Tuple[Unpack[Ts]]): ... +[builtins fixtures/tuple.pyi] From 11f1c7637a5544d5a226bbb2a4ef5399c8837a4a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Tue, 12 Sep 2023 21:50:13 +0100 Subject: [PATCH 11/14] Fix couple more crashes --- mypy/constraints.py | 9 +++++++-- mypy/fixup.py | 7 ++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index ca43cdc49c5f..0524e38f9643 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -1271,8 +1271,13 @@ def visit_type_alias_type(self, template: TypeAliasType) -> list[Constraint]: def infer_against_any(self, types: Iterable[Type], any_type: AnyType) -> list[Constraint]: res: list[Constraint] = [] for t in types: - if isinstance(t, UnpackType) and isinstance(t.type, TypeVarTupleType): - res.append(Constraint(t.type, self.direction, any_type)) + if isinstance(t, UnpackType): + if isinstance(t.type, TypeVarTupleType): + res.append(Constraint(t.type, self.direction, any_type)) + else: + unpacked = get_proper_type(t.type) + assert isinstance(unpacked, Instance) + res.extend(infer_constraints(unpacked, any_type, self.direction)) else: # Note that we ignore variance and simply always use the # original direction. This is because for Any targets direction is diff --git a/mypy/fixup.py b/mypy/fixup.py index cb11416b78b7..5ffc47120734 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -172,11 +172,7 @@ def visit_decorator(self, d: Decorator) -> None: def visit_class_def(self, c: ClassDef) -> None: for v in c.type_vars: - if isinstance(v, TypeVarType): - for value in v.values: - value.accept(self.type_fixer) - v.upper_bound.accept(self.type_fixer) - v.default.accept(self.type_fixer) + v.accept(self.type_fixer) def visit_type_var_expr(self, tv: TypeVarExpr) -> None: for value in tv.values: @@ -190,6 +186,7 @@ def visit_paramspec_expr(self, p: ParamSpecExpr) -> None: def visit_type_var_tuple_expr(self, tv: TypeVarTupleExpr) -> None: tv.upper_bound.accept(self.type_fixer) + tv.tuple_fallback.accept(self.type_fixer) tv.default.accept(self.type_fixer) def visit_var(self, v: Var) -> None: From ab3c2cc3741ea3b69082f2e6c38849a381368b74 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2023 09:31:44 +0100 Subject: [PATCH 12/14] Add a test --- test-data/unit/check-typevar-tuple.test | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 832769317bdf..77b08c444170 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1366,3 +1366,14 @@ class A: def f(self, x: Any) -> Any: ... [builtins fixtures/tuple.pyi] + +[case testTypeVarTupleInferAgainstAny] +from typing import Any, Tuple, TypeVar +from typing_extensions import Unpack + +T = TypeVar("T") + +def test(x: int, t: Tuple[T, ...]) -> Tuple[int, Unpack[Tuple[T, ...]]]: + ... +a: Any = test(42, ()) +[builtins fixtures/tuple.pyi] From 6946a49ec0f29dcdbffba4b14eabfef3e94511fb Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2023 22:12:08 +0100 Subject: [PATCH 13/14] Add (and clean-up) tests --- mypy/test/testconstraints.py | 13 +-- mypy/test/testsubtypes.py | 33 ------ test-data/unit/check-typevar-tuple.test | 139 ++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 45 deletions(-) diff --git a/mypy/test/testconstraints.py b/mypy/test/testconstraints.py index 43ccb370b202..5ec292f07056 100644 --- a/mypy/test/testconstraints.py +++ b/mypy/test/testconstraints.py @@ -20,8 +20,6 @@ def test_basic_type_variable(self) -> None: Constraint(type_var=fx.t, op=direction, target=fx.a) ] - # TODO: some tests here use non-normalized variadic forms (mostly fixed tuple unpacks). - # We may want to either replace these with normalized ones, or add matching normalized tests. def test_basic_type_var_tuple_subtype(self) -> None: fx = self.fx assert infer_constraints( @@ -103,16 +101,7 @@ def test_unpack_tuple_length_non_match(self) -> None: fx = self.fx assert set( infer_constraints( - Instance( - fx.gv2i, - [ - fx.u, - UnpackType( - TupleType([fx.t, fx.s], fallback=Instance(fx.std_tuplei, [fx.o])) - ), - fx.u, - ], - ), + Instance(fx.gv2i, [fx.u, fx.t, fx.s, fx.u]), Instance(fx.gv2i, [fx.a, fx.b, fx.d]), SUPERTYPE_OF, ) diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index 541ba47ced4a..6fd1c0b206f3 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -198,8 +198,6 @@ def test_type_callable_subtyping(self) -> None: self.fx.callable_type(self.fx.a, self.fx.b), self.fx.callable(self.fx.a, self.fx.b) ) - # TODO: some tests here use non-normalized variadic forms (mostly fixed tuple unpacks). - # We may want to either replace these with normalized ones, or add matching normalized tests. def test_type_var_tuple(self) -> None: self.assert_subtype(Instance(self.fx.gvi, []), Instance(self.fx.gvi, [])) self.assert_subtype( @@ -270,37 +268,6 @@ def test_type_var_tuple_with_prefix_suffix(self) -> None: Instance(self.fx.gvi, [self.fx.a, UnpackType(self.fx.ss), self.fx.b, self.fx.c]), ) - def test_type_var_tuple_unpacked_tuple(self) -> None: - self.assert_not_subtype( - Instance( - self.fx.gvi, - [ - UnpackType( - TupleType( - [self.fx.a, self.fx.b], - fallback=Instance(self.fx.std_tuplei, [self.fx.o]), - ) - ) - ], - ), - Instance(self.fx.gvi, [self.fx.a]), - ) - self.assert_not_subtype( - Instance( - self.fx.gvi, - [ - UnpackType( - TupleType( - [self.fx.a, self.fx.b], - fallback=Instance(self.fx.std_tuplei, [self.fx.o]), - ) - ) - ], - ), - # Order flipped here. - Instance(self.fx.gvi, [self.fx.b, self.fx.a]), - ) - def test_type_var_tuple_unpacked_variable_length_tuple(self) -> None: self.assert_subtype( Instance(self.fx.gvi, [self.fx.a, self.fx.a]), diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 77b08c444170..d38d492fe9b2 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -1257,30 +1257,45 @@ t3: Tuple[Unpack[Tuple[int, ...]], int] t4: Tuple[int, Unpack[Tuple[int, ...]], int] t5: Tuple[int, ...] +tl: Tuple[int, int, Unpack[Tuple[int, ...]]] +tr: Tuple[Unpack[Tuple[int, ...]], int, int] + f1(t1) f1(t2) f1(t3) f1(t4) f1(t5) +f1(tl) +f1(tr) + f2(t1) f2(t2) f2(t3) f2(t4) f2(t5) # E: Argument 1 to "f2" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]]]" +f2(tl) +f2(tr) + f3(t1) f3(t2) f3(t3) f3(t4) f3(t5) # E: Argument 1 to "f3" has incompatible type "Tuple[int, ...]"; expected "Tuple[Unpack[Tuple[float, ...]], float]" +f3(tl) +f3(tr) + f4(t1) f4(t2) # E: Argument 1 to "f4" has incompatible type "Tuple[int, Unpack[Tuple[int, ...]]]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" f4(t3) # E: Argument 1 to "f4" has incompatible type "Tuple[Unpack[Tuple[int, ...]], int]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" f4(t4) f4(t5) # E: Argument 1 to "f4" has incompatible type "Tuple[int, ...]"; expected "Tuple[float, Unpack[Tuple[float, ...]], float]" +f4(tl) +f4(tr) + t5_verbose: Tuple[Unpack[Tuple[int, ...]]] t5 = t5_verbose # OK [builtins fixtures/tuple.pyi] @@ -1321,6 +1336,130 @@ class Array(Generic[Unpack[Ts]]): self._close() [builtins fixtures/tuple.pyi] +[case testVariadicSubclassFixed] +from typing import Generic, Tuple +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C(B[int, str]): ... +class D(B[Unpack[Tuple[int, ...]]]): ... + +def fii(x: B[int, int]) -> None: ... +def fis(x: B[int, str]) -> None: ... +def fiv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +fii(C()) # E: Argument 1 to "fii" has incompatible type "C"; expected "B[int, int]" +fii(D()) # E: Argument 1 to "fii" has incompatible type "D"; expected "B[int, int]" +fis(C()) +fis(D()) # E: Argument 1 to "fis" has incompatible type "D"; expected "B[int, str]" +fiv(C()) # E: Argument 1 to "fiv" has incompatible type "C"; expected "B[Unpack[Tuple[int, ...]]]" +fiv(D()) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassSame] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +class C(B[Unpack[Ts]]): ... + +def fii(x: B[int, int]) -> None: ... +def fis(x: B[int, str]) -> None: ... +def fiv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +cii: C[int, int] +cis: C[int, str] +civ: C[Unpack[Tuple[int, ...]]] + +fii(cii) +fii(cis) # E: Argument 1 to "fii" has incompatible type "C[int, str]"; expected "B[int, int]" +fii(civ) # E: Argument 1 to "fii" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, int]" + +fis(cii) # E: Argument 1 to "fis" has incompatible type "C[int, int]"; expected "B[int, str]" +fis(cis) +fis(civ) # E: Argument 1 to "fis" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, str]" + +fiv(cii) +fiv(cis) # E: Argument 1 to "fiv" has incompatible type "C[int, str]"; expected "B[Unpack[Tuple[int, ...]]]" +fiv(civ) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassExtra] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... + +T = TypeVar("T") +class C(B[int, Unpack[Ts], T]): ... + +def ff(x: B[int, int, int]) -> None: ... +def fv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +cii: C[int, int] +cis: C[int, str] +civ: C[Unpack[Tuple[int, ...]]] + +ff(cii) +ff(cis) # E: Argument 1 to "ff" has incompatible type "C[int, str]"; expected "B[int, int, int]" +ff(civ) # E: Argument 1 to "ff" has incompatible type "C[Unpack[Tuple[int, ...]]]"; expected "B[int, int, int]" + +fv(cii) +fv(cis) # E: Argument 1 to "fv" has incompatible type "C[int, str]"; expected "B[Unpack[Tuple[int, ...]]]" +fv(civ) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassVariadic] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): ... +T = TypeVar("T") +class C(B[Unpack[Tuple[T, ...]]]): ... + +def ff(x: B[int, int]) -> None: ... +def fv(x: B[Unpack[Tuple[int, ...]]]) -> None: ... + +ci: C[int] +ff(ci) # E: Argument 1 to "ff" has incompatible type "C[int]"; expected "B[int, int]" +fv(ci) +[builtins fixtures/tuple.pyi] + +[case testVariadicSubclassMethodAccess] +from typing import Generic, Tuple, TypeVar +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +class B(Generic[Unpack[Ts]]): + def meth(self) -> Tuple[Unpack[Ts]]: ... + +class C1(B[int, str]): ... +class C2(B[Unpack[Ts]]): ... +T = TypeVar("T") +class C3(B[int, Unpack[Ts], T]): ... +class C4(B[Unpack[Tuple[T, ...]]]): ... + +c1: C1 +reveal_type(c1.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.str]" + +c2f: C2[int, str] +c2v: C2[Unpack[Tuple[int, ...]]] +reveal_type(c2f.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.str]" +reveal_type(c2v.meth()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +c3f: C3[int, str] +c3v: C3[Unpack[Tuple[int, ...]]] +reveal_type(c3f.meth()) # N: Revealed type is "Tuple[builtins.int, builtins.int, builtins.str]" +reveal_type(c3v.meth()) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.int]" + +c4: C4[int] +reveal_type(c4.meth()) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +[builtins fixtures/tuple.pyi] + [case testVariadicTupleAnySubtype] from typing import Any, Generic, Tuple from typing_extensions import TypeVarTuple, Unpack From 7917beaaba81fe85edced22010ac6aca1c3a589e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 13 Sep 2023 23:00:14 +0100 Subject: [PATCH 14/14] Meets/joins tests; fix silly bug; handle some more edge cases --- mypy/join.py | 8 +++- mypy/meet.py | 2 +- mypy/test/testsubtypes.py | 2 +- mypy/test/testtypes.py | 77 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 3 deletions(-) diff --git a/mypy/join.py b/mypy/join.py index 48d8ef3dc8dd..e4429425d98a 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -527,6 +527,7 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: if suffix_len: for si, ti in zip(s.items[-suffix_len:], t.items[-suffix_len:]): items.append(join_types(si, ti)) + return items if s.length() == 1 or t.length() == 1: # Another case we can handle is when one of tuple is purely variadic # (i.e. a non-normalized form of tuple[X, ...]), in this case the join @@ -541,7 +542,7 @@ def join_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: other_joined = join_type_list(s_other + t_other) mid_joined = join_types(mid_joined, other_joined) return [UnpackType(s_unpacked.copy_modified(args=[mid_joined]))] - # TODO: are there other case we can handle? + # TODO: are there other case we can handle (e.g. both prefix/suffix are shorter)? return None if s_unpack_index is not None: variadic = s @@ -601,6 +602,11 @@ def visit_tuple_type(self, t: TupleType) -> ProperType: if items is not None: return TupleType(items, fallback) else: + # TODO: should this be a default fallback behaviour like for meet? + if is_proper_subtype(self.s, t): + return t + if is_proper_subtype(t, self.s): + return self.s return fallback else: return join_types(self.s, mypy.typeops.tuple_fallback(t)) diff --git a/mypy/meet.py b/mypy/meet.py index cceb34354314..0fa500d32c30 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -870,7 +870,7 @@ def meet_tuples(self, s: TupleType, t: TupleType) -> list[Type] | None: # a variadic item is effectively a union of tuples of all length, thus # potentially causing overlap between a suffix in `s` and a prefix # in `t` (see how this is handled in is_subtype() for details). - # TODO: handle more cases (like when both prefix/suffix are strictly shorter). + # TODO: handle more cases (like when both prefix/suffix are shorter in s or t). if s.length() == 1 and t.length() == 1: s_unpack = s.items[0] assert isinstance(s_unpack, UnpackType) diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index 6fd1c0b206f3..480fe38a90a7 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -4,7 +4,7 @@ from mypy.subtypes import is_subtype from mypy.test.helpers import Suite from mypy.test.typefixture import InterfaceTypeFixture, TypeFixture -from mypy.types import Instance, TupleType, Type, UnpackType +from mypy.types import Instance, Type, UnpackType class SubtypingSuite(Suite): diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index 59457dfa5d3b..e8dd623bec53 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -47,6 +47,7 @@ UnboundType, UninhabitedType, UnionType, + UnpackType, get_proper_type, has_recursive_types, ) @@ -986,6 +987,54 @@ def test_literal_type(self) -> None: UnionType([lit2, lit3]), UnionType([lit1, lit2]), UnionType([lit2, lit3, lit1]) ) + def test_variadic_tuple_joins(self) -> None: + # These tests really test just the "arity", to be sure it is handled correctly. + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + ) + self.assert_join( + self.tuple(self.fx.a, self.fx.a), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple( + self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a + ), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_join( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple( + self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b])), self.fx.b + ), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + ) + # There are additional test cases in check-inference.test. # TODO: Function types + varargs and default args. @@ -1221,6 +1270,34 @@ def assert_meet_uninhabited(self, s: Type, t: Type) -> None: with state.strict_optional_set(True): self.assert_meet(s, t, self.fx.uninhabited) + def test_variadic_tuple_meets(self) -> None: + # These tests really test just the "arity", to be sure it is handled correctly. + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(self.fx.a, self.fx.a), + self.tuple(self.fx.a, UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(self.fx.a, self.fx.a), + ) + self.assert_meet( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a]))), + ) + self.assert_meet( + self.tuple(UnpackType(Instance(self.fx.std_tuplei, [self.fx.a])), self.fx.a), + self.tuple(self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b]))), + self.tuple(self.fx.b, UnpackType(Instance(self.fx.std_tuplei, [self.fx.b]))), + ) + def assert_meet(self, s: Type, t: Type, meet: Type) -> None: self.assert_simple_meet(s, t, meet) self.assert_simple_meet(t, s, meet)