From a74ff66ef7f64379484c759d0769dac9a5fc088a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 20 Aug 2023 00:59:03 +0100 Subject: [PATCH 1/7] Re-work some of TypeVarTuple and TypeAlias internals --- mypy/checkexpr.py | 2 - mypy/expandtype.py | 4 -- mypy/mixedtraverser.py | 2 +- mypy/nodes.py | 17 +------ mypy/semanal.py | 13 ++++- mypy/semanal_typeargs.py | 32 +++++++++--- mypy/server/astmerge.py | 5 -- mypy/server/deps.py | 2 +- mypy/strconv.py | 2 +- mypy/typeanal.py | 66 +++++++++++++++++++++---- mypy/typeops.py | 8 ++- mypy/types.py | 4 -- mypy/types_utils.py | 10 +++- test-data/unit/check-generics.test | 1 - test-data/unit/check-typevar-tuple.test | 38 ++++++++++++-- test-data/unit/check-varargs.test | 2 +- test-data/unit/semanal-errors.test | 9 ++-- 17 files changed, 148 insertions(+), 69 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 420cfd990820..00d03d24fd35 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -168,7 +168,6 @@ UninhabitedType, UnionType, UnpackType, - flatten_nested_tuples, flatten_nested_unions, get_proper_type, get_proper_types, @@ -4448,7 +4447,6 @@ class C(Generic[T, Unpack[Ts]]): ... prefix = next(i for (i, v) in enumerate(vars) if isinstance(v, TypeVarTupleType)) suffix = len(vars) - prefix - 1 - args = flatten_nested_tuples(args) if len(args) < len(vars) - 1: self.msg.incompatible_type_application(len(vars), len(args), ctx) return [AnyType(TypeOfAny.from_error)] * len(vars) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 6f69e09936db..7bd1e5363271 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -35,7 +35,6 @@ UninhabitedType, UnionType, UnpackType, - flatten_nested_tuples, flatten_nested_unions, get_proper_type, split_with_prefix_and_suffix, @@ -460,9 +459,6 @@ def expand_types_with_unpack( indicates use of Any or some error occurred earlier. In this case callers should simply propagate the resulting type. """ - # TODO: this will cause a crash on aliases like A = Tuple[int, Unpack[A]]. - # Although it is unlikely anyone will write this, we should fail gracefully. - typs = flatten_nested_tuples(typs) items: list[Type] = [] for item in typs: if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType): diff --git a/mypy/mixedtraverser.py b/mypy/mixedtraverser.py index 771f87fc6bd6..dfde41859c67 100644 --- a/mypy/mixedtraverser.py +++ b/mypy/mixedtraverser.py @@ -49,7 +49,7 @@ def visit_class_def(self, o: ClassDef) -> None: def visit_type_alias_expr(self, o: TypeAliasExpr) -> None: super().visit_type_alias_expr(o) self.in_type_alias_expr = True - o.type.accept(self) + o.node.target.accept(self) self.in_type_alias_expr = False def visit_type_var_expr(self, o: TypeVarExpr) -> None: diff --git a/mypy/nodes.py b/mypy/nodes.py index 452a4f643255..7efb01c1b18e 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2625,27 +2625,14 @@ def deserialize(cls, data: JsonDict) -> TypeVarTupleExpr: class TypeAliasExpr(Expression): """Type alias expression (rvalue).""" - __slots__ = ("type", "tvars", "no_args", "node") + __slots__ = ("node",) - __match_args__ = ("type", "tvars", "no_args", "node") + __match_args__ = ("node",) - # The target type. - type: mypy.types.Type - # Names of type variables used to define the alias - tvars: list[str] - # Whether this alias was defined in bare form. Used to distinguish - # between - # A = List - # and - # A = List[Any] - no_args: bool node: TypeAlias def __init__(self, node: TypeAlias) -> None: super().__init__() - self.type = node.target - self.tvars = [v.name for v in node.alias_tvars] - self.no_args = node.no_args self.node = node def accept(self, visitor: ExpressionVisitor[T]) -> T: diff --git a/mypy/semanal.py b/mypy/semanal.py index ef66c9276664..381759ab01d9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -3680,7 +3680,10 @@ def disable_invalid_recursive_aliases( """Prohibit and fix recursive type aliases that are invalid/unsupported.""" messages = [] if is_invalid_recursive_alias({current_node}, current_node.target): - messages.append("Invalid recursive alias: a union item of itself") + target = ( + "tuple" if isinstance(get_proper_type(current_node.target), TupleType) else "union" + ) + messages.append(f"Invalid recursive alias: a {target} item of itself") if detect_diverging_alias( current_node, current_node.target, self.lookup_qualified, self.tvar_scope ): @@ -5289,6 +5292,7 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: # Probably always allow Parameters literals, and validate in semanal_typeargs.py base = expr.base if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias): + allow_unpack = base.node.tvar_tuple_index is not None alias = base.node if any(isinstance(t, ParamSpecType) for t in alias.alias_tvars): has_param_spec = True @@ -5297,9 +5301,11 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: has_param_spec = False num_args = -1 elif isinstance(base, RefExpr) and isinstance(base.node, TypeInfo): + allow_unpack = base.node.has_type_var_tuple_type has_param_spec = base.node.has_param_spec_type num_args = len(base.node.type_vars) else: + allow_unpack = False has_param_spec = False num_args = -1 @@ -5317,6 +5323,7 @@ def analyze_type_application_args(self, expr: IndexExpr) -> list[Type] | None: allow_unbound_tvars=self.allow_unbound_tvars, allow_placeholder=True, allow_param_spec_literals=has_param_spec, + allow_unpack=allow_unpack, ) if analyzed is None: return None @@ -6537,6 +6544,7 @@ def type_analyzer( allow_placeholder: bool = False, allow_required: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, report_invalid_types: bool = True, prohibit_self_type: str | None = None, allow_type_any: bool = False, @@ -6555,6 +6563,7 @@ def type_analyzer( allow_placeholder=allow_placeholder, allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, prohibit_self_type=prohibit_self_type, allow_type_any=allow_type_any, ) @@ -6575,6 +6584,7 @@ def anal_type( allow_placeholder: bool = False, allow_required: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, report_invalid_types: bool = True, prohibit_self_type: str | None = None, allow_type_any: bool = False, @@ -6612,6 +6622,7 @@ def anal_type( allow_placeholder=allow_placeholder, allow_required=allow_required, allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, report_invalid_types=report_invalid_types, prohibit_self_type=prohibit_self_type, allow_type_any=allow_type_any, diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index e188955dabbb..27381b0d9681 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -14,13 +14,14 @@ from mypy.errors import Errors from mypy.messages import format_type from mypy.mixedtraverser import MixedTraverserVisitor -from mypy.nodes import Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile +from mypy.nodes import ARG_STAR, Block, ClassDef, Context, FakeInfo, FuncItem, MypyFile from mypy.options import Options from mypy.scope import Scope from mypy.subtypes import is_same_type, is_subtype from mypy.typeanal import set_any_tvars from mypy.types import ( AnyType, + CallableType, Instance, Parameters, ParamSpecType, @@ -116,20 +117,40 @@ def visit_type_alias_type(self, t: TypeAliasType) -> None: # the expansion, most likely it will result in the same kind of error. get_proper_type(t).accept(self) + def visit_tuple_type(self, t: TupleType) -> None: + t.items = flatten_nested_tuples(t.items) + # We could also normalize Tuple[*tuple[X, ...]] -> tuple[X, ...] like in + # expand_type() but we can't do this here since it is not a translator visitor, + # and we need to return an Instance instead of TupleType. + super().visit_tuple_type(t) + + def visit_callable_type(self, t: CallableType) -> None: + super().visit_callable_type(t) + # Normalize trivial unpack in var args as *args: *tuple[X, ...] -> *args: X + if t.is_var_arg: + star_index = t.arg_kinds.index(ARG_STAR) + star_type = t.arg_types[star_index] + if isinstance(star_type, UnpackType): + p_type = get_proper_type(star_type.type) + if isinstance(p_type, Instance): + assert p_type.type.fullname == "builtins.tuple" + t.arg_types[star_index] = p_type.args[0] + def visit_instance(self, t: Instance) -> None: # Type argument counts were checked in the main semantic analyzer pass. We assume # that the counts are correct here. info = t.type if isinstance(info, FakeInfo): return # https://github.com/python/mypy/issues/11079 + # TODO: we can also normalize tuple[*tuple[X, ...], ...] -> tuple[X, ...] + # bit this looks quite rare corner case (and we should be able to handle it). + t.args = tuple(flatten_nested_tuples(t.args)) self.validate_args(info.name, t.args, info.defn.type_vars, t) super().visit_instance(t) def validate_args( self, name: str, args: Sequence[Type], type_vars: list[TypeVarLikeType], ctx: Context ) -> bool: - # TODO: we need to do flatten_nested_tuples and validate arg count for instances - # similar to how do we do this for type aliases above, but this may have perf penalty. if any(isinstance(v, TypeVarTupleType) for v in type_vars): prefix = next(i for (i, v) in enumerate(type_vars) if isinstance(v, TypeVarTupleType)) tvt = type_vars[prefix] @@ -198,6 +219,7 @@ def validate_args( return is_error def visit_unpack_type(self, typ: UnpackType) -> None: + super().visit_unpack_type(typ) proper_type = get_proper_type(typ.type) if isinstance(proper_type, TupleType): return @@ -211,12 +233,10 @@ def visit_unpack_type(self, typ: UnpackType) -> None: and proper_type.type_of_any == TypeOfAny.from_error ): return - - # TODO: Infer something when it can't be unpacked to allow rest of - # typechecking to work. self.fail( message_registry.INVALID_UNPACK.format(format_type(proper_type, self.options)), typ ) + typ.type = AnyType(TypeOfAny.from_error) def check_type_var_values( self, name: str, actuals: list[Type], arg_name: str, valids: list[Type], context: Context diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index f58a4eedabc8..862c3898a383 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -73,7 +73,6 @@ SymbolNode, SymbolTable, TypeAlias, - TypeAliasExpr, TypedDictExpr, TypeInfo, Var, @@ -326,10 +325,6 @@ def visit_enum_call_expr(self, node: EnumCallExpr) -> None: self.process_synthetic_type_info(node.info) super().visit_enum_call_expr(node) - def visit_type_alias_expr(self, node: TypeAliasExpr) -> None: - self.fixup_type(node.type) - super().visit_type_alias_expr(node) - # Others def visit_var(self, node: Var) -> None: diff --git a/mypy/server/deps.py b/mypy/server/deps.py index ed85b74f2206..9ed2d4549629 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -472,7 +472,7 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None: self.add_dependency(make_trigger(class_name + ".__init__")) self.add_dependency(make_trigger(class_name + ".__new__")) if isinstance(rvalue, IndexExpr) and isinstance(rvalue.analyzed, TypeAliasExpr): - self.add_type_dependencies(rvalue.analyzed.type) + self.add_type_dependencies(rvalue.analyzed.node.target) elif typ: self.add_type_dependencies(typ) else: diff --git a/mypy/strconv.py b/mypy/strconv.py index c428addd43aa..42a07c7f62fa 100644 --- a/mypy/strconv.py +++ b/mypy/strconv.py @@ -511,7 +511,7 @@ def visit_type_var_tuple_expr(self, o: mypy.nodes.TypeVarTupleExpr) -> str: return self.dump(a, o) def visit_type_alias_expr(self, o: mypy.nodes.TypeAliasExpr) -> str: - return f"TypeAliasExpr({self.stringify_type(o.type)})" + return f"TypeAliasExpr({self.stringify_type(o.node.target)})" def visit_namedtuple_expr(self, o: mypy.nodes.NamedTupleExpr) -> str: return f"NamedTupleExpr:{o.line}({o.info.name}, {self.stringify_type(o.info.tuple_type) if o.info.tuple_type is not None else None})" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b15b5c7654ba..5e744a013be5 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -195,6 +195,7 @@ def __init__( allow_placeholder: bool = False, allow_required: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, report_invalid_types: bool = True, prohibit_self_type: str | None = None, allowed_alias_tvars: list[TypeVarLikeType] | None = None, @@ -241,6 +242,8 @@ def __init__( self.prohibit_self_type = prohibit_self_type # Allow variables typed as Type[Any] and type (useful for base classes). self.allow_type_any = allow_type_any + self.allow_type_var_tuple = False + self.allow_unpack = allow_unpack def lookup_qualified( self, name: str, ctx: Context, suppress_errors: bool = False @@ -277,7 +280,10 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) return PlaceholderType( node.fullname, self.anal_array( - t.args, allow_param_spec=True, allow_param_spec_literals=True + t.args, + allow_param_spec=True, + allow_param_spec_literals=True, + allow_unpack=True, ), t.line, ) @@ -365,6 +371,13 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) self.fail(f'TypeVarTuple "{t.name}" is unbound', t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) assert isinstance(tvar_def, TypeVarTupleType) + if not self.allow_type_var_tuple: + self.fail( + f'TypeVarTuple "{t.name}" is only valid with an unpack', + t, + code=codes.VALID_TYPE, + ) + return AnyType(TypeOfAny.from_error) if len(t.args) > 0: self.fail( f'Type variable "{t.name}" used with arguments', t, code=codes.VALID_TYPE @@ -390,6 +403,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) t.args, allow_param_spec=True, allow_param_spec_literals=node.has_param_spec_type, + allow_unpack=node.tvar_tuple_index is not None, ) if node.has_param_spec_type and len(node.alias_tvars) == 1: an_args = self.pack_paramspec_args(an_args) @@ -531,7 +545,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ instance = self.named_type("builtins.tuple", [self.anal_type(t.args[0])]) instance.line = t.line return instance - return self.tuple_type(self.anal_array(t.args)) + return self.tuple_type(self.anal_array(t.args, allow_unpack=True)) elif fullname == "typing.Union": items = self.anal_array(t.args) return UnionType.make_union(items) @@ -631,7 +645,13 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ if len(t.args) != 1: self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) - return UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) + if not self.allow_unpack: + self.fail("Unpack is only valid in a variadic position", t) + return AnyType(TypeOfAny.from_error) + self.allow_type_var_tuple = True + result = UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) + self.allow_type_var_tuple = False + return result elif fullname in SELF_TYPE_NAMES: if t.args: self.fail("Self type cannot have type arguments", t) @@ -666,7 +686,7 @@ def analyze_type_with_type_info( if len(args) > 0 and info.fullname == "builtins.tuple": fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) - return TupleType(self.anal_array(args), fallback, ctx.line) + return TupleType(self.anal_array(args, allow_unpack=True), fallback, ctx.line) # Analyze arguments and (usually) construct Instance type. The # number of type arguments and their values are @@ -679,7 +699,10 @@ def analyze_type_with_type_info( instance = Instance( info, self.anal_array( - args, allow_param_spec=True, allow_param_spec_literals=info.has_param_spec_type + args, + allow_param_spec=True, + allow_param_spec_literals=info.has_param_spec_type, + allow_unpack=info.has_type_var_tuple_type, ), ctx.line, ctx.column, @@ -715,7 +738,7 @@ def analyze_type_with_type_info( if info.special_alias: return instantiate_type_alias( info.special_alias, - # TODO: should we allow NamedTuples generic in ParamSpec? + # TODO: should we allow NamedTuples generic in ParamSpec and TypeVarTuple? self.anal_array(args), self.fail, False, @@ -723,7 +746,9 @@ def analyze_type_with_type_info( self.options, use_standard_error=True, ) - return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) + return tup.copy_modified( + items=self.anal_array(tup.items, allow_unpack=True), fallback=instance + ) td = info.typeddict_type if td is not None: # The class has a TypedDict[...] base class so it will be @@ -940,7 +965,21 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: self.anal_star_arg_type(t.arg_types[-1], ARG_STAR2, nested=nested), ] else: - arg_types = self.anal_array(t.arg_types, nested=nested) + arg_types = self.anal_array(t.arg_types, nested=nested, allow_unpack=True) + star_index = None + if ARG_STAR in arg_kinds: + star_index = arg_kinds.index(ARG_STAR) + star2_index = None + if ARG_STAR2 in arg_kinds: + star2_index = arg_kinds.index(ARG_STAR2) + validated_args: list[Type] = [] + for i, at in enumerate(arg_types): + if isinstance(at, UnpackType) and i not in (star_index, star2_index): + self.fail("Unpack is only valid in a variadic position", at) + validated_args.append(AnyType(TypeOfAny.from_error)) + else: + validated_args.append(at) + arg_types = validated_args # If there were multiple (invalid) unpacks, the arg types list will become shorter, # we need to trim the kinds/names as well to avoid crashes. arg_kinds = t.arg_kinds[: len(arg_types)] @@ -1012,7 +1051,10 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: line=t.line, column=t.column, ) - return self.anal_type(t, nested=nested) + self.allow_unpack = True + result = self.anal_type(t, nested=nested) + self.allow_unpack = False + return result def visit_overloaded(self, t: Overloaded) -> Type: # Overloaded types are manually constructed in semanal.py by analyzing the @@ -1051,7 +1093,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: if t.partial_fallback.type else self.named_type("builtins.tuple", [any_type]) ) - return TupleType(self.anal_array(t.items), fallback, t.line) + return TupleType(self.anal_array(t.items, allow_unpack=True), fallback, t.line) def visit_typeddict_type(self, t: TypedDictType) -> Type: items = { @@ -1534,13 +1576,17 @@ def anal_array( *, allow_param_spec: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, ) -> list[Type]: old_allow_param_spec_literals = self.allow_param_spec_literals self.allow_param_spec_literals = allow_param_spec_literals + old_allow_unpack = self.allow_unpack + self.allow_unpack = allow_unpack res: list[Type] = [] for t in a: res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) self.allow_param_spec_literals = old_allow_param_spec_literals + self.allow_unpack = old_allow_unpack return self.check_unpacks_in_list(res) def anal_type( diff --git a/mypy/typeops.py b/mypy/typeops.py index e01aad950573..b6af5572f8b4 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -105,17 +105,15 @@ def tuple_fallback(typ: TupleType) -> Instance: unpacked_type = get_proper_type(item.type) if isinstance(unpacked_type, TypeVarTupleType): items.append(unpacked_type.upper_bound) - elif isinstance(unpacked_type, TupleType): - # TODO: might make sense to do recursion here to support nested unpacks - # of tuple constants - items.extend(unpacked_type.items) elif ( isinstance(unpacked_type, Instance) and unpacked_type.type.fullname == "builtins.tuple" ): items.append(unpacked_type.args[0]) + elif isinstance(unpacked_type, (AnyType, UninhabitedType)): + continue else: - raise NotImplementedError + raise NotImplementedError(unpacked_type) else: items.append(item) return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) diff --git a/mypy/types.py b/mypy/types.py index d4e2fc7cb63c..73a67b835082 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2261,10 +2261,6 @@ def __init__( ) -> None: super().__init__(line, column) self.partial_fallback = fallback - # TODO: flatten/normalize unpack items (very similar to unions) here. - # Probably also for instances, type aliases, callables, and Unpack itself. For example, - # tuple[*tuple[X, ...], ...] -> tuple[X, ...] and Tuple[*tuple[X, ...]] -> tuple[X, ...]. - # Currently normalization happens in expand_type() et al., which is sub-optimal. self.items = items self.implicit = implicit diff --git a/mypy/types_utils.py b/mypy/types_utils.py index 7f2e38ef3753..a6b5baa9bdfc 100644 --- a/mypy/types_utils.py +++ b/mypy/types_utils.py @@ -64,9 +64,15 @@ def is_invalid_recursive_alias(seen_nodes: set[TypeAlias], target: Type) -> bool assert target.alias, f"Unfixed type alias {target.type_ref}" return is_invalid_recursive_alias(seen_nodes | {target.alias}, get_proper_type(target)) assert isinstance(target, ProperType) - if not isinstance(target, UnionType): + if not isinstance(target, (UnionType, TupleType)): return False - return any(is_invalid_recursive_alias(seen_nodes, item) for item in target.items) + if isinstance(target, UnionType): + return any(is_invalid_recursive_alias(seen_nodes, item) for item in target.items) + for item in target.items: + if isinstance(item, UnpackType): + if is_invalid_recursive_alias(seen_nodes, item.type): + return True + return False def is_bad_type_type_item(item: Type) -> bool: diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 95a7bdd2b2cd..93674c0c2d5c 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -3360,7 +3360,6 @@ class Foo(Generic[Unpack[Ts]]): ... class Bar(Generic[Unpack[Ts], T]): ... def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], List[T]]: ... -# TODO: do not crash on Foo[Us] (with missing Unpack), instead give an error. def f(*args: Unpack[Us]) -> Foo[Unpack[Us]]: ... reveal_type(dec(f)) # N: Revealed type is "def [Ts] (*Unpack[Ts`1]) -> builtins.list[__main__.Foo[Unpack[Ts`1]]]" g: Callable[[Unpack[Us]], Foo[Unpack[Us]]] diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index b28b2ead45e7..8ea6cbdb602c 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -123,9 +123,7 @@ reveal_type(empty) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tup bad: Variadic[Unpack[Tuple[int, ...]], str, Unpack[Tuple[bool, ...]]] # E: More than one Unpack in a type is not allowed reveal_type(bad) # N: Revealed type is "__main__.Variadic[Unpack[builtins.tuple[builtins.int, ...]], builtins.str]" -# TODO: This is tricky to fix because we need typeanal to know whether the current -# location is valid for an Unpack or not. -# bad2: Unpack[Tuple[int, ...]] +bad2: Unpack[Tuple[int, ...]] # E: Unpack is only valid in a variadic position m1: Mixed1[int, str, bool] reveal_type(m1) # N: Revealed type is "__main__.Mixed1[builtins.int, builtins.str, builtins.bool]" @@ -443,8 +441,7 @@ def foo(*args: Unpack[Tuple[int, ...]]) -> None: reveal_type(args) # N: Revealed type is "builtins.tuple[builtins.int, ...]" foo(0, 1, 2) -# TODO: this should say 'expected "int"' rather than the unpack -foo(0, 1, "bar") # E: Argument 3 to "foo" has incompatible type "str"; expected "Unpack[Tuple[int, ...]]" +foo(0, 1, "bar") # E: Argument 3 to "foo" has incompatible type "str"; expected "int" def foo2(*args: Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]) -> None: @@ -805,3 +802,34 @@ reveal_type(x) # N: Revealed type is "Tuple[builtins.int, Unpack[builtins.tuple y: A[Unpack[Tuple[bool, ...]]] reveal_type(y) # N: Revealed type is "Tuple[builtins.bool, Unpack[builtins.tuple[builtins.bool, ...]], builtins.bool, builtins.bool]" [builtins fixtures/tuple.pyi] + +[case testBanPathologicalRecursiveTuples] +from typing import Tuple +from typing_extensions import Unpack +A = Tuple[int, Unpack[A]] # E: Invalid recursive alias: a tuple item of itself +B = Tuple[int, Unpack[C]] # E: Invalid recursive alias: a tuple item of itself \ + # E: Name "C" is used before definition +C = Tuple[int, Unpack[B]] +x: A +y: B +z: C +reveal_type(x) # N: Revealed type is "Any" +reveal_type(y) # N: Revealed type is "Any" +reveal_type(z) # N: Revealed type is "Tuple[builtins.int, Unpack[Any]]" +[builtins fixtures/tuple.pyi] + +[case testInferenceAgainstGenericVariadicWithBadType] +# flags: --new-type-inference +from typing import TypeVar, Callable, Generic +from typing_extensions import Unpack, TypeVarTuple + +T = TypeVar("T") +Ts = TypeVarTuple("Ts") +Us = TypeVarTuple("Us") + +class Foo(Generic[Unpack[Ts]]): ... + +def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], T]: ... +def f(*args: Unpack[Us]) -> Foo[Us]: ... # E: TypeVarTuple "Us" is only valid with an unpack +dec(f) # No crash +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-varargs.test b/test-data/unit/check-varargs.test index 6e118597551f..fe09fb43c97c 100644 --- a/test-data/unit/check-varargs.test +++ b/test-data/unit/check-varargs.test @@ -775,7 +775,7 @@ class Person(TypedDict): name: str age: int -def foo(x: Unpack[Person]) -> None: # E: "Person" cannot be unpacked (must be tuple or TypeVarTuple) +def foo(x: Unpack[Person]) -> None: # E: Unpack is only valid in a variadic position ... def bar(x: int, *args: Unpack[Person]) -> None: # E: "Person" cannot be unpacked (must be tuple or TypeVarTuple) ... diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 09d4da54bff3..f21ba5253437 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1457,7 +1457,7 @@ homogenous_tuple: Tuple[Unpack[Tuple[int, ...]]] bad: Tuple[Unpack[int]] # E: "int" cannot be unpacked (must be tuple or TypeVarTuple) [builtins fixtures/tuple.pyi] -[case testTypeVarTuple] +[case testTypeVarTupleErrors] from typing import Generic from typing_extensions import TypeVarTuple, Unpack @@ -1471,15 +1471,14 @@ TP5 = TypeVarTuple(t='TP5') # E: TypeVarTuple() expects a string literal as fir TP6 = TypeVarTuple('TP6', bound=int) # E: Unexpected keyword argument "bound" for "TypeVarTuple" x: TVariadic # E: TypeVarTuple "TVariadic" is unbound -y: Unpack[TVariadic] # E: TypeVarTuple "TVariadic" is unbound +y: Unpack[TVariadic] # E: Unpack is only valid in a variadic position class Variadic(Generic[Unpack[TVariadic], Unpack[TVariadic2]]): # E: Can only use one type var tuple in a class def pass -# TODO: this should generate an error -#def bad_args(*args: TVariadic): -# pass +def bad_args(*args: TVariadic): # E: TypeVarTuple "TVariadic" is only valid with an unpack + pass def bad_kwargs(**kwargs: Unpack[TVariadic]): # E: Unpack item in ** argument must be a TypedDict pass From 5524bc182413b7484fcce250a4a76481afcdc926 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 20 Aug 2023 15:17:59 +0100 Subject: [PATCH 2/7] Some more progress + type analyzer fixes --- mypy/checker.py | 5 +-- mypy/checkexpr.py | 8 ++++ mypy/constraints.py | 41 +++++++++++++++---- mypy/semanal.py | 5 +++ mypy/subtypes.py | 28 ++++++++++++- mypy/typeanal.py | 18 +++++---- mypy/typeops.py | 1 + test-data/unit/check-typevar-tuple.test | 54 ++++++++++++------------- 8 files changed, 110 insertions(+), 50 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 87dff91758f5..a44601b83e21 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4665,10 +4665,7 @@ def analyze_iterable_item_type(self, expr: Expression) -> tuple[Type, Type]: isinstance(iterable, TupleType) and iterable.partial_fallback.type.fullname == "builtins.tuple" ): - joined: Type = UninhabitedType() - for item in iterable.items: - joined = join_types(joined, item) - return iterator, joined + return iterator, tuple_fallback(iterable).args[0] else: # Non-tuple iterable. return iterator, echk.check_method_call_by_name("__next__", iterator, [], [], expr)[0] diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 00d03d24fd35..9ef5527788b9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1639,6 +1639,11 @@ def check_callable_call( callee.type_object().name, abstract_attributes, context ) + callee_star = callee.var_arg() + if callee_star is not None and isinstance(callee_star.typ, UnpackType): + # TODO: factor out normalization code to avoid weird call. + callee = expand_type(callee, {}) + formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, @@ -2408,6 +2413,9 @@ def check_argument_types( + unpacked_type.items[inner_unpack_index + 1 :] ) callee_arg_kinds = [ARG_POS] * len(actuals) + elif isinstance(unpacked_type, TypeVarTupleType): + callee_arg_types = [orig_callee_arg_type] + callee_arg_kinds = [ARG_STAR] else: assert isinstance(unpacked_type, Instance) assert unpacked_type.type.fullname == "builtins.tuple" diff --git a/mypy/constraints.py b/mypy/constraints.py index 26504ed06b3e..4619dbe53bbb 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -155,16 +155,31 @@ def infer_constraints_for_callable( # not to hold we can always handle the prefixes too. inner_unpack = unpacked_type.items[0] assert isinstance(inner_unpack, UnpackType) - inner_unpacked_type = inner_unpack.type - assert isinstance(inner_unpacked_type, TypeVarTupleType) + inner_unpacked_type = get_proper_type(inner_unpack.type) suffix_len = len(unpacked_type.items) - 1 - constraints.append( - Constraint( - inner_unpacked_type, - SUPERTYPE_OF, - TupleType(actual_types[:-suffix_len], inner_unpacked_type.tuple_fallback), + if isinstance(inner_unpacked_type, TypeVarTupleType): + constraints.append( + Constraint( + inner_unpacked_type, + SUPERTYPE_OF, + TupleType( + actual_types[:-suffix_len], inner_unpacked_type.tuple_fallback + ), + ) ) - ) + else: + assert ( + isinstance(inner_unpacked_type, Instance) + and inner_unpacked_type.type.fullname == "builtins.tuple" + ) + for at in actual_types[:-suffix_len]: + constraints.extend( + infer_constraints(inner_unpacked_type.args[0], at, SUPERTYPE_OF) + ) + # Now handle the suffix (if any). + if suffix_len: + for tt, at in zip(unpacked_type.items[1:], actual_types[-suffix_len:]): + constraints.extend(infer_constraints(tt, at, SUPERTYPE_OF)) else: assert False, "mypy bug: unhandled constraint inference case" else: @@ -865,6 +880,16 @@ def visit_instance(self, template: Instance) -> list[Constraint]: and self.direction == SUPERTYPE_OF ): for item in actual.items: + if isinstance(item, UnpackType): + unpacked = get_proper_type(item.type) + if isinstance(unpacked, TypeVarType): + # Cannot infer anything for T from [T, ...] <: *Ts + continue + assert ( + isinstance(unpacked, Instance) + and unpacked.type.fullname == "builtins.tuple" + ) + item = unpacked.args[0] cb = infer_constraints(template.args[0], item, SUPERTYPE_OF) res.extend(cb) return res diff --git a/mypy/semanal.py b/mypy/semanal.py index 381759ab01d9..55d4e6a3f506 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4216,6 +4216,7 @@ def get_typevarlike_argument( *, allow_unbound_tvars: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, report_invalid_typevar_arg: bool = True, ) -> ProperType | None: try: @@ -4227,6 +4228,7 @@ def get_typevarlike_argument( report_invalid_types=False, allow_unbound_tvars=allow_unbound_tvars, allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, ) if analyzed is None: # Type variables are special: we need to place them in the symbol table @@ -4378,6 +4380,7 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: s, allow_unbound_tvars=True, report_invalid_typevar_arg=False, + allow_unpack=True, ) default = tv_arg or AnyType(TypeOfAny.from_error) if not isinstance(default, UnpackType): @@ -6493,6 +6496,7 @@ def expr_to_analyzed_type( allow_type_any: bool = False, allow_unbound_tvars: bool = False, allow_param_spec_literals: bool = False, + allow_unpack: bool = False, ) -> Type | None: if isinstance(expr, CallExpr): # This is a legacy syntax intended mostly for Python 2, we keep it for @@ -6523,6 +6527,7 @@ def expr_to_analyzed_type( allow_type_any=allow_type_any, allow_unbound_tvars=allow_unbound_tvars, allow_param_spec_literals=allow_param_spec_literals, + allow_unpack=allow_unpack, ) def analyze_type_expr(self, expr: Expression) -> None: diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 11847858c62c..299779402432 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,7 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_self_type, expand_type_by_instance +from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -659,6 +659,8 @@ def visit_type_var_tuple(self, left: TypeVarTupleType) -> bool: return self._is_subtype(left.upper_bound, self.right) def visit_unpack_type(self, left: UnpackType) -> bool: + # TODO: Ideally we should not need this (since it is not a real type). + # Instead callers (prevous level types) handle it when it appears in type list. if isinstance(self.right, UnpackType): return self._is_subtype(left.type, self.right.type) if isinstance(self.right, Instance) and self.right.type.fullname == "builtins.object": @@ -752,7 +754,15 @@ def visit_tuple_type(self, left: TupleType) -> bool: # TODO: We shouldn't need this special case. This is currently needed # for isinstance(x, tuple), though it's unclear why. return True - return all(self._is_subtype(li, iter_type) for li in left.items) + for li in left.items: + if isinstance(li, UnpackType): + unpack = get_proper_type(li.type) + if isinstance(unpack, Instance): + assert unpack.type.fullname == "builtins.tuple" + li = unpack.args[0] + if not self._is_subtype(li, iter_type): + return False + return True elif self._is_subtype(left.partial_fallback, right) and self._is_subtype( mypy.typeops.tuple_fallback(left), right ): @@ -760,6 +770,7 @@ def visit_tuple_type(self, left: TupleType) -> bool: return False elif isinstance(right, TupleType): if len(left.items) != len(right.items): + # TODO: handle tuple with variadic items better. return False if any(not self._is_subtype(l, r) for l, r in zip(left.items, right.items)): return False @@ -1493,6 +1504,18 @@ def are_parameters_compatible( right_star = right.var_arg() right_star2 = right.kw_arg() + if right_star and isinstance(right_star.typ, UnpackType): + # TODO: factor out normalization code to avoid the import. + expanded = expand_type(right, {}) + assert isinstance(expanded, (Parameters, CallableType)) + right = cast(NormalizedCallableType, expanded) + right_star = right.var_arg() + if left_star and isinstance(left_star.typ, UnpackType): + expanded = expand_type(left, {}) + assert isinstance(expanded, (Parameters, CallableType)) + left = cast(NormalizedCallableType, expanded) + left_star = left.var_arg() + # Treat "def _(*a: Any, **kw: Any) -> X" similarly to "Callable[..., X]" if are_trivial_parameters(right): return True @@ -1549,6 +1572,7 @@ def _incompatible(left_arg: FormalArgument | None, right_arg: FormalArgument | N # Phase 1c: Check var args. Right has an infinite series of optional positional # arguments. Get all further positional args of left, and make sure # they're more general then the corresponding member in right. + # TODO: are we handling UnpackType correctly here? if right_star is not None: # Synthesize an anonymous formal argument for the right right_by_position = right.try_synthesizing_arg_from_vararg(None) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 5e744a013be5..dd0b15cc26da 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1051,10 +1051,7 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type: line=t.line, column=t.column, ) - self.allow_unpack = True - result = self.anal_type(t, nested=nested) - self.allow_unpack = False - return result + return self.anal_type(t, nested=nested, allow_unpack=True) def visit_overloaded(self, t: Overloaded) -> Type: # Overloaded types are manually constructed in semanal.py by analyzing the @@ -1580,13 +1577,14 @@ def anal_array( ) -> list[Type]: old_allow_param_spec_literals = self.allow_param_spec_literals self.allow_param_spec_literals = allow_param_spec_literals - old_allow_unpack = self.allow_unpack - self.allow_unpack = allow_unpack res: list[Type] = [] for t in a: - res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) + res.append( + self.anal_type( + t, nested, allow_param_spec=allow_param_spec, allow_unpack=allow_unpack + ) + ) self.allow_param_spec_literals = old_allow_param_spec_literals - self.allow_unpack = old_allow_unpack return self.check_unpacks_in_list(res) def anal_type( @@ -1595,6 +1593,7 @@ def anal_type( nested: bool = True, *, allow_param_spec: bool = False, + allow_unpack: bool = False, allow_ellipsis: bool = False, ) -> Type: if nested: @@ -1603,6 +1602,8 @@ def anal_type( self.allow_required = False old_allow_ellipsis = self.allow_ellipsis self.allow_ellipsis = allow_ellipsis + old_allow_unpack = self.allow_unpack + self.allow_unpack = allow_unpack try: analyzed = t.accept(self) finally: @@ -1610,6 +1611,7 @@ def anal_type( self.nesting_level -= 1 self.allow_required = old_allow_required self.allow_ellipsis = old_allow_ellipsis + self.allow_unpack = old_allow_unpack if ( not allow_param_spec and isinstance(analyzed, ParamSpecType) diff --git a/mypy/typeops.py b/mypy/typeops.py index b6af5572f8b4..0e0bc348942e 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -116,6 +116,7 @@ def tuple_fallback(typ: TupleType) -> Instance: raise NotImplementedError(unpacked_type) else: items.append(item) + # TODO: we should really use a union here, tuple types are special. return Instance(info, [join_type_list(items)], extra_attrs=typ.partial_fallback.extra_attrs) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 8ea6cbdb602c..3ad34935b38b 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -60,9 +60,10 @@ reveal_type(f(f_args2)) # N: Revealed type is "Tuple[builtins.str]" reveal_type(f(f_args3)) # N: Revealed type is "Tuple[builtins.str, builtins.str, builtins.bool]" f(empty) # E: Argument 1 to "f" has incompatible type "Tuple[()]"; expected "Tuple[int]" f(bad_args) # E: Argument 1 to "f" has incompatible type "Tuple[str, str]"; expected "Tuple[int, str]" -# TODO: This hits a crash where we assert len(templates.items) == 1. See visit_tuple_type -# in mypy/constraints.py. -#f(var_len_tuple) + +# The reason for error in subtle: actual can be empty, formal cannot. +reveal_type(f(var_len_tuple)) # N: Revealed type is "Tuple[builtins.str, Unpack[builtins.tuple[builtins.int, ...]]]" \ + # E: Argument 1 to "f" has incompatible type "Tuple[int, ...]"; expected "Tuple[int, Unpack[Tuple[int, ...]]]" g_args: Tuple[str, int] reveal_type(g(g_args)) # N: Revealed type is "Tuple[builtins.str, builtins.str]" @@ -127,7 +128,6 @@ bad2: Unpack[Tuple[int, ...]] # E: Unpack is only valid in a variadic position m1: Mixed1[int, str, bool] reveal_type(m1) # N: Revealed type is "__main__.Mixed1[builtins.int, builtins.str, builtins.bool]" - [builtins fixtures/tuple.pyi] [case testTypeVarTupleGenericClassWithFunctions] @@ -146,7 +146,6 @@ def foo(t: Variadic[int, Unpack[Ts], object]) -> Tuple[int, Unpack[Ts]]: v: Variadic[int, str, bool, object] reveal_type(foo(v)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" - [builtins fixtures/tuple.pyi] [case testTypeVarTupleGenericClassWithMethods] @@ -166,7 +165,6 @@ class Variadic(Generic[T, Unpack[Ts], S]): v: Variadic[float, str, bool, object] reveal_type(v.foo(0)) # N: Revealed type is "Tuple[builtins.int, builtins.str, builtins.bool]" - [builtins fixtures/tuple.pyi] [case testTypeVarTupleIsNotValidAliasTarget] @@ -209,8 +207,8 @@ shape = (Height(480), Width(640)) x: Array[Height, Width] = Array(shape) reveal_type(abs(x)) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" reveal_type(x + x) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" - [builtins fixtures/tuple.pyi] + [case testTypeVarTuplePep646ArrayExampleWithDType] from typing import Generic, Tuple, TypeVar, Protocol, NewType from typing_extensions import TypeVarTuple, Unpack @@ -245,7 +243,6 @@ shape = (Height(480), Width(640)) x: Array[float, Height, Width] = Array(shape) reveal_type(abs(x)) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" reveal_type(x + x) # N: Revealed type is "__main__.Array[builtins.float, __main__.Height, __main__.Width]" - [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646ArrayExampleInfer] @@ -291,8 +288,8 @@ c = del_batch_axis(b) reveal_type(c) # N: Revealed type is "__main__.Array[__main__.Height, __main__.Width]" d = add_batch_channels(a) reveal_type(d) # N: Revealed type is "__main__.Array[__main__.Batch, __main__.Height, __main__.Width, __main__.Channels]" - [builtins fixtures/tuple.pyi] + [case testTypeVarTuplePep646TypeVarConcatenation] from typing import Generic, TypeVar, NewType, Tuple from typing_extensions import TypeVarTuple, Unpack @@ -309,6 +306,7 @@ def prefix_tuple( z = prefix_tuple(x=0, y=(True, 'a')) reveal_type(z) # N: Revealed type is "Tuple[builtins.int, builtins.bool, builtins.str]" [builtins fixtures/tuple.pyi] + [case testTypeVarTuplePep646TypeVarTupleUnpacking] from typing import Generic, TypeVar, NewType, Any, Tuple from typing_extensions import TypeVarTuple, Unpack @@ -361,8 +359,6 @@ reveal_type(bad) # N: Revealed type is "def [Ts, Ts2] (x: Tuple[builtins.int, U def bad2(x: Tuple[int, Unpack[Tuple[int, ...]], str, Unpack[Tuple[str, ...]]]) -> None: # E: More than one Unpack in a type is not allowed ... reveal_type(bad2) # N: Revealed type is "def (x: Tuple[builtins.int, Unpack[builtins.tuple[builtins.int, ...]], builtins.str])" - - [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646TypeVarStarArgsBasic] @@ -378,8 +374,8 @@ def args_to_tuple(*args: Unpack[Ts]) -> Tuple[Unpack[Ts]]: return args reveal_type(args_to_tuple(1, 'a')) # N: Revealed type is "Tuple[Literal[1]?, Literal['a']?]" - [builtins fixtures/tuple.pyi] + [case testTypeVarTuplePep646TypeVarStarArgs] from typing import Tuple from typing_extensions import TypeVarTuple, Unpack @@ -408,8 +404,6 @@ with_prefix_suffix(*bad_t) # E: Too few arguments for "with_prefix_suffix" def foo(*args: Unpack[Ts]) -> None: reveal_type(with_prefix_suffix(True, "bar", *args, 5)) # N: Revealed type is "Tuple[builtins.bool, builtins.str, Unpack[Ts`-1], builtins.int]" - - [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646TypeVarStarArgsFixedLengthTuple] @@ -420,17 +414,23 @@ def foo(*args: Unpack[Tuple[int, str]]) -> None: reveal_type(args) # N: Revealed type is "Tuple[builtins.int, builtins.str]" foo(0, "foo") -foo(0, 1) # E: Argument 2 to "foo" has incompatible type "int"; expected "Unpack[Tuple[int, str]]" -foo("foo", "bar") # E: Argument 1 to "foo" has incompatible type "str"; expected "Unpack[Tuple[int, str]]" -foo(0, "foo", 1) # E: Invalid number of arguments -foo(0) # E: Invalid number of arguments -foo() # E: Invalid number of arguments +foo(0, 1) # E: Argument 2 to "foo" has incompatible type "int"; expected "str" +foo("foo", "bar") # E: Argument 1 to "foo" has incompatible type "str"; expected "int" +foo(0, "foo", 1) # E: Too many arguments for "foo" +foo(0) # E: Too few arguments for "foo" +foo() # E: Too few arguments for "foo" foo(*(0, "foo")) -# TODO: fix this case to do something sensible. -#def foo2(*args: Unpack[Tuple[bool, Unpack[Tuple[int, str]], bool]]) -> None: -# reveal_type(args) +def foo2(*args: Unpack[Tuple[bool, Unpack[Tuple[int, str]], bool]]) -> None: + reveal_type(args) # N: Revealed type is "Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]" +# It is hard to normalize callable types in definition, because there is deep relation between `FuncDef.type` +# and `FuncDef.arguments`, therefore various typeops need to be sure to normalize Callable types before using them. +reveal_type(foo2) # N: Revealed type is "def (*args: Unpack[Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]])" + +class C: + def foo2(self, *args: Unpack[Tuple[bool, Unpack[Tuple[int, str]], bool]]) -> None: ... +reveal_type(C().foo2) # N: Revealed type is "def (*args: Unpack[Tuple[builtins.bool, builtins.int, builtins.str, builtins.bool]])" [builtins fixtures/tuple.pyi] [case testTypeVarTuplePep646TypeVarStarArgsVariableLengthTuple] @@ -450,9 +450,9 @@ def foo2(*args: Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]) -> None # reveal_type(args[1]) foo2("bar", 1, 2, 3, False, True) -foo2(0, 1, 2, 3, False, True) # E: Argument 1 to "foo2" has incompatible type "int"; expected "Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]" -foo2("bar", "bar", 2, 3, False, True) # E: Argument 2 to "foo2" has incompatible type "str"; expected "Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]" -foo2("bar", 1, 2, 3, 4, True) # E: Argument 5 to "foo2" has incompatible type "int"; expected "Unpack[Tuple[str, Unpack[Tuple[int, ...]], bool, bool]]" +foo2(0, 1, 2, 3, False, True) # E: Argument 1 to "foo2" has incompatible type "int"; expected "str" +foo2("bar", "bar", 2, 3, False, True) # E: Argument 2 to "foo2" has incompatible type "str"; expected "Unpack[Tuple[Unpack[Tuple[int, ...]], bool, bool]]" +foo2("bar", 1, 2, 3, 4, True) # E: Argument 5 to "foo2" has incompatible type "int"; expected "Unpack[Tuple[Unpack[Tuple[int, ...]], bool, bool]]" foo2(*("bar", 1, 2, 3, False, True)) [builtins fixtures/tuple.pyi] @@ -547,8 +547,7 @@ def call( *args: Unpack[Ts], ) -> None: ... - # TODO: exposes unhandled case in checkexpr - # target(*args) + target(*args) class A: def func(self, arg1: int, arg2: str) -> None: ... @@ -566,7 +565,6 @@ call(A().func, 0, 1) # E: Argument 1 to "call" has incompatible type "Callable[ call(A().func2, 0, 0) call(A().func3, 0, 1, 2) call(A().func3) - [builtins fixtures/tuple.pyi] [case testVariadicAliasBasicTuple] From 321bde95956b097e467556902de096c42906d440 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 20 Aug 2023 23:14:46 +0100 Subject: [PATCH 3/7] Refactor Callable normalization --- mypy/checkexpr.py | 9 +--- mypy/constraints.py | 3 +- mypy/expandtype.py | 102 +++++---------------------------------- mypy/semanal_typeargs.py | 3 +- mypy/subtypes.py | 18 ++----- mypy/types.py | 76 +++++++++++++++++++++++++++++ mypy/typevartuples.py | 15 +----- 7 files changed, 98 insertions(+), 128 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 9ef5527788b9..789a5e2fb0a3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -168,6 +168,7 @@ UninhabitedType, UnionType, UnpackType, + find_unpack_in_list, flatten_nested_unions, get_proper_type, get_proper_types, @@ -184,7 +185,6 @@ ) from mypy.typestate import type_state from mypy.typevars import fill_typevars -from mypy.typevartuples import find_unpack_in_list from mypy.util import split_module_names from mypy.visitor import ExpressionVisitor @@ -1599,7 +1599,7 @@ def check_callable_call( See the docstring of check_call for more information. """ # Always unpack **kwargs before checking a call. - callee = callee.with_unpacked_kwargs() + callee = callee.with_unpacked_kwargs().with_normalized_var_args() if callable_name is None and callee.name: callable_name = callee.name ret_type = get_proper_type(callee.ret_type) @@ -1639,11 +1639,6 @@ def check_callable_call( callee.type_object().name, abstract_attributes, context ) - callee_star = callee.var_arg() - if callee_star is not None and isinstance(callee_star.typ, UnpackType): - # TODO: factor out normalization code to avoid weird call. - callee = expand_type(callee, {}) - formal_to_actual = map_actuals_to_formals( arg_kinds, arg_names, diff --git a/mypy/constraints.py b/mypy/constraints.py index 4619dbe53bbb..f0bfe3e87aa4 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -49,6 +49,7 @@ UninhabitedType, UnionType, UnpackType, + find_unpack_in_list, get_proper_type, has_recursive_types, has_type_vars, @@ -57,7 +58,7 @@ ) from mypy.types_utils import is_union_with_any from mypy.typestate import type_state -from mypy.typevartuples import extract_unpack, find_unpack_in_list, split_with_mapped_and_template +from mypy.typevartuples import extract_unpack, split_with_mapped_and_template if TYPE_CHECKING: from mypy.infer import ArgumentInferContext diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 7bd1e5363271..915044848e57 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -2,7 +2,7 @@ from typing import Final, Iterable, Mapping, Sequence, TypeVar, cast, overload -from mypy.nodes import ARG_POS, ARG_STAR, ArgKind, Var +from mypy.nodes import ARG_STAR, Var from mypy.state import state from mypy.types import ( ANY_STRATEGY, @@ -39,7 +39,7 @@ get_proper_type, split_with_prefix_and_suffix, ) -from mypy.typevartuples import find_unpack_in_list, split_with_instance +from mypy.typevartuples import split_with_instance # Solving the import cycle: import mypy.type_visitor # ruff: isort: skip @@ -293,11 +293,10 @@ def expand_unpack(self, t: UnpackType) -> list[Type] | AnyType | UninhabitedType def visit_parameters(self, t: Parameters) -> Type: return t.copy_modified(arg_types=self.expand_types(t.arg_types)) - # TODO: can we simplify this method? It is too long. - def interpolate_args_for_unpack( - self, t: CallableType, var_arg: UnpackType - ) -> tuple[list[str | None], list[ArgKind], list[Type]]: + def interpolate_args_for_unpack(self, t: CallableType, var_arg: UnpackType) -> list[Type]: star_index = t.arg_kinds.index(ARG_STAR) + prefix = self.expand_types(t.arg_types[:star_index]) + suffix = self.expand_types(t.arg_types[star_index + 1 :]) var_arg_type = get_proper_type(var_arg.type) # We have something like Unpack[Tuple[Unpack[Ts], X1, X2]] @@ -305,89 +304,19 @@ def interpolate_args_for_unpack( expanded_tuple = var_arg_type.accept(self) assert isinstance(expanded_tuple, ProperType) and isinstance(expanded_tuple, TupleType) expanded_items = expanded_tuple.items + fallback = var_arg_type.partial_fallback else: # We have plain Unpack[Ts] + assert isinstance(var_arg_type, TypeVarTupleType) + fallback = var_arg_type.tuple_fallback expanded_items_res = self.expand_unpack(var_arg) if isinstance(expanded_items_res, list): expanded_items = expanded_items_res else: # We got Any or - arg_types = ( - t.arg_types[:star_index] + [expanded_items_res] + t.arg_types[star_index + 1 :] - ) - return t.arg_names, t.arg_kinds, arg_types - - expanded_unpack_index = find_unpack_in_list(expanded_items) - # This is the case where we just have Unpack[Tuple[X1, X2, X3]] - # (for example if either the tuple had no unpacks, or the unpack in the - # tuple got fully expanded to something with fixed length) - if expanded_unpack_index is None: - arg_names = ( - t.arg_names[:star_index] - + [None] * len(expanded_items) - + t.arg_names[star_index + 1 :] - ) - arg_kinds = ( - t.arg_kinds[:star_index] - + [ARG_POS] * len(expanded_items) - + t.arg_kinds[star_index + 1 :] - ) - arg_types = ( - self.expand_types(t.arg_types[:star_index]) - + expanded_items - + self.expand_types(t.arg_types[star_index + 1 :]) - ) - else: - # If Unpack[Ts] simplest form still has an unpack or is a - # homogenous tuple, then only the prefix can be represented as - # positional arguments, and we pass Tuple[Unpack[Ts-1], Y1, Y2] - # as the star arg, for example. - expanded_unpack = expanded_items[expanded_unpack_index] - assert isinstance(expanded_unpack, UnpackType) - - # Extract the TypeVarTuple, so we can get a tuple fallback from it. - expanded_unpacked_tvt = expanded_unpack.type - if isinstance(expanded_unpacked_tvt, TypeVarTupleType): - fallback = expanded_unpacked_tvt.tuple_fallback - else: - # This can happen when tuple[Any, ...] is used to "patch" a variadic - # generic type without type arguments provided, or when substitution is - # homogeneous tuple. - assert isinstance(expanded_unpacked_tvt, ProperType) - assert isinstance(expanded_unpacked_tvt, Instance) - assert expanded_unpacked_tvt.type.fullname == "builtins.tuple" - fallback = expanded_unpacked_tvt - - prefix_len = expanded_unpack_index - arg_names = t.arg_names[:star_index] + [None] * prefix_len + t.arg_names[star_index:] - arg_kinds = ( - t.arg_kinds[:star_index] + [ARG_POS] * prefix_len + t.arg_kinds[star_index:] - ) - if ( - len(expanded_items) == 1 - and isinstance(expanded_unpack.type, ProperType) - and isinstance(expanded_unpack.type, Instance) - ): - assert expanded_unpack.type.type.fullname == "builtins.tuple" - # Normalize *args: *tuple[X, ...] -> *args: X - arg_types = ( - self.expand_types(t.arg_types[:star_index]) - + [expanded_unpack.type.args[0]] - + self.expand_types(t.arg_types[star_index + 1 :]) - ) - else: - arg_types = ( - self.expand_types(t.arg_types[:star_index]) - + expanded_items[:prefix_len] - # Constructing the Unpack containing the tuple without the prefix. - + [ - UnpackType(TupleType(expanded_items[prefix_len:], fallback)) - if len(expanded_items) - prefix_len > 1 - else expanded_items[prefix_len] - ] - + self.expand_types(t.arg_types[star_index + 1 :]) - ) - return arg_names, arg_kinds, arg_types + return prefix + [expanded_items_res] + suffix + new_unpack = UnpackType(TupleType(expanded_items, fallback)) + return prefix + [new_unpack] + suffix def visit_callable_type(self, t: CallableType) -> CallableType: param_spec = t.param_spec() @@ -427,19 +356,14 @@ def visit_callable_type(self, t: CallableType) -> CallableType: var_arg = t.var_arg() if var_arg is not None and isinstance(var_arg.typ, UnpackType): - arg_names, arg_kinds, arg_types = self.interpolate_args_for_unpack(t, var_arg.typ) + arg_types = self.interpolate_args_for_unpack(t, var_arg.typ) else: - arg_names = t.arg_names - arg_kinds = t.arg_kinds arg_types = self.expand_types(t.arg_types) - return t.copy_modified( arg_types=arg_types, - arg_names=arg_names, - arg_kinds=arg_kinds, ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), - ) + ).with_normalized_var_args() def visit_overloaded(self, t: Overloaded) -> Type: items: list[CallableType] = [] diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index 27381b0d9681..f82e60923a50 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -142,9 +142,8 @@ def visit_instance(self, t: Instance) -> None: info = t.type if isinstance(info, FakeInfo): return # https://github.com/python/mypy/issues/11079 - # TODO: we can also normalize tuple[*tuple[X, ...], ...] -> tuple[X, ...] - # bit this looks quite rare corner case (and we should be able to handle it). t.args = tuple(flatten_nested_tuples(t.args)) + # TODO: fix #15410 and #15411. self.validate_args(info.name, t.args, info.defn.type_vars, t) super().visit_instance(t) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 299779402432..938a810b6a8a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -8,7 +8,7 @@ import mypy.constraints import mypy.typeops from mypy.erasetype import erase_type -from mypy.expandtype import expand_self_type, expand_type, expand_type_by_instance +from mypy.expandtype import expand_self_type, expand_type_by_instance from mypy.maptype import map_instance_to_supertype # Circular import; done in the function instead. @@ -1404,8 +1404,8 @@ def g(x: int) -> int: ... whether or not we check the args covariantly. """ # Normalize both types before comparing them. - left = left.with_unpacked_kwargs() - right = right.with_unpacked_kwargs() + left = left.with_unpacked_kwargs().with_normalized_var_args() + right = right.with_unpacked_kwargs().with_normalized_var_args() if is_compat_return is None: is_compat_return = is_compat @@ -1504,18 +1504,6 @@ def are_parameters_compatible( right_star = right.var_arg() right_star2 = right.kw_arg() - if right_star and isinstance(right_star.typ, UnpackType): - # TODO: factor out normalization code to avoid the import. - expanded = expand_type(right, {}) - assert isinstance(expanded, (Parameters, CallableType)) - right = cast(NormalizedCallableType, expanded) - right_star = right.var_arg() - if left_star and isinstance(left_star.typ, UnpackType): - expanded = expand_type(left, {}) - assert isinstance(expanded, (Parameters, CallableType)) - left = cast(NormalizedCallableType, expanded) - left_star = left.var_arg() - # Treat "def _(*a: Any, **kw: Any) -> X" similarly to "Callable[..., X]" if are_trivial_parameters(right): return True diff --git a/mypy/types.py b/mypy/types.py index 73a67b835082..5feee128156b 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2077,6 +2077,68 @@ def with_unpacked_kwargs(self) -> NormalizedCallableType: ) ) + def with_normalized_var_args(self) -> Self: + var_arg = self.var_arg() + if not var_arg or not isinstance(var_arg.typ, UnpackType): + return self + unpacked = get_proper_type(var_arg.typ.type) + if not isinstance(unpacked, TupleType): + # Note that we don't normalize *args: *tuple[X, ...] -> *args: X, + # this should be done once in semanal_typeargs.py for user-defined types, + # and we ourselves should never construct such type. + return self + unpack_index = find_unpack_in_list(unpacked.items) + if unpack_index == 0 and len(unpacked.items) > 1: + # Already normalized. + return self + + # Boilerplate: + var_arg_index = self.arg_kinds.index(ARG_STAR) + types_prefix = self.arg_types[:var_arg_index] + kinds_prefix = self.arg_kinds[:var_arg_index] + names_prefix = self.arg_names[:var_arg_index] + types_suffix = self.arg_types[var_arg_index + 1 :] + kinds_suffix = self.arg_kinds[var_arg_index + 1 :] + names_suffix = self.arg_names[var_arg_index + 1 :] + no_name: str | None = None # to silence mypy + + # Now we have something non-trivial to do. + if unpack_index is None: + # Plain *Tuple[X, Y, Z] -> replace with ARG_POS completely + types_middle = unpacked.items + kinds_middle = [ARG_POS] * len(unpacked.items) + names_middle = [no_name] * len(unpacked.items) + else: + # *Tuple[X, *Ts, Y, Z] or *Tuple[X, *tuple[T, ...], X, Z], here + # we replace the prefix by ARG_POS (this is how some places expect + # Callables to be represented) + nested_unpack = unpacked.items[unpack_index] + assert isinstance(nested_unpack, UnpackType) + nested_unpacked = get_proper_type(nested_unpack.type) + if unpack_index == len(unpacked.items) - 1: + # Normalize also single item tuples like + # *args: *Tuple[*tuple[X, ...]] -> *args: X + # *args: *Tuple[*Ts] -> *args: *Ts + # This may be not strictly necessary, but these are very verbose. + if isinstance(nested_unpacked, Instance): + assert nested_unpacked.type.fullname == "builtins.tuple" + new_unpack = nested_unpacked.args[0] + else: + assert isinstance(nested_unpacked, TypeVarTupleType) + new_unpack = nested_unpack + else: + new_unpack = UnpackType( + unpacked.copy_modified(items=unpacked.items[unpack_index:]) + ) + types_middle = unpacked.items[:unpack_index] + [new_unpack] + kinds_middle = [ARG_POS] * unpack_index + [ARG_STAR] + names_middle = [no_name] * unpack_index + [self.arg_names[var_arg_index]] + return self.copy_modified( + arg_types=types_prefix + types_middle + types_suffix, + arg_kinds=kinds_prefix + kinds_middle + kinds_suffix, + arg_names=names_prefix + names_middle + names_suffix, + ) + def __hash__(self) -> int: # self.is_type_obj() will fail if self.fallback.type is a FakeInfo if isinstance(self.fallback.type, FakeInfo): @@ -3424,6 +3486,20 @@ def flatten_nested_unions( return flat_items +def find_unpack_in_list(items: Sequence[Type]) -> int | None: + unpack_index: int | None = None + for i, item in enumerate(items): + if isinstance(item, UnpackType): + # We cannot fail here, so we must check this in an earlier + # semanal phase. + # Funky code here avoids mypyc narrowing the type of unpack_index. + old_index = unpack_index + assert old_index is None + # Don't return so that we can also sanity check there is only one. + unpack_index = i + return unpack_index + + def flatten_nested_tuples(types: Sequence[Type]) -> list[Type]: """Recursively flatten TupleTypes nested with Unpack. diff --git a/mypy/typevartuples.py b/mypy/typevartuples.py index 29c800140eec..bcb5e96b615c 100644 --- a/mypy/typevartuples.py +++ b/mypy/typevartuples.py @@ -9,25 +9,12 @@ ProperType, Type, UnpackType, + find_unpack_in_list, get_proper_type, split_with_prefix_and_suffix, ) -def find_unpack_in_list(items: Sequence[Type]) -> int | None: - unpack_index: int | None = None - for i, item in enumerate(items): - if isinstance(item, UnpackType): - # We cannot fail here, so we must check this in an earlier - # semanal phase. - # Funky code here avoids mypyc narrowing the type of unpack_index. - old_index = unpack_index - assert old_index is None - # Don't return so that we can also sanity check there is only one. - unpack_index = i - return unpack_index - - def split_with_instance( typ: Instance, ) -> tuple[tuple[Type, ...], tuple[Type, ...], tuple[Type, ...]]: From d8c00bbb02b71f57cbbc8a51d50e1be165bebc6c Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sun, 20 Aug 2023 23:47:56 +0100 Subject: [PATCH 4/7] Add tests for some of crashes --- test-data/unit/check-typevar-tuple.test | 53 +++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 3ad34935b38b..9ac3ffe24a09 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -831,3 +831,56 @@ def dec(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], T]: ... def f(*args: Unpack[Us]) -> Foo[Us]: ... # E: TypeVarTuple "Us" is only valid with an unpack dec(f) # No crash [builtins fixtures/tuple.pyi] + +[case testHomogeneousGenericTupleUnpackInferenceNoCrash1] +from typing import Any, TypeVar, Tuple, Type, Optional +from typing_extensions import Unpack + +T = TypeVar("T") +def convert(obj: Any, *to_classes: Unpack[Tuple[Type[T], ...]]) -> Optional[T]: + ... + +x = convert(1, int, float) +reveal_type(x) # N: Revealed type is "Union[builtins.float, None]" +[builtins fixtures/tuple.pyi] + +[case testHomogeneousGenericTupleUnpackInferenceNoCrash2] +from typing import TypeVar, Tuple, Callable, Iterable +from typing_extensions import Unpack + +T = TypeVar("T") +def combine(x: T, y: T) -> T: ... +def reduce(fn: Callable[[T, T], T], xs: Iterable[T]) -> T: ... + +def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[str, ...]], bool]]) -> None: + reduce(combine, xs) +[builtins fixtures/tuple.pyi] + +[case testVariadicStarArgsCallNoCrash] +from typing import TypeVar, Callable, Tuple +from typing_extensions import TypeVarTuple, Unpack + +X = TypeVar("X") +Y = TypeVar("Y") +Xs = TypeVarTuple("Xs") +Ys = TypeVarTuple("Ys") + +def nil() -> Tuple[()]: + return () + +def cons( + f: Callable[[X], Y], + g: Callable[[Unpack[Xs]], Tuple[Unpack[Ys]]], +) -> Callable[[X, Unpack[Xs]], Tuple[Y, Unpack[Ys]]]: + def wrapped(x: X, *xs: Unpack[Xs]) -> Tuple[Y, Unpack[Ys]]: + y, ys = f(x), g(*xs) + return y, *ys + return wrapped + +def star(f: Callable[[X], Y]) -> Callable[[Unpack[Tuple[X, ...]]], Tuple[Y, ...]]: + def wrapped(*xs: X): + if not xs: + return nil() + return cons(f, star(f))(*xs) + return wrapped +[builtins fixtures/tuple.pyi] From dc154274c42608116b73ad8d66bcc84c166265c6 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 21 Aug 2023 00:18:36 +0100 Subject: [PATCH 5/7] Add tests for rest of the crashes --- mypy/checkexpr.py | 3 ++ test-data/unit/check-typevar-tuple.test | 38 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 789a5e2fb0a3..35e50046597d 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2412,6 +2412,9 @@ def check_argument_types( callee_arg_types = [orig_callee_arg_type] callee_arg_kinds = [ARG_STAR] else: + # TODO: Any and can appear in Unpack (as a result of user error), + # fail gracefully here and elsewhere (and/or normalize them away). + # TODO: figure out how UnboundType can leak here. assert isinstance(unpacked_type, Instance) assert unpacked_type.type.fullname == "builtins.tuple" callee_arg_types = [unpacked_type.args[0]] * len(actuals) diff --git a/test-data/unit/check-typevar-tuple.test b/test-data/unit/check-typevar-tuple.test index 9ac3ffe24a09..58fc1265ae99 100644 --- a/test-data/unit/check-typevar-tuple.test +++ b/test-data/unit/check-typevar-tuple.test @@ -884,3 +884,41 @@ def star(f: Callable[[X], Y]) -> Callable[[Unpack[Tuple[X, ...]]], Tuple[Y, ...] return cons(f, star(f))(*xs) return wrapped [builtins fixtures/tuple.pyi] + +[case testInvalidTypeVarTupleUseNoCrash] +from typing_extensions import TypeVarTuple + +Ts = TypeVarTuple("Ts") + +def f(x: Ts) -> Ts: # E: TypeVarTuple "Ts" is only valid with an unpack + return x + +v = f(1, 2, "A") # E: Too many arguments for "f" +reveal_type(v) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] + +[case testTypeVarTupleSimpleDecoratorWorks] +from typing import TypeVar, Callable +from typing_extensions import TypeVarTuple, Unpack + +Ts = TypeVarTuple("Ts") +T = TypeVar("T") + +def decorator(f: Callable[[Unpack[Ts]], T]) -> Callable[[Unpack[Ts]], T]: + def wrapper(*args: Unpack[Ts]) -> T: + return f(*args) + return wrapper + +@decorator +def f(a: int, b: int) -> int: ... +reveal_type(f) # N: Revealed type is "def (builtins.int, builtins.int) -> builtins.int" +[builtins fixtures/tuple.pyi] + +[case testTupleWithUnpackIterator] +from typing import Tuple +from typing_extensions import Unpack + +def pipeline(*xs: Unpack[Tuple[int, Unpack[Tuple[float, ...]], bool]]) -> None: + for x in xs: + reveal_type(x) # N: Revealed type is "builtins.float" +[builtins fixtures/tuple.pyi] From e48aa658cfa8d0cded79bdaa3b1afe2ed3f8a8ea Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 21 Aug 2023 21:05:37 +0100 Subject: [PATCH 6/7] Small cleanups --- mypy/constraints.py | 2 ++ mypy/expandtype.py | 9 +++++++-- mypy/message_registry.py | 3 ++- mypy/subtypes.py | 2 +- mypy/typeanal.py | 6 ++++-- mypy/types_utils.py | 2 +- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index f0bfe3e87aa4..4a2a9c83dd99 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -159,6 +159,7 @@ def infer_constraints_for_callable( inner_unpacked_type = get_proper_type(inner_unpack.type) suffix_len = len(unpacked_type.items) - 1 if isinstance(inner_unpacked_type, TypeVarTupleType): + # Variadic item can be either *Ts... constraints.append( Constraint( inner_unpacked_type, @@ -169,6 +170,7 @@ def infer_constraints_for_callable( ) ) else: + # ...or it can be a homogeneous tuple. assert ( isinstance(inner_unpacked_type, Instance) and inner_unpacked_type.type.fullname == "builtins.tuple" diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 915044848e57..e71f6429d9c0 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -355,15 +355,20 @@ def visit_callable_type(self, t: CallableType) -> CallableType: ) var_arg = t.var_arg() + needs_normalization = False if var_arg is not None and isinstance(var_arg.typ, UnpackType): + needs_normalization = True arg_types = self.interpolate_args_for_unpack(t, var_arg.typ) else: arg_types = self.expand_types(t.arg_types) - return t.copy_modified( + expanded = t.copy_modified( arg_types=arg_types, ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None), - ).with_normalized_var_args() + ) + if needs_normalization: + return expanded.with_normalized_var_args() + return expanded def visit_overloaded(self, t: Overloaded) -> Type: items: list[CallableType] = [] diff --git a/mypy/message_registry.py b/mypy/message_registry.py index bd3b8571b69e..713ec2e3c759 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -171,7 +171,8 @@ def with_additional_msg(self, info: str) -> ErrorMessage: IMPLICIT_GENERIC_ANY_BUILTIN: Final = ( 'Implicit generic "Any". Use "{}" and specify generic parameters' ) -INVALID_UNPACK = "{} cannot be unpacked (must be tuple or TypeVarTuple)" +INVALID_UNPACK: Final = "{} cannot be unpacked (must be tuple or TypeVarTuple)" +INVALID_UNPACK_POSITION: Final = "Unpack is only valid in a variadic position" # TypeVar INCOMPATIBLE_TYPEVAR_VALUE: Final = 'Value of type variable "{}" of {} cannot be {}' diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 938a810b6a8a..1531be744768 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -660,7 +660,7 @@ def visit_type_var_tuple(self, left: TypeVarTupleType) -> bool: def visit_unpack_type(self, left: UnpackType) -> bool: # TODO: Ideally we should not need this (since it is not a real type). - # Instead callers (prevous level types) handle it when it appears in type list. + # Instead callers (upper level types) should handle it when it appears in type list. if isinstance(self.right, UnpackType): return self._is_subtype(left.type, self.right.type) if isinstance(self.right, Instance) and self.right.type.fullname == "builtins.object": diff --git a/mypy/typeanal.py b/mypy/typeanal.py index dd0b15cc26da..14b37539afea 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -646,7 +646,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ self.fail("Unpack[...] requires exactly one type argument", t) return AnyType(TypeOfAny.from_error) if not self.allow_unpack: - self.fail("Unpack is only valid in a variadic position", t) + self.fail(message_registry.INVALID_UNPACK_POSITION, t, code=codes.VALID_TYPE) return AnyType(TypeOfAny.from_error) self.allow_type_var_tuple = True result = UnpackType(self.anal_type(t.args[0]), line=t.line, column=t.column) @@ -975,7 +975,9 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: validated_args: list[Type] = [] for i, at in enumerate(arg_types): if isinstance(at, UnpackType) and i not in (star_index, star2_index): - self.fail("Unpack is only valid in a variadic position", at) + self.fail( + message_registry.INVALID_UNPACK_POSITION, at, code=codes.VALID_TYPE + ) validated_args.append(AnyType(TypeOfAny.from_error)) else: validated_args.append(at) diff --git a/mypy/types_utils.py b/mypy/types_utils.py index a6b5baa9bdfc..f289ac3e9ed1 100644 --- a/mypy/types_utils.py +++ b/mypy/types_utils.py @@ -54,7 +54,7 @@ def strip_type(typ: Type) -> Type: def is_invalid_recursive_alias(seen_nodes: set[TypeAlias], target: Type) -> bool: - """Flag aliases like A = Union[int, A] (and similar mutual aliases). + """Flag aliases like A = Union[int, A], T = tuple[int, *T] (and similar mutual aliases). Such aliases don't make much sense, and cause problems in later phases. """ From 80b909913dfc44cab2bd478d94d42e00a4824cd3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Mon, 21 Aug 2023 21:51:54 +0100 Subject: [PATCH 7/7] Fix leaking of unbound types --- mypy/checkexpr.py | 1 - mypy/semanal_typeargs.py | 14 ++++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 35e50046597d..8e16b93910f3 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2414,7 +2414,6 @@ def check_argument_types( else: # TODO: Any and can appear in Unpack (as a result of user error), # fail gracefully here and elsewhere (and/or normalize them away). - # TODO: figure out how UnboundType can leak here. assert isinstance(unpacked_type, Instance) assert unpacked_type.type.fullname == "builtins.tuple" callee_arg_types = [unpacked_type.args[0]] * len(actuals) diff --git a/mypy/semanal_typeargs.py b/mypy/semanal_typeargs.py index f82e60923a50..8d8ef66b5c69 100644 --- a/mypy/semanal_typeargs.py +++ b/mypy/semanal_typeargs.py @@ -226,15 +226,13 @@ def visit_unpack_type(self, typ: UnpackType) -> None: return if isinstance(proper_type, Instance) and proper_type.type.fullname == "builtins.tuple": return - if ( - isinstance(proper_type, UnboundType) - or isinstance(proper_type, AnyType) - and proper_type.type_of_any == TypeOfAny.from_error - ): + if isinstance(proper_type, AnyType) and proper_type.type_of_any == TypeOfAny.from_error: return - self.fail( - message_registry.INVALID_UNPACK.format(format_type(proper_type, self.options)), typ - ) + if not isinstance(proper_type, UnboundType): + # Avoid extra errors if there were some errors already. + self.fail( + message_registry.INVALID_UNPACK.format(format_type(proper_type, self.options)), typ + ) typ.type = AnyType(TypeOfAny.from_error) def check_type_var_values(