From ef3268077de939ac96493246bd948f48a539bbd1 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Fri, 24 Dec 2021 16:29:00 +0900 Subject: [PATCH 01/41] Add ParamSpec literals --- mypy/constraints.py | 5 +- mypy/erasetype.py | 6 +- mypy/expandtype.py | 5 +- mypy/fixup.py | 8 +- mypy/indirection.py | 3 + mypy/join.py | 5 +- mypy/meet.py | 5 +- mypy/sametypes.py | 8 +- mypy/server/astdiff.py | 9 ++- mypy/server/astmerge.py | 6 +- mypy/server/deps.py | 8 +- mypy/subtypes.py | 34 +++++++- mypy/type_visitor.py | 12 ++- mypy/typeanal.py | 27 +++++-- mypy/typeops.py | 6 +- mypy/types.py | 174 ++++++++++++++++++++++++++++++++++++++++ mypy/typetraverser.py | 5 +- 17 files changed, 305 insertions(+), 21 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 9c1bfb0eba53..64751a2168e7 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -8,7 +8,7 @@ TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, TypeQuery, is_named_instance, TypeOfAny, LiteralType, ProperType, ParamSpecType, get_proper_type, TypeAliasType, is_union_with_any, - callable_with_ellipsis + callable_with_ellipsis, Parameters ) from mypy.maptype import map_instance_to_supertype import mypy.subtypes @@ -403,6 +403,9 @@ def visit_param_spec(self, template: ParamSpecType) -> List[Constraint]: # Can't infer ParamSpecs from component values (only via Callable[P, T]). return [] + def visit_parameters(self, template: Parameters) -> List[Constraint]: + raise RuntimeError("Parameters cannot be constrained to") + # Non-leaf types def visit_instance(self, template: Instance) -> List[Constraint]: diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 8acebbd783d8..ce5d92b71642 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -4,7 +4,7 @@ Type, TypeVisitor, UnboundType, AnyType, NoneType, TypeVarId, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType, TypeTranslator, UninhabitedType, TypeType, TypeOfAny, LiteralType, ProperType, - get_proper_type, TypeAliasType, ParamSpecType + get_proper_type, TypeAliasType, ParamSpecType, Parameters ) from mypy.nodes import ARG_STAR, ARG_STAR2 @@ -60,6 +60,10 @@ def visit_type_var(self, t: TypeVarType) -> ProperType: def visit_param_spec(self, t: ParamSpecType) -> ProperType: return AnyType(TypeOfAny.special_form) + def visit_parameters(self, t: Parameters) -> ProperType: + raise RuntimeError("Parameters should have been bound to a class") + + def visit_callable_type(self, t: CallableType) -> ProperType: # We must preserve the fallback type for overload resolution to work. any_type = AnyType(TypeOfAny.special_form) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ca1bac71cc52..2bf03824f376 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -5,7 +5,7 @@ NoneType, Overloaded, TupleType, TypedDictType, UnionType, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, FunctionLike, TypeVarType, LiteralType, get_proper_type, ProperType, - TypeAliasType, ParamSpecType, TypeVarLikeType + TypeAliasType, ParamSpecType, TypeVarLikeType, Parameters ) @@ -111,6 +111,9 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: else: return repl + def visit_parameters(self, t: Parameters) -> Type: + return t.copy_modified(arg_types=self.expand_types(t.arg_types)) + def visit_callable_type(self, t: CallableType) -> Type: param_spec = t.param_spec() if param_spec is not None: diff --git a/mypy/fixup.py b/mypy/fixup.py index da54c40e733f..1cfc1b7c00b9 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -10,7 +10,8 @@ from mypy.types import ( CallableType, Instance, Overloaded, TupleType, TypedDictType, TypeVarType, UnboundType, UnionType, TypeVisitor, LiteralType, - TypeType, NOT_READY, TypeAliasType, AnyType, TypeOfAny, ParamSpecType + TypeType, NOT_READY, TypeAliasType, AnyType, TypeOfAny, ParamSpecType, + Parameters ) from mypy.visitor import NodeVisitor from mypy.lookup import lookup_fully_qualified @@ -251,6 +252,11 @@ def visit_type_var(self, tvt: TypeVarType) -> None: def visit_param_spec(self, p: ParamSpecType) -> None: p.upper_bound.accept(self) + def visit_parameters(self, p: Parameters) -> None: + for argt in p.arg_types: + if argt is not None: + argt.accept(self) + def visit_unbound_type(self, o: UnboundType) -> None: for a in o.args: a.accept(self) diff --git a/mypy/indirection.py b/mypy/indirection.py index 238f46c8830f..e6a602236f12 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -67,6 +67,9 @@ def visit_type_var(self, t: types.TypeVarType) -> Set[str]: def visit_param_spec(self, t: types.ParamSpecType) -> Set[str]: return set() + def visit_parameters(self, t: types.Parameters) -> Set[str]: + return self._visit(t.arg_types) + def visit_instance(self, t: types.Instance) -> Set[str]: out = self._visit(t.args) if t.type: diff --git a/mypy/join.py b/mypy/join.py index e0d926f3fcf4..b0b781f39149 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -7,7 +7,7 @@ Type, AnyType, NoneType, TypeVisitor, Instance, UnboundType, TypeVarType, CallableType, TupleType, TypedDictType, ErasedType, UnionType, FunctionLike, Overloaded, LiteralType, PartialType, DeletedType, UninhabitedType, TypeType, TypeOfAny, get_proper_type, - ProperType, get_proper_types, TypeAliasType, PlaceholderType, ParamSpecType + ProperType, get_proper_types, TypeAliasType, PlaceholderType, ParamSpecType, Parameters ) from mypy.maptype import map_instance_to_supertype from mypy.subtypes import ( @@ -256,6 +256,9 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType: return t return self.default(self.s) + def visit_parameters(self, t: Parameters) -> ProperType: + raise NotImplementedError("joining two paramspec literals is not supported yet") + def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): if self.instance_joiner is None: diff --git a/mypy/meet.py b/mypy/meet.py index 644b57afbcbe..a3ab78290157 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -6,7 +6,7 @@ TupleType, TypedDictType, ErasedType, UnionType, PartialType, DeletedType, UninhabitedType, TypeType, TypeOfAny, Overloaded, FunctionLike, LiteralType, ProperType, get_proper_type, get_proper_types, TypeAliasType, TypeGuardedType, - ParamSpecType + ParamSpecType, Parameters ) from mypy.subtypes import is_equivalent, is_subtype, is_callable_compatible, is_proper_subtype from mypy.erasetype import erase_type @@ -506,6 +506,9 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType: else: return self.default(self.s) + def visit_parameters(self, t: Parameters) -> ProperType: + raise NotImplementedError("meeting two paramspec literals is not supported yet") + def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): si = self.s diff --git a/mypy/sametypes.py b/mypy/sametypes.py index 33cd7f0606cf..3ce5aa010b94 100644 --- a/mypy/sametypes.py +++ b/mypy/sametypes.py @@ -4,7 +4,7 @@ Type, UnboundType, AnyType, NoneType, TupleType, TypedDictType, UnionType, CallableType, TypeVarType, Instance, TypeVisitor, ErasedType, Overloaded, PartialType, DeletedType, UninhabitedType, TypeType, LiteralType, - ProperType, get_proper_type, TypeAliasType, ParamSpecType + ProperType, get_proper_type, TypeAliasType, ParamSpecType, Parameters ) from mypy.typeops import tuple_fallback, make_simplified_union @@ -102,6 +102,12 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: return (isinstance(self.right, ParamSpecType) and left.id == self.right.id and left.flavor == self.right.flavor) + def visit_parameters(self, left: Parameters) -> bool: + return (isinstance(self.right, Parameters) and + left.arg_names == self.right.arg_names and + is_same_types(left.arg_types, self.right.arg_types) and + left.arg_kinds == self.right.arg_kinds) + def visit_callable_type(self, left: CallableType) -> bool: # FIX generics if isinstance(self.right, CallableType): diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 12add8efcb3a..2d21d9c4a53e 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -59,7 +59,8 @@ class level -- these are handled at attribute level (say, 'mod.Cls.method' from mypy.types import ( Type, TypeVisitor, UnboundType, AnyType, NoneType, UninhabitedType, ErasedType, DeletedType, Instance, TypeVarType, CallableType, TupleType, TypedDictType, - UnionType, Overloaded, PartialType, TypeType, LiteralType, TypeAliasType, ParamSpecType + UnionType, Overloaded, PartialType, TypeType, LiteralType, TypeAliasType, ParamSpecType, + Parameters ) from mypy.util import get_prefix @@ -317,6 +318,12 @@ def visit_param_spec(self, typ: ParamSpecType) -> SnapshotItem: typ.flavor, snapshot_type(typ.upper_bound)) + def visit_parameters(self, typ: Parameters) -> SnapshotItem: + return ('Parameters', + snapshot_types(typ.arg_types), + tuple(encode_optional_str(name) for name in typ.arg_names), + tuple(typ.arg_kinds)) + def visit_callable_type(self, typ: CallableType) -> SnapshotItem: # FIX generics return ('CallableType', diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 8db2b302b844..ae8578536fb8 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -59,7 +59,7 @@ Type, SyntheticTypeVisitor, Instance, AnyType, NoneType, CallableType, ErasedType, DeletedType, TupleType, TypeType, TypedDictType, UnboundType, UninhabitedType, UnionType, Overloaded, TypeVarType, TypeList, CallableArgument, EllipsisType, StarType, LiteralType, - RawExpressionType, PartialType, PlaceholderType, TypeAliasType, ParamSpecType + RawExpressionType, PartialType, PlaceholderType, TypeAliasType, ParamSpecType, Parameters ) from mypy.util import get_prefix, replace_object_state from mypy.typestate import TypeState @@ -411,6 +411,10 @@ def visit_type_var(self, typ: TypeVarType) -> None: def visit_param_spec(self, typ: ParamSpecType) -> None: pass + def visit_parameters(self, typ: Parameters) -> None: + for arg in typ.arg_types: + arg.accept(self) + def visit_typeddict_type(self, typ: TypedDictType) -> None: for value_type in typ.items.values(): value_type.accept(self) diff --git a/mypy/server/deps.py b/mypy/server/deps.py index cee12c9f8aab..5f5ee4ff50fd 100644 --- a/mypy/server/deps.py +++ b/mypy/server/deps.py @@ -99,7 +99,7 @@ class 'mod.Cls'. This can also refer to an attribute inherited from a Type, Instance, AnyType, NoneType, TypeVisitor, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, FunctionLike, Overloaded, TypeOfAny, LiteralType, ErasedType, get_proper_type, ProperType, - TypeAliasType, ParamSpecType + TypeAliasType, ParamSpecType, Parameters ) from mypy.server.trigger import make_trigger, make_wildcard_trigger from mypy.util import correct_relative_import @@ -961,6 +961,12 @@ def visit_param_spec(self, typ: ParamSpecType) -> List[str]: triggers.extend(self.get_type_triggers(typ.upper_bound)) return triggers + def visit_parameters(self, typ: Parameters) -> List[str]: + triggers = [] + for arg in typ.arg_types: + triggers.extend(self.get_type_triggers(arg)) + return triggers + def visit_typeddict_type(self, typ: TypedDictType) -> List[str]: triggers = [] for item in typ.items.values(): diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 26054a057540..a2331aa5d020 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -7,7 +7,8 @@ Type, AnyType, UnboundType, TypeVisitor, FormalArgument, NoneType, Instance, TypeVarType, CallableType, TupleType, TypedDictType, UnionType, Overloaded, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, is_named_instance, - FunctionLike, TypeOfAny, LiteralType, get_proper_type, TypeAliasType, ParamSpecType + FunctionLike, TypeOfAny, LiteralType, get_proper_type, TypeAliasType, ParamSpecType, + Parameters ) import mypy.applytype import mypy.constraints @@ -327,6 +328,16 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: return True return self._is_subtype(left.upper_bound, self.right) + def visit_parameters(self, left: Parameters) -> bool: + right = self.right + if isinstance(right, Parameters): + return are_parameters_compatible( + left, right, + is_compat=self._is_subtype, + ignore_pos_arg_names=self.ignore_pos_arg_names) + else: + return False + def visit_callable_type(self, left: CallableType) -> bool: right = self.right if isinstance(right, CallableType): @@ -918,6 +929,19 @@ def g(x: int) -> int: ... if right.is_ellipsis_args: return True + return are_parameters_compatible(left, right, is_compat=is_compat, + ignore_pos_arg_names=ignore_pos_arg_names, + check_args_covariantly=check_args_covariantly, + allow_partial_overlap=allow_partial_overlap) + +def are_parameters_compatible(left: Union[Parameters, CallableType], + right: Union[Parameters, CallableType], + *, + is_compat: Callable[[Type, Type], bool], + ignore_pos_arg_names: bool = False, + check_args_covariantly: bool = False, + allow_partial_overlap: bool = False) -> bool: + """Helper function for is_callable_compatible, used for Parameter compatibility""" left_star = left.var_arg() left_star2 = left.kw_arg() right_star = right.var_arg() @@ -1360,6 +1384,14 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: return True return self._is_proper_subtype(left.upper_bound, self.right) + def visit_parameters(self, left: Parameters) -> bool: + right = self.right + if isinstance(right, Parameters): + return are_parameters_compatible(left, right, is_compat=self._is_proper_subtype) + else: + # TODO: should this work against callables too? + return False + def visit_callable_type(self, left: CallableType) -> bool: right = self.right if isinstance(right, CallableType): diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 99821fa62640..80ddee56154f 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -20,7 +20,7 @@ from mypy.types import ( Type, AnyType, CallableType, Overloaded, TupleType, TypedDictType, LiteralType, - RawExpressionType, Instance, NoneType, TypeType, + Parameters, RawExpressionType, Instance, NoneType, TypeType, UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarLikeType, UnboundType, ErasedType, StarType, EllipsisType, TypeList, CallableArgument, PlaceholderType, TypeAliasType, ParamSpecType, get_proper_type @@ -67,6 +67,10 @@ def visit_type_var(self, t: TypeVarType) -> T: def visit_param_spec(self, t: ParamSpecType) -> T: pass + @abstractmethod + def visit_parameters(self, t: Parameters) -> T: + pass + @abstractmethod def visit_instance(self, t: Instance) -> T: pass @@ -186,6 +190,9 @@ def visit_type_var(self, t: TypeVarType) -> Type: def visit_param_spec(self, t: ParamSpecType) -> Type: return t + def visit_parameters(self, t: Parameters) -> Type: + return t.copy_modified(arg_types=self.translate_types(t.arg_types)) + def visit_partial_type(self, t: PartialType) -> Type: return t @@ -301,6 +308,9 @@ def visit_type_var(self, t: TypeVarType) -> T: def visit_param_spec(self, t: ParamSpecType) -> T: return self.strategy([]) + def visit_parameters(self, t: Parameters) -> T: + return self.query_types(t.arg_types) + def visit_partial_type(self, t: PartialType) -> T: return self.strategy([]) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f7b584eadae8..a71f9009a87f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -15,7 +15,7 @@ Type, UnboundType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneType, ErasedType, DeletedType, TypeList, TypeVarType, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, CallableArgument, - TypeQuery, union_items, TypeOfAny, LiteralType, RawExpressionType, + Parameters, TypeQuery, union_items, TypeOfAny, LiteralType, RawExpressionType, PlaceholderType, Overloaded, get_proper_type, TypeAliasType, RequiredType, TypeVarLikeType, ParamSpecType, ParamSpecFlavor, callable_with_ellipsis ) @@ -396,12 +396,17 @@ def analyze_type_with_type_info( if len(args) > 0 and info.fullname == 'builtins.tuple': fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) return TupleType(self.anal_array(args), fallback, ctx.line) + # Only allow ParamSpec literals if there's a ParamSpec arg type: + # This might not be necessary. + allow_param_spec_literal = any(isinstance(tvar, ParamSpecType) for tvar in info.defn.type_vars) + # Analyze arguments and (usually) construct Instance type. The # number of type arguments and their values are # checked only later, since we do not always know the # valid count at this point. Thus we may construct an # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(args, allow_param_spec=True), + instance = Instance(info, self.anal_array(args, allow_param_spec=True, + allow_param_spec_literal=allow_param_spec_literal), ctx.line, ctx.column) # Check type argument count. if len(instance.args) != len(info.type_vars) and not self.defining_alias: @@ -555,6 +560,9 @@ def visit_type_var(self, t: TypeVarType) -> Type: def visit_param_spec(self, t: ParamSpecType) -> Type: return t + def visit_parameters(self, t: Parameters) -> Type: + raise NotImplementedError("ParamSpec literals cannot have unbound TypeVars") + def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: # Every Callable can bind its own type variables, if they're not in the outer scope with self.tvar_scope_frame(): @@ -1005,10 +1013,19 @@ def is_defined_type_var(self, tvar: str, context: Context) -> bool: def anal_array(self, a: Iterable[Type], nested: bool = True, *, - allow_param_spec: bool = False) -> List[Type]: + allow_param_spec: bool = False, + allow_param_spec_literal: bool = False) -> List[Type]: res: List[Type] = [] for t in a: - res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) + if allow_param_spec_literal and isinstance(t, TypeList): + # paramspec literal (Z[[int, str, Whatever]]) + params = self.analyze_callable_args(t) + if params: + res.append(Parameters(*params)) + else: + res.append(AnyType(TypeOfAny.from_error)) + else: + res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) return res def anal_type(self, t: Type, nested: bool = True, *, allow_param_spec: bool = False) -> Type: @@ -1266,7 +1283,7 @@ def __init__(self, def _seems_like_callable(self, type: UnboundType) -> bool: if not type.args: return False - if isinstance(type.args[0], (EllipsisType, TypeList)): + if isinstance(type.args[0], (EllipsisType, TypeList, ParamSpecType)): return True return False diff --git a/mypy/typeops.py b/mypy/typeops.py index e7e9b098eb43..5733287737c8 100644 --- a/mypy/typeops.py +++ b/mypy/typeops.py @@ -5,7 +5,7 @@ since these may assume that MROs are ready. """ -from typing import cast, Optional, List, Sequence, Set, Iterable, TypeVar, Dict, Tuple, Any +from typing import cast, Optional, List, Sequence, Set, Iterable, TypeVar, Dict, Tuple, Any, Union from typing_extensions import Type as TypingType import itertools import sys @@ -14,7 +14,7 @@ TupleType, Instance, FunctionLike, Type, CallableType, TypeVarLikeType, Overloaded, TypeVarType, UninhabitedType, FormalArgument, UnionType, NoneType, TypedDictType, AnyType, TypeOfAny, TypeType, ProperType, LiteralType, get_proper_type, get_proper_types, - copy_type, TypeAliasType, TypeQuery, ParamSpecType + copy_type, TypeAliasType, TypeQuery, ParamSpecType, Parameters ) from mypy.nodes import ( FuncBase, FuncItem, FuncDef, OverloadedFuncDef, TypeInfo, ARG_STAR, ARG_STAR2, ARG_POS, @@ -286,7 +286,7 @@ def erase_to_bound(t: Type) -> Type: return t -def callable_corresponding_argument(typ: CallableType, +def callable_corresponding_argument(typ: Union[CallableType, Parameters], model: FormalArgument) -> Optional[FormalArgument]: """Return the argument a function that corresponds to `model`""" diff --git a/mypy/types.py b/mypy/types.py index 14eefea7dd81..86bc9c1bc179 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1062,6 +1062,150 @@ def get_name(self) -> Optional[str]: pass ('required', bool)]) +# TODO: should this take bound typevars too? what would this take? +# ex: class Z(Generic[P, T]): ...; Z[[V], V] +# What does a typevar even mean in this context? +class Parameters(ProperType): + """Type that represents the parameters to a function. + + Used for ParamSpec analysis.""" + __slots__ = ('arg_types', + 'arg_kinds', + 'arg_names', + 'min_args') + + def __init__(self, + arg_types: Sequence[Type], + arg_kinds: List[ArgKind], + arg_names: Sequence[Optional[str]], + ) -> None: + #print(f"Parameters.__init__({arg_types=}, {arg_kinds=}, {arg_names=})") + self.arg_types = list(arg_types) + self.arg_kinds = arg_kinds + self.arg_names = list(arg_names) + self.min_args = arg_kinds.count(ARG_POS) + assert len(arg_types) == len(arg_kinds) == len(arg_names) + + def copy_modified(self, + arg_types: Bogus[Sequence[Type]] = _dummy, + arg_kinds: Bogus[List[ArgKind]] = _dummy, + arg_names: Bogus[Sequence[Optional[str]]] = _dummy + ) -> 'Parameters': + return Parameters( + arg_types=arg_types if arg_types is not _dummy else self.arg_types, + arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, + arg_names=arg_names if arg_names is not _dummy else self.arg_names, + ) + + def var_arg(self) -> Optional[FormalArgument]: + """The formal argument for *args.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR: + return FormalArgument(None, position, type, False) + return None + + def kw_arg(self) -> Optional[FormalArgument]: + """The formal argument for **kwargs.""" + for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): + if kind == ARG_STAR2: + return FormalArgument(None, position, type, False) + return None + + def formal_arguments(self, include_star_args: bool = False) -> List[FormalArgument]: + """Yields the formal arguments corresponding to this callable, ignoring *arg and **kwargs. + + To handle *args and **kwargs, use the 'callable.var_args' and 'callable.kw_args' fields, + if they are not None. + + If you really want to include star args in the yielded output, set the + 'include_star_args' parameter to 'True'.""" + args = [] + done_with_positional = False + for i in range(len(self.arg_types)): + kind = self.arg_kinds[i] + if kind.is_named() or kind.is_star(): + done_with_positional = True + if not include_star_args and kind.is_star(): + continue + + required = kind.is_required() + pos = None if done_with_positional else i + arg = FormalArgument( + self.arg_names[i], + pos, + self.arg_types[i], + required + ) + args.append(arg) + return args + + def argument_by_name(self, name: Optional[str]) -> Optional[FormalArgument]: + if name is None: + return None + seen_star = False + for i, (arg_name, kind, typ) in enumerate( + zip(self.arg_names, self.arg_kinds, self.arg_types)): + # No more positional arguments after these. + if kind.is_named() or kind.is_star(): + seen_star = True + if kind.is_star(): + continue + if arg_name == name: + position = None if seen_star else i + return FormalArgument(name, position, typ, kind.is_required()) + return self.try_synthesizing_arg_from_kwarg(name) + + def argument_by_position(self, position: Optional[int]) -> Optional[FormalArgument]: + if position is None: + return None + if position >= len(self.arg_names): + return self.try_synthesizing_arg_from_vararg(position) + name, kind, typ = ( + self.arg_names[position], + self.arg_kinds[position], + self.arg_types[position], + ) + if kind.is_positional(): + return FormalArgument(name, position, typ, kind == ARG_POS) + else: + return self.try_synthesizing_arg_from_vararg(position) + + def try_synthesizing_arg_from_kwarg(self, + name: Optional[str]) -> Optional[FormalArgument]: + kw_arg = self.kw_arg() + if kw_arg is not None: + return FormalArgument(name, None, kw_arg.typ, False) + else: + return None + + def try_synthesizing_arg_from_vararg(self, + position: Optional[int]) -> Optional[FormalArgument]: + var_arg = self.var_arg() + if var_arg is not None: + return FormalArgument(None, position, var_arg.typ, False) + else: + return None + + def accept(self, visitor: 'TypeVisitor[T]') -> T: + return visitor.visit_parameters(self) + + def serialize(self) -> JsonDict: + return {'.class': 'Parameters', + 'arg_types': [t.serialize() for t in self.arg_types], + 'arg_kinds': [int(x.value) for x in self.arg_kinds], + 'arg_names': self.arg_names, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'Parameters': + assert data['.class'] == 'Parameters' + return Parameters( + [deserialize_type(t) for t in data['arg_types']], + [ArgKind(x) for x in data['arg_kinds']], + data['arg_names'], + ) + + class CallableType(FunctionLike): """Type of a non-overloaded callable object (such as function).""" @@ -1091,6 +1235,7 @@ class CallableType(FunctionLike): ) def __init__(self, + # maybe this should be refactored to take a Parameters object arg_types: Sequence[Type], arg_kinds: List[ArgKind], arg_names: Sequence[Optional[str]], @@ -2228,6 +2373,35 @@ def visit_param_spec(self, t: ParamSpecType) -> str: s = f'{t.name_with_suffix()}`{t.id}' return s + def visit_parameters(self, t: Parameters) -> str: + s = '' + bare_asterisk = False + for i in range(len(t.arg_types)): + if s != '': + s += ', ' + if t.arg_kinds[i].is_named() and not bare_asterisk: + s += '*, ' + bare_asterisk = True + if t.arg_kinds[i] == ARG_STAR: + s += '*' + if t.arg_kinds[i] == ARG_STAR2: + s += '**' + name = t.arg_names[i] + if name: + s += name + ': ' + r = t.arg_types[i].accept(self) + + # TODO: why are these treated differently than callable args? + if isinstance(t.arg_types[i], UnboundType): + s += r[:-1] + else: + s += r + + if t.arg_kinds[i].is_optional(): + s += ' =' + + return '({})'.format(s) + def visit_callable_type(self, t: CallableType) -> str: param_spec = t.param_spec() if param_spec is not None: diff --git a/mypy/typetraverser.py b/mypy/typetraverser.py index a03784b0406e..14ae9d1d1e7a 100644 --- a/mypy/typetraverser.py +++ b/mypy/typetraverser.py @@ -6,7 +6,7 @@ Type, SyntheticTypeVisitor, AnyType, UninhabitedType, NoneType, ErasedType, DeletedType, TypeVarType, LiteralType, Instance, CallableType, TupleType, TypedDictType, UnionType, Overloaded, TypeType, CallableArgument, UnboundType, TypeList, StarType, EllipsisType, - PlaceholderType, PartialType, RawExpressionType, TypeAliasType, ParamSpecType + PlaceholderType, PartialType, RawExpressionType, TypeAliasType, ParamSpecType, Parameters ) @@ -40,6 +40,9 @@ def visit_type_var(self, t: TypeVarType) -> None: def visit_param_spec(self, t: ParamSpecType) -> None: pass + def visit_parameters(self, t: Parameters) -> None: + self.traverse_types(t.arg_types) + def visit_literal_type(self, t: LiteralType) -> None: t.fallback.accept(self) From 816f3cd2fdc86a8e0af31ff5d5cf97f4ecf5c5ab Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 25 Dec 2021 14:45:10 +0900 Subject: [PATCH 02/41] Improve ParamSpec and Parameters checking Now this program works: ```py from typing_extensions import ParamSpec from typing import Generic, Callable P = ParamSpec("P") class Z(Generic[P]): def call(self, *args: P.args, **kwargs: P.kwargs) -> None: pass n: Z[[int]] reveal_type(n) # N: Revealed type is "repro.Z[[builtins.int]]" def f(n: Z[P]) -> Z[P]: ... reveal_type(f) # N: Revealed type is "def [P] (n: repro.Z[P`-1]) -> repro.Z[P`-1]" reveal_type(f(n)) # N: Revealed type is "repro.Z[[builtins.int]]" ``` --- mypy/checker.py | 2 +- mypy/constraints.py | 12 ++++++++++-- mypy/messages.py | 5 ++++- mypy/typeanal.py | 14 +++++++++++++- mypy/types.py | 10 +++------- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 1a8b22aaa79e..5282b2aa8c1e 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4994,7 +4994,7 @@ def check_subtype(self, if subtype_label is not None: extra_info.append(subtype_label + ' ' + subtype_str) if supertype_label is not None: - extra_info.append(supertype_label + ' ' + supertype_str) + extra_info.append(f'{supertype_label} {supertype_str}') note_msg = make_inferred_type_note(outer_context or context, subtype, supertype, supertype_str) if isinstance(subtype, Instance) and isinstance(supertype, Instance): diff --git a/mypy/constraints.py b/mypy/constraints.py index 64751a2168e7..68912c9fe0fd 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -446,7 +446,6 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. for tvar, mapped_arg, instance_arg in zip(tvars, mapped.args, instance.args): - # TODO: ParamSpecType if isinstance(tvar, TypeVarType): # The constraints for generic type parameters depend on variance. # Include constraints from both directions if invariant. @@ -456,6 +455,11 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if tvar.variance != COVARIANT: res.extend(infer_constraints( mapped_arg, instance_arg, neg_op(self.direction))) + elif isinstance(tvar, ParamSpecType): + # no such thing as variance for ParamSpecs + # TODO: is there a case I am missing? + # TODO: what is setting meta_level to 0? + res.append(Constraint(TypeVarId(tvar.id.raw_id, 1), SUPERTYPE_OF, mapped_arg)) return res elif (self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname)): @@ -464,7 +468,6 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. for tvar, mapped_arg, template_arg in zip(tvars, mapped.args, template.args): - # TODO: ParamSpecType if isinstance(tvar, TypeVarType): # The constraints for generic type parameters depend on variance. # Include constraints from both directions if invariant. @@ -474,6 +477,11 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if tvar.variance != COVARIANT: res.extend(infer_constraints( template_arg, mapped_arg, neg_op(self.direction))) + elif isinstance(tvar, ParamSpecType): + # no such thing as variance for ParamSpecs + # TODO: is there a case I am missing? + # TODO: what is setting meta_level to 0? + res.append(Constraint(TypeVarId(tvar.id.raw_id, 1), SUPERTYPE_OF, mapped_arg)) return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking diff --git a/mypy/messages.py b/mypy/messages.py index da284cc88ba4..9e4466ebc675 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -24,7 +24,7 @@ Type, CallableType, Instance, TypeVarType, TupleType, TypedDictType, LiteralType, UnionType, NoneType, AnyType, Overloaded, FunctionLike, DeletedType, TypeType, UninhabitedType, TypeOfAny, UnboundType, PartialType, get_proper_type, ProperType, - ParamSpecType, get_proper_types + ParamSpecType, Parameters, get_proper_types ) from mypy.typetraverser import TypeTraverserVisitor from mypy.nodes import ( @@ -1792,6 +1792,9 @@ def format(typ: Type) -> str: return 'overloaded function' elif isinstance(typ, UnboundType): return str(typ) + elif isinstance(typ, Parameters): + # TODO: technically this is not the right way to format (there could be non-pos). + return '[{}]'.format(', '.join(map(format, typ.arg_types))) elif typ is None: raise RuntimeError('Type is None') else: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a71f9009a87f..628efca328e6 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -264,6 +264,9 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) return self.analyze_type_with_type_info(node, t.args, t) elif node.fullname in ("typing_extensions.TypeAlias", "typing.TypeAlias"): return AnyType(TypeOfAny.special_form) + # Concatenate is an operator, no need for a proper type + elif node.fullname in ("typing_extensions.Concatenate", "typing.Concatenate"): + return self.apply_concatenate_operator(t) else: return self.analyze_unbound_type_without_type_info(t, sym, defining_literal) else: # sym is None @@ -277,6 +280,13 @@ def cannot_resolve_type(self, t: UnboundType) -> None: 'Cannot resolve name "{}" (possible cyclic definition)'.format(t.name), t) + def apply_concatenate_operator(self, t: UnboundType) -> Optional[Type]: + if len(t.args) == 0: + self.api.fail('Concatenate needs type arguments', t) + return AnyType(TypeOfAny.from_error) + + raise RuntimeError("TODO") + def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Optional[Type]: """Bind special type that is recognized through magic name such as 'typing.Any'. @@ -1021,7 +1031,9 @@ def anal_array(self, # paramspec literal (Z[[int, str, Whatever]]) params = self.analyze_callable_args(t) if params: - res.append(Parameters(*params)) + ts, kinds, names = params + # bind these types + res.append(Parameters(self.anal_array(ts), kinds, names)) else: res.append(AnyType(TypeOfAny.from_error)) else: diff --git a/mypy/types.py b/mypy/types.py index 86bc9c1bc179..facdecda44c1 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2388,19 +2388,15 @@ def visit_parameters(self, t: Parameters) -> str: s += '**' name = t.arg_names[i] if name: - s += name + ': ' + s += f'{name}: ' r = t.arg_types[i].accept(self) - # TODO: why are these treated differently than callable args? - if isinstance(t.arg_types[i], UnboundType): - s += r[:-1] - else: - s += r + s += r if t.arg_kinds[i].is_optional(): s += ' =' - return '({})'.format(s) + return f'[{s}]' def visit_callable_type(self, t: CallableType) -> str: param_spec = t.param_spec() From d9b352faa2d013259849952cb078a3b874f34932 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 10:46:19 +0900 Subject: [PATCH 03/41] Get basic Concatenate features working This program: ```py from typing_extensions import ParamSpec, Concatenate from typing import Generic P = ParamSpec("P") class Z(Generic[P]): ... n: Z[[int]] reveal_type(n) def f(n: Z[P]) -> Z[Concatenate[bool, Concatenate[str, P]]]: ... reveal_type(f) reveal_type(f(n)) ``` outputs: ``` repro.py:10: note: Revealed type is "repro.Z[[builtins.int]]" repro.py:14: note: Revealed type is "def [P] (n: repro.Z[P`-1]) -> repro.Z[Concatenate[builtins.bool, builtins.str, P`-1]]" repro.py:15: note: Revealed type is "repro.Z[[builtins.bool, builtins.str, builtins.int]]" ``` Next up: checking inputs match prefixes and why does it only work when cache exists? --- mypy/expandtype.py | 12 +++++++++++- mypy/typeanal.py | 17 +++++++++++++++-- mypy/types.py | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 11 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 2bf03824f376..60e00420ca91 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -104,11 +104,21 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: if isinstance(repl, Instance): inst = repl # Return copy of instance with type erasure flag on. + # TODO: what does prefix mean in this case? + # TODO: why does this case even happen? Instances aren't plural. return Instance(inst.type, inst.args, line=inst.line, column=inst.column, erased=True) elif isinstance(repl, ParamSpecType): - return repl.with_flavor(t.flavor) + # TODO: what if both have prefixes??? + # (realistically, `repl` is the unification variable for `t` so this is fine) + return repl.copy_modified(flavor=t.flavor, prefix=t.prefix) + elif isinstance(repl, Parameters): + return repl.copy_modified(t.prefix.arg_types + repl.arg_types, + t.prefix.arg_kinds + repl.arg_kinds, + t.prefix.arg_names + repl.arg_names) else: + # returning parameters + # TODO: should this branch be removed? better not to fail silently return repl def visit_parameters(self, t: Parameters) -> Type: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 628efca328e6..30a3fff86b79 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -266,6 +266,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) return AnyType(TypeOfAny.special_form) # Concatenate is an operator, no need for a proper type elif node.fullname in ("typing_extensions.Concatenate", "typing.Concatenate"): + # TODO: detect valid locations (`allow_param_spec` is not here.) return self.apply_concatenate_operator(t) else: return self.analyze_unbound_type_without_type_info(t, sym, defining_literal) @@ -280,12 +281,24 @@ def cannot_resolve_type(self, t: UnboundType) -> None: 'Cannot resolve name "{}" (possible cyclic definition)'.format(t.name), t) - def apply_concatenate_operator(self, t: UnboundType) -> Optional[Type]: + def apply_concatenate_operator(self, t: UnboundType) -> Optional[ParamSpecType]: if len(t.args) == 0: self.api.fail('Concatenate needs type arguments', t) return AnyType(TypeOfAny.from_error) - raise RuntimeError("TODO") + # last argument has to be ParamSpec (or Concatenate) + ps = self.anal_type(t.args[-1], allow_param_spec=True) + if not isinstance(ps, ParamSpecType): + print(ps) + self.api.fail('The last parameter to Concatenate needs to be a ParamSpec', t) + return AnyType(TypeOfAny.from_error) + + args = self.anal_array(t.args[:-1]) + pre = ps.prefix + pre = Parameters(args + pre.arg_types, + [ARG_POS] * len(args) + pre.arg_kinds, + [None] * len(args) + pre.arg_names) + return ps.copy_modified(prefix=pre) def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Optional[Type]: """Bind special type that is recognized through magic name such as 'typing.Any'. diff --git a/mypy/types.py b/mypy/types.py index facdecda44c1..800ffc7799a3 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -489,26 +489,43 @@ class ParamSpecType(TypeVarLikeType): always just 'object'). """ - __slots__ = ('flavor',) + __slots__ = ('flavor', 'prefix') flavor: int + prefix: 'Parameters' def __init__( self, name: str, fullname: str, id: Union[TypeVarId, int], flavor: int, - upper_bound: Type, *, line: int = -1, column: int = -1 + upper_bound: Type, *, line: int = -1, column: int = -1, prefix: Optional['Parameters'] = None ) -> None: super().__init__(name, fullname, id, upper_bound, line=line, column=column) self.flavor = flavor + self.prefix = prefix or Parameters([], [], []) @staticmethod def new_unification_variable(old: 'ParamSpecType') -> 'ParamSpecType': new_id = TypeVarId.new(meta_level=1) return ParamSpecType(old.name, old.fullname, new_id, old.flavor, old.upper_bound, - line=old.line, column=old.column) + line=old.line, column=old.column, prefix=old.prefix) def with_flavor(self, flavor: int) -> 'ParamSpecType': return ParamSpecType(self.name, self.fullname, self.id, flavor, - upper_bound=self.upper_bound) + upper_bound=self.upper_bound, prefix=self.prefix) + + def copy_modified(self, *, + id: Bogus[Union[TypeVarId, int]] = _dummy, + flavor: Bogus[int] = _dummy, + prefix: Bogus['Parameters'] = _dummy) -> 'ParamSpecType': + return ParamSpecType( + self.name, + self.fullname, + id if id is not _dummy else self.id, + flavor if flavor is not _dummy else self.flavor, + self.upper_bound, + line=self.line, + column=self.column, + prefix=prefix if prefix is not _dummy else self.prefix, + ) def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_param_spec(self) @@ -539,6 +556,7 @@ def serialize(self) -> JsonDict: 'id': self.id.raw_id, 'flavor': self.flavor, 'upper_bound': self.upper_bound.serialize(), + 'prefix': self.prefix.serialize() } @classmethod @@ -550,6 +568,7 @@ def deserialize(cls, data: JsonDict) -> 'ParamSpecType': data['id'], data['flavor'], deserialize_type(data['upper_bound']), + prefix=deserialize_type(data['prefix']) ) @@ -1079,7 +1098,6 @@ def __init__(self, arg_kinds: List[ArgKind], arg_names: Sequence[Optional[str]], ) -> None: - #print(f"Parameters.__init__({arg_types=}, {arg_kinds=}, {arg_names=})") self.arg_types = list(arg_types) self.arg_kinds = arg_kinds self.arg_names = list(arg_names) @@ -1493,7 +1511,7 @@ def param_spec(self) -> Optional[ParamSpecType]: if not isinstance(arg_type, ParamSpecType): return None return ParamSpecType(arg_type.name, arg_type.fullname, arg_type.id, ParamSpecFlavor.BARE, - arg_type.upper_bound) + arg_type.upper_bound, prefix=arg_type.prefix) def expand_param_spec(self, c: 'CallableType') -> 'CallableType': return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, @@ -2365,12 +2383,18 @@ def visit_type_var(self, t: TypeVarType) -> str: return s def visit_param_spec(self, t: ParamSpecType) -> str: + # prefixes are displayed as Concatenate + s = '' + if t.prefix.arg_types: + s += f'Concatenate[{self.list_str(t.prefix.arg_types)}, ' if t.name is None: # Anonymous type variable type (only numeric id). - s = f'`{t.id}' + s += f'`{t.id}' else: # Named type variable type. - s = f'{t.name_with_suffix()}`{t.id}' + s += f'{t.name_with_suffix()}`{t.id}' + if t.prefix.arg_types: + s += ']' return s def visit_parameters(self, t: Parameters) -> str: From 58e6dbe1bdfeaadc3b7cbfa56cf16261da943364 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 11:06:02 +0900 Subject: [PATCH 04/41] Fix "cache" bug It wasn't actually cache... --- mypy/constraints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 68912c9fe0fd..811932603560 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -459,7 +459,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: what is setting meta_level to 0? - res.append(Constraint(TypeVarId(tvar.id.raw_id, 1), SUPERTYPE_OF, mapped_arg)) + res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, mapped_arg)) return res elif (self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname)): @@ -481,7 +481,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: what is setting meta_level to 0? - res.append(Constraint(TypeVarId(tvar.id.raw_id, 1), SUPERTYPE_OF, mapped_arg)) + res.append(Constraint(template_arg.id, SUPERTYPE_OF, mapped_arg)) return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking From 51ba4ea3fbbf0480445646141b3383d6e95e1b31 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 12:09:50 +0900 Subject: [PATCH 05/41] Check Concatenate prefixes --- mypy/constraints.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 811932603560..18be8eef810e 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -459,7 +459,12 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: what is setting meta_level to 0? - res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, mapped_arg)) + suffix = template_arg + prefix = mapped_arg.prefix + suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) + res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) return res elif (self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname)): @@ -481,7 +486,12 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: what is setting meta_level to 0? - res.append(Constraint(template_arg.id, SUPERTYPE_OF, mapped_arg)) + suffix = mapped_arg + prefix = template_arg.prefix + suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) + res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking From 24432ee4afdfebd23de1e5e4c14a95a88578ff9a Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 12:23:37 +0900 Subject: [PATCH 06/41] Polish work --- mypy/constraints.py | 43 +++++++++++++++++++++++-------------------- mypy/typeanal.py | 8 ++++++-- mypy/types.py | 2 +- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 18be8eef810e..da0c6aea0c44 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -446,6 +446,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # N.B: We use zip instead of indexing because the lengths might have # mismatches during daemon reprocessing. for tvar, mapped_arg, instance_arg in zip(tvars, mapped.args, instance.args): + # TODO(PEP612): More ParamSpec work (or is Parameters the only thing accepted) if isinstance(tvar, TypeVarType): # The constraints for generic type parameters depend on variance. # Include constraints from both directions if invariant. @@ -455,16 +456,17 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if tvar.variance != COVARIANT: res.extend(infer_constraints( mapped_arg, instance_arg, neg_op(self.direction))) - elif isinstance(tvar, ParamSpecType): - # no such thing as variance for ParamSpecs - # TODO: is there a case I am missing? - # TODO: what is setting meta_level to 0? - suffix = template_arg - prefix = mapped_arg.prefix - suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], - suffix.arg_kinds[len(prefix.arg_kinds):], - suffix.arg_names[len(prefix.arg_names):]) - res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) + elif isinstance(tvar, ParamSpecType) and isinstance(mapped_arg, ParamSpecType): + suffix = get_proper_type(instance_arg) + if isinstance(suffix, Parameters): + # no such thing as variance for ParamSpecs + # TODO: is there a case I am missing? + # TODO: what is setting meta_level to 0? + prefix = mapped_arg.prefix + suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) + res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) return res elif (self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname)): @@ -482,16 +484,17 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if tvar.variance != COVARIANT: res.extend(infer_constraints( template_arg, mapped_arg, neg_op(self.direction))) - elif isinstance(tvar, ParamSpecType): - # no such thing as variance for ParamSpecs - # TODO: is there a case I am missing? - # TODO: what is setting meta_level to 0? - suffix = mapped_arg - prefix = template_arg.prefix - suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], - suffix.arg_kinds[len(prefix.arg_kinds):], - suffix.arg_names[len(prefix.arg_names):]) - res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) + elif isinstance(tvar, ParamSpecType) and isinstance(template_arg, ParamSpecType): + suffix = get_proper_type(mapped_arg) + if isinstance(suffix, Parameters): + # no such thing as variance for ParamSpecs + # TODO: is there a case I am missing? + # TODO: what is setting meta_level to 0? + prefix = template_arg.prefix + suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) + res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 30a3fff86b79..dfac035aec5a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -281,7 +281,7 @@ def cannot_resolve_type(self, t: UnboundType) -> None: 'Cannot resolve name "{}" (possible cyclic definition)'.format(t.name), t) - def apply_concatenate_operator(self, t: UnboundType) -> Optional[ParamSpecType]: + def apply_concatenate_operator(self, t: UnboundType) -> Type: if len(t.args) == 0: self.api.fail('Concatenate needs type arguments', t) return AnyType(TypeOfAny.from_error) @@ -295,9 +295,13 @@ def apply_concatenate_operator(self, t: UnboundType) -> Optional[ParamSpecType]: args = self.anal_array(t.args[:-1]) pre = ps.prefix + + # mypy can't infer this :( + names: List[Optional[str]] = [None] * len(args) + pre = Parameters(args + pre.arg_types, [ARG_POS] * len(args) + pre.arg_kinds, - [None] * len(args) + pre.arg_names) + names + pre.arg_names) return ps.copy_modified(prefix=pre) def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Optional[Type]: diff --git a/mypy/types.py b/mypy/types.py index 800ffc7799a3..55d396bd3af0 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -568,7 +568,7 @@ def deserialize(cls, data: JsonDict) -> 'ParamSpecType': data['id'], data['flavor'], deserialize_type(data['upper_bound']), - prefix=deserialize_type(data['prefix']) + prefix=Parameters.deserialize(data['prefix']) ) From d202d1e9ae7a0b77ba054ac98e9305a0e473a5cf Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 13:18:34 +0900 Subject: [PATCH 07/41] Tests for literals --- mypy/typeanal.py | 34 +++++-------- .../unit/check-parameter-specification.test | 51 +++++++++++++++++++ 2 files changed, 64 insertions(+), 21 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 2953e6a83822..23498e6957a3 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -423,18 +423,15 @@ def analyze_type_with_type_info( if len(args) > 0 and info.fullname == 'builtins.tuple': fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) return TupleType(self.anal_array(args), fallback, ctx.line) - # Only allow ParamSpec literals if there's a ParamSpec arg type: - # This might not be necessary. - allow_param_spec_literal = any(isinstance(tvar, ParamSpecType) for tvar in info.defn.type_vars) # Analyze arguments and (usually) construct Instance type. The # number of type arguments and their values are # checked only later, since we do not always know the # valid count at this point. Thus we may construct an # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(args, allow_param_spec=True, - allow_param_spec_literal=allow_param_spec_literal), + instance = Instance(info, self.anal_array(args, allow_param_spec=True), ctx.line, ctx.column) + # Check type argument count. if len(instance.args) != len(info.type_vars) and not self.defining_alias: fix_instance(instance, self.fail, self.note, @@ -566,9 +563,15 @@ def visit_deleted_type(self, t: DeletedType) -> Type: return t def visit_type_list(self, t: TypeList) -> Type: - self.fail('Bracketed expression "[...]" is not valid as a type', t) - self.note('Did you mean "List[...]"?', t) - return AnyType(TypeOfAny.from_error) + # paramspec literal (Z[[int, str, Whatever]]) + # TODO: invalid usage restrictions + params = self.analyze_callable_args(t) + if params: + ts, kinds, names = params + # bind these types + return Parameters(self.anal_array(ts), kinds, names) + else: + return AnyType(TypeOfAny.from_error) def visit_callable_argument(self, t: CallableArgument) -> Type: self.fail('Invalid type', t) @@ -1040,21 +1043,10 @@ def is_defined_type_var(self, tvar: str, context: Context) -> bool: def anal_array(self, a: Iterable[Type], nested: bool = True, *, - allow_param_spec: bool = False, - allow_param_spec_literal: bool = False) -> List[Type]: + allow_param_spec: bool = False) -> List[Type]: res: List[Type] = [] for t in a: - if allow_param_spec_literal and isinstance(t, TypeList): - # paramspec literal (Z[[int, str, Whatever]]) - params = self.analyze_callable_args(t) - if params: - ts, kinds, names = params - # bind these types - res.append(Parameters(self.anal_array(ts), kinds, names)) - else: - res.append(AnyType(TypeOfAny.from_error)) - else: - res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) + res.append(self.anal_type(t, nested, allow_param_spec=allow_param_spec)) return res def anal_type(self, t: Type, nested: bool = True, *, allow_param_spec: bool = False) -> Type: diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index f6123915aada..fb6dd4e7728f 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -406,3 +406,54 @@ with f() as x: pass [builtins fixtures/dict.pyi] [typing fixtures/typing-full.pyi] + +[case testParamSpecLiterals] +from typing_extensions import ParamSpec, TypeAlias +from typing import Generic, TypeVar + +P = ParamSpec("P") +T = TypeVar("T") + +class Z(Generic[P]): ... + +# literals can be applied +n: Z[[int]] + +# type aliases too +nt1 = Z[[int]] +nt2: TypeAlias = Z[[int]] + +unt1: nt1 +unt2: nt2 + +# literals actually keep types +reveal_type(n) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt1) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt2) # N: Revealed type is "__main__.Z[[builtins.int]]" + +# passing into a function keeps the type +def fT(a: T) -> T: ... +def fP(a: Z[P]) -> Z[P]: ... + +reveal_type(fT(n)) # N: Revealed type is "__main__.Z*[[builtins.int]]" +reveal_type(fP(n)) # N: Revealed type is "__main__.Z[[builtins.int]]" + +# literals can be in function args and return type +def k(a: Z[[int]]) -> Z[[str]]: ... + +# functions work +reveal_type(k(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" + +# literals can be matched in arguments +def kb(a: Z[[bytes]]) -> Z[[str]]: ... + +# TODO: return type is a bit weird, return Any +reveal_type(kb(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" \ + # E: Argument 1 to "kb" has incompatible type "Z[[int]]"; expected "Z[[bytes]]" + + +# TODO(PEP612): fancy "aesthetic" syntax defined in PEP +# n2: Z[bytes] +# +# reveal_type(kb(n2)) # N: Revealed type is "__main__.Z[[builtins.str]]" +[builtins fixtures/tuple.pyi] From 9ed983062227541e0bcc129b592e4bf87640eb81 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 14:52:07 +0900 Subject: [PATCH 08/41] Tests for Concatenate --- mypy/constraints.py | 9 +- mypy/typeanal.py | 39 +++++++++ .../unit/check-parameter-specification.test | 84 ++++++++++++++++++- 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index c84447e209b0..48dacdb51fd8 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -586,9 +586,16 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: else: # TODO: Direction # TODO: Deal with arguments that come before param spec ones? + # TODO: check the prefixes match + prefix = param_spec.prefix + prefix_len = len(prefix.arg_types) res.append(Constraint(param_spec.id, SUBTYPE_OF, - cactual.copy_modified(ret_type=NoneType()))) + cactual.copy_modified( + arg_types=cactual.arg_types[prefix_len:], + arg_kinds=cactual.arg_kinds[prefix_len:], + arg_names=cactual.arg_names[prefix_len:], + ret_type=NoneType()))) template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type if template.type_guard is not None: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 23498e6957a3..4b26ad784885 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -797,6 +797,41 @@ def analyze_callable_args_for_paramspec( fallback=fallback, ) + def analyze_callable_args_for_concatenate( + self, + callable_args: Type, + ret_type: Type, + fallback: Instance, + ) -> Optional[CallableType]: + """Construct a 'Callable[C, RET]', where C is Concatenate[..., P], return None if we cannot.""" + if not isinstance(callable_args, UnboundType): + return None + sym = self.lookup_qualified(callable_args.name, callable_args) + if sym is None: + return None + if sym.node.fullname not in ("typing_extensions.Concatenate", "typing.Concatenate"): + return None + + tvar_def = self.anal_type(callable_args, allow_param_spec=True) + if not isinstance(tvar_def, ParamSpecType): + return None + + # TODO: Use tuple[...] or Mapping[..] instead? + obj = self.named_type('builtins.object') + # ick, CallableType should take ParamSpecType + prefix = tvar_def.prefix + return CallableType( + [*prefix.arg_types, + ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.ARGS, + upper_bound=obj, prefix=tvar_def.prefix), + ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.KWARGS, + upper_bound=obj)], + [*prefix.arg_kinds, nodes.ARG_STAR, nodes.ARG_STAR2], + [*prefix.arg_names, None, None], + ret_type=ret_type, + fallback=fallback, + ) + def analyze_callable_type(self, t: UnboundType) -> Type: fallback = self.named_type('builtins.function') if len(t.args) == 0: @@ -828,6 +863,10 @@ def analyze_callable_type(self, t: UnboundType) -> Type: callable_args, ret_type, fallback + ) or self.analyze_callable_args_for_concatenate( + callable_args, + ret_type, + fallback ) if maybe_ret is None: # Callable[?, RET] (where ? is something invalid) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index fb6dd4e7728f..6e4010132ff0 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -15,8 +15,9 @@ def foo1(x: Callable[P, int]) -> Callable[P, str]: ... def foo2(x: P) -> P: ... # E: Invalid location for ParamSpec "P" \ # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' -# TODO(PEP612): uncomment once we have support for Concatenate -# def foo3(x: Concatenate[int, P]) -> int: ... $ E: Invalid location for Concatenate +# TODO: Better error message +def foo3(x: Concatenate[int, P]) -> int: ... # E: Invalid location for ParamSpec "P" \ + # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' def foo4(x: List[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' @@ -455,5 +456,82 @@ reveal_type(kb(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" \ # TODO(PEP612): fancy "aesthetic" syntax defined in PEP # n2: Z[bytes] # -# reveal_type(kb(n2)) # N: Revealed type is "__main__.Z[[builtins.str]]" +# reveal_type(kb(n2)) $ N: Revealed type is "__main__.Z[[builtins.str]]" [builtins fixtures/tuple.pyi] + +[case testParamSpecConcatenateFromPep] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar, Generic + +P = ParamSpec("P") +R = TypeVar("R") + +# CASE 1 +class Request: + ... + +def with_request(f: Callable[Concatenate[Request, P], R]) -> Callable[P, R]: + def inner(*args: P.args, **kwargs: P.kwargs) -> R: + return f(Request(), *args, **kwargs) + return inner + +@with_request +def takes_int_str(request: Request, x: int, y: str) -> int: + # use request + return x + 7 + +reveal_type(takes_int_str) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int*" + +takes_int_str(1, "A") # Accepted +takes_int_str("B", 2) # E: Argument 1 to "takes_int_str" has incompatible type "str"; expected "int" \ + # E: Argument 2 to "takes_int_str" has incompatible type "int"; expected "str" + +# CASE 2 +T = TypeVar("T") +P_2 = ParamSpec("P_2") + +class X(Generic[T, P]): + f: Callable[P, int] + x: T + +def f1(x: X[int, P_2]) -> str: ... # Accepted +def f2(x: X[int, Concatenate[int, P_2]]) -> str: ... # Accepted +def f3(x: X[int, [int, bool]]) -> str: ... # Accepted +# Is ellipsis allowed by PEP? This shows up: +# def f4(x: X[int, ...]) -> str: ... # Accepted +# TODO: this is not rejected: +# def f5(x: X[int, int]) -> str: ... # Rejected + +# CASE 3 +def bar(x: int, *args: bool) -> int: ... +def add(x: Callable[P, int]) -> Callable[Concatenate[str, P], bool]: ... + +reveal_type(add(bar)) # N: Revealed type is "def (builtins.str, x: builtins.int, *args: builtins.bool) -> builtins.bool" + +def remove(x: Callable[Concatenate[int, P], int]) -> Callable[P, bool]: ... + +reveal_type(remove(bar)) # N: Revealed type is "def (*args: builtins.bool) -> builtins.bool" + +def transform( + x: Callable[Concatenate[int, P], int] +) -> Callable[Concatenate[str, P], bool]: ... + +# In the PEP, "__a" appears. What is that? Autogenerated names? To what spec? +reveal_type(transform(bar)) # N: Revealed type is "def (builtins.str, *args: builtins.bool) -> builtins.bool" + +# CASE 4 +def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" +def one(x: str) -> int: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int], int]" +def two(*, x: int) -> int: ... + +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[KwArg(int)], int]"; expected "Callable[[int], int]" +def three(**kwargs: int) -> int: ... + +@expects_int_first # Accepted +def four(*args: int) -> int: ... +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] From 3ffc34301843c3c6b82e6a0ace5c72d2cb3354eb Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 15:16:20 +0900 Subject: [PATCH 09/41] Appease CI --- mypy/constraints.py | 3 ++- mypy/erasetype.py | 1 - mypy/subtypes.py | 1 + mypy/typeanal.py | 6 +++++- mypy/types.py | 3 ++- test-data/unit/check-literal.test | 6 ++++-- test-data/unit/semanal-errors.test | 18 ++++++++++-------- 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 48dacdb51fd8..ac0e416d57b9 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -484,7 +484,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if tvar.variance != COVARIANT: res.extend(infer_constraints( template_arg, mapped_arg, neg_op(self.direction))) - elif isinstance(tvar, ParamSpecType) and isinstance(template_arg, ParamSpecType): + elif (isinstance(tvar, ParamSpecType) and + isinstance(template_arg, ParamSpecType)): suffix = get_proper_type(mapped_arg) if isinstance(suffix, Parameters): # no such thing as variance for ParamSpecs diff --git a/mypy/erasetype.py b/mypy/erasetype.py index ce5d92b71642..31c4c6d8b59b 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -63,7 +63,6 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType: def visit_parameters(self, t: Parameters) -> ProperType: raise RuntimeError("Parameters should have been bound to a class") - def visit_callable_type(self, t: CallableType) -> ProperType: # We must preserve the fallback type for overload resolution to work. any_type = AnyType(TypeOfAny.special_form) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c3bbb3fbef1a..91e4eeb7026f 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -937,6 +937,7 @@ def g(x: int) -> int: ... check_args_covariantly=check_args_covariantly, allow_partial_overlap=allow_partial_overlap) + def are_parameters_compatible(left: Union[Parameters, CallableType], right: Union[Parameters, CallableType], *, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 4b26ad784885..953cac068423 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -803,12 +803,16 @@ def analyze_callable_args_for_concatenate( ret_type: Type, fallback: Instance, ) -> Optional[CallableType]: - """Construct a 'Callable[C, RET]', where C is Concatenate[..., P], return None if we cannot.""" + """Construct a 'Callable[C, RET]', where C is Concatenate[..., P], returning None if we + cannot. + """ if not isinstance(callable_args, UnboundType): return None sym = self.lookup_qualified(callable_args.name, callable_args) if sym is None: return None + if sym.node is None: + return None if sym.node.fullname not in ("typing_extensions.Concatenate", "typing.Concatenate"): return None diff --git a/mypy/types.py b/mypy/types.py index 55d396bd3af0..101d3a312269 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -496,7 +496,8 @@ class ParamSpecType(TypeVarLikeType): def __init__( self, name: str, fullname: str, id: Union[TypeVarId, int], flavor: int, - upper_bound: Type, *, line: int = -1, column: int = -1, prefix: Optional['Parameters'] = None + upper_bound: Type, *, line: int = -1, column: int = -1, + prefix: Optional['Parameters'] = None ) -> None: super().__init__(name, fullname, id, upper_bound, line=line, column=column) self.flavor = flavor diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 37ae12419151..dff85960ed64 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -942,8 +942,10 @@ from typing_extensions import Literal a: (1, 2, 3) # E: Syntax error in type annotation \ # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid -c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type \ - # N: Did you mean "List[...]"? +# TODO: fix this +# c: [1, 2, 3] $ E: Bracketed expression "[...]" is not valid as a type \ +# $ N: Did you mean "List[...]"? + [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index f73de6470926..ce2186f969d3 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -807,8 +807,9 @@ class C(Generic[t]): pass cast(str + str, None) # E: Cast target is not a type cast(C[str][str], None) # E: Cast target is not a type cast(C[str + str], None) # E: Cast target is not a type -cast([int, str], None) # E: Bracketed expression "[...]" is not valid as a type \ - # N: Did you mean "List[...]"? +# TODO: fix this +# cast([int, str], None) $ E: Bracketed expression "[...]" is not valid as a type \ +# $ N: Did you mean "List[...]"? [out] [case testInvalidCastTargetType] @@ -845,12 +846,13 @@ Any(str, None) # E: Any(...) is no longer supported. Use cast(Any, ...) instead Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead [out] -[case testTypeListAsType] - -def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ - # N: Did you mean "List[...]"? - pass -[out] +# TODO: fix this +# [case testTypeListAsType] +# +# def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ +# # N: Did you mean "List[...]"? +# pass +# [out] [case testInvalidFunctionType] from typing import Callable From ae8ac73d150d15947611dc51849b81e1ee83a5dc Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 26 Dec 2021 15:44:44 +0900 Subject: [PATCH 10/41] Forgot to comment out the directives... --- test-data/unit/semanal-errors.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index ce2186f969d3..ba8ee6e44307 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -844,13 +844,13 @@ cast(str, target=None) # E: "cast" must be called with 2 positional arguments from typing import Any Any(str, None) # E: Any(...) is no longer supported. Use cast(Any, ...) instead Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead -[out] +#[out] # TODO: fix this # [case testTypeListAsType] # -# def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ -# # N: Did you mean "List[...]"? +# def f(x:[int, str]) -> None: $ E: Bracketed expression "[...]" is not valid as a type \ +# $ N: Did you mean "List[...]"? # pass # [out] From 9c849cc453d0e6e911e7ca6f9427c07c0edfcffa Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 27 Dec 2021 10:50:23 +0900 Subject: [PATCH 11/41] Improve literal TODOs --- mypy/nodes.py | 13 +++- mypy/semanal.py | 3 +- mypy/typeanal.py | 73 ++++++++++++++----- test-data/unit/check-literal.test | 5 +- .../unit/check-parameter-specification.test | 25 +++---- test-data/unit/semanal-errors.test | 22 +++--- 6 files changed, 90 insertions(+), 51 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 78a018f94a78..6e1f86cbfb17 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1018,6 +1018,7 @@ def serialize(self) -> JsonDict: @classmethod def deserialize(self, data: JsonDict) -> 'ClassDef': + # TODO: Does this have to work with ParamSpecType too? assert data['.class'] == 'ClassDef' res = ClassDef(data['name'], Block([]), @@ -2469,8 +2470,8 @@ class is generic then it will be a type constructor of higher kind. 'declared_metaclass', 'metaclass_type', 'names', 'is_abstract', 'is_protocol', 'runtime_protocol', 'abstract_attributes', 'deletable_attributes', 'slots', 'assuming', 'assuming_proper', - 'inferring', 'is_enum', 'fallback_to_any', 'type_vars', 'bases', - '_promote', 'tuple_type', 'is_named_tuple', 'typeddict_type', + 'inferring', 'is_enum', 'fallback_to_any', 'type_vars', 'has_param_spec_type', + 'bases', '_promote', 'tuple_type', 'is_named_tuple', 'typeddict_type', 'is_newtype', 'is_intersection', 'metadata', ) @@ -2553,6 +2554,9 @@ class is generic then it will be a type constructor of higher kind. # Generic type variable names (full names) type_vars: List[str] + # Whether this class has a ParamSpec type variable + has_param_spec_type: bool + # Direct base classes. bases: List["mypy.types.Instance"] @@ -2600,6 +2604,7 @@ def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> No self.defn = defn self.module_name = module_name self.type_vars = [] + self.has_param_spec_type = False self.bases = [] self.mro = [] self._mro_refs = None @@ -2630,6 +2635,8 @@ def __init__(self, names: 'SymbolTable', defn: ClassDef, module_name: str) -> No def add_type_vars(self) -> None: if self.defn.type_vars: for vd in self.defn.type_vars: + if isinstance(vd, mypy.types.ParamSpecType): + self.has_param_spec_type = True self.type_vars.append(vd.fullname) @property @@ -2794,6 +2801,7 @@ def serialize(self) -> JsonDict: 'defn': self.defn.serialize(), 'abstract_attributes': self.abstract_attributes, 'type_vars': self.type_vars, + 'has_param_spec_type': self.has_param_spec_type, 'bases': [b.serialize() for b in self.bases], 'mro': [c.fullname for c in self.mro], '_promote': None if self._promote is None else self._promote.serialize(), @@ -2819,6 +2827,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeInfo': # TODO: Is there a reason to reconstruct ti.subtypes? ti.abstract_attributes = data['abstract_attributes'] ti.type_vars = data['type_vars'] + ti.has_param_spec_type = data['has_param_spec_type'] ti.bases = [mypy.types.Instance.deserialize(b) for b in data['bases']] ti._promote = (None if data['_promote'] is None else mypy.types.deserialize_type(data['_promote'])) diff --git a/mypy/semanal.py b/mypy/semanal.py index a9226d2cdd0c..bd3cd307655d 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1170,7 +1170,8 @@ def analyze_class(self, defn: ClassDef) -> None: self.prepare_class_def(defn) defn.type_vars = tvar_defs - defn.info.type_vars = [tvar.name for tvar in tvar_defs] + defn.info.type_vars = [] + defn.info.add_type_vars() if base_error: defn.info.fallback_to_any = True diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 953cac068423..c283ed7c52be 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -152,6 +152,8 @@ def __init__(self, self.allow_placeholder = allow_placeholder # Are we in a context where Required[] is allowed? self.allow_required = allow_required + # Are we in a context where ParamSpec literals are allowed? + self.allow_param_spec_literals = False # Should we report an error whenever we encounter a RawExpressionType outside # of a Literal context: e.g. whenever we encounter an invalid type? Normally, # we want to report an error, but the caller may want to do more specialized @@ -424,13 +426,30 @@ def analyze_type_with_type_info( fallback = Instance(info, [AnyType(TypeOfAny.special_form)], ctx.line) return TupleType(self.anal_array(args), fallback, ctx.line) - # Analyze arguments and (usually) construct Instance type. The - # number of type arguments and their values are - # checked only later, since we do not always know the - # valid count at this point. Thus we may construct an - # Instance with an invalid number of type arguments. - instance = Instance(info, self.anal_array(args, allow_param_spec=True), - ctx.line, ctx.column) + # This is a heuristic: it will be checked later anyways but the error + # message may be worse. + with self.set_allow_param_spec_literals(info.has_param_spec_type): + # Analyze arguments and (usually) construct Instance type. The + # number of type arguments and their values are + # checked only later, since we do not always know the + # valid count at this point. Thus we may construct an + # Instance with an invalid number of type arguments. + instance = Instance(info, self.anal_array(args, allow_param_spec=True), + ctx.line, ctx.column) + + # "aesthetic" paramspec literals + # these do not support mypy_extensions VarArgs, etc. as they were already analyzed + # TODO: should these be re-analyzed to get rid of this inconsistency? + # another inconsistency is with empty type args (Z[] is more possibly an error imo) + if len(info.type_vars) == 1 and info.has_param_spec_type and len(instance.args) > 0: + first_arg = get_proper_type(instance.args[0]) + + # TODO: can I use tuple syntax to isinstance multiple in 3.6? + if not (len(instance.args) == 1 and (isinstance(first_arg, Parameters) or + isinstance(first_arg, ParamSpecType) or + isinstance(first_arg, AnyType))): + args = instance.args + instance.args = (Parameters(args, [ARG_POS] * len(args), [None] * len(args)),) # Check type argument count. if len(instance.args) != len(info.type_vars) and not self.defining_alias: @@ -564,13 +583,17 @@ def visit_deleted_type(self, t: DeletedType) -> Type: def visit_type_list(self, t: TypeList) -> Type: # paramspec literal (Z[[int, str, Whatever]]) - # TODO: invalid usage restrictions - params = self.analyze_callable_args(t) - if params: - ts, kinds, names = params - # bind these types - return Parameters(self.anal_array(ts), kinds, names) + if self.allow_param_spec_literals: + params = self.analyze_callable_args(t) + if params: + ts, kinds, names = params + # bind these types + return Parameters(self.anal_array(ts), kinds, names) + else: + return AnyType(TypeOfAny.from_error) else: + self.fail('Bracketed expression "[...]" is not valid as a type', t) + self.note('Did you mean "List[...]"?', t) return AnyType(TypeOfAny.from_error) def visit_callable_argument(self, t: CallableArgument) -> Type: @@ -1106,12 +1129,15 @@ def anal_type(self, t: Type, nested: bool = True, *, allow_param_spec: bool = Fa if (not allow_param_spec and isinstance(analyzed, ParamSpecType) and analyzed.flavor == ParamSpecFlavor.BARE): - self.fail('Invalid location for ParamSpec "{}"'.format(analyzed.name), t) - self.note( - 'You can use ParamSpec as the first argument to Callable, e.g., ' - "'Callable[{}, int]'".format(analyzed.name), - t - ) + if analyzed.prefix.arg_types: + self.fail('Invalid location for Concatenate', t) + else: + self.fail('Invalid location for ParamSpec "{}"'.format(analyzed.name), t) + self.note( + 'You can use ParamSpec as the first argument to Callable, e.g., ' + "'Callable[{}, int]'".format(analyzed.name), + t + ) return analyzed def anal_var_def(self, var_def: TypeVarLikeType) -> TypeVarLikeType: @@ -1156,6 +1182,15 @@ def tuple_type(self, items: List[Type]) -> TupleType: any_type = AnyType(TypeOfAny.special_form) return TupleType(items, fallback=self.named_type('builtins.tuple', [any_type])) + @contextmanager + def set_allow_param_spec_literals(self, to: bool) -> Iterator[None]: + old = self.allow_param_spec_literals + try: + self.allow_param_spec_literals = to + yield + finally: + self.allow_param_spec_literals = old + TypeVarLikeList = List[Tuple[str, TypeVarLikeExpr]] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index dff85960ed64..7c30cbab4fd5 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -942,9 +942,8 @@ from typing_extensions import Literal a: (1, 2, 3) # E: Syntax error in type annotation \ # N: Suggestion: Use Tuple[T1, ..., Tn] instead of (T1, ..., Tn) b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid -# TODO: fix this -# c: [1, 2, 3] $ E: Bracketed expression "[...]" is not valid as a type \ -# $ N: Did you mean "List[...]"? +c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 6e4010132ff0..6a9e0c01552f 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -15,9 +15,7 @@ def foo1(x: Callable[P, int]) -> Callable[P, str]: ... def foo2(x: P) -> P: ... # E: Invalid location for ParamSpec "P" \ # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' -# TODO: Better error message -def foo3(x: Concatenate[int, P]) -> int: ... # E: Invalid location for ParamSpec "P" \ - # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' +def foo3(x: Concatenate[int, P]) -> int: ... # E: Invalid location for Concatenate def foo4(x: List[P]) -> None: ... # E: Invalid location for ParamSpec "P" \ # N: You can use ParamSpec as the first argument to Callable, e.g., 'Callable[P, int]' @@ -420,17 +418,17 @@ class Z(Generic[P]): ... # literals can be applied n: Z[[int]] -# type aliases too -nt1 = Z[[int]] -nt2: TypeAlias = Z[[int]] +# TODO: type aliases too +# nt1 = Z[[int]] +# nt2: TypeAlias = Z[[int]] -unt1: nt1 -unt2: nt2 +# unt1: nt1 +# unt2: nt2 # literals actually keep types reveal_type(n) # N: Revealed type is "__main__.Z[[builtins.int]]" -reveal_type(unt1) # N: Revealed type is "__main__.Z[[builtins.int]]" -reveal_type(unt2) # N: Revealed type is "__main__.Z[[builtins.int]]" +# reveal_type(unt1) $ N: Revealed type is "__main__.Z[[builtins.int]]" +# reveal_type(unt2) $ N: Revealed type is "__main__.Z[[builtins.int]]" # passing into a function keeps the type def fT(a: T) -> T: ... @@ -453,10 +451,9 @@ reveal_type(kb(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" \ # E: Argument 1 to "kb" has incompatible type "Z[[int]]"; expected "Z[[bytes]]" -# TODO(PEP612): fancy "aesthetic" syntax defined in PEP -# n2: Z[bytes] -# -# reveal_type(kb(n2)) $ N: Revealed type is "__main__.Z[[builtins.str]]" +n2: Z[bytes] + +reveal_type(kb(n2)) # N: Revealed type is "__main__.Z[[builtins.str]]" [builtins fixtures/tuple.pyi] [case testParamSpecConcatenateFromPep] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index ba8ee6e44307..b9197453bf5f 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -807,9 +807,8 @@ class C(Generic[t]): pass cast(str + str, None) # E: Cast target is not a type cast(C[str][str], None) # E: Cast target is not a type cast(C[str + str], None) # E: Cast target is not a type -# TODO: fix this -# cast([int, str], None) $ E: Bracketed expression "[...]" is not valid as a type \ -# $ N: Did you mean "List[...]"? +cast([int, str], None) # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? [out] [case testInvalidCastTargetType] @@ -844,15 +843,14 @@ cast(str, target=None) # E: "cast" must be called with 2 positional arguments from typing import Any Any(str, None) # E: Any(...) is no longer supported. Use cast(Any, ...) instead Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead -#[out] - -# TODO: fix this -# [case testTypeListAsType] -# -# def f(x:[int, str]) -> None: $ E: Bracketed expression "[...]" is not valid as a type \ -# $ N: Did you mean "List[...]"? -# pass -# [out] +[out] + +[case testTypeListAsType] + +def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? + pass +[out] [case testInvalidFunctionType] from typing import Callable From d9dcc76b1b084e7224d23a966c3b03dec7cab28e Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 28 Dec 2021 12:33:55 +0900 Subject: [PATCH 12/41] Add more tests --- mypy/messages.py | 81 +++++++++++++------ mypy/types.py | 8 +- .../unit/check-parameter-specification.test | 50 ++++++++++++ 3 files changed, 113 insertions(+), 26 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 9e4466ebc675..bd1192b657b2 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -15,7 +15,9 @@ import difflib from textwrap import dedent -from typing import cast, List, Dict, Any, Sequence, Iterable, Iterator, Tuple, Set, Optional, Union +from typing import ( + cast, List, Dict, Any, Sequence, Iterable, Iterator, Tuple, Set, Optional, Union, Callable +) from typing_extensions import Final from mypy.erasetype import erase_type @@ -1646,6 +1648,32 @@ def quote_type_string(type_string: str) -> str: return '"{}"'.format(type_string) +def format_callable_args(arg_types: List[Type], arg_kinds: List[ArgKind], + arg_names: List[Optional[str]], format: Callable[[Type], str], + verbosity: int) -> str: + """Format a bunch of Callable arguments into a string""" + arg_strings = [] + for arg_name, arg_type, arg_kind in zip( + arg_names, arg_types, arg_kinds): + if (arg_kind == ARG_POS and arg_name is None + or verbosity == 0 and arg_kind.is_positional()): + + arg_strings.append(format(arg_type)) + else: + constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] + if arg_kind.is_star() or arg_name is None: + arg_strings.append("{}({})".format( + constructor, + format(arg_type))) + else: + arg_strings.append("{}({}, {})".format( + constructor, + format(arg_type), + repr(arg_name))) + + return ", ".join(arg_strings) + + def format_type_inner(typ: Type, verbosity: int, fullnames: Optional[Set[str]]) -> str: @@ -1694,7 +1722,18 @@ def format(typ: Type) -> str: # This is similar to non-generic instance types. return typ.name elif isinstance(typ, ParamSpecType): - return typ.name_with_suffix() + # Concatenate[..., P] + if typ.prefix.arg_types: + args = format_callable_args( + typ.prefix.arg_types, + typ.prefix.arg_kinds, + typ.prefix.arg_names, + format, + verbosity) + + return f'Concatenate[{args}, {typ.name_with_suffix()}]' + else: + return typ.name_with_suffix() elif isinstance(typ, TupleType): # Prefer the name of the fallback class (if not tuple), as it's more informative. if typ.partial_fallback.type.fullname != 'builtins.tuple': @@ -1764,27 +1803,14 @@ def format(typ: Type) -> str: return 'Callable[..., {}]'.format(return_type) param_spec = func.param_spec() if param_spec is not None: - return f'Callable[{param_spec.name}, {return_type}]' - arg_strings = [] - for arg_name, arg_type, arg_kind in zip( - func.arg_names, func.arg_types, func.arg_kinds): - if (arg_kind == ARG_POS and arg_name is None - or verbosity == 0 and arg_kind.is_positional()): - - arg_strings.append(format(arg_type)) - else: - constructor = ARG_CONSTRUCTOR_NAMES[arg_kind] - if arg_kind.is_star() or arg_name is None: - arg_strings.append("{}({})".format( - constructor, - format(arg_type))) - else: - arg_strings.append("{}({}, {})".format( - constructor, - format(arg_type), - repr(arg_name))) - - return 'Callable[[{}], {}]'.format(", ".join(arg_strings), return_type) + return f'Callable[{format(param_spec)}, {return_type}]' + args = format_callable_args( + func.arg_types, + func.arg_kinds, + func.arg_names, + format, + verbosity) + return 'Callable[[{}], {}]'.format(args, return_type) else: # Use a simple representation for function types; proper # function types may result in long and difficult-to-read @@ -1793,8 +1819,13 @@ def format(typ: Type) -> str: elif isinstance(typ, UnboundType): return str(typ) elif isinstance(typ, Parameters): - # TODO: technically this is not the right way to format (there could be non-pos). - return '[{}]'.format(', '.join(map(format, typ.arg_types))) + args = format_callable_args( + typ.arg_types, + typ.arg_kinds, + typ.arg_names, + format, + verbosity) + return f'[{args}]' elif typ is None: raise RuntimeError('Type is None') else: diff --git a/mypy/types.py b/mypy/types.py index 101d3a312269..fad52f546460 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1511,8 +1511,14 @@ def param_spec(self) -> Optional[ParamSpecType]: arg_type = self.arg_types[-2] if not isinstance(arg_type, ParamSpecType): return None + # sometimes paramspectypes are analyzed in from mysterious places, + # e.g. def f(prefix..., *args: P.args, **kwargs: P.kwargs) -> ...: ... + prefix = arg_type.prefix + if not prefix.arg_types: + # TODO: confirm that all arg kinds are positional + prefix = Parameters(self.arg_types[:-2], self.arg_kinds[:-2], self.arg_names[:-2]) return ParamSpecType(arg_type.name, arg_type.fullname, arg_type.id, ParamSpecFlavor.BARE, - arg_type.upper_bound, prefix=arg_type.prefix) + arg_type.upper_bound, prefix=prefix) def expand_param_spec(self, c: 'CallableType') -> 'CallableType': return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 6a9e0c01552f..e543aa325a57 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -532,3 +532,53 @@ def three(**kwargs: int) -> int: ... def four(*args: int) -> int: ... [builtins fixtures/tuple.pyi] [builtins fixtures/dict.pyi] + +[case testParamSpecTwiceSolving] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f(one: Callable[Concatenate[int, P], R], two: Callable[Concatenate[str, P], R]) -> Callable[P, R]: ... + +a: Callable[[int, bytes], str] +b: Callable[[str, bytes], str] + +reveal_type(f(a, b)) # N: Revealed type is "def (builtins.bytes) -> builtins.str*" +[builtins fixtures/tuple.pyi] + +[case testParamSpecConcatenateInReturn] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, Protocol + +P = ParamSpec("P") + +def f(i: Callable[Concatenate[int, P], str]) -> Callable[Concatenate[int, P], str]: ... + +n: Callable[[int, bytes], str] + +reveal_type(f(n)) # N: Revealed type is "def (builtins.int, builtins.bytes) -> builtins.str" +[builtins fixtures/tuple.pyi] + +[case testParamSpecConcatenateNamedArgs] +# this is one noticeable deviation from PEP but I believe it is for the better +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Accepted + +def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # E: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") + +# reason for rejection: +f2(lambda x: 42)(42, x=42) +[builtins fixtures/tuple.pyi] From 0e2b207b19bebe7a76f84baa015d133a8558256d Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 28 Dec 2021 13:20:37 +0900 Subject: [PATCH 13/41] Allow TypeVars in Concatenate --- mypy/constraints.py | 15 ++++++++++++--- mypy/expandtype.py | 13 ++++++++----- mypy/typeanal.py | 4 +++- mypy/types.py | 13 +++++++++---- .../unit/check-parameter-specification.test | 18 ++++++++++++++++++ 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index ac0e416d57b9..c999becdf566 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -461,7 +461,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if isinstance(suffix, Parameters): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? - # TODO: what is setting meta_level to 0? + # TODO: constraints between prefixes prefix = mapped_arg.prefix suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], suffix.arg_kinds[len(prefix.arg_kinds):], @@ -490,7 +490,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: if isinstance(suffix, Parameters): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? - # TODO: what is setting meta_level to 0? + # TODO: constraints between prefixes prefix = template_arg.prefix suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], suffix.arg_kinds[len(prefix.arg_kinds):], @@ -586,7 +586,6 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: res.extend(infer_constraints(t, a, neg_op(self.direction))) else: # TODO: Direction - # TODO: Deal with arguments that come before param spec ones? # TODO: check the prefixes match prefix = param_spec.prefix prefix_len = len(prefix.arg_types) @@ -597,6 +596,16 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: arg_kinds=cactual.arg_kinds[prefix_len:], arg_names=cactual.arg_names[prefix_len:], ret_type=NoneType()))) + # compare prefixes + cactual_prefix = cactual.copy_modified( + arg_types=cactual.arg_types[:prefix_len], + arg_kinds=cactual.arg_kinds[:prefix_len], + arg_names=cactual.arg_names[:prefix_len]) + + # TODO: see above "FIX" comments for param_spec is None case + # TODO: this assume positional arguments + for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): + res.extend(infer_constraints(t, a, neg_op(self.direction))) template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type if template.type_guard is not None: diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 60e00420ca91..ae73d1469efa 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -137,11 +137,14 @@ def visit_callable_type(self, t: CallableType) -> Type: # the replacement is ignored. if isinstance(repl, CallableType): # Substitute *args: P.args, **kwargs: P.kwargs - t = t.expand_param_spec(repl) - # TODO: Substitute remaining arg types - return t.copy_modified(ret_type=t.ret_type.accept(self), - type_guard=(t.type_guard.accept(self) - if t.type_guard is not None else None)) + prefix = param_spec.prefix + t = t.expand_param_spec(repl, no_prefix=True) + return t.copy_modified( + arg_types=self.expand_types(prefix.arg_types) + t.arg_types, + arg_kinds=prefix.arg_kinds + t.arg_kinds, + arg_names=prefix.arg_names + t.arg_names, + ret_type=t.ret_type.accept(self), + type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None)) return t.copy_modified(arg_types=self.expand_types(t.arg_types), ret_type=t.ret_type.accept(self), diff --git a/mypy/typeanal.py b/mypy/typeanal.py index c283ed7c52be..f30cc476a00a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -847,10 +847,12 @@ def analyze_callable_args_for_concatenate( obj = self.named_type('builtins.object') # ick, CallableType should take ParamSpecType prefix = tvar_def.prefix + # we don't set the prefix here as generic arguments will get updated at some point + # in the future. CallableType.param_spec() accounts for this. return CallableType( [*prefix.arg_types, ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.ARGS, - upper_bound=obj, prefix=tvar_def.prefix), + upper_bound=obj), ParamSpecType(tvar_def.name, tvar_def.fullname, tvar_def.id, ParamSpecFlavor.KWARGS, upper_bound=obj)], [*prefix.arg_kinds, nodes.ARG_STAR, nodes.ARG_STAR2], diff --git a/mypy/types.py b/mypy/types.py index fad52f546460..5556619be5e5 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1520,10 +1520,15 @@ def param_spec(self) -> Optional[ParamSpecType]: return ParamSpecType(arg_type.name, arg_type.fullname, arg_type.id, ParamSpecFlavor.BARE, arg_type.upper_bound, prefix=prefix) - def expand_param_spec(self, c: 'CallableType') -> 'CallableType': - return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, - arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, - arg_names=self.arg_names[:-2] + c.arg_names) + def expand_param_spec(self, c: 'CallableType', no_prefix: bool = False) -> 'CallableType': + if no_prefix: + return self.copy_modified(arg_types=c.arg_types, + arg_kinds=c.arg_kinds, + arg_names=c.arg_names) + else: + return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, + arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, + arg_names=self.arg_names[:-2] + c.arg_names) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e543aa325a57..eca2080956f4 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -582,3 +582,21 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: # reason for rejection: f2(lambda x: 42)(42, x=42) [builtins fixtures/tuple.pyi] + +[case testParamSpecConcatenateWithTypeVar] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") +S = TypeVar("S") + +def f(c: Callable[Concatenate[S, P], R]) -> Callable[Concatenate[S, P], R]: ... + +def a(n: int) -> None: ... + +n = f(a) + +reveal_type(n) # N: Revealed type is "def (builtins.int*)" +reveal_type(n(42)) # N: Revealed type is "None" +[builtins fixtures/tuple.pyi] From bd445e5105ad2c96ef29dd8dd135673b780aebd2 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 28 Dec 2021 14:07:01 +0900 Subject: [PATCH 14/41] Fix a couple of dumb oversights --- mypy/typeanal.py | 1 - test-data/unit/check-parameter-specification.test | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index f30cc476a00a..846335252e09 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -291,7 +291,6 @@ def apply_concatenate_operator(self, t: UnboundType) -> Type: # last argument has to be ParamSpec (or Concatenate) ps = self.anal_type(t.args[-1], allow_param_spec=True) if not isinstance(ps, ParamSpecType): - print(ps) self.api.fail('The last parameter to Concatenate needs to be a ParamSpec', t) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index eca2080956f4..62a89c0ed376 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -569,10 +569,11 @@ from typing import Callable, TypeVar P = ParamSpec("P") R = TypeVar("R") -def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: - def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... - - return result # Accepted +# TODO: figure out how to only run this under >= 3.8 +# def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: +# def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... +# +# return result # Accepted def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... From 604c3040bb6d994b13ae636283f668c6b31ccce5 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 28 Dec 2021 15:00:45 +0900 Subject: [PATCH 15/41] Allow Callables along with Parameters --- mypy/constraints.py | 4 ++-- mypy/expandtype.py | 2 +- .../unit/check-parameter-specification.test | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index c999becdf566..0113e70eb520 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -458,7 +458,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mapped_arg, instance_arg, neg_op(self.direction))) elif isinstance(tvar, ParamSpecType) and isinstance(mapped_arg, ParamSpecType): suffix = get_proper_type(instance_arg) - if isinstance(suffix, Parameters): + if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: constraints between prefixes @@ -487,7 +487,7 @@ def visit_instance(self, template: Instance) -> List[Constraint]: elif (isinstance(tvar, ParamSpecType) and isinstance(template_arg, ParamSpecType)): suffix = get_proper_type(mapped_arg) - if isinstance(suffix, Parameters): + if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: constraints between prefixes diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ae73d1469efa..6a526c4a73cc 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -112,7 +112,7 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: # TODO: what if both have prefixes??? # (realistically, `repl` is the unification variable for `t` so this is fine) return repl.copy_modified(flavor=t.flavor, prefix=t.prefix) - elif isinstance(repl, Parameters): + elif isinstance(repl, Parameters) or isinstance(repl, CallableType): return repl.copy_modified(t.prefix.arg_types + repl.arg_types, t.prefix.arg_kinds + repl.arg_kinds, t.prefix.arg_names + repl.arg_names) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 62a89c0ed376..4658b5693a2b 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -601,3 +601,20 @@ n = f(a) reveal_type(n) # N: Revealed type is "def (builtins.int*)" reveal_type(n(42)) # N: Revealed type is "None" [builtins fixtures/tuple.pyi] + +[case testCallablesAsParameters] +# credits to https://github.com/microsoft/pyright/issues/2705 +from typing import ParamSpec, Concatenate, Generic, Callable, Any + +P = ParamSpec("P") + +class Foo(Generic[P]): + def __init__(self, func: Callable[P, Any]) -> None: ... +def bar(baz: Foo[Concatenate[int, P]]) -> Foo[P]: ... + +def test(a: int, b: str) -> str: ... + +abc = Foo(test) +reveal_type(abc) +bar(abc) +[builtins fixtures/tuple.pyi] From f8004ec121565aa977cf7c014e38403d2fcb7774 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 28 Dec 2021 15:33:01 +0900 Subject: [PATCH 16/41] Fix tests --- test-data/unit/check-parameter-specification.test | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 4658b5693a2b..7511de654cb7 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -604,7 +604,8 @@ reveal_type(n(42)) # N: Revealed type is "None" [case testCallablesAsParameters] # credits to https://github.com/microsoft/pyright/issues/2705 -from typing import ParamSpec, Concatenate, Generic, Callable, Any +from typing_extensions import ParamSpec, Concatenate +from typing import Generic, Callable, Any P = ParamSpec("P") @@ -612,9 +613,10 @@ class Foo(Generic[P]): def __init__(self, func: Callable[P, Any]) -> None: ... def bar(baz: Foo[Concatenate[int, P]]) -> Foo[P]: ... -def test(a: int, b: str) -> str: ... +# TODO: how to mark this as py>=3.8 +# def test(a: int, /, b: str) -> str: ... -abc = Foo(test) -reveal_type(abc) -bar(abc) +#abc = Foo(test) +#reveal_type(abc) +#bar(abc) [builtins fixtures/tuple.pyi] From 9e75481f92e3cfb5f5e877af0eec04e9bb065ecf Mon Sep 17 00:00:00 2001 From: A5rocks Date: Wed, 29 Dec 2021 12:22:34 +0900 Subject: [PATCH 17/41] Misc changes --- mypy/checker.py | 2 +- mypy/expandtype.py | 3 +- mypy/semanal.py | 1 + mypy/subtypes.py | 3 +- mypy/typeanal.py | 6 ++-- mypy/types.py | 2 ++ .../unit/check-parameter-specification.test | 28 +++++++++++-------- 7 files changed, 27 insertions(+), 18 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index f4e3da56acab..b90221a0a5a5 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4926,7 +4926,7 @@ def check_subtype(self, if subtype_label is not None: extra_info.append(subtype_label + ' ' + subtype_str) if supertype_label is not None: - extra_info.append(f'{supertype_label} {supertype_str}') + extra_info.append(supertype_label + ' ' + supertype_str) note_msg = make_inferred_type_note(outer_context or context, subtype, supertype, supertype_str) if isinstance(subtype, Instance) and isinstance(supertype, Instance): diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 6a526c4a73cc..6cbae62149d7 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -117,7 +117,6 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: t.prefix.arg_kinds + repl.arg_kinds, t.prefix.arg_names + repl.arg_names) else: - # returning parameters # TODO: should this branch be removed? better not to fail silently return repl @@ -138,6 +137,8 @@ def visit_callable_type(self, t: CallableType) -> Type: if isinstance(repl, CallableType): # Substitute *args: P.args, **kwargs: P.kwargs prefix = param_spec.prefix + # we need to expand the types in the prefix, so might as well + # not get them in the first place t = t.expand_param_spec(repl, no_prefix=True) return t.copy_modified( arg_types=self.expand_types(prefix.arg_types) + t.arg_types, diff --git a/mypy/semanal.py b/mypy/semanal.py index bd3cd307655d..3468819e09fc 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1171,6 +1171,7 @@ def analyze_class(self, defn: ClassDef) -> None: defn.type_vars = tvar_defs defn.info.type_vars = [] + # we want to make sure any additional logic in add_type_vars gets run defn.info.add_type_vars() if base_error: defn.info.fallback_to_any = True diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 91e4eeb7026f..78e4cb0b316b 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1390,10 +1390,9 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: def visit_parameters(self, left: Parameters) -> bool: right = self.right - if isinstance(right, Parameters): + if isinstance(right, Parameters) or isinstance(right, CallableType): return are_parameters_compatible(left, right, is_compat=self._is_proper_subtype) else: - # TODO: should this work against callables too? return False def visit_callable_type(self, left: CallableType) -> bool: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 846335252e09..bf4d0235b781 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -267,8 +267,8 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) elif node.fullname in ("typing_extensions.TypeAlias", "typing.TypeAlias"): return AnyType(TypeOfAny.special_form) # Concatenate is an operator, no need for a proper type - elif node.fullname in ("typing_extensions.Concatenate", "typing.Concatenate"): - # TODO: detect valid locations (`allow_param_spec` is not here.) + elif node.fullname in ('typing_extensions.Concatenate', 'typing.Concatenate'): + # We check the return type further up the stack for valid use locations return self.apply_concatenate_operator(t) else: return self.analyze_unbound_type_without_type_info(t, sym, defining_literal) @@ -835,7 +835,7 @@ def analyze_callable_args_for_concatenate( return None if sym.node is None: return None - if sym.node.fullname not in ("typing_extensions.Concatenate", "typing.Concatenate"): + if sym.node.fullname not in ('typing_extensions.Concatenate', 'typing.Concatenate'): return None tvar_def = self.anal_type(callable_args, allow_param_spec=True) diff --git a/mypy/types.py b/mypy/types.py index 5556619be5e5..6cd73324eac2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1116,6 +1116,7 @@ def copy_modified(self, arg_names=arg_names if arg_names is not _dummy else self.arg_names, ) + # the following are copied from CallableType. Is there a way to decrease code duplication? def var_arg(self) -> Optional[FormalArgument]: """The formal argument for *args.""" for position, (type, kind) in enumerate(zip(self.arg_types, self.arg_kinds)): @@ -2410,6 +2411,7 @@ def visit_param_spec(self, t: ParamSpecType) -> str: return s def visit_parameters(self, t: Parameters) -> str: + # This is copied from visit_callable -- is there a way to decrease duplication? s = '' bare_asterisk = False for i in range(len(t.arg_types)): diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 7511de654cb7..b6158c283fce 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -569,20 +569,23 @@ from typing import Callable, TypeVar P = ParamSpec("P") R = TypeVar("R") -# TODO: figure out how to only run this under >= 3.8 -# def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: -# def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... -# -# return result # Accepted +def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Accepted def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... - return result # E: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") + return result # Rejected # reason for rejection: f2(lambda x: 42)(42, x=42) [builtins fixtures/tuple.pyi] +[out] +main:9: error: invalid syntax +[out version>=3.8] +main:16: error: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") [case testParamSpecConcatenateWithTypeVar] from typing_extensions import ParamSpec, Concatenate @@ -613,10 +616,13 @@ class Foo(Generic[P]): def __init__(self, func: Callable[P, Any]) -> None: ... def bar(baz: Foo[Concatenate[int, P]]) -> Foo[P]: ... -# TODO: how to mark this as py>=3.8 -# def test(a: int, /, b: str) -> str: ... +def test(a: int, /, b: str) -> str: ... -#abc = Foo(test) -#reveal_type(abc) -#bar(abc) +abc = Foo(test) +reveal_type(abc) +bar(abc) [builtins fixtures/tuple.pyi] +[out] +main:11: error: invalid syntax +[out version>=3.8] +main:14: note: Revealed type is "__main__.Foo[def (builtins.int, b: builtins.str)]" From 7b89f0626c1d5cf1231862b820f4c6ec5fd19c9b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 1 Jan 2022 14:14:22 +0900 Subject: [PATCH 18/41] Solve with self types --- mypy/expandtype.py | 2 +- mypy/meet.py | 8 +++++++- mypy/types.py | 4 +++- test-data/unit/check-parameter-specification.test | 15 +++++++++++++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 6cbae62149d7..ca615135a425 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -134,7 +134,7 @@ def visit_callable_type(self, t: CallableType) -> Type: # must expand both of them with all the argument types, # kinds and names in the replacement. The return type in # the replacement is ignored. - if isinstance(repl, CallableType): + if isinstance(repl, CallableType) or isinstance(repl, Parameters): # Substitute *args: P.args, **kwargs: P.kwargs prefix = param_spec.prefix # we need to expand the types in the prefix, so might as well diff --git a/mypy/meet.py b/mypy/meet.py index a3ab78290157..90f09dc00332 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -507,7 +507,13 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType: return self.default(self.s) def visit_parameters(self, t: Parameters) -> ProperType: - raise NotImplementedError("meeting two paramspec literals is not supported yet") + # TODO: is this the right variance? + if isinstance(self.s, Parameters) or isinstance(self.s, CallableType): + if len(t.arg_types) != len(self.s.arg_types): + return self.default(self.s) + return t.copy_modified( + arg_types=[meet_types(s_a, t_a) for s_a, t_a in zip(self.s.arg_types, t.arg_types)] + ) def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): diff --git a/mypy/types.py b/mypy/types.py index 6cd73324eac2..699cddce9c18 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1521,7 +1521,9 @@ def param_spec(self) -> Optional[ParamSpecType]: return ParamSpecType(arg_type.name, arg_type.fullname, arg_type.id, ParamSpecFlavor.BARE, arg_type.upper_bound, prefix=prefix) - def expand_param_spec(self, c: 'CallableType', no_prefix: bool = False) -> 'CallableType': + def expand_param_spec(self, + c: Union['CallableType', Parameters], + no_prefix: bool = False) -> 'CallableType': if no_prefix: return self.copy_modified(arg_types=c.arg_types, arg_kinds=c.arg_kinds, diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index b6158c283fce..c38c73ce675e 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -626,3 +626,18 @@ bar(abc) main:11: error: invalid syntax [out version>=3.8] main:14: note: Revealed type is "__main__.Foo[def (builtins.int, b: builtins.str)]" + +[case testSolveLiteralsWithSelfType] +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, Generic + +P = ParamSpec("P") + +class Foo(Generic[P]): + def foo(self: 'Foo[P]', other: Callable[P, None]) -> None: ... + +n: Foo[[int]] +def f(x: int) -> None: ... + +n.foo(f) +[builtins fixtures/tuple.pyi] From f24cf4f862d10e644bbf5e5b9c1f434decba72f4 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 1 Jan 2022 14:50:15 +0900 Subject: [PATCH 19/41] Add fallback return to meeting paramspec literals --- mypy/meet.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/meet.py b/mypy/meet.py index 90f09dc00332..8a4837633d83 100644 --- a/mypy/meet.py +++ b/mypy/meet.py @@ -514,6 +514,8 @@ def visit_parameters(self, t: Parameters) -> ProperType: return t.copy_modified( arg_types=[meet_types(s_a, t_a) for s_a, t_a in zip(self.s.arg_types, t.arg_types)] ) + else: + return self.default(self.s) def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): From 472b20ce7bd5050410ad96dda03b31442d1e6c9d Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 3 Jan 2022 10:30:07 +0900 Subject: [PATCH 20/41] Type application of ParamSpec literals --- mypy/semanal.py | 38 +++++++++++++- mypy/typeanal.py | 3 +- .../unit/check-parameter-specification.test | 50 +++++++++++++++---- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 3468819e09fc..e6d484c9ff4e 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -92,7 +92,7 @@ FunctionLike, UnboundType, TypeVarType, TupleType, UnionType, StarType, CallableType, Overloaded, Instance, Type, AnyType, LiteralType, LiteralValue, TypeTranslator, TypeOfAny, TypeType, NoneType, PlaceholderType, TPDICT_NAMES, ProperType, - get_proper_type, get_proper_types, TypeAliasType, TypeVarLikeType + get_proper_type, get_proper_types, TypeAliasType, TypeVarLikeType, Parameters, ParamSpecType ) from mypy.typeops import function_type, get_type_vars from mypy.type_visitor import TypeQuery @@ -4113,6 +4113,27 @@ def analyze_type_application_args(self, expr: IndexExpr) -> Optional[List[Type]] items = items[:-1] else: items = [index] + + # whether param spec literals be allowed here + # TODO: should this be computed once and passed in? + # or is there a better way to do this? + base = expr.base + if isinstance(base, RefExpr) and isinstance(base.node, TypeAlias): + alias = base.node + target = get_proper_type(alias.target) + if isinstance(target, Instance): + has_param_spec = target.type.has_param_spec_type + num_args = len(target.type.type_vars) + else: + has_param_spec = False + num_args = -1 + elif isinstance(base, NameExpr) and isinstance(base.node, TypeInfo): + has_param_spec = base.node.has_param_spec_type + num_args = len(base.node.type_vars) + else: + has_param_spec = False + num_args = -1 + for item in items: try: typearg = self.expr_to_unanalyzed_type(item) @@ -4123,10 +4144,19 @@ def analyze_type_application_args(self, expr: IndexExpr) -> Optional[List[Type]] # may be analysing a type alias definition rvalue. The error will be # reported elsewhere if it is not the case. analyzed = self.anal_type(typearg, allow_unbound_tvars=True, - allow_placeholder=True) + allow_placeholder=True, + allow_param_spec_literals=has_param_spec) if analyzed is None: return None types.append(analyzed) + + if has_param_spec and num_args == 1 and len(types) > 0: + first_arg = get_proper_type(types[0]) + if not (len(types) == 1 and (isinstance(first_arg, Parameters) or + isinstance(first_arg, ParamSpecType) or + isinstance(first_arg, AnyType))): + types = [Parameters(types, [ARG_POS] * len(types), [None] * len(types))] + return types def visit_slice_expr(self, expr: SliceExpr) -> None: @@ -5173,6 +5203,7 @@ def type_analyzer(self, *, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, allow_required: bool = False, + allow_param_spec_literals: bool = False, report_invalid_types: bool = True) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope @@ -5186,6 +5217,7 @@ def type_analyzer(self, *, report_invalid_types=report_invalid_types, allow_placeholder=allow_placeholder, allow_required=allow_required, + allow_param_spec_literals=allow_param_spec_literals, allow_new_syntax=self.is_stub_file) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) tpan.global_scope = not self.type and not self.function_stack @@ -5201,6 +5233,7 @@ def anal_type(self, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, allow_required: bool = False, + allow_param_spec_literals: bool = False, report_invalid_types: bool = True, third_pass: bool = False) -> Optional[Type]: """Semantically analyze a type. @@ -5228,6 +5261,7 @@ def anal_type(self, allow_tuple_literal=allow_tuple_literal, allow_placeholder=allow_placeholder, allow_required=allow_required, + allow_param_spec_literals=allow_param_spec_literals, report_invalid_types=report_invalid_types) tag = self.track_incomplete_refs() typ = typ.accept(a) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index bf4d0235b781..d89689c4cb62 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -131,6 +131,7 @@ def __init__(self, allow_unbound_tvars: bool = False, allow_placeholder: bool = False, allow_required: bool = False, + allow_param_spec_literals: bool = False, report_invalid_types: bool = True) -> None: self.api = api self.lookup_qualified = api.lookup_qualified @@ -153,7 +154,7 @@ def __init__(self, # Are we in a context where Required[] is allowed? self.allow_required = allow_required # Are we in a context where ParamSpec literals are allowed? - self.allow_param_spec_literals = False + self.allow_param_spec_literals = allow_param_spec_literals # Should we report an error whenever we encounter a RawExpressionType outside # of a Literal context: e.g. whenever we encounter an invalid type? Normally, # we want to report an error, but the caller may want to do more specialized diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index c38c73ce675e..489da1937444 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -419,16 +419,16 @@ class Z(Generic[P]): ... n: Z[[int]] # TODO: type aliases too -# nt1 = Z[[int]] -# nt2: TypeAlias = Z[[int]] +nt1 = Z[[int]] +nt2: TypeAlias = Z[[int]] -# unt1: nt1 -# unt2: nt2 +unt1: nt1 +unt2: nt2 # literals actually keep types reveal_type(n) # N: Revealed type is "__main__.Z[[builtins.int]]" -# reveal_type(unt1) $ N: Revealed type is "__main__.Z[[builtins.int]]" -# reveal_type(unt2) $ N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt1) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(unt2) # N: Revealed type is "__main__.Z[[builtins.int]]" # passing into a function keeps the type def fT(a: T) -> T: ... @@ -494,8 +494,8 @@ class X(Generic[T, P]): def f1(x: X[int, P_2]) -> str: ... # Accepted def f2(x: X[int, Concatenate[int, P_2]]) -> str: ... # Accepted def f3(x: X[int, [int, bool]]) -> str: ... # Accepted -# Is ellipsis allowed by PEP? This shows up: -# def f4(x: X[int, ...]) -> str: ... # Accepted +# ellipsis only show up here, but I can assume it works like Callable[..., R] +def f4(x: X[int, ...]) -> str: ... # Accepted # TODO: this is not rejected: # def f5(x: X[int, int]) -> str: ... # Rejected @@ -627,7 +627,7 @@ main:11: error: invalid syntax [out version>=3.8] main:14: note: Revealed type is "__main__.Foo[def (builtins.int, b: builtins.str)]" -[case testSolveLiteralsWithSelfType] +[case testSolveParamSpecWithSelfType] from typing_extensions import ParamSpec, Concatenate from typing import Callable, Generic @@ -641,3 +641,35 @@ def f(x: int) -> None: ... n.foo(f) [builtins fixtures/tuple.pyi] + +[case testParamSpecLiteralsTypeApplication] +from typing_extensions import ParamSpec +from typing import Generic, Callable + +P = ParamSpec("P") + +class Z(Generic[P]): + def __init__(self, c: Callable[P, None]) -> None: + ... + +# it allows valid functions +reveal_type(Z[[int]](lambda x: None)) # N: Revealed type is "__main__.Z[[builtins.int]]" +reveal_type(Z[[]](lambda: None)) # N: Revealed type is "__main__.Z[[]]" +reveal_type(Z[bytes, str](lambda b, s: None)) # N: Revealed type is "__main__.Z[[builtins.bytes, builtins.str]]" + +# it disallows invalid functions +def f1(n: str) -> None: ... +def f2(b: bytes, i: int) -> None: ... + +Z[[int]](lambda one, two: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any, Any], None]"; expected "Callable[[int], None]" +Z[[int]](f1) # E: Argument 1 to "Z" has incompatible type "Callable[[str], None]"; expected "Callable[[int], None]" + +Z[[]](lambda one: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any], None]"; expected "Callable[[], None]" + +Z[bytes, str](lambda one: None) # E: Cannot infer type of lambda \ + # E: Argument 1 to "Z" has incompatible type "Callable[[Any], None]"; expected "Callable[[bytes, str], None]" +Z[bytes, str](f2) # E: Argument 1 to "Z" has incompatible type "Callable[[bytes, int], None]"; expected "Callable[[bytes, str], None]" + +[builtins fixtures/tuple.pyi] From 14ecfb9753514e4e3448bf8bb5345044b6015af1 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 3 Jan 2022 11:06:29 +0900 Subject: [PATCH 21/41] Ellipsis paramspec literals --- mypy/subtypes.py | 6 ++-- mypy/typeanal.py | 11 +++++-- mypy/types.py | 23 ++++++++++--- .../unit/check-parameter-specification.test | 32 +++++++++++++++++++ 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 78e4cb0b316b..f078b8e9530b 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -929,9 +929,6 @@ def g(x: int) -> int: ... if check_args_covariantly: is_compat = flip_compat_check(is_compat) - if right.is_ellipsis_args: - return True - return are_parameters_compatible(left, right, is_compat=is_compat, ignore_pos_arg_names=ignore_pos_arg_names, check_args_covariantly=check_args_covariantly, @@ -945,6 +942,9 @@ def are_parameters_compatible(left: Union[Parameters, CallableType], ignore_pos_arg_names: bool = False, check_args_covariantly: bool = False, allow_partial_overlap: bool = False) -> bool: + if right.is_ellipsis_args: + return True + """Helper function for is_callable_compatible, used for Parameter compatibility""" left_star = left.var_arg() left_star2 = left.kw_arg() diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d89689c4cb62..b45a75e4d5eb 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -775,8 +775,15 @@ def visit_partial_type(self, t: PartialType) -> Type: assert False, "Internal error: Unexpected partial type" def visit_ellipsis_type(self, t: EllipsisType) -> Type: - self.fail('Unexpected "..."', t) - return AnyType(TypeOfAny.from_error) + if self.allow_param_spec_literals: + any_type = AnyType(TypeOfAny.explicit) + return Parameters([any_type, any_type], + [ARG_STAR, ARG_STAR2], + [None, None], + is_ellipsis_args=True) + else: + self.fail('Unexpected "..."', t) + return AnyType(TypeOfAny.from_error) def visit_type_type(self, t: TypeType) -> Type: return TypeType.make_normalized(self.anal_type(t.item), line=t.line) diff --git a/mypy/types.py b/mypy/types.py index 699cddce9c18..6bbba5318287 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1092,28 +1092,36 @@ class Parameters(ProperType): __slots__ = ('arg_types', 'arg_kinds', 'arg_names', - 'min_args') + 'min_args', + 'is_ellipsis_args') def __init__(self, arg_types: Sequence[Type], arg_kinds: List[ArgKind], arg_names: Sequence[Optional[str]], + *, + is_ellipsis_args: bool = False ) -> None: self.arg_types = list(arg_types) self.arg_kinds = arg_kinds self.arg_names = list(arg_names) - self.min_args = arg_kinds.count(ARG_POS) assert len(arg_types) == len(arg_kinds) == len(arg_names) + self.min_args = arg_kinds.count(ARG_POS) + self.is_ellipsis_args = is_ellipsis_args def copy_modified(self, arg_types: Bogus[Sequence[Type]] = _dummy, arg_kinds: Bogus[List[ArgKind]] = _dummy, - arg_names: Bogus[Sequence[Optional[str]]] = _dummy + arg_names: Bogus[Sequence[Optional[str]]] = _dummy, + *, + is_ellipsis_args: Bogus[bool] = _dummy ) -> 'Parameters': return Parameters( arg_types=arg_types if arg_types is not _dummy else self.arg_types, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, arg_names=arg_names if arg_names is not _dummy else self.arg_names, + is_ellipsis_args=is_ellipsis_args + if is_ellipsis_args is not _dummy else self.is_ellipsis_args ) # the following are copied from CallableType. Is there a way to decrease code duplication? @@ -1527,11 +1535,13 @@ def expand_param_spec(self, if no_prefix: return self.copy_modified(arg_types=c.arg_types, arg_kinds=c.arg_kinds, - arg_names=c.arg_names) + arg_names=c.arg_names, + is_ellipsis_args=c.is_ellipsis_args) else: return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, - arg_names=self.arg_names[:-2] + c.arg_names) + arg_names=self.arg_names[:-2] + c.arg_names, + is_ellipsis_args=c.is_ellipsis_args) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), @@ -2414,6 +2424,9 @@ def visit_param_spec(self, t: ParamSpecType) -> str: def visit_parameters(self, t: Parameters) -> str: # This is copied from visit_callable -- is there a way to decrease duplication? + if t.is_ellipsis_args: + return '...' + s = '' bare_asterisk = False for i in range(len(t.arg_types)): diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 489da1937444..7bc645e27022 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -673,3 +673,35 @@ Z[bytes, str](lambda one: None) # E: Cannot infer type of lambda \ Z[bytes, str](f2) # E: Argument 1 to "Z" has incompatible type "Callable[[bytes, int], None]"; expected "Callable[[bytes, str], None]" [builtins fixtures/tuple.pyi] + +[case testParamSpecLiteralEllipsis] +from typing_extensions import ParamSpec +from typing import Generic, Callable + +P = ParamSpec("P") + +class Z(Generic[P]): + def __init__(self: 'Z[P]', c: Callable[P, None]) -> None: + ... + +def f1() -> None: ... +def f2(*args: int) -> None: ... +def f3(a: int, *, b: bytes) -> None: ... + +def f4(b: bytes) -> None: ... + +argh: Callable[..., None] = f4 + +# check it works +Z[...](f1) +Z[...](f2) +Z[...](f3) + +# check subtyping works +n: Z[...] +n = Z(f1) +n = Z(f2) +n = Z(f3) + +[builtins fixtures/tuple.pyi] + From 45c80576aa16ae5d42a8fd04299c96cee3f3e6e0 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 3 Jan 2022 11:13:37 +0900 Subject: [PATCH 22/41] Appease flake8 --- mypy/semanal.py | 2 +- mypy/types.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 6d01925ce75e..3dc3e020b6d1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4128,7 +4128,7 @@ def analyze_type_application_args(self, expr: IndexExpr) -> Optional[List[Type]] num_args = len(target.type.type_vars) else: has_param_spec = False - num_args = -1 + num_args = -1 elif isinstance(base, NameExpr) and isinstance(base.node, TypeInfo): has_param_spec = base.node.has_param_spec_type num_args = len(base.node.type_vars) diff --git a/mypy/types.py b/mypy/types.py index 592ecb05fe0c..09d279613ba6 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1146,8 +1146,8 @@ def copy_modified(self, arg_types=arg_types if arg_types is not _dummy else self.arg_types, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, arg_names=arg_names if arg_names is not _dummy else self.arg_names, - is_ellipsis_args=is_ellipsis_args - if is_ellipsis_args is not _dummy else self.is_ellipsis_args + is_ellipsis_args=(is_ellipsis_args if is_ellipsis_args is not _dummy + else self.is_ellipsis_args) ) # the following are copied from CallableType. Is there a way to decrease code duplication? From c46feec706d62d188352826bbc4502e7d8259309 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 9 Jan 2022 13:10:52 +0900 Subject: [PATCH 23/41] Minor code cleanup --- mypy/expandtype.py | 11 ++++------- mypy/types.py | 17 +++++------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ca615135a425..69c2cbda92f5 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -136,14 +136,11 @@ def visit_callable_type(self, t: CallableType) -> Type: # the replacement is ignored. if isinstance(repl, CallableType) or isinstance(repl, Parameters): # Substitute *args: P.args, **kwargs: P.kwargs - prefix = param_spec.prefix - # we need to expand the types in the prefix, so might as well - # not get them in the first place - t = t.expand_param_spec(repl, no_prefix=True) + t = t.expand_param_spec(repl) return t.copy_modified( - arg_types=self.expand_types(prefix.arg_types) + t.arg_types, - arg_kinds=prefix.arg_kinds + t.arg_kinds, - arg_names=prefix.arg_names + t.arg_names, + arg_types=self.expand_types(t.arg_types), + arg_kinds=t.arg_kinds, + arg_names=t.arg_names, ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None)) diff --git a/mypy/types.py b/mypy/types.py index 09d279613ba6..eea7eacd9139 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1556,18 +1556,11 @@ def param_spec(self) -> Optional[ParamSpecType]: arg_type.upper_bound, prefix=prefix) def expand_param_spec(self, - c: Union['CallableType', Parameters], - no_prefix: bool = False) -> 'CallableType': - if no_prefix: - return self.copy_modified(arg_types=c.arg_types, - arg_kinds=c.arg_kinds, - arg_names=c.arg_names, - is_ellipsis_args=c.is_ellipsis_args) - else: - return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, - arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, - arg_names=self.arg_names[:-2] + c.arg_names, - is_ellipsis_args=c.is_ellipsis_args) + c: Union['CallableType', Parameters]) -> 'CallableType': + return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, + arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, + arg_names=self.arg_names[:-2] + c.arg_names, + is_ellipsis_args=c.is_ellipsis_args) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), From 6a9cd71aec89a3aed95ee5e8ad156993f2c4b3f2 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 9 Jan 2022 13:59:25 +0900 Subject: [PATCH 24/41] Error notes and better subtyping for paramspec literals --- mypy/checker.py | 1 + mypy/messages.py | 28 +++++++++++++++++++ mypy/subtypes.py | 8 +++++- mypy/typeanal.py | 1 + mypy/types.py | 9 ++++++ .../unit/check-parameter-specification.test | 4 ++- 6 files changed, 49 insertions(+), 2 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 51aa7a3942ff..01c6660a5675 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4938,6 +4938,7 @@ def check_subtype(self, self.msg.note(note, context, code=code) if note_msg: self.note(note_msg, context, code=code) + self.msg.maybe_note_concatenate_pos_args(subtype, supertype, context, code=code) if (isinstance(supertype, Instance) and supertype.type.is_protocol and isinstance(subtype, (Instance, TupleType, TypedDictType))): self.msg.report_protocol_problems(subtype, supertype, context, code=code) diff --git a/mypy/messages.py b/mypy/messages.py index 2981011279e7..96eecfc33cc1 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -617,6 +617,34 @@ def incompatible_argument_note(self, if call: self.note_call(original_caller_type, call, context, code=code) + self.maybe_note_concatenate_pos_args(original_caller_type, callee_type, context, code) + + + def maybe_note_concatenate_pos_args(self, + original_caller_type: ProperType, + callee_type: ProperType, + context: Context, + code: Optional[ErrorCode] = None) -> None: + # pos-only vs positional can be confusing, with Concatenate + if (isinstance(callee_type, CallableType) and + isinstance(original_caller_type, CallableType) and + (original_caller_type.from_concatenate or callee_type.from_concatenate)): + names = [] + for c, o in zip( + callee_type.formal_arguments(), + original_caller_type.formal_arguments()): + if None in (c.pos, o.pos): + #non-positional + continue + if c.name != o.name and None in (c.name, o.name): + names.append(o.name) + + if names: + missing_arguments = '"' + '", "'.join(names) + '"' + self.note(f'This may be because "{original_caller_type.name}" has arguments ' + f'named: {missing_arguments}', context, code=code) + + def invalid_index_type(self, index_type: Type, expected_type: Type, base_str: str, context: Context, *, code: ErrorCode) -> None: index_str, expected_str = format_type_distinctly(index_type, expected_type) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 5d019fb1949b..d309f0404348 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -329,7 +329,7 @@ def visit_param_spec(self, left: ParamSpecType) -> bool: def visit_parameters(self, left: Parameters) -> bool: right = self.right - if isinstance(right, Parameters): + if isinstance(right, Parameters) or isinstance(right, CallableType): return are_parameters_compatible( left, right, is_compat=self._is_subtype, @@ -365,6 +365,12 @@ def visit_callable_type(self, left: CallableType) -> bool: elif isinstance(right, TypeType): # This is unsound, we don't check the __init__ signature. return left.is_type_obj() and self._is_subtype(left.ret_type, right.item) + elif isinstance(right, Parameters): + # this doesn't check return types.... but is needed for is_equivalent + return are_parameters_compatible( + left, right, + is_compat=self._is_subtype, + ignore_pos_arg_names=self.ignore_pos_arg_names) else: return False diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0a64b171efad..cdf72979a35a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -864,6 +864,7 @@ def analyze_callable_args_for_concatenate( [*prefix.arg_names, None, None], ret_type=ret_type, fallback=fallback, + from_concatenate=True, ) def analyze_callable_type(self, t: UnboundType) -> Type: diff --git a/mypy/types.py b/mypy/types.py index eea7eacd9139..c360da9074f8 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1286,6 +1286,8 @@ class CallableType(FunctionLike): 'def_extras', # Information about original definition we want to serialize. # This is used for more detailed error messages. 'type_guard', # T, if -> TypeGuard[T] (ret_type is bool in this case). + 'from_concatenate', # whether this callable is from a concatenate object + # (this is used for error messages) ) def __init__(self, @@ -1307,6 +1309,7 @@ def __init__(self, bound_args: Sequence[Optional[Type]] = (), def_extras: Optional[Dict[str, Any]] = None, type_guard: Optional[Type] = None, + from_concatenate: bool = False ) -> None: super().__init__(line, column) assert len(arg_types) == len(arg_kinds) == len(arg_names) @@ -1326,6 +1329,7 @@ def __init__(self, self.implicit = implicit self.special_sig = special_sig self.from_type_type = from_type_type + self.from_concatenate = from_concatenate if not bound_args: bound_args = () self.bound_args = bound_args @@ -1368,6 +1372,7 @@ def copy_modified(self, bound_args: Bogus[List[Optional[Type]]] = _dummy, def_extras: Bogus[Dict[str, Any]] = _dummy, type_guard: Bogus[Optional[Type]] = _dummy, + from_concatenate: Bogus[bool] = _dummy, ) -> 'CallableType': return CallableType( arg_types=arg_types if arg_types is not _dummy else self.arg_types, @@ -1388,6 +1393,8 @@ def copy_modified(self, bound_args=bound_args if bound_args is not _dummy else self.bound_args, def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), type_guard=type_guard if type_guard is not _dummy else self.type_guard, + from_concatenate=(from_concatenate if from_concatenate is not _dummy + else self.from_concatenate), ) def var_arg(self) -> Optional[FormalArgument]: @@ -1597,6 +1604,7 @@ def serialize(self) -> JsonDict: for t in self.bound_args], 'def_extras': dict(self.def_extras), 'type_guard': self.type_guard.serialize() if self.type_guard is not None else None, + 'from_concatenate': self.from_concatenate, } @classmethod @@ -1617,6 +1625,7 @@ def deserialize(cls, data: JsonDict) -> 'CallableType': def_extras=data['def_extras'], type_guard=(deserialize_type(data['type_guard']) if data['type_guard'] is not None else None), + from_concatenate=data['from_concatenate'], ) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 7bc645e27022..a9f7209a1b3b 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -519,7 +519,8 @@ reveal_type(transform(bar)) # N: Revealed type is "def (builtins.str, *args: bui # CASE 4 def expects_int_first(x: Callable[Concatenate[int, P], int]) -> None: ... -@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" +@expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[str], int]"; expected "Callable[[int], int]" \ + # N: This may be because "one" has arguments named: "x" def one(x: str) -> int: ... @expects_int_first # E: Argument 1 to "expects_int_first" has incompatible type "Callable[[NamedArg(int, 'x')], int]"; expected "Callable[[int], int]" @@ -586,6 +587,7 @@ f2(lambda x: 42)(42, x=42) main:9: error: invalid syntax [out version>=3.8] main:16: error: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") +main:16: note: This may be because "result" has arguments named: "x" [case testParamSpecConcatenateWithTypeVar] from typing_extensions import ParamSpec, Concatenate From afc1a5765ebfee27244c9f2004e255232be71cef Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sun, 9 Jan 2022 14:09:22 +0900 Subject: [PATCH 25/41] Appease CI --- mypy/messages.py | 8 +++----- mypy/types.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 96eecfc33cc1..81df44737a79 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -619,7 +619,6 @@ def incompatible_argument_note(self, self.maybe_note_concatenate_pos_args(original_caller_type, callee_type, context, code) - def maybe_note_concatenate_pos_args(self, original_caller_type: ProperType, callee_type: ProperType, @@ -629,14 +628,14 @@ def maybe_note_concatenate_pos_args(self, if (isinstance(callee_type, CallableType) and isinstance(original_caller_type, CallableType) and (original_caller_type.from_concatenate or callee_type.from_concatenate)): - names = [] + names: List[str] = [] for c, o in zip( callee_type.formal_arguments(), original_caller_type.formal_arguments()): if None in (c.pos, o.pos): - #non-positional + # non-positional continue - if c.name != o.name and None in (c.name, o.name): + if c.name != o.name and c.name is None and o.name is not None: names.append(o.name) if names: @@ -644,7 +643,6 @@ def maybe_note_concatenate_pos_args(self, self.note(f'This may be because "{original_caller_type.name}" has arguments ' f'named: {missing_arguments}', context, code=code) - def invalid_index_type(self, index_type: Type, expected_type: Type, base_str: str, context: Context, *, code: ErrorCode) -> None: index_str, expected_str = format_type_distinctly(index_type, expected_type) diff --git a/mypy/types.py b/mypy/types.py index c360da9074f8..f96cf1e1bbfb 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1394,7 +1394,7 @@ def copy_modified(self, def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), type_guard=type_guard if type_guard is not _dummy else self.type_guard, from_concatenate=(from_concatenate if from_concatenate is not _dummy - else self.from_concatenate), + else self.from_concatenate), ) def var_arg(self) -> Optional[FormalArgument]: From 86e23c2a30cae75ced26c8f087fb1f41e4371fea Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 27 Jan 2022 11:05:55 +0900 Subject: [PATCH 26/41] Fix something I assumed incorrectly I ran into this while testing an interface with ParamSpec :( --- mypy/expandtype.py | 8 +++-- .../unit/check-parameter-specification.test | 29 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 69c2cbda92f5..021e19851838 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -109,9 +109,11 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: return Instance(inst.type, inst.args, line=inst.line, column=inst.column, erased=True) elif isinstance(repl, ParamSpecType): - # TODO: what if both have prefixes??? - # (realistically, `repl` is the unification variable for `t` so this is fine) - return repl.copy_modified(flavor=t.flavor, prefix=t.prefix) + return repl.copy_modified(flavor=t.flavor, prefix=t.prefix.copy_modified( + arg_types=t.prefix.arg_types + repl.prefix.arg_types, + arg_kinds=t.prefix.arg_kinds + repl.prefix.arg_kinds, + arg_names=t.prefix.arg_names + repl.prefix.arg_names, + )) elif isinstance(repl, Parameters) or isinstance(repl, CallableType): return repl.copy_modified(t.prefix.arg_types + repl.arg_types, t.prefix.arg_kinds + repl.arg_kinds, diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index a9f7209a1b3b..515d25d466be 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -707,3 +707,32 @@ n = Z(f3) [builtins fixtures/tuple.pyi] +[case testParamSpecApplyConcatenateTwice] +from typing_extensions import ParamSpec, Concatenate +from typing import Generic, Callable, Optional + +P = ParamSpec("P") + +class C(Generic[P]): + # think PhantomData from rust + phantom: Optional[Callable[P, None]] + + def add_str(self) -> C[Concatenate[int, P]]: + return C[Concatenate[int, P]]() + + def add_int(self) -> C[Concatenate[str, P]]: + return C[Concatenate[str, P]]() + +def f(c: C[P]) -> None: + reveal_type(c) # N: Revealed type is "__main__.C[P`-1]" + + n1 = c.add_str() + reveal_type(n1) # N: Revealed type is "__main__.C[Concatenate[builtins.int, P`-1]]" + n2 = n1.add_int() + reveal_type(n2) # N: Revealed type is "__main__.C[Concatenate[builtins.str, builtins.int, P`-1]]" + + p1 = c.add_int() + reveal_type(p1) # N: Revealed type is "__main__.C[Concatenate[builtins.str, P`-1]]" + p2 = p1.add_str() + reveal_type(p2) # N: Revealed type is "__main__.C[Concatenate[builtins.int, builtins.str, P`-1]]" +[builtins fixtures/tuple.pyi] From 61b00cd2beea369a4ec8d6dde93a6e17759b8fe5 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 29 Jan 2022 16:09:45 +0900 Subject: [PATCH 27/41] Revert "Minor code cleanup" This reverts commit c46feec706d62d188352826bbc4502e7d8259309. This commit caused bugs for some reason.... --- mypy/expandtype.py | 11 +++++++---- mypy/types.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 021e19851838..589a1f308df0 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -138,11 +138,14 @@ def visit_callable_type(self, t: CallableType) -> Type: # the replacement is ignored. if isinstance(repl, CallableType) or isinstance(repl, Parameters): # Substitute *args: P.args, **kwargs: P.kwargs - t = t.expand_param_spec(repl) + prefix = param_spec.prefix + # we need to expand the types in the prefix, so might as well + # not get them in the first place + t = t.expand_param_spec(repl, no_prefix=True) return t.copy_modified( - arg_types=self.expand_types(t.arg_types), - arg_kinds=t.arg_kinds, - arg_names=t.arg_names, + arg_types=self.expand_types(prefix.arg_types) + t.arg_types, + arg_kinds=prefix.arg_kinds + t.arg_kinds, + arg_names=prefix.arg_names + t.arg_names, ret_type=t.ret_type.accept(self), type_guard=(t.type_guard.accept(self) if t.type_guard is not None else None)) diff --git a/mypy/types.py b/mypy/types.py index 91875f4b4e74..b7553edc9a95 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1611,11 +1611,18 @@ def param_spec(self) -> Optional[ParamSpecType]: arg_type.upper_bound, prefix=prefix) def expand_param_spec(self, - c: Union['CallableType', Parameters]) -> 'CallableType': - return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, - arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, - arg_names=self.arg_names[:-2] + c.arg_names, - is_ellipsis_args=c.is_ellipsis_args) + c: Union['CallableType', Parameters], + no_prefix: bool = False) -> 'CallableType': + if no_prefix: + return self.copy_modified(arg_types=c.arg_types, + arg_kinds=c.arg_kinds, + arg_names=c.arg_names, + is_ellipsis_args=c.is_ellipsis_args) + else: + return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, + arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, + arg_names=self.arg_names[:-2] + c.arg_names, + is_ellipsis_args=c.is_ellipsis_args) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), From a44937bfc4a41e7f33175f858b1294ce1d8ea394 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 1 Mar 2022 12:41:54 +0900 Subject: [PATCH 28/41] Fixed raised bugs Main things left: - Concatenate with mypy_extensions args - Flag for non-strict Concatenate --- mypy/applytype.py | 4 +- mypy/expandtype.py | 13 ++-- mypy/join.py | 5 +- mypy/nodes.py | 4 +- mypy/types.py | 20 +++++- .../unit/check-parameter-specification.test | 71 +++++++++++++++++++ 6 files changed, 107 insertions(+), 10 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index 5b803a4aaa0b..a967d834f1a2 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -5,7 +5,7 @@ from mypy.expandtype import expand_type from mypy.types import ( Type, TypeVarId, TypeVarType, CallableType, AnyType, PartialType, get_proper_types, - TypeVarLikeType, ProperType, ParamSpecType, get_proper_type + TypeVarLikeType, ProperType, ParamSpecType, Parameters, get_proper_type ) from mypy.nodes import Context @@ -94,7 +94,7 @@ def apply_generic_arguments( nt = id_to_type.get(param_spec.id) if nt is not None: nt = get_proper_type(nt) - if isinstance(nt, CallableType): + if isinstance(nt, CallableType) or isinstance(nt, Parameters): callable = callable.expand_param_spec(nt) # Apply arguments to argument types. diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 589a1f308df0..053425dbf890 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -5,7 +5,7 @@ NoneType, Overloaded, TupleType, TypedDictType, UnionType, ErasedType, PartialType, DeletedType, UninhabitedType, TypeType, TypeVarId, FunctionLike, TypeVarType, LiteralType, get_proper_type, ProperType, - TypeAliasType, ParamSpecType, TypeVarLikeType, Parameters + TypeAliasType, ParamSpecType, TypeVarLikeType, Parameters, ParamSpecFlavor ) @@ -115,9 +115,14 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: arg_names=t.prefix.arg_names + repl.prefix.arg_names, )) elif isinstance(repl, Parameters) or isinstance(repl, CallableType): - return repl.copy_modified(t.prefix.arg_types + repl.arg_types, - t.prefix.arg_kinds + repl.arg_kinds, - t.prefix.arg_names + repl.arg_names) + # if the paramspec is *P.args or **P.kwargs: + if t.flavor != ParamSpecFlavor.BARE: + # Is this always the right thing to do? + return repl.param_spec().with_flavor(t.flavor) + else: + return repl.copy_modified(t.prefix.arg_types + repl.arg_types, + t.prefix.arg_kinds + repl.arg_kinds, + t.prefix.arg_names + repl.arg_names) else: # TODO: should this branch be removed? better not to fail silently return repl diff --git a/mypy/join.py b/mypy/join.py index c4c29a01f255..016990e9e744 100644 --- a/mypy/join.py +++ b/mypy/join.py @@ -257,7 +257,10 @@ def visit_param_spec(self, t: ParamSpecType) -> ProperType: return self.default(self.s) def visit_parameters(self, t: Parameters) -> ProperType: - raise NotImplementedError("joining two paramspec literals is not supported yet") + if self.s == t: + return t + else: + return self.default(self.s) def visit_instance(self, t: Instance) -> ProperType: if isinstance(self.s, Instance): diff --git a/mypy/nodes.py b/mypy/nodes.py index 6fa24146126b..692264186f47 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1030,11 +1030,11 @@ def serialize(self) -> JsonDict: @classmethod def deserialize(self, data: JsonDict) -> 'ClassDef': - # TODO: Does this have to work with ParamSpecType too? assert data['.class'] == 'ClassDef' res = ClassDef(data['name'], Block([]), - [mypy.types.TypeVarType.deserialize(v) for v in data['type_vars']], + # https://github.com/python/mypy/issues/12257 + [mypy.types.deserialize_type(v) for v in data['type_vars']], ) res.fullname = data['fullname'] return res diff --git a/mypy/types.py b/mypy/types.py index 0ba38b1142a4..d98708bab6d4 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1196,8 +1196,11 @@ def __init__(self, arg_kinds: List[ArgKind], arg_names: Sequence[Optional[str]], *, - is_ellipsis_args: bool = False + is_ellipsis_args: bool = False, + line: int = -1, + column: int = -1 ) -> None: + super().__init__(line, column) self.arg_types = list(arg_types) self.arg_kinds = arg_kinds self.arg_names = list(arg_names) @@ -1329,6 +1332,21 @@ def deserialize(cls, data: JsonDict) -> 'Parameters': data['arg_names'], ) + def __hash__(self) -> int: + return hash((self.is_ellipsis_args, tuple(self.arg_types), + tuple(self.arg_names), tuple(self.arg_kinds))) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Parameters) or isinstance(other, CallableType): + return ( + self.arg_types == other.arg_types and + self.arg_names == other.arg_names and + self.arg_kinds == other.arg_kinds and + self.is_ellipsis_args == other.is_ellipsis_args + ) + else: + return NotImplemented + class CallableType(FunctionLike): """Type of a non-overloaded callable object (such as function).""" diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 9d16212afa57..f0edf068edcb 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -746,3 +746,74 @@ def f(c: C[P]) -> None: p2 = p1.add_str() reveal_type(p2) # N: Revealed type is "__main__.C[Concatenate[builtins.int, builtins.str, P`-1]]" [builtins fixtures/tuple.pyi] + +[case testParamSpecLiteralJoin] +from typing import Generic, Callable, Union +from typing_extensions import ParamSpec + + +_P = ParamSpec("_P") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: + self.target = target + +def func( + action: Union[Job[int], Callable[[int], None]], +) -> None: + job = action if isinstance(action, Job) else Job(action) + reveal_type(job) # N: Revealed type is "__main__.Job[[builtins.int]]" +[builtins fixtures/tuple.pyi] + +[case testApplyParamSpecToParamSpecLiterals] +from typing import TypeVar, Generic, Callable +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +class Job(Generic[_P, _R_co]): + def __init__(self, target: Callable[_P, _R_co]) -> None: + self.target = target + +def run_job(job: Job[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: + ... + + +def func(job: Job[[int, str], None]) -> None: + run_job(job, 42, "Hello") + run_job(job, "Hello", 42) # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ + # E: Argument 3 to "run_job" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] + +[case testExpandNonBareParamSpecAgainstCallable] +from typing import Callable, TypeVar, Any +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") +_R = TypeVar("_R") + +def simple_decorator(callable: CallableT) -> CallableT: + # set some attribute on 'callable' + return callable + + +class A: + @simple_decorator + def func(self, action: Callable[_P, _R], *args: _P.args, **kwargs: _P.kwargs) -> _R: + ... + +reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, action: def (*_P.args, **_P.kwargs) -> _R`-2, *_P.args, **_P.kwargs) -> _R`-2" + +# TODO: _R` keeps flip-flopping between 5 (?), 13, 14, 15. Spooky. +# reveal_type(A().func) $ N: Revealed type is "def [_P, _R] (action: def (*_P.args, **_P.kwargs) -> _R`13, *_P.args, **_P.kwargs) -> _R`13" + +def f(x: int) -> int: + ... + +reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int*" + +# TODO: this should reveal `int` +reveal_type(A().func(lambda x: x + x, 42)) # N: Revealed type is "Any" +[builtins fixtures/tuple.pyi] From bbabbf1046acb74f6261bd7e2231b21fd52e972f Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 1 Mar 2022 12:52:36 +0900 Subject: [PATCH 29/41] Fix CI errors --- mypy/expandtype.py | 7 ++++++- mypy/nodes.py | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 053425dbf890..2180ed5994a9 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -117,8 +117,13 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: elif isinstance(repl, Parameters) or isinstance(repl, CallableType): # if the paramspec is *P.args or **P.kwargs: if t.flavor != ParamSpecFlavor.BARE: + assert isinstance(repl, CallableType), "Should not be able to get here." # Is this always the right thing to do? - return repl.param_spec().with_flavor(t.flavor) + param_spec = repl.param_spec() + if param_spec: + return param_spec.with_flavor(t.flavor) + else: + return repl else: return repl.copy_modified(t.prefix.arg_types + repl.arg_types, t.prefix.arg_kinds + repl.arg_kinds, diff --git a/mypy/nodes.py b/mypy/nodes.py index 692264186f47..2ee936e9066d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1031,10 +1031,18 @@ def serialize(self) -> JsonDict: @classmethod def deserialize(self, data: JsonDict) -> 'ClassDef': assert data['.class'] == 'ClassDef' + + tvs_ = [mypy.types.deserialize_type(v) for v in data['type_vars']] + tvs = [] + for tv in tvs_: + # mypy tells me to expand `tv`, which isn't necessary I believe? + assert isinstance(tv, mypy.types.TypeVarLikeType) # type: ignore[misc] + tvs.append(tv) + res = ClassDef(data['name'], Block([]), # https://github.com/python/mypy/issues/12257 - [mypy.types.deserialize_type(v) for v in data['type_vars']], + tvs, ) res.fullname = data['fullname'] return res From ddfd34a346aa7edcb572467da10ce319ae31c97b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 5 Mar 2022 12:47:14 +0900 Subject: [PATCH 30/41] Squash some more bugs Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- mypy/constraints.py | 52 +++++++++------ mypy/nodes.py | 10 +-- mypy/types.py | 11 +++- .../unit/check-parameter-specification.test | 65 +++++++++++++++++++ 4 files changed, 106 insertions(+), 32 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index f5ae71cea77d..091e78f4d10d 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -467,6 +467,9 @@ def visit_instance(self, template: Instance) -> List[Constraint]: suffix.arg_kinds[len(prefix.arg_kinds):], suffix.arg_names[len(prefix.arg_names):]) res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) + elif isinstance(suffix, ParamSpecType): + res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) + return res elif (self.direction == SUPERTYPE_OF and instance.type.has_base(template.type.fullname)): @@ -496,6 +499,8 @@ def visit_instance(self, template: Instance) -> List[Constraint]: suffix.arg_kinds[len(prefix.arg_kinds):], suffix.arg_names[len(prefix.arg_names):]) res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) + elif isinstance(suffix, ParamSpecType): + res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) return res if (template.type.is_protocol and self.direction == SUPERTYPE_OF and # We avoid infinite recursion for structural subtypes by checking @@ -586,27 +591,32 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: # Negate direction due to function argument type contravariance. res.extend(infer_constraints(t, a, neg_op(self.direction))) else: - # TODO: Direction - # TODO: check the prefixes match - prefix = param_spec.prefix - prefix_len = len(prefix.arg_types) - res.append(Constraint(param_spec.id, - SUBTYPE_OF, - cactual.copy_modified( - arg_types=cactual.arg_types[prefix_len:], - arg_kinds=cactual.arg_kinds[prefix_len:], - arg_names=cactual.arg_names[prefix_len:], - ret_type=NoneType()))) - # compare prefixes - cactual_prefix = cactual.copy_modified( - arg_types=cactual.arg_types[:prefix_len], - arg_kinds=cactual.arg_kinds[:prefix_len], - arg_names=cactual.arg_names[:prefix_len]) - - # TODO: see above "FIX" comments for param_spec is None case - # TODO: this assume positional arguments - for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): - res.extend(infer_constraints(t, a, neg_op(self.direction))) + # sometimes, it appears we try to get constraints between two paramspec callables? + cactual_ps = cactual.param_spec() + if cactual_ps: + res.append(Constraint(param_spec.id, SUBTYPE_OF, cactual_ps)) + else: + # TODO: Direction + # TODO: check the prefixes match + prefix = param_spec.prefix + prefix_len = len(prefix.arg_types) + res.append(Constraint(param_spec.id, + SUBTYPE_OF, + cactual.copy_modified( + arg_types=cactual.arg_types[prefix_len:], + arg_kinds=cactual.arg_kinds[prefix_len:], + arg_names=cactual.arg_names[prefix_len:], + ret_type=NoneType()))) + # compare prefixes + cactual_prefix = cactual.copy_modified( + arg_types=cactual.arg_types[:prefix_len], + arg_kinds=cactual.arg_kinds[:prefix_len], + arg_names=cactual.arg_names[:prefix_len]) + + # TODO: see above "FIX" comments for param_spec is None case + # TODO: this assume positional arguments + for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): + res.extend(infer_constraints(t, a, neg_op(self.direction))) template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type if template.type_guard is not None: diff --git a/mypy/nodes.py b/mypy/nodes.py index 2ee936e9066d..f3c3c4aab9b2 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1031,18 +1031,10 @@ def serialize(self) -> JsonDict: @classmethod def deserialize(self, data: JsonDict) -> 'ClassDef': assert data['.class'] == 'ClassDef' - - tvs_ = [mypy.types.deserialize_type(v) for v in data['type_vars']] - tvs = [] - for tv in tvs_: - # mypy tells me to expand `tv`, which isn't necessary I believe? - assert isinstance(tv, mypy.types.TypeVarLikeType) # type: ignore[misc] - tvs.append(tv) - res = ClassDef(data['name'], Block([]), # https://github.com/python/mypy/issues/12257 - tvs, + [cast(mypy.types.TypeVarLikeType, mypy.types.deserialize_type(v)) for v in data['type_vars']], ) res.fullname = data['fullname'] return res diff --git a/mypy/types.py b/mypy/types.py index d98708bab6d4..713fc9f10e3d 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1653,16 +1653,23 @@ def param_spec(self) -> Optional[ParamSpecType]: def expand_param_spec(self, c: Union['CallableType', Parameters], no_prefix: bool = False) -> 'CallableType': + if isinstance(c, CallableType): + variables = c.variables + else: + variables = [] + if no_prefix: return self.copy_modified(arg_types=c.arg_types, arg_kinds=c.arg_kinds, arg_names=c.arg_names, - is_ellipsis_args=c.is_ellipsis_args) + is_ellipsis_args=c.is_ellipsis_args, + variables=variables + self.variables) else: return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, arg_names=self.arg_names[:-2] + c.arg_names, - is_ellipsis_args=c.is_ellipsis_args) + is_ellipsis_args=c.is_ellipsis_args, + variables=variables + self.variables) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index f0edf068edcb..180ca19a24ce 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -817,3 +817,68 @@ reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int*" # TODO: this should reveal `int` reveal_type(A().func(lambda x: x + x, 42)) # N: Revealed type is "Any" [builtins fixtures/tuple.pyi] + +[case testParamSpecConstraintOnOtherParamSpec] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +def simple_decorator(callable: CallableT) -> CallableT: + ... + +class Job(Generic[_P, _R_co]): + def __init__(self, target: Callable[_P, _R_co]) -> None: + ... + + +class A: + @simple_decorator + def func(self, action: Job[_P, None]) -> Job[_P, None]: + ... + +reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`-1, None]) -> __main__.Job[_P`-1, None]" +reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" +reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[def (x: Any), None]" + +def f(x: int, y: int) -> None: ... +reveal_type(A().func(Job(f))) # N: Revealed type is "__main__.Job[def (x: builtins.int, y: builtins.int), None]" +[builtins fixtures/tuple.pyi] + +[case testConstraintBetweenParamSpecFunctions1] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_R_co = TypeVar("_R_co", covariant=True) + +def simple_decorator(callable: Callable[_P, _R_co]) -> Callable[_P, _R_co]: ... +class Job(Generic[_P]): ... + + +@simple_decorator +def func(action: Job[_P], /) -> Callable[_P, None]: + ... + +reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" +[builtins fixtures/tuple.pyi] + +[case testConstraintBetweenParamSpecFunctions2] +from typing import Callable, TypeVar, Any, Generic +from typing_extensions import ParamSpec + +CallableT = TypeVar("CallableT", bound=Callable[..., Any]) +_P = ParamSpec("_P") + +def simple_decorator(callable: CallableT) -> CallableT: ... +class Job(Generic[_P]): ... + + +@simple_decorator +def func(action: Job[_P], /) -> Callable[_P, None]: + ... + +reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" +[builtins fixtures/tuple.pyi] From 2d54ac4223718a7ef93dbc37713159d7ebd03505 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 5 Mar 2022 13:07:40 +0900 Subject: [PATCH 31/41] Concatenate flag Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- docs/source/config_file.rst | 7 ++++ mypy/checker.py | 2 +- mypy/checkexpr.py | 2 +- mypy/constraints.py | 24 +++++++++---- mypy/main.py | 4 +++ mypy/options.py | 4 +++ mypy/subtypes.py | 70 +++++++++++++++++++++++++------------ 7 files changed, 83 insertions(+), 30 deletions(-) diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index 0b53f1ca5370..f398c8b55e3c 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -668,6 +668,13 @@ section of the command line docs. from foo import bar __all__ = ['bar'] +.. confval:: strict_concatenate + + :type: boolean + :default: False + + Make arguments prepended via ``Concatenate`` be truly positional-only. + .. confval:: strict_equality :type: boolean diff --git a/mypy/checker.py b/mypy/checker.py index 1953ca541930..31320cf4dcc7 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -5156,7 +5156,7 @@ def check_subtype(self, code: Optional[ErrorCode] = None, outer_context: Optional[Context] = None) -> bool: """Generate an error if the subtype is not compatible with supertype.""" - if is_subtype(subtype, supertype): + if is_subtype(subtype, supertype, options=self.options): return True if isinstance(msg, ErrorMessage): diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 4ece9ac4c94b..0cefb3abe1f7 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -1556,7 +1556,7 @@ def check_arg(self, isinstance(callee_type.item, Instance) and (callee_type.item.type.is_abstract or callee_type.item.type.is_protocol)): self.msg.concrete_only_call(callee_type, context) - elif not is_subtype(caller_type, callee_type): + elif not is_subtype(caller_type, callee_type, options=self.chk.options): if self.chk.should_suppress_optional_error([caller_type, callee_type]): return code = messages.incompatible_argument(n, diff --git a/mypy/constraints.py b/mypy/constraints.py index 091e78f4d10d..8746a4940c15 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -463,9 +463,15 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # TODO: is there a case I am missing? # TODO: constraints between prefixes prefix = mapped_arg.prefix - suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], - suffix.arg_kinds[len(prefix.arg_kinds):], - suffix.arg_names[len(prefix.arg_names):]) + + if isinstance(suffix, CallableType): + from_concatenate = bool(prefix.arg_types) or suffix.from_concatenate + suffix = suffix.copy_modified(from_concatenate=from_concatenate) + + suffix = suffix.copy_modified( + suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) elif isinstance(suffix, ParamSpecType): res.append(Constraint(mapped_arg.id, SUPERTYPE_OF, suffix)) @@ -495,9 +501,15 @@ def visit_instance(self, template: Instance) -> List[Constraint]: # TODO: is there a case I am missing? # TODO: constraints between prefixes prefix = template_arg.prefix - suffix = suffix.copy_modified(suffix.arg_types[len(prefix.arg_types):], - suffix.arg_kinds[len(prefix.arg_kinds):], - suffix.arg_names[len(prefix.arg_names):]) + + if isinstance(suffix, CallableType): + from_concatenate = bool(prefix.arg_types) or suffix.from_concatenate + suffix = suffix.copy_modified(from_concatenate=from_concatenate) + + suffix = suffix.copy_modified( + suffix.arg_types[len(prefix.arg_types):], + suffix.arg_kinds[len(prefix.arg_kinds):], + suffix.arg_names[len(prefix.arg_names):]) res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) elif isinstance(suffix, ParamSpecType): res.append(Constraint(template_arg.id, SUPERTYPE_OF, suffix)) diff --git a/mypy/main.py b/mypy/main.py index d765781838cf..d50ff4894436 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -677,6 +677,10 @@ def add_invertible_flag(flag: str, " non-overlapping types", group=strictness_group) + add_invertible_flag('--strict-concatenate', default=False, strict_flag=True, + help="Make arguments prepended via Concatenate be truly positional-only", + group=strictness_group) + strict_help = "Strict mode; enables the following flags: {}".format( ", ".join(strict_flag_names)) strictness_group.add_argument( diff --git a/mypy/options.py b/mypy/options.py index 58278b1580e8..b9bc6d60b9be 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -46,6 +46,7 @@ class BuildType: "mypyc", "no_implicit_optional", "show_none_errors", + "strict_concatenate", "strict_equality", "strict_optional", "strict_optional_whitelist", @@ -183,6 +184,9 @@ def __init__(self) -> None: # This makes 1 == '1', 1 in ['1'], and 1 is '1' errors. self.strict_equality = False + # Make arguments prepended via Concatenate be truly positional-only. + self.strict_concatenate = False + # Report an error for any branches inferred to be unreachable as a result of # type analysis. self.warn_unreachable = False diff --git a/mypy/subtypes.py b/mypy/subtypes.py index c44a10281a66..160f640fba79 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -24,6 +24,7 @@ from mypy.maptype import map_instance_to_supertype from mypy.expandtype import expand_type_by_instance from mypy.typestate import TypeState, SubtypeKind +from mypy.options import Options from mypy import state # Flags for detected protocol members @@ -52,7 +53,8 @@ def is_subtype(left: Type, right: Type, ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, - ignore_promotions: bool = False) -> bool: + ignore_promotions: bool = False, + options: Optional[Options] = None) -> bool: """Is 'left' subtype of 'right'? Also consider Any to be a subtype of any type, and vice versa. This @@ -90,12 +92,14 @@ def is_subtype(left: Type, right: Type, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, - ignore_promotions=ignore_promotions) + ignore_promotions=ignore_promotions, + options=options) return _is_subtype(left, right, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, - ignore_promotions=ignore_promotions) + ignore_promotions=ignore_promotions, + options=options) def _is_subtype(left: Type, right: Type, @@ -103,7 +107,8 @@ def _is_subtype(left: Type, right: Type, ignore_type_params: bool = False, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, - ignore_promotions: bool = False) -> bool: + ignore_promotions: bool = False, + options: Optional[Options] = None) -> bool: orig_right = right orig_left = left left = get_proper_type(left) @@ -120,7 +125,8 @@ def _is_subtype(left: Type, right: Type, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, - ignore_promotions=ignore_promotions) + ignore_promotions=ignore_promotions, + options=options) for item in right.items) # Recombine rhs literal types, to make an enum type a subtype # of a union of all enum items as literal types. Only do it if @@ -135,7 +141,8 @@ def _is_subtype(left: Type, right: Type, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, - ignore_promotions=ignore_promotions) + ignore_promotions=ignore_promotions, + options=options) for item in right.items) # However, if 'left' is a type variable T, T might also have # an upper bound which is itself a union. This case will be @@ -152,19 +159,21 @@ def _is_subtype(left: Type, right: Type, ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, ignore_declared_variance=ignore_declared_variance, - ignore_promotions=ignore_promotions)) + ignore_promotions=ignore_promotions, + options=options)) def is_equivalent(a: Type, b: Type, *, ignore_type_params: bool = False, - ignore_pos_arg_names: bool = False + ignore_pos_arg_names: bool = False, + options: Optional[Options] = None ) -> bool: return ( is_subtype(a, b, ignore_type_params=ignore_type_params, - ignore_pos_arg_names=ignore_pos_arg_names) + ignore_pos_arg_names=ignore_pos_arg_names, options=options) and is_subtype(b, a, ignore_type_params=ignore_type_params, - ignore_pos_arg_names=ignore_pos_arg_names)) + ignore_pos_arg_names=ignore_pos_arg_names, options=options)) class SubtypeVisitor(TypeVisitor[bool]): @@ -174,7 +183,8 @@ def __init__(self, right: Type, ignore_type_params: bool, ignore_pos_arg_names: bool = False, ignore_declared_variance: bool = False, - ignore_promotions: bool = False) -> None: + ignore_promotions: bool = False, + options: Optional[Options] = None) -> None: self.right = get_proper_type(right) self.orig_right = right self.ignore_type_params = ignore_type_params @@ -183,6 +193,7 @@ def __init__(self, right: Type, self.ignore_promotions = ignore_promotions self.check_type_parameter = (ignore_type_parameter if ignore_type_params else check_type_parameter) + self.options = options self._subtype_kind = SubtypeVisitor.build_subtype_kind( ignore_type_params=ignore_type_params, ignore_pos_arg_names=ignore_pos_arg_names, @@ -206,7 +217,8 @@ def _is_subtype(self, left: Type, right: Type) -> bool: ignore_type_params=self.ignore_type_params, ignore_pos_arg_names=self.ignore_pos_arg_names, ignore_declared_variance=self.ignore_declared_variance, - ignore_promotions=self.ignore_promotions) + ignore_promotions=self.ignore_promotions, + options=self.options) # visit_x(left) means: is left (which is an instance of X) a subtype of # right? @@ -278,7 +290,7 @@ def visit_instance(self, left: Instance) -> bool: if not self.check_type_parameter(lefta, righta, tvar.variance): nominal = False else: - if not is_equivalent(lefta, righta): + if not is_equivalent(lefta, righta, options=self.options): nominal = False if nominal: TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) @@ -350,7 +362,8 @@ def visit_callable_type(self, left: CallableType) -> bool: return is_callable_compatible( left, right, is_compat=self._is_subtype, - ignore_pos_arg_names=self.ignore_pos_arg_names) + ignore_pos_arg_names=self.ignore_pos_arg_names, + strict_concatenate=self.options.strict_concatenate if self.options else True) elif isinstance(right, Overloaded): return all(self._is_subtype(left, item) for item in right.items) elif isinstance(right, Instance): @@ -417,7 +430,8 @@ def visit_typeddict_type(self, left: TypedDictType) -> bool: return False for name, l, r in left.zip(right): if not is_equivalent(l, r, - ignore_type_params=self.ignore_type_params): + ignore_type_params=self.ignore_type_params, + options=self.options): return False # Non-required key is not compatible with a required key since # indexing may fail unexpectedly if a required key is missing. @@ -484,12 +498,15 @@ def visit_overloaded(self, left: Overloaded) -> bool: else: # If this one overlaps with the supertype in any way, but it wasn't # an exact match, then it's a potential error. + strict_concat = self.options.strict_concatenate if self.options else True if (is_callable_compatible(left_item, right_item, is_compat=self._is_subtype, ignore_return=True, - ignore_pos_arg_names=self.ignore_pos_arg_names) or + ignore_pos_arg_names=self.ignore_pos_arg_names, + strict_concatenate=strict_concat) or is_callable_compatible(right_item, left_item, is_compat=self._is_subtype, ignore_return=True, - ignore_pos_arg_names=self.ignore_pos_arg_names)): + ignore_pos_arg_names=self.ignore_pos_arg_names, + strict_concatenate=strict_concat)): # If this is an overload that's already been matched, there's no # problem. if left_item not in matched_overloads: @@ -791,7 +808,8 @@ def is_callable_compatible(left: CallableType, right: CallableType, ignore_return: bool = False, ignore_pos_arg_names: bool = False, check_args_covariantly: bool = False, - allow_partial_overlap: bool = False) -> bool: + allow_partial_overlap: bool = False, + strict_concatenate: bool = False) -> bool: """Is the left compatible with the right, using the provided compatibility check? is_compat: @@ -927,10 +945,16 @@ def g(x: int) -> int: ... if check_args_covariantly: is_compat = flip_compat_check(is_compat) + if not strict_concatenate and (left.from_concatenate or right.from_concatenate): + strict_concatenate_check = False + else: + strict_concatenate_check = True + return are_parameters_compatible(left, right, is_compat=is_compat, ignore_pos_arg_names=ignore_pos_arg_names, check_args_covariantly=check_args_covariantly, - allow_partial_overlap=allow_partial_overlap) + allow_partial_overlap=allow_partial_overlap, + strict_concatenate_check=strict_concatenate_check) def are_parameters_compatible(left: Union[Parameters, CallableType], @@ -939,7 +963,8 @@ def are_parameters_compatible(left: Union[Parameters, CallableType], is_compat: Callable[[Type, Type], bool], ignore_pos_arg_names: bool = False, check_args_covariantly: bool = False, - allow_partial_overlap: bool = False) -> bool: + allow_partial_overlap: bool = False, + strict_concatenate_check: bool = True) -> bool: if right.is_ellipsis_args: return True @@ -1028,7 +1053,7 @@ def _incompatible(left_arg: Optional[FormalArgument], right_names = {name for name in right.arg_names if name is not None} left_only_names = set() for name, kind in zip(left.arg_names, left.arg_kinds): - if name is None or kind.is_star() or name in right_names: + if name is None or kind.is_star() or name in right_names or not strict_concatenate_check: continue left_only_names.add(name) @@ -1064,7 +1089,8 @@ def _incompatible(left_arg: Optional[FormalArgument], if (right_by_name is not None and right_by_pos is not None and right_by_name != right_by_pos - and (right_by_pos.required or right_by_name.required)): + and (right_by_pos.required or right_by_name.required) + and strict_concatenate_check): return False # All *required* left-hand arguments must have a corresponding From bba91e5966df0073324910e39646f379b4b91825 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 5 Mar 2022 13:25:53 +0900 Subject: [PATCH 32/41] Prepare for GitHub Actions --- mypy/constraints.py | 21 +++++++----- mypy/nodes.py | 3 +- mypy/subtypes.py | 4 ++- mypy/types.py | 4 +-- .../unit/check-parameter-specification.test | 33 +++++++++++++++++-- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 8746a4940c15..37aadbda1809 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -458,16 +458,17 @@ def visit_instance(self, template: Instance) -> List[Constraint]: mapped_arg, instance_arg, neg_op(self.direction))) elif isinstance(tvar, ParamSpecType) and isinstance(mapped_arg, ParamSpecType): suffix = get_proper_type(instance_arg) + + if isinstance(suffix, CallableType): + prefix = mapped_arg.prefix + from_concat = bool(prefix.arg_types) or suffix.from_concatenate + suffix = suffix.copy_modified(from_concatenate=from_concat) + if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: constraints between prefixes prefix = mapped_arg.prefix - - if isinstance(suffix, CallableType): - from_concatenate = bool(prefix.arg_types) or suffix.from_concatenate - suffix = suffix.copy_modified(from_concatenate=from_concatenate) - suffix = suffix.copy_modified( suffix.arg_types[len(prefix.arg_types):], suffix.arg_kinds[len(prefix.arg_kinds):], @@ -496,16 +497,18 @@ def visit_instance(self, template: Instance) -> List[Constraint]: elif (isinstance(tvar, ParamSpecType) and isinstance(template_arg, ParamSpecType)): suffix = get_proper_type(mapped_arg) + + if isinstance(suffix, CallableType): + prefix = template_arg.prefix + from_concat = bool(prefix.arg_types) or suffix.from_concatenate + suffix = suffix.copy_modified(from_concatenate=from_concat) + if isinstance(suffix, Parameters) or isinstance(suffix, CallableType): # no such thing as variance for ParamSpecs # TODO: is there a case I am missing? # TODO: constraints between prefixes prefix = template_arg.prefix - if isinstance(suffix, CallableType): - from_concatenate = bool(prefix.arg_types) or suffix.from_concatenate - suffix = suffix.copy_modified(from_concatenate=from_concatenate) - suffix = suffix.copy_modified( suffix.arg_types[len(prefix.arg_types):], suffix.arg_kinds[len(prefix.arg_kinds):], diff --git a/mypy/nodes.py b/mypy/nodes.py index f3c3c4aab9b2..8846d568b839 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1034,7 +1034,8 @@ def deserialize(self, data: JsonDict) -> 'ClassDef': res = ClassDef(data['name'], Block([]), # https://github.com/python/mypy/issues/12257 - [cast(mypy.types.TypeVarLikeType, mypy.types.deserialize_type(v)) for v in data['type_vars']], + [cast(mypy.types.TypeVarLikeType, mypy.types.deserialize_type(v)) + for v in data['type_vars']], ) res.fullname = data['fullname'] return res diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 160f640fba79..fa978c66881a 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1053,7 +1053,9 @@ def _incompatible(left_arg: Optional[FormalArgument], right_names = {name for name in right.arg_names if name is not None} left_only_names = set() for name, kind in zip(left.arg_names, left.arg_kinds): - if name is None or kind.is_star() or name in right_names or not strict_concatenate_check: + if (name is None or kind.is_star() + or name in right_names + or not strict_concatenate_check): continue left_only_names.add(name) diff --git a/mypy/types.py b/mypy/types.py index 713fc9f10e3d..88425058efed 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1663,13 +1663,13 @@ def expand_param_spec(self, arg_kinds=c.arg_kinds, arg_names=c.arg_names, is_ellipsis_args=c.is_ellipsis_args, - variables=variables + self.variables) + variables=[*variables, *self.variables]) else: return self.copy_modified(arg_types=self.arg_types[:-2] + c.arg_types, arg_kinds=self.arg_kinds[:-2] + c.arg_kinds, arg_names=self.arg_names[:-2] + c.arg_names, is_ellipsis_args=c.is_ellipsis_args, - variables=variables + self.variables) + variables=[*variables, *self.variables]) def __hash__(self) -> int: return hash((self.ret_type, self.is_type_obj(), diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 180ca19a24ce..ad7571f0f25f 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -573,6 +573,7 @@ reveal_type(f(n)) # N: Revealed type is "def (builtins.int, builtins.bytes) -> [builtins fixtures/tuple.pyi] [case testParamSpecConcatenateNamedArgs] +# flags: --strict-concatenate # this is one noticeable deviation from PEP but I believe it is for the better from typing_extensions import ParamSpec, Concatenate from typing import Callable, TypeVar @@ -590,14 +591,39 @@ def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: return result # Rejected +# reason for rejection: +f2(lambda x: 42)(42, x=42) +[builtins fixtures/tuple.pyi] +[out] +main:10: error: invalid syntax +[out version>=3.8] +main:17: error: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") +main:17: note: This may be because "result" has arguments named: "x" + +[case testNonStrictParamSpecConcatenateNamedArgs] +# this is one noticeable deviation from PEP but I believe it is for the better +from typing_extensions import ParamSpec, Concatenate +from typing import Callable, TypeVar + +P = ParamSpec("P") +R = TypeVar("R") + +def f1(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, /, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Accepted + +def f2(c: Callable[P, R]) -> Callable[Concatenate[int, P], R]: + def result(x: int, *args: P.args, **kwargs: P.kwargs) -> R: ... + + return result # Rejected -> Accepted + # reason for rejection: f2(lambda x: 42)(42, x=42) [builtins fixtures/tuple.pyi] [out] main:9: error: invalid syntax [out version>=3.8] -main:16: error: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") -main:16: note: This may be because "result" has arguments named: "x" [case testParamSpecConcatenateWithTypeVar] from typing_extensions import ParamSpec, Concatenate @@ -840,7 +866,8 @@ class A: ... reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`-1, None]) -> __main__.Job[_P`-1, None]" -reveal_type(A().func) # N: Revealed type is "def [_P] (action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" +# TODO: flakey, _P`4 alternates around. +# reveal_type(A().func) $ N: Revealed type is "def [_P] (action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[def (x: Any), None]" def f(x: int, y: int) -> None: ... From 0363803017126078fa51c964d8c5c9d7aa723f4c Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 7 Mar 2022 11:03:55 +0900 Subject: [PATCH 33/41] Bug report with nested decorators and Concatenate --- mypy/constraints.py | 36 ++++++++++--------- .../unit/check-parameter-specification.test | 20 +++++++++++ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 37aadbda1809..2d04cd222e93 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -607,14 +607,13 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: res.extend(infer_constraints(t, a, neg_op(self.direction))) else: # sometimes, it appears we try to get constraints between two paramspec callables? + # TODO: Direction + # TODO: check the prefixes match + prefix = param_spec.prefix + prefix_len = len(prefix.arg_types) cactual_ps = cactual.param_spec() - if cactual_ps: - res.append(Constraint(param_spec.id, SUBTYPE_OF, cactual_ps)) - else: - # TODO: Direction - # TODO: check the prefixes match - prefix = param_spec.prefix - prefix_len = len(prefix.arg_types) + + if not cactual_ps: res.append(Constraint(param_spec.id, SUBTYPE_OF, cactual.copy_modified( @@ -622,16 +621,19 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: arg_kinds=cactual.arg_kinds[prefix_len:], arg_names=cactual.arg_names[prefix_len:], ret_type=NoneType()))) - # compare prefixes - cactual_prefix = cactual.copy_modified( - arg_types=cactual.arg_types[:prefix_len], - arg_kinds=cactual.arg_kinds[:prefix_len], - arg_names=cactual.arg_names[:prefix_len]) - - # TODO: see above "FIX" comments for param_spec is None case - # TODO: this assume positional arguments - for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): - res.extend(infer_constraints(t, a, neg_op(self.direction))) + else: + res.append(Constraint(param_spec.id, SUBTYPE_OF, cactual_ps)) + + # compare prefixes + cactual_prefix = cactual.copy_modified( + arg_types=cactual.arg_types[:prefix_len], + arg_kinds=cactual.arg_kinds[:prefix_len], + arg_names=cactual.arg_names[:prefix_len]) + + # TODO: see above "FIX" comments for param_spec is None case + # TODO: this assume positional arguments + for t, a in zip(prefix.arg_types, cactual_prefix.arg_types): + res.extend(infer_constraints(t, a, neg_op(self.direction))) template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type if template.type_guard is not None: diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index ad7571f0f25f..92e8c79cb46c 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -909,3 +909,23 @@ def func(action: Job[_P], /) -> Callable[_P, None]: reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" [builtins fixtures/tuple.pyi] + +[case testConstraintsBetweenConcatenatePrefixes] +from typing import Any, Callable, Coroutine, TypeVar +from typing_extensions import Concatenate, ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + + +def adds_await() -> Callable[ + [Callable[Concatenate[_T, _P], None]], + Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], +]: + def decorator( + func: Callable[Concatenate[_T, _P], None], + ) -> Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]]: + ... + + return decorator # we want `_T` and `_P` to refer to the same things. +[builtins fixtures/tuple.pyi] From 278b8c45a59bbcc35fcd140ba21c6cbc57c82cf7 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 7 Mar 2022 11:18:21 +0900 Subject: [PATCH 34/41] Switch over to using Parameters instead of CallableType --- mypy/expandtype.py | 6 +++--- .../unit/check-parameter-specification.test | 17 +++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 2180ed5994a9..ee08e9c9306d 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -125,9 +125,9 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: else: return repl else: - return repl.copy_modified(t.prefix.arg_types + repl.arg_types, - t.prefix.arg_kinds + repl.arg_kinds, - t.prefix.arg_names + repl.arg_names) + return Parameters(t.prefix.arg_types + repl.arg_types, + t.prefix.arg_kinds + repl.arg_kinds, + t.prefix.arg_names + repl.arg_names) else: # TODO: should this branch be removed? better not to fail silently return repl diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 92e8c79cb46c..049f61140189 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -99,7 +99,7 @@ class C(Generic[P]): def f(x: int, y: str) -> None: ... -reveal_type(C(f)) # N: Revealed type is "__main__.C[def (x: builtins.int, y: builtins.str)]" +reveal_type(C(f)) # N: Revealed type is "__main__.C[[x: builtins.int, y: builtins.str]]" reveal_type(C(f).m) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int" [builtins fixtures/dict.pyi] @@ -141,7 +141,7 @@ def dec() -> Callable[[Callable[P, R]], W[P, R]]: @dec() def f(a: int, b: str) -> None: ... -reveal_type(f) # N: Revealed type is "__main__.W[def (a: builtins.int, b: builtins.str), None]" +reveal_type(f) # N: Revealed type is "__main__.W[[a: builtins.int, b: builtins.str], None]" reveal_type(f(1, '')) # N: Revealed type is "None" reveal_type(f.x) # N: Revealed type is "builtins.int" @@ -663,7 +663,7 @@ bar(abc) [out] main:11: error: invalid syntax [out version>=3.8] -main:14: note: Revealed type is "__main__.Foo[def (builtins.int, b: builtins.str)]" +main:14: note: Revealed type is "__main__.Foo[[builtins.int, b: builtins.str]]" [case testSolveParamSpecWithSelfType] from typing_extensions import ParamSpec, Concatenate @@ -868,10 +868,10 @@ class A: reveal_type(A.func) # N: Revealed type is "def [_P] (self: __main__.A, action: __main__.Job[_P`-1, None]) -> __main__.Job[_P`-1, None]" # TODO: flakey, _P`4 alternates around. # reveal_type(A().func) $ N: Revealed type is "def [_P] (action: __main__.Job[_P`4, None]) -> __main__.Job[_P`4, None]" -reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[def (x: Any), None]" +reveal_type(A().func(Job(lambda x: x))) # N: Revealed type is "__main__.Job[[x: Any], None]" def f(x: int, y: int) -> None: ... -reveal_type(A().func(Job(f))) # N: Revealed type is "__main__.Job[def (x: builtins.int, y: builtins.int), None]" +reveal_type(A().func(Job(f))) # N: Revealed type is "__main__.Job[[x: builtins.int, y: builtins.int], None]" [builtins fixtures/tuple.pyi] [case testConstraintBetweenParamSpecFunctions1] @@ -911,20 +911,21 @@ reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def [builtins fixtures/tuple.pyi] [case testConstraintsBetweenConcatenatePrefixes] -from typing import Any, Callable, Coroutine, TypeVar +from typing import Any, Callable, Generic, TypeVar from typing_extensions import Concatenate, ParamSpec _P = ParamSpec("_P") _T = TypeVar("_T") +class Awaitable(Generic[_T]): ... def adds_await() -> Callable[ [Callable[Concatenate[_T, _P], None]], - Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]], + Callable[Concatenate[_T, _P], Awaitable[None]], ]: def decorator( func: Callable[Concatenate[_T, _P], None], - ) -> Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]]: + ) -> Callable[Concatenate[_T, _P], Awaitable[None]]: ... return decorator # we want `_T` and `_P` to refer to the same things. From c2b7628aa5817de778d0fa7a563428b1a3fbc36e Mon Sep 17 00:00:00 2001 From: A5rocks Date: Mon, 7 Mar 2022 11:45:47 +0900 Subject: [PATCH 35/41] Add variance to paramspecs This is required to allow positional arguments to be replaced with named arguments. --- mypy/subtypes.py | 2 +- .../unit/check-parameter-specification.test | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index fa978c66881a..eac2c21008a3 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -290,7 +290,7 @@ def visit_instance(self, left: Instance) -> bool: if not self.check_type_parameter(lefta, righta, tvar.variance): nominal = False else: - if not is_equivalent(lefta, righta, options=self.options): + if not self.check_type_parameter(lefta, righta, COVARIANT): nominal = False if nominal: TypeState.record_subtype_cache_entry(self._subtype_kind, left, right) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 049f61140189..bc39bff203a4 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -930,3 +930,49 @@ def adds_await() -> Callable[ return decorator # we want `_T` and `_P` to refer to the same things. [builtins fixtures/tuple.pyi] + +[case testParamSpecVariance] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: ... + def into_callable(self) -> Callable[_P, None]: ... + +class A: + def func(self, var: int) -> None: ... + def other_func(self, job: Job[[int]]) -> None: ... + + +job = Job(A().func) +reveal_type(job) # N: Revealed type is "__main__.Job[[var: builtins.int]]" +A().other_func(job) # This should NOT error (despite the keyword) + +# and yet the keyword should remain +job.into_callable()(var=42) +job.into_callable()(x=42) # E: Unexpected keyword argument "x" + +# similar for other functions +def f1(n: object) -> None: ... +def f2(n: int) -> None: ... +def f3(n: bool) -> None: ... + +# just like how this is legal... +a1: Callable[[bool], None] +a1 = f3 +a1 = f2 +a1 = f1 + +# ... this is also legal +a2: Job[[bool]] +a2 = Job(f3) +a2 = Job(f2) +a2 = Job(f1) + +# and this is not legal +def f4(n: bytes) -> None: ... +a1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[bytes], None]", variable has type "Callable[[bool], None]") +a2 = Job(f4) # E: Argument 1 to "Job" has incompatible type "Callable[[bytes], None]"; expected "Callable[[bool], None]" +[builtins fixtures/tuple.pyi] From 0b1fdfbf5f0b3f6ccd113bd58e044c40b870b445 Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Thu, 10 Mar 2022 11:09:20 +0900 Subject: [PATCH 36/41] Apply suggestions from code review Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- test-data/unit/check-literal.test | 1 - test-data/unit/check-parameter-specification.test | 9 +++++++++ test-data/unit/semanal-errors.test | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index 891affadfa20..3886d3c39edd 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -944,7 +944,6 @@ a: (1, 2, 3) # E: Syntax error in type annotation \ b: Literal[[1, 2, 3]] # E: Parameter 1 of Literal[...] is invalid c: [1, 2, 3] # E: Bracketed expression "[...]" is not valid as a type \ # N: Did you mean "List[...]"? - [builtins fixtures/tuple.pyi] [out] diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index bc39bff203a4..e19ab77c7d1e 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -810,6 +810,15 @@ def func(job: Job[[int, str], None]) -> None: run_job(job, 42, "Hello") run_job(job, "Hello", 42) # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ # E: Argument 3 to "run_job" has incompatible type "int"; expected "str" + run_job(job, 42, msg="Hello") # E: Unexpected keyword argument "msg" for "run_job" + run_job(job, "Hello") # Too few arguments for "run_job" + # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ + +def func2(job: Job[..., None]) -> None: + run_job(job, 42, "Hello") + run_job(job, "Hello", 42) + run_job(job, 42, msg="Hello") + run_job(job, x=42, msg="Hello") [builtins fixtures/tuple.pyi] [case testExpandNonBareParamSpecAgainstCallable] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 6c6656209782..4b1f4ce00da7 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -847,8 +847,8 @@ Any(arg=str) # E: Any(...) is no longer supported. Use cast(Any, ...) instead [case testTypeListAsType] -def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ - # N: Did you mean "List[...]"? +def f(x:[int, str]) -> None: # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? pass [out] From 0fff609705734e5474c609f1de22221630528a13 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Thu, 10 Mar 2022 11:38:28 +0900 Subject: [PATCH 37/41] Update tests --- mypy/expandtype.py | 3 +- mypy/types.py | 16 +++--- .../unit/check-parameter-specification.test | 51 +++++++++++++++---- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index ee08e9c9306d..a9c71caa98fa 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -127,7 +127,8 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: else: return Parameters(t.prefix.arg_types + repl.arg_types, t.prefix.arg_kinds + repl.arg_kinds, - t.prefix.arg_names + repl.arg_names) + t.prefix.arg_names + repl.arg_names, + variables=t.prefix.variables + repl.variables) else: # TODO: should this branch be removed? better not to fail silently return repl diff --git a/mypy/types.py b/mypy/types.py index 88425058efed..1cbd802be6eb 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1189,13 +1189,15 @@ class Parameters(ProperType): 'arg_kinds', 'arg_names', 'min_args', - 'is_ellipsis_args') + 'is_ellipsis_args', + 'variables') def __init__(self, arg_types: Sequence[Type], arg_kinds: List[ArgKind], arg_names: Sequence[Optional[str]], *, + variables: Optional[Sequence[TypeVarLikeType]] = None, is_ellipsis_args: bool = False, line: int = -1, column: int = -1 @@ -1207,12 +1209,14 @@ def __init__(self, assert len(arg_types) == len(arg_kinds) == len(arg_names) self.min_args = arg_kinds.count(ARG_POS) self.is_ellipsis_args = is_ellipsis_args + self.variables = variables or [] def copy_modified(self, arg_types: Bogus[Sequence[Type]] = _dummy, arg_kinds: Bogus[List[ArgKind]] = _dummy, arg_names: Bogus[Sequence[Optional[str]]] = _dummy, *, + variables: Bogus[Sequence[TypeVarLikeType]] = _dummy, is_ellipsis_args: Bogus[bool] = _dummy ) -> 'Parameters': return Parameters( @@ -1220,7 +1224,8 @@ def copy_modified(self, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, arg_names=arg_names if arg_names is not _dummy else self.arg_names, is_ellipsis_args=(is_ellipsis_args if is_ellipsis_args is not _dummy - else self.is_ellipsis_args) + else self.is_ellipsis_args), + variables=variables if variables is not _dummy else self.variables ) # the following are copied from CallableType. Is there a way to decrease code duplication? @@ -1321,6 +1326,7 @@ def serialize(self) -> JsonDict: 'arg_types': [t.serialize() for t in self.arg_types], 'arg_kinds': [int(x.value) for x in self.arg_kinds], 'arg_names': self.arg_names, + 'variables': [tv.serialize() for tv in self.variables], } @classmethod @@ -1330,6 +1336,7 @@ def deserialize(cls, data: JsonDict) -> 'Parameters': [deserialize_type(t) for t in data['arg_types']], [ArgKind(x) for x in data['arg_kinds']], data['arg_names'], + variables=[deserialize_type(tv) for tv in data['variables']], ) def __hash__(self) -> int: @@ -1653,10 +1660,7 @@ def param_spec(self) -> Optional[ParamSpecType]: def expand_param_spec(self, c: Union['CallableType', Parameters], no_prefix: bool = False) -> 'CallableType': - if isinstance(c, CallableType): - variables = c.variables - else: - variables = [] + variables = c.variables if no_prefix: return self.copy_modified(arg_types=c.arg_types, diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index e19ab77c7d1e..694ba1c7e102 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -456,7 +456,6 @@ reveal_type(k(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" # literals can be matched in arguments def kb(a: Z[[bytes]]) -> Z[[str]]: ... -# TODO: return type is a bit weird, return Any reveal_type(kb(n)) # N: Revealed type is "__main__.Z[[builtins.str]]" \ # E: Argument 1 to "kb" has incompatible type "Z[[int]]"; expected "Z[[bytes]]" @@ -802,17 +801,17 @@ class Job(Generic[_P, _R_co]): def __init__(self, target: Callable[_P, _R_co]) -> None: self.target = target -def run_job(job: Job[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: +def run_job(job: Job[_P, None], *args: _P.args, **kwargs: _P.kwargs) -> None: # N: "run_job" defined here ... def func(job: Job[[int, str], None]) -> None: run_job(job, 42, "Hello") - run_job(job, "Hello", 42) # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ - # E: Argument 3 to "run_job" has incompatible type "int"; expected "str" + run_job(job, "Hello", 42) # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ + # E: Argument 3 to "run_job" has incompatible type "int"; expected "str" run_job(job, 42, msg="Hello") # E: Unexpected keyword argument "msg" for "run_job" - run_job(job, "Hello") # Too few arguments for "run_job" - # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" \ + run_job(job, "Hello") # E: Too few arguments for "run_job" \ + # E: Argument 2 to "run_job" has incompatible type "str"; expected "int" def func2(job: Job[..., None]) -> None: run_job(job, 42, "Hello") @@ -895,7 +894,7 @@ class Job(Generic[_P]): ... @simple_decorator -def func(action: Job[_P], /) -> Callable[_P, None]: +def func(__action: Job[_P]) -> Callable[_P, None]: ... reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" @@ -913,7 +912,7 @@ class Job(Generic[_P]): ... @simple_decorator -def func(action: Job[_P], /) -> Callable[_P, None]: +def func(__action: Job[_P]) -> Callable[_P, None]: ... reveal_type(func) # N: Revealed type is "def [_P] (__main__.Job[_P`-1]) -> def (*_P.args, **_P.kwargs)" @@ -941,7 +940,7 @@ def adds_await() -> Callable[ [builtins fixtures/tuple.pyi] [case testParamSpecVariance] -from typing import Callable, TypeVar, Generic +from typing import Callable, Generic from typing_extensions import ParamSpec _P = ParamSpec("_P") @@ -984,4 +983,38 @@ a2 = Job(f1) def f4(n: bytes) -> None: ... a1 = f4 # E: Incompatible types in assignment (expression has type "Callable[[bytes], None]", variable has type "Callable[[bool], None]") a2 = Job(f4) # E: Argument 1 to "Job" has incompatible type "Callable[[bytes], None]"; expected "Callable[[bool], None]" + +# nor is this: +a4: Job[[int]] +a4 = Job(f3) # E: Argument 1 to "Job" has incompatible type "Callable[[bool], None]"; expected "Callable[[int], None]" +a4 = Job(f2) +a4 = Job(f1) + +# just like this: +a3: Callable[[int], None] +a3 = f3 # E: Incompatible types in assignment (expression has type "Callable[[bool], None]", variable has type "Callable[[int], None]") +a3 = f2 +a3 = f1 +[builtins fixtures/tuple.pyi] + +[case testGenericsInInferredParamspec] +from typing import Callable, TypeVar, Generic +from typing_extensions import ParamSpec + +_P = ParamSpec("_P") +_T = TypeVar("_T") + +class Job(Generic[_P]): + def __init__(self, target: Callable[_P, None]) -> None: ... + def into_callable(self) -> Callable[_P, None]: ... + +def generic_f(x: _T) -> None: ... + +j = Job(generic_f) +reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" + +jf = j.into_callable() +reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`-1)" +reveal_type(jf(1)) # N: Revealed type is "None" + [builtins fixtures/tuple.pyi] From 4475515656428c14de0d9131cb5106801503a761 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 26 Mar 2022 08:54:51 +0900 Subject: [PATCH 38/41] Some of the PR feedback --- mypy/subtypes.py | 2 +- mypy/typeanal.py | 7 ++++++- test-data/unit/check-parameter-specification.test | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index eac2c21008a3..c718e41d7a80 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -965,10 +965,10 @@ def are_parameters_compatible(left: Union[Parameters, CallableType], check_args_covariantly: bool = False, allow_partial_overlap: bool = False, strict_concatenate_check: bool = True) -> bool: + """Helper function for is_callable_compatible, used for Parameter compatibility""" if right.is_ellipsis_args: return True - """Helper function for is_callable_compatible, used for Parameter compatibility""" left_star = left.var_arg() left_star2 = left.kw_arg() right_star = right.var_arg() diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 3913ae8ed31c..f0383b546c86 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -288,12 +288,17 @@ def apply_concatenate_operator(self, t: UnboundType) -> Type: self.api.fail('Concatenate needs type arguments', t) return AnyType(TypeOfAny.from_error) - # last argument has to be ParamSpec (or Concatenate) + # last argument has to be ParamSpec ps = self.anal_type(t.args[-1], allow_param_spec=True) if not isinstance(ps, ParamSpecType): self.api.fail('The last parameter to Concatenate needs to be a ParamSpec', t) return AnyType(TypeOfAny.from_error) + # TODO: this may not work well with aliases, if those worked. + # Those should be special-cased. + elif ps.prefix.arg_types: + self.api.fail('Nested Concatenates are invalid', t) + args = self.anal_array(t.args[:-1]) pre = ps.prefix diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 694ba1c7e102..b236243551da 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -1016,5 +1016,13 @@ reveal_type(j) # N: Revealed type is "__main__.Job[[x: _T`-1]]" jf = j.into_callable() reveal_type(jf) # N: Revealed type is "def [_T] (x: _T`-1)" reveal_type(jf(1)) # N: Revealed type is "None" +[builtins fixtures/tuple.pyi] + +[case testStackedConcatenateIsIllegal] +from typing_extensions import Concatenate, ParamSpec +from typing import Callable + +P = ParamSpec("P") +def x(f: Callable[Concatenate[int, Concatenate[int, P]], None]) -> None: ... # E: Nested Concatenates are invalid [builtins fixtures/tuple.pyi] From 81994f13510f78c8fad61a8d35658a99727e976b Mon Sep 17 00:00:00 2001 From: A5rocks Date: Sat, 26 Mar 2022 10:19:41 +0900 Subject: [PATCH 39/41] Prepare for GitHub actions --- mypy/expandtype.py | 2 +- mypy/nodes.py | 2 +- mypy/types.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 6667942c23f6..2c0e815202e2 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -129,7 +129,7 @@ def visit_param_spec(self, t: ParamSpecType) -> Type: return Parameters(t.prefix.arg_types + repl.arg_types, t.prefix.arg_kinds + repl.arg_kinds, t.prefix.arg_names + repl.arg_names, - variables=t.prefix.variables + repl.variables) + variables=[*t.prefix.variables, *repl.variables]) else: # TODO: should this branch be removed? better not to fail silently return repl diff --git a/mypy/nodes.py b/mypy/nodes.py index df2604776f1f..7fcf5d85673c 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2676,7 +2676,7 @@ def add_type_vars(self) -> None: for vd in self.defn.type_vars: if isinstance(vd, mypy.types.ParamSpecType): self.has_param_spec_type = True - self.type_vars.append(vd.fullname) + self.type_vars.append(vd.name) @property def name(self) -> str: diff --git a/mypy/types.py b/mypy/types.py index 60a1c72939c3..40eba023fd5c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1365,7 +1365,7 @@ def deserialize(cls, data: JsonDict) -> 'Parameters': [deserialize_type(t) for t in data['arg_types']], [ArgKind(x) for x in data['arg_kinds']], data['arg_names'], - variables=[deserialize_type(tv) for tv in data['variables']], + variables=[cast(TypeVarLikeType, deserialize_type(v)) for v in data['variables']], ) def __hash__(self) -> int: From c79918e0dc37763ffc01f56a07488a6df5662be7 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 5 Apr 2022 10:17:24 +0900 Subject: [PATCH 40/41] Fix tests to latest output Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- test-data/unit/check-parameter-specification.test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 16cdf43c6d0c..1c1284df0d84 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -444,7 +444,7 @@ reveal_type(unt2) # N: Revealed type is "__main__.Z[[builtins.int]]" def fT(a: T) -> T: ... def fP(a: Z[P]) -> Z[P]: ... -reveal_type(fT(n)) # N: Revealed type is "__main__.Z*[[builtins.int]]" +reveal_type(fT(n)) # N: Revealed type is "__main__.Z[[builtins.int]]" reveal_type(fP(n)) # N: Revealed type is "__main__.Z[[builtins.int]]" # literals can be in function args and return type @@ -486,7 +486,7 @@ def takes_int_str(request: Request, x: int, y: str) -> int: # use request return x + 7 -reveal_type(takes_int_str) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int*" +reveal_type(takes_int_str) # N: Revealed type is "def (x: builtins.int, y: builtins.str) -> builtins.int" takes_int_str(1, "A") # Accepted takes_int_str("B", 2) # E: Argument 1 to "takes_int_str" has incompatible type "str"; expected "int" \ @@ -555,7 +555,7 @@ def f(one: Callable[Concatenate[int, P], R], two: Callable[Concatenate[str, P], a: Callable[[int, bytes], str] b: Callable[[str, bytes], str] -reveal_type(f(a, b)) # N: Revealed type is "def (builtins.bytes) -> builtins.str*" +reveal_type(f(a, b)) # N: Revealed type is "def (builtins.bytes) -> builtins.str" [builtins fixtures/tuple.pyi] [case testParamSpecConcatenateInReturn] @@ -638,7 +638,7 @@ def a(n: int) -> None: ... n = f(a) -reveal_type(n) # N: Revealed type is "def (builtins.int*)" +reveal_type(n) # N: Revealed type is "def (builtins.int)" reveal_type(n(42)) # N: Revealed type is "None" [builtins fixtures/tuple.pyi] @@ -846,7 +846,7 @@ reveal_type(A.func) # N: Revealed type is "def [_P, _R] (self: __main__.A, acti def f(x: int) -> int: ... -reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int*" +reveal_type(A().func(f, 42)) # N: Revealed type is "builtins.int" # TODO: this should reveal `int` reveal_type(A().func(lambda x: x + x, 42)) # N: Revealed type is "Any" From 9b1fc750a46b21b9316bedcb6df2c284be951f79 Mon Sep 17 00:00:00 2001 From: A5rocks Date: Tue, 5 Apr 2022 10:35:29 +0900 Subject: [PATCH 41/41] Copy pyright's representation of Concatenate pros: - more readable IMO - more concise - standardization of output cons: - can be easy to miss double backets - I feel like this is a small deviation from normal mypy representations... but I have nothing to back that up. Overall, I think it's a good thing to try at least. --- mypy/messages.py | 2 +- mypy/types.py | 2 +- test-data/unit/check-parameter-specification.test | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index 58c5a63ab6fd..0e9a59ea4016 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -1768,7 +1768,7 @@ def format_literal_value(typ: LiteralType) -> str: format, verbosity) - return f'Concatenate[{args}, {typ.name_with_suffix()}]' + return f'[{args}, **{typ.name_with_suffix()}]' else: return typ.name_with_suffix() elif isinstance(typ, TupleType): diff --git a/mypy/types.py b/mypy/types.py index 9ccb24e53b07..06cf3f9e9dff 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -2591,7 +2591,7 @@ def visit_param_spec(self, t: ParamSpecType) -> str: # prefixes are displayed as Concatenate s = '' if t.prefix.arg_types: - s += f'Concatenate[{self.list_str(t.prefix.arg_types)}, ' + s += f'[{self.list_str(t.prefix.arg_types)}, **' if t.name is None: # Anonymous type variable type (only numeric id). s += f'`{t.id}' diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index 1c1284df0d84..fe2354612fbb 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -596,7 +596,7 @@ f2(lambda x: 42)(42, x=42) [out] main:10: error: invalid syntax [out version>=3.8] -main:17: error: Incompatible return value type (got "Callable[Concatenate[Arg(int, 'x'), P], R]", expected "Callable[Concatenate[int, P], R]") +main:17: error: Incompatible return value type (got "Callable[[Arg(int, 'x'), **P], R]", expected "Callable[[int, **P], R]") main:17: note: This may be because "result" has arguments named: "x" [case testNonStrictParamSpecConcatenateNamedArgs] @@ -762,14 +762,14 @@ def f(c: C[P]) -> None: reveal_type(c) # N: Revealed type is "__main__.C[P`-1]" n1 = c.add_str() - reveal_type(n1) # N: Revealed type is "__main__.C[Concatenate[builtins.int, P`-1]]" + reveal_type(n1) # N: Revealed type is "__main__.C[[builtins.int, **P`-1]]" n2 = n1.add_int() - reveal_type(n2) # N: Revealed type is "__main__.C[Concatenate[builtins.str, builtins.int, P`-1]]" + reveal_type(n2) # N: Revealed type is "__main__.C[[builtins.str, builtins.int, **P`-1]]" p1 = c.add_int() - reveal_type(p1) # N: Revealed type is "__main__.C[Concatenate[builtins.str, P`-1]]" + reveal_type(p1) # N: Revealed type is "__main__.C[[builtins.str, **P`-1]]" p2 = p1.add_str() - reveal_type(p2) # N: Revealed type is "__main__.C[Concatenate[builtins.int, builtins.str, P`-1]]" + reveal_type(p2) # N: Revealed type is "__main__.C[[builtins.int, builtins.str, **P`-1]]" [builtins fixtures/tuple.pyi] [case testParamSpecLiteralJoin]