From 2ab8849f6460fe586a1a1add3a006ce4edd3fb75 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Wed, 7 Jun 2023 01:40:57 +0200 Subject: [PATCH] Update semantic analyzer for TypeVar defaults (PEP 696) (#14873) This PR updates the semantic analyzer to support most forms of TypeVars with defaults while also providing basic argument validation. Ref: #14851 --- mypy/message_registry.py | 2 +- mypy/semanal.py | 152 ++++++++++++++---- mypy/types.py | 22 ++- .../unit/check-parameter-specification.test | 10 +- test-data/unit/check-typevar-defaults.test | 74 +++++++++ test-data/unit/semanal-errors.test | 3 +- 6 files changed, 224 insertions(+), 39 deletions(-) create mode 100644 test-data/unit/check-typevar-defaults.test diff --git a/mypy/message_registry.py b/mypy/message_registry.py index 776d0bfef213..c5164d48fd13 100644 --- a/mypy/message_registry.py +++ b/mypy/message_registry.py @@ -181,7 +181,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage: INVALID_TYPEVAR_ARG_BOUND: Final = 'Type argument {} of "{}" must be a subtype of {}' INVALID_TYPEVAR_ARG_VALUE: Final = 'Invalid type argument value for "{}"' TYPEVAR_VARIANCE_DEF: Final = 'TypeVar "{}" may only be a literal bool' -TYPEVAR_BOUND_MUST_BE_TYPE: Final = 'TypeVar "bound" must be a type' +TYPEVAR_ARG_MUST_BE_TYPE: Final = '{} "{}" must be a type' TYPEVAR_UNEXPECTED_ARGUMENT: Final = 'Unexpected argument to "TypeVar()"' UNBOUND_TYPEVAR: Final = ( "A function returning TypeVar should receive at least " diff --git a/mypy/semanal.py b/mypy/semanal.py index 3ca758ad4eb1..c5a6989f4f61 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -4135,28 +4135,15 @@ def process_typevar_parameters( if has_values: self.fail("TypeVar cannot have both values and an upper bound", context) return None - try: - # We want to use our custom error message below, so we suppress - # the default error message for invalid types here. - analyzed = self.expr_to_analyzed_type( - param_value, allow_placeholder=True, report_invalid_types=False - ) - if analyzed is None: - # Type variables are special: we need to place them in the symbol table - # soon, even if upper bound is not ready yet. Otherwise avoiding - # a "deadlock" in this common pattern would be tricky: - # T = TypeVar('T', bound=Custom[Any]) - # class Custom(Generic[T]): - # ... - analyzed = PlaceholderType(None, [], context.line) - upper_bound = get_proper_type(analyzed) - if isinstance(upper_bound, AnyType) and upper_bound.is_from_error: - self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value) - # Note: we do not return 'None' here -- we want to continue - # using the AnyType as the upper bound. - except TypeTranslationError: - self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value) + tv_arg = self.get_typevarlike_argument("TypeVar", param_name, param_value, context) + if tv_arg is None: return None + upper_bound = tv_arg + elif param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVar", param_name, param_value, context, allow_unbound_tvars=True + ) + default = tv_arg or AnyType(TypeOfAny.from_error) elif param_name == "values": # Probably using obsolete syntax with values=(...). Explain the current syntax. self.fail('TypeVar "values" argument not supported', context) @@ -4184,6 +4171,52 @@ def process_typevar_parameters( variance = INVARIANT return variance, upper_bound, default + def get_typevarlike_argument( + self, + typevarlike_name: str, + param_name: str, + param_value: Expression, + context: Context, + *, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, + report_invalid_typevar_arg: bool = True, + ) -> ProperType | None: + try: + # We want to use our custom error message below, so we suppress + # the default error message for invalid types here. + analyzed = self.expr_to_analyzed_type( + param_value, + allow_placeholder=True, + report_invalid_types=False, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, + ) + if analyzed is None: + # Type variables are special: we need to place them in the symbol table + # soon, even if upper bound is not ready yet. Otherwise avoiding + # a "deadlock" in this common pattern would be tricky: + # T = TypeVar('T', bound=Custom[Any]) + # class Custom(Generic[T]): + # ... + analyzed = PlaceholderType(None, [], context.line) + typ = get_proper_type(analyzed) + if report_invalid_typevar_arg and isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + # Note: we do not return 'None' here -- we want to continue + # using the AnyType. + return typ + except TypeTranslationError: + if report_invalid_typevar_arg: + self.fail( + message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name), + param_value, + ) + return None + def extract_typevarlike_name(self, s: AssignmentStmt, call: CallExpr) -> str | None: if not call: return None @@ -4216,13 +4249,50 @@ def process_paramspec_declaration(self, s: AssignmentStmt) -> bool: if name is None: return False - # ParamSpec is different from a regular TypeVar: - # arguments are not semantically valid. But, allowed in runtime. - # So, we need to warn users about possible invalid usage. - if len(call.args) > 1: - self.fail("Only the first argument to ParamSpec has defined semantics", s) + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "ParamSpec"', s) default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "ParamSpec", + param_name, + param_value, + s, + allow_unbound_tvars=True, + allow_param_spec_literals=True, + report_invalid_typevar_arg=False, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + if isinstance(tv_arg, Parameters): + for i, arg_type in enumerate(tv_arg.arg_types): + typ = get_proper_type(arg_type) + if isinstance(typ, AnyType) and typ.is_from_error: + self.fail( + f"Argument {i} of ParamSpec default must be a type", param_value + ) + elif ( + isinstance(default, AnyType) + and default.is_from_error + or not isinstance(default, (AnyType, UnboundType)) + ): + self.fail( + "The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec", + param_value, + ) + default = AnyType(TypeOfAny.from_error) + else: + # ParamSpec is different from a regular TypeVar: + # arguments are not semantically valid. But, allowed in runtime. + # So, we need to warn users about possible invalid usage. + self.fail( + "The variance and bound arguments to ParamSpec do not have defined semantics yet", + s, + ) # PEP 612 reserves the right to define bound, covariant and contravariant arguments to # ParamSpec in a later PEP. If and when that happens, we should do something @@ -4256,10 +4326,32 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool: if not call: return False - if len(call.args) > 1: - self.fail("Only the first argument to TypeVarTuple has defined semantics", s) + n_values = call.arg_kinds[1:].count(ARG_POS) + if n_values != 0: + self.fail('Too many positional arguments for "TypeVarTuple"', s) default: Type = AnyType(TypeOfAny.from_omitted_generics) + for param_value, param_name in zip( + call.args[1 + n_values :], call.arg_names[1 + n_values :] + ): + if param_name == "default": + tv_arg = self.get_typevarlike_argument( + "TypeVarTuple", + param_name, + param_value, + s, + allow_unbound_tvars=True, + report_invalid_typevar_arg=False, + ) + default = tv_arg or AnyType(TypeOfAny.from_error) + if not isinstance(default, UnpackType): + self.fail( + "The default argument to TypeVarTuple must be an Unpacked tuple", + param_value, + ) + default = AnyType(TypeOfAny.from_error) + else: + self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s) if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s): return False @@ -6359,6 +6451,8 @@ def expr_to_analyzed_type( report_invalid_types: bool = True, allow_placeholder: bool = False, allow_type_any: bool = False, + allow_unbound_tvars: bool = False, + allow_param_spec_literals: bool = False, ) -> Type | None: if isinstance(expr, CallExpr): # This is a legacy syntax intended mostly for Python 2, we keep it for @@ -6387,6 +6481,8 @@ def expr_to_analyzed_type( report_invalid_types=report_invalid_types, allow_placeholder=allow_placeholder, allow_type_any=allow_type_any, + allow_unbound_tvars=allow_unbound_tvars, + allow_param_spec_literals=allow_param_spec_literals, ) def analyze_type_expr(self, expr: Expression) -> None: diff --git a/mypy/types.py b/mypy/types.py index 53f21e8c0222..5fbdd385826c 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -3049,6 +3049,8 @@ def visit_type_var(self, t: TypeVarType) -> str: s = f"{t.name}`{t.id}" if self.id_mapper and t.upper_bound: s += f"(upper_bound={t.upper_bound.accept(self)})" + if t.has_default(): + s += f" = {t.default.accept(self)}" return s def visit_param_spec(self, t: ParamSpecType) -> str: @@ -3064,6 +3066,8 @@ def visit_param_spec(self, t: ParamSpecType) -> str: s += f"{t.name_with_suffix()}`{t.id}" if t.prefix.arg_types: s += "]" + if t.has_default(): + s += f" = {t.default.accept(self)}" return s def visit_parameters(self, t: Parameters) -> str: @@ -3102,6 +3106,8 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> str: else: # Named type variable type. s = f"{t.name}`{t.id}" + if t.has_default(): + s += f" = {t.default.accept(self)}" return s def visit_callable_type(self, t: CallableType) -> str: @@ -3138,6 +3144,8 @@ def visit_callable_type(self, t: CallableType) -> str: if s: s += ", " s += f"*{n}.args, **{n}.kwargs" + if param_spec.has_default(): + s += f" = {param_spec.default.accept(self)}" s = f"({s})" @@ -3156,12 +3164,18 @@ def visit_callable_type(self, t: CallableType) -> str: vals = f"({', '.join(val.accept(self) for val in var.values)})" vs.append(f"{var.name} in {vals}") elif not is_named_instance(var.upper_bound, "builtins.object"): - vs.append(f"{var.name} <: {var.upper_bound.accept(self)}") + vs.append( + f"{var.name} <: {var.upper_bound.accept(self)}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) else: - vs.append(var.name) + vs.append( + f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) else: - # For other TypeVarLikeTypes, just use the name - vs.append(var.name) + # For other TypeVarLikeTypes, use the name and default + vs.append( + f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}" + ) s = f"[{', '.join(vs)}] {s}" return f"def {s}" diff --git a/test-data/unit/check-parameter-specification.test b/test-data/unit/check-parameter-specification.test index fe66b18fbfea..901e73008d56 100644 --- a/test-data/unit/check-parameter-specification.test +++ b/test-data/unit/check-parameter-specification.test @@ -6,11 +6,11 @@ P = ParamSpec('P') [case testInvalidParamSpecDefinitions] from typing import ParamSpec -P1 = ParamSpec("P1", covariant=True) # E: Only the first argument to ParamSpec has defined semantics -P2 = ParamSpec("P2", contravariant=True) # E: Only the first argument to ParamSpec has defined semantics -P3 = ParamSpec("P3", bound=int) # E: Only the first argument to ParamSpec has defined semantics -P4 = ParamSpec("P4", int, str) # E: Only the first argument to ParamSpec has defined semantics -P5 = ParamSpec("P5", covariant=True, bound=int) # E: Only the first argument to ParamSpec has defined semantics +P1 = ParamSpec("P1", covariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P2 = ParamSpec("P2", contravariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P3 = ParamSpec("P3", bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet +P4 = ParamSpec("P4", int, str) # E: Too many positional arguments for "ParamSpec" +P5 = ParamSpec("P5", covariant=True, bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet [builtins fixtures/paramspec.pyi] [case testParamSpecLocations] diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test new file mode 100644 index 000000000000..7bc2d4089ecd --- /dev/null +++ b/test-data/unit/check-typevar-defaults.test @@ -0,0 +1,74 @@ +[case testTypeVarDefaultsBasic] +import builtins +from typing import Generic, TypeVar, ParamSpec, Callable, Tuple, List +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1", default=int) +P1 = ParamSpec("P1", default=[int, str]) +Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]]) + +def f1(a: T1) -> List[T1]: ... +reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]" + +def f2(a: Callable[P1, None] ) -> Callable[P1, None]: ... +reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)" + +def f3(a: Tuple[Unpack[Ts1]]) -> Tuple[Unpack[Ts1]]: ... +reveal_type(f3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]" + + +class ClassA1(Generic[T1]): ... +class ClassA2(Generic[P1]): ... +class ClassA3(Generic[Unpack[Ts1]]): ... + +reveal_type(ClassA1) # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]" +reveal_type(ClassA2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]" +reveal_type(ClassA3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]" +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsValid] +from typing import TypeVar, ParamSpec, Any, List, Tuple +from typing_extensions import TypeVarTuple, Unpack + +S0 = TypeVar("S0") +S1 = TypeVar("S1", bound=int) + +P0 = ParamSpec("P0") +Ts0 = TypeVarTuple("Ts0") + +T1 = TypeVar("T1", default=int) +T2 = TypeVar("T2", bound=float, default=int) +T3 = TypeVar("T3", bound=List[Any], default=List[int]) +T4 = TypeVar("T4", int, str, default=int) +T5 = TypeVar("T5", default=S0) +T6 = TypeVar("T6", bound=float, default=S1) +# T7 = TypeVar("T7", bound=List[Any], default=List[S0]) # TODO + +P1 = ParamSpec("P1", default=[]) +P2 = ParamSpec("P2", default=...) +P3 = ParamSpec("P3", default=[int, str]) +P4 = ParamSpec("P4", default=P0) + +Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int]]) +Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, ...]]) +# Ts3 = TypeVarTuple("Ts3", default=Unpack[Ts0]) # TODO +[builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsInvalid] +from typing import TypeVar, ParamSpec, Tuple +from typing_extensions import TypeVarTuple, Unpack + +T1 = TypeVar("T1", default=2) # E: TypeVar "default" must be a type +T2 = TypeVar("T2", default=[int, str]) # E: Bracketed expression "[...]" is not valid as a type \ + # N: Did you mean "List[...]"? \ + # E: TypeVar "default" must be a type + +P1 = ParamSpec("P1", default=int) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P2 = ParamSpec("P2", default=2) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P3 = ParamSpec("P3", default=(2, int)) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec +P4 = ParamSpec("P4", default=[2, int]) # E: Argument 0 of ParamSpec default must be a type + +Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple +Ts2 = TypeVarTuple("Ts2", default=int) # E: The default argument to TypeVarTuple must be an Unpacked tuple +Ts3 = TypeVarTuple("Ts3", default=Tuple[int]) # E: The default argument to TypeVarTuple must be an Unpacked tuple +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index d09ed87d3afc..0c3de312cdfa 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -1465,8 +1465,9 @@ TVariadic2 = TypeVarTuple('TVariadic2') TP = TypeVarTuple('?') # E: String argument 1 "?" to TypeVarTuple(...) does not match variable name "TP" TP2: int = TypeVarTuple('TP2') # E: Cannot declare the type of a TypeVar or similar construct TP3 = TypeVarTuple() # E: Too few arguments for TypeVarTuple() -TP4 = TypeVarTuple('TP4', 'TP4') # E: Only the first argument to TypeVarTuple has defined semantics +TP4 = TypeVarTuple('TP4', 'TP4') # E: Too many positional arguments for "TypeVarTuple" TP5 = TypeVarTuple(t='TP5') # E: TypeVarTuple() expects a string literal as first argument +TP6 = TypeVarTuple('TP6', bound=int) # E: Unexpected keyword argument "bound" for "TypeVarTuple" x: TVariadic # E: TypeVarTuple "TVariadic" is unbound y: Unpack[TVariadic] # E: TypeVarTuple "TVariadic" is unbound