From 1df0dd269c7d442f858b7803068d05a368fe0155 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Tue, 20 Feb 2024 01:02:49 +0100 Subject: [PATCH] Error handling for recursive TypeVar defaults (PEP 696) (#16925) This PR adds some additional error handling for recursive TypeVar defaults. Open issue for future PRs: - Expanding nested recursive defaults, e.g. `T2 = list[T1 = str]` - Scope binding, especially for TypeAliasTypes Ref: https://github.com/python/mypy/issues/14851 --- mypy/messages.py | 9 ++ mypy/semanal.py | 47 ++++++-- mypy/typeanal.py | 36 +++++- mypy/types.py | 9 ++ test-data/unit/check-typevar-defaults.test | 134 ++++++++++++++++++++- 5 files changed, 223 insertions(+), 12 deletions(-) diff --git a/mypy/messages.py b/mypy/messages.py index c107e874f4fc0..db6c91ba90083 100644 --- a/mypy/messages.py +++ b/mypy/messages.py @@ -2059,6 +2059,15 @@ def impossible_intersection( template.format(formatted_base_class_list, reason), context, code=codes.UNREACHABLE ) + def tvar_without_default_type( + self, tvar_name: str, last_tvar_name_with_default: str, context: Context + ) -> None: + self.fail( + f'"{tvar_name}" cannot appear after "{last_tvar_name_with_default}" ' + "in type parameter list because it has no default type", + context, + ) + def report_protocol_problems( self, subtype: Instance | TupleType | TypedDictType | TypeType | CallableType, diff --git a/mypy/semanal.py b/mypy/semanal.py index 055640c7e906d..3f7d1f78e3bed 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -226,6 +226,7 @@ SELF_TYPE_NAMES, FindTypeVarVisitor, TypeAnalyser, + TypeVarDefaultTranslator, TypeVarLikeList, analyze_type_alias, check_for_explicit_any, @@ -252,6 +253,7 @@ TPDICT_NAMES, TYPE_ALIAS_NAMES, TYPE_CHECK_ONLY_NAMES, + TYPE_VAR_LIKE_NAMES, TYPED_NAMEDTUPLE_NAMES, AnyType, CallableType, @@ -1953,17 +1955,19 @@ class Foo(Bar, Generic[T]): ... defn.removed_base_type_exprs.append(defn.base_type_exprs[i]) del base_type_exprs[i] tvar_defs: list[TypeVarLikeType] = [] + last_tvar_name_with_default: str | None = None for name, tvar_expr in declared_tvars: - tvar_expr_default = tvar_expr.default - if isinstance(tvar_expr_default, UnboundType): - # TODO: - detect out of order and self-referencing TypeVars - # - nested default types, e.g. list[T1] - n = self.lookup_qualified( - tvar_expr_default.name, tvar_expr_default, suppress_errors=True - ) - if n is not None and (default := self.tvar_scope.get_binding(n)) is not None: - tvar_expr.default = default + tvar_expr.default = tvar_expr.default.accept( + TypeVarDefaultTranslator(self, tvar_expr.name, context) + ) tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + if last_tvar_name_with_default is not None and not tvar_def.has_default(): + self.msg.tvar_without_default_type( + tvar_def.name, last_tvar_name_with_default, context + ) + tvar_def.default = AnyType(TypeOfAny.from_error) + elif tvar_def.has_default(): + last_tvar_name_with_default = tvar_def.name tvar_defs.append(tvar_def) return base_type_exprs, tvar_defs, is_protocol @@ -2857,6 +2861,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: with self.allow_unbound_tvars_set(): s.rvalue.accept(self) self.basic_type_applications = old_basic_type_applications + elif self.can_possibly_be_typevarlike_declaration(s): + # Allow unbound tvars inside TypeVarLike defaults to be evaluated later + with self.allow_unbound_tvars_set(): + s.rvalue.accept(self) else: s.rvalue.accept(self) @@ -3033,6 +3041,16 @@ def can_possibly_be_type_form(self, s: AssignmentStmt) -> bool: # Something that looks like Foo = Bar[Baz, ...] return True + def can_possibly_be_typevarlike_declaration(self, s: AssignmentStmt) -> bool: + """Check if r.h.s. can be a TypeVarLike declaration.""" + if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr): + return False + if not isinstance(s.rvalue, CallExpr) or not isinstance(s.rvalue.callee, NameExpr): + return False + ref = s.rvalue.callee + ref.accept(self) + return ref.fullname in TYPE_VAR_LIKE_NAMES + def is_type_ref(self, rv: Expression, bare: bool = False) -> bool: """Does this expression refer to a type? @@ -3522,9 +3540,20 @@ def analyze_alias( tvar_defs: list[TypeVarLikeType] = [] namespace = self.qualified_name(name) alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars + last_tvar_name_with_default: str | None = None with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)): for name, tvar_expr in alias_type_vars: + tvar_expr.default = tvar_expr.default.accept( + TypeVarDefaultTranslator(self, tvar_expr.name, typ) + ) tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + if last_tvar_name_with_default is not None and not tvar_def.has_default(): + self.msg.tvar_without_default_type( + tvar_def.name, last_tvar_name_with_default, typ + ) + tvar_def.default = AnyType(TypeOfAny.from_error) + elif tvar_def.has_default(): + last_tvar_name_with_default = tvar_def.name tvar_defs.append(tvar_def) analyzed, depends_on = analyze_type_alias( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b542b5d5e3cc5..9ede8b1090acf 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -38,7 +38,12 @@ ) from mypy.options import Options from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface -from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs +from mypy.semanal_shared import ( + SemanticAnalyzerCoreInterface, + SemanticAnalyzerInterface, + paramspec_args, + paramspec_kwargs, +) from mypy.state import state from mypy.tvar_scope import TypeVarLikeScope from mypy.types import ( @@ -2520,3 +2525,32 @@ def process_types(self, types: list[Type] | tuple[Type, ...]) -> None: else: for t in types: t.accept(self) + + +class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator): + """Type translate visitor that replaces UnboundTypes with in-scope TypeVars.""" + + def __init__( + self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context + ) -> None: + self.api = api + self.tvar_expr_name = tvar_expr_name + self.context = context + + def visit_unbound_type(self, t: UnboundType) -> Type: + sym = self.api.lookup_qualified(t.name, t, suppress_errors=True) + if sym is not None: + if type_var := self.api.tvar_scope.get_binding(sym): + return type_var + if isinstance(sym.node, TypeVarLikeExpr): + self.api.fail( + f'Type parameter "{self.tvar_expr_name}" has a default type ' + "that refers to one or more type variables that are out of scope", + self.context, + ) + return AnyType(TypeOfAny.from_error) + return super().visit_unbound_type(t) + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + # TypeAliasTypes are analyzed separately already, just return it + return t diff --git a/mypy/types.py b/mypy/types.py index b1119c9447e2d..f76e35784d8fe 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -85,6 +85,15 @@ TypeVisitor as TypeVisitor, ) +TYPE_VAR_LIKE_NAMES: Final = ( + "typing.TypeVar", + "typing_extensions.TypeVar", + "typing.ParamSpec", + "typing_extensions.ParamSpec", + "typing.TypeVarTuple", + "typing_extensions.TypeVarTuple", +) + TYPED_NAMEDTUPLE_NAMES: Final = ("typing.NamedTuple", "typing_extensions.NamedTuple") # Supported names of TypedDict type constructors. diff --git a/test-data/unit/check-typevar-defaults.test b/test-data/unit/check-typevar-defaults.test index 1a08823cb692a..9ca67376da26a 100644 --- a/test-data/unit/check-typevar-defaults.test +++ b/test-data/unit/check-typevar-defaults.test @@ -82,6 +82,74 @@ T3 = TypeVar("T3", int, str, default=bytes) # E: TypeVar default must be one of T4 = TypeVar("T4", int, str, default=Union[int, str]) # E: TypeVar default must be one of the constraint types T5 = TypeVar("T5", float, str, default=int) # E: TypeVar default must be one of the constraint types +[case testTypeVarDefaultsInvalid3] +from typing import Dict, Generic, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=T3) # E: Name "T3" is used before definition +T3 = TypeVar("T3", default=str) +T4 = TypeVar("T4", default=T3) + +class ClassError1(Generic[T3, T1]): ... # E: "T1" cannot appear after "T3" in type parameter list because it has no default type + +def func_error1( + a: ClassError1, + b: ClassError1[int], + c: ClassError1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]" + reveal_type(b) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]" + reveal_type(c) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]" + + k = ClassError1() + reveal_type(k) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]" + l = ClassError1[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]" + m = ClassError1[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]" + +class ClassError2(Generic[T4, T3]): ... # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope + +def func_error2( + a: ClassError2, + b: ClassError2[int], + c: ClassError2[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]" + reveal_type(b) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]" + + k = ClassError2() + reveal_type(k) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]" + l = ClassError2[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]" + m = ClassError2[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]" + +TERR1 = Dict[T3, T1] # E: "T1" cannot appear after "T3" in type parameter list because it has no default type + +def func_error_alias1( + a: TERR1, + b: TERR1[int], + c: TERR1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[builtins.str, Any]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, Any]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" + +TERR2 = Dict[T4, T3] # TODO should be an error \ + # Type parameter "T4" has a default type that refers to one or more type variables that are out of scope + +def func_error_alias2( + a: TERR2, + b: TERR2[int], + c: TERR2[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[Any, builtins.str]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" +[builtins fixtures/dict.pyi] + [case testTypeVarDefaultsFunctions] from typing import TypeVar, ParamSpec, List, Union, Callable, Tuple from typing_extensions import TypeVarTuple, Unpack @@ -351,11 +419,12 @@ def func_c4( [case testTypeVarDefaultsClassRecursive1] # flags: --disallow-any-generics -from typing import Generic, TypeVar +from typing import Generic, TypeVar, List T1 = TypeVar("T1", default=str) T2 = TypeVar("T2", default=T1) T3 = TypeVar("T3", default=T2) +T4 = TypeVar("T4", default=List[T1]) class ClassD1(Generic[T1, T2]): ... @@ -397,12 +466,30 @@ def func_d2( n = ClassD2[int, float, str]() reveal_type(n) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.str]" +class ClassD3(Generic[T1, T4]): ... + +def func_d3( + a: ClassD3, + b: ClassD3[int], + c: ClassD3[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" + reveal_type(b) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]" + reveal_type(c) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]" + + # k = ClassD3() + # reveal_type(k) # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" # TODO + l = ClassD3[int]() + reveal_type(l) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]" + m = ClassD3[int, float]() + reveal_type(m) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]" + [case testTypeVarDefaultsClassRecursiveMultipleFiles] # flags: --disallow-any-generics from typing import Generic, TypeVar from file2 import T as T2 -T = TypeVar('T', default=T2) +T = TypeVar("T", default=T2) class ClassG1(Generic[T2, T]): pass @@ -587,3 +674,46 @@ def func_c4( # reveal_type(b) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]" [builtins fixtures/tuple.pyi] + +[case testTypeVarDefaultsTypeAliasRecursive1] +# flags: --disallow-any-generics +from typing import Dict, List, TypeVar + +T1 = TypeVar("T1") +T2 = TypeVar("T2", default=T1) + +TD1 = Dict[T1, T2] + +def func_d1( + a: TD1, # E: Missing type parameters for generic type "TD1" + b: TD1[int], + c: TD1[int, float], +) -> None: + reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]" + reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.int]" + reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]" +[builtins fixtures/dict.pyi] + +[case testTypeVarDefaultsTypeAliasRecursive2] +from typing import Any, Dict, Generic, TypeVar + +T1 = TypeVar("T1", default=str) +T2 = TypeVar("T2", default=T1) +Alias1 = Dict[T1, T2] +T3 = TypeVar("T3") +class A(Generic[T3]): ... + +T4 = TypeVar("T4", default=A[Alias1]) +class B(Generic[T4]): ... + +def func_d3( + a: B, + b: B[A[Alias1[int]]], + c: B[A[Alias1[int, float]]], + d: B[int], +) -> None: + reveal_type(a) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.str, builtins.str]]]" + reveal_type(b) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.int]]]" + reveal_type(c) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.float]]]" + reveal_type(d) # N: Revealed type is "__main__.B[builtins.int]" +[builtins fixtures/dict.pyi]