Skip to content

Commit

Permalink
Error handling for recursive TypeVar defaults (PEP 696) (python#16925)
Browse files Browse the repository at this point in the history
This PR adds some additional error handling for recursive TypeVar
defaults.
Open issue for future PRs:
- Expanding nested recursive defaults, e.g. `T2 = list[T1 = str]`
- Scope binding, especially for TypeAliasTypes

Ref: python#14851
  • Loading branch information
cdce8p authored and hamdanal committed Feb 20, 2024
1 parent 1ee3e0b commit 1df0dd2
Show file tree
Hide file tree
Showing 5 changed files with 223 additions and 12 deletions.
9 changes: 9 additions & 0 deletions mypy/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -2059,6 +2059,15 @@ def impossible_intersection(
template.format(formatted_base_class_list, reason), context, code=codes.UNREACHABLE
)

def tvar_without_default_type(
self, tvar_name: str, last_tvar_name_with_default: str, context: Context
) -> None:
self.fail(
f'"{tvar_name}" cannot appear after "{last_tvar_name_with_default}" '
"in type parameter list because it has no default type",
context,
)

def report_protocol_problems(
self,
subtype: Instance | TupleType | TypedDictType | TypeType | CallableType,
Expand Down
47 changes: 38 additions & 9 deletions mypy/semanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@
SELF_TYPE_NAMES,
FindTypeVarVisitor,
TypeAnalyser,
TypeVarDefaultTranslator,
TypeVarLikeList,
analyze_type_alias,
check_for_explicit_any,
Expand All @@ -252,6 +253,7 @@
TPDICT_NAMES,
TYPE_ALIAS_NAMES,
TYPE_CHECK_ONLY_NAMES,
TYPE_VAR_LIKE_NAMES,
TYPED_NAMEDTUPLE_NAMES,
AnyType,
CallableType,
Expand Down Expand Up @@ -1953,17 +1955,19 @@ class Foo(Bar, Generic[T]): ...
defn.removed_base_type_exprs.append(defn.base_type_exprs[i])
del base_type_exprs[i]
tvar_defs: list[TypeVarLikeType] = []
last_tvar_name_with_default: str | None = None
for name, tvar_expr in declared_tvars:
tvar_expr_default = tvar_expr.default
if isinstance(tvar_expr_default, UnboundType):
# TODO: - detect out of order and self-referencing TypeVars
# - nested default types, e.g. list[T1]
n = self.lookup_qualified(
tvar_expr_default.name, tvar_expr_default, suppress_errors=True
)
if n is not None and (default := self.tvar_scope.get_binding(n)) is not None:
tvar_expr.default = default
tvar_expr.default = tvar_expr.default.accept(
TypeVarDefaultTranslator(self, tvar_expr.name, context)
)
tvar_def = self.tvar_scope.bind_new(name, tvar_expr)
if last_tvar_name_with_default is not None and not tvar_def.has_default():
self.msg.tvar_without_default_type(
tvar_def.name, last_tvar_name_with_default, context
)
tvar_def.default = AnyType(TypeOfAny.from_error)
elif tvar_def.has_default():
last_tvar_name_with_default = tvar_def.name
tvar_defs.append(tvar_def)
return base_type_exprs, tvar_defs, is_protocol

Expand Down Expand Up @@ -2857,6 +2861,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
with self.allow_unbound_tvars_set():
s.rvalue.accept(self)
self.basic_type_applications = old_basic_type_applications
elif self.can_possibly_be_typevarlike_declaration(s):
# Allow unbound tvars inside TypeVarLike defaults to be evaluated later
with self.allow_unbound_tvars_set():
s.rvalue.accept(self)
else:
s.rvalue.accept(self)

Expand Down Expand Up @@ -3033,6 +3041,16 @@ def can_possibly_be_type_form(self, s: AssignmentStmt) -> bool:
# Something that looks like Foo = Bar[Baz, ...]
return True

def can_possibly_be_typevarlike_declaration(self, s: AssignmentStmt) -> bool:
"""Check if r.h.s. can be a TypeVarLike declaration."""
if len(s.lvalues) != 1 or not isinstance(s.lvalues[0], NameExpr):
return False
if not isinstance(s.rvalue, CallExpr) or not isinstance(s.rvalue.callee, NameExpr):
return False
ref = s.rvalue.callee
ref.accept(self)
return ref.fullname in TYPE_VAR_LIKE_NAMES

def is_type_ref(self, rv: Expression, bare: bool = False) -> bool:
"""Does this expression refer to a type?
Expand Down Expand Up @@ -3522,9 +3540,20 @@ def analyze_alias(
tvar_defs: list[TypeVarLikeType] = []
namespace = self.qualified_name(name)
alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars
last_tvar_name_with_default: str | None = None
with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
for name, tvar_expr in alias_type_vars:
tvar_expr.default = tvar_expr.default.accept(
TypeVarDefaultTranslator(self, tvar_expr.name, typ)
)
tvar_def = self.tvar_scope.bind_new(name, tvar_expr)
if last_tvar_name_with_default is not None and not tvar_def.has_default():
self.msg.tvar_without_default_type(
tvar_def.name, last_tvar_name_with_default, typ
)
tvar_def.default = AnyType(TypeOfAny.from_error)
elif tvar_def.has_default():
last_tvar_name_with_default = tvar_def.name
tvar_defs.append(tvar_def)

analyzed, depends_on = analyze_type_alias(
Expand Down
36 changes: 35 additions & 1 deletion mypy/typeanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@
)
from mypy.options import Options
from mypy.plugin import AnalyzeTypeContext, Plugin, TypeAnalyzerPluginInterface
from mypy.semanal_shared import SemanticAnalyzerCoreInterface, paramspec_args, paramspec_kwargs
from mypy.semanal_shared import (
SemanticAnalyzerCoreInterface,
SemanticAnalyzerInterface,
paramspec_args,
paramspec_kwargs,
)
from mypy.state import state
from mypy.tvar_scope import TypeVarLikeScope
from mypy.types import (
Expand Down Expand Up @@ -2520,3 +2525,32 @@ def process_types(self, types: list[Type] | tuple[Type, ...]) -> None:
else:
for t in types:
t.accept(self)


class TypeVarDefaultTranslator(TrivialSyntheticTypeTranslator):
"""Type translate visitor that replaces UnboundTypes with in-scope TypeVars."""

def __init__(
self, api: SemanticAnalyzerInterface, tvar_expr_name: str, context: Context
) -> None:
self.api = api
self.tvar_expr_name = tvar_expr_name
self.context = context

def visit_unbound_type(self, t: UnboundType) -> Type:
sym = self.api.lookup_qualified(t.name, t, suppress_errors=True)
if sym is not None:
if type_var := self.api.tvar_scope.get_binding(sym):
return type_var
if isinstance(sym.node, TypeVarLikeExpr):
self.api.fail(
f'Type parameter "{self.tvar_expr_name}" has a default type '
"that refers to one or more type variables that are out of scope",
self.context,
)
return AnyType(TypeOfAny.from_error)
return super().visit_unbound_type(t)

def visit_type_alias_type(self, t: TypeAliasType) -> Type:
# TypeAliasTypes are analyzed separately already, just return it
return t
9 changes: 9 additions & 0 deletions mypy/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,15 @@
TypeVisitor as TypeVisitor,
)

TYPE_VAR_LIKE_NAMES: Final = (
"typing.TypeVar",
"typing_extensions.TypeVar",
"typing.ParamSpec",
"typing_extensions.ParamSpec",
"typing.TypeVarTuple",
"typing_extensions.TypeVarTuple",
)

TYPED_NAMEDTUPLE_NAMES: Final = ("typing.NamedTuple", "typing_extensions.NamedTuple")

# Supported names of TypedDict type constructors.
Expand Down
134 changes: 132 additions & 2 deletions test-data/unit/check-typevar-defaults.test
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,74 @@ T3 = TypeVar("T3", int, str, default=bytes) # E: TypeVar default must be one of
T4 = TypeVar("T4", int, str, default=Union[int, str]) # E: TypeVar default must be one of the constraint types
T5 = TypeVar("T5", float, str, default=int) # E: TypeVar default must be one of the constraint types

[case testTypeVarDefaultsInvalid3]
from typing import Dict, Generic, TypeVar

T1 = TypeVar("T1")
T2 = TypeVar("T2", default=T3) # E: Name "T3" is used before definition
T3 = TypeVar("T3", default=str)
T4 = TypeVar("T4", default=T3)

class ClassError1(Generic[T3, T1]): ... # E: "T1" cannot appear after "T3" in type parameter list because it has no default type

def func_error1(
a: ClassError1,
b: ClassError1[int],
c: ClassError1[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]"
reveal_type(b) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]"
reveal_type(c) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]"

k = ClassError1()
reveal_type(k) # N: Revealed type is "__main__.ClassError1[builtins.str, Any]"
l = ClassError1[int]()
reveal_type(l) # N: Revealed type is "__main__.ClassError1[builtins.int, Any]"
m = ClassError1[int, float]()
reveal_type(m) # N: Revealed type is "__main__.ClassError1[builtins.int, builtins.float]"

class ClassError2(Generic[T4, T3]): ... # E: Type parameter "T4" has a default type that refers to one or more type variables that are out of scope

def func_error2(
a: ClassError2,
b: ClassError2[int],
c: ClassError2[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]"
reveal_type(b) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]"
reveal_type(c) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]"

k = ClassError2()
reveal_type(k) # N: Revealed type is "__main__.ClassError2[Any, builtins.str]"
l = ClassError2[int]()
reveal_type(l) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.str]"
m = ClassError2[int, float]()
reveal_type(m) # N: Revealed type is "__main__.ClassError2[builtins.int, builtins.float]"

TERR1 = Dict[T3, T1] # E: "T1" cannot appear after "T3" in type parameter list because it has no default type

def func_error_alias1(
a: TERR1,
b: TERR1[int],
c: TERR1[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "builtins.dict[builtins.str, Any]"
reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, Any]"
reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"

TERR2 = Dict[T4, T3] # TODO should be an error \
# Type parameter "T4" has a default type that refers to one or more type variables that are out of scope

def func_error_alias2(
a: TERR2,
b: TERR2[int],
c: TERR2[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "builtins.dict[Any, builtins.str]"
reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.str]"
reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
[builtins fixtures/dict.pyi]

[case testTypeVarDefaultsFunctions]
from typing import TypeVar, ParamSpec, List, Union, Callable, Tuple
from typing_extensions import TypeVarTuple, Unpack
Expand Down Expand Up @@ -351,11 +419,12 @@ def func_c4(

[case testTypeVarDefaultsClassRecursive1]
# flags: --disallow-any-generics
from typing import Generic, TypeVar
from typing import Generic, TypeVar, List

T1 = TypeVar("T1", default=str)
T2 = TypeVar("T2", default=T1)
T3 = TypeVar("T3", default=T2)
T4 = TypeVar("T4", default=List[T1])

class ClassD1(Generic[T1, T2]): ...

Expand Down Expand Up @@ -397,12 +466,30 @@ def func_d2(
n = ClassD2[int, float, str]()
reveal_type(n) # N: Revealed type is "__main__.ClassD2[builtins.int, builtins.float, builtins.str]"

class ClassD3(Generic[T1, T4]): ...

def func_d3(
a: ClassD3,
b: ClassD3[int],
c: ClassD3[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]"
reveal_type(b) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
reveal_type(c) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"

# k = ClassD3()
# reveal_type(k) # Revealed type is "__main__.ClassD3[builtins.str, builtins.list[builtins.str]]" # TODO
l = ClassD3[int]()
reveal_type(l) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.list[builtins.int]]"
m = ClassD3[int, float]()
reveal_type(m) # N: Revealed type is "__main__.ClassD3[builtins.int, builtins.float]"

[case testTypeVarDefaultsClassRecursiveMultipleFiles]
# flags: --disallow-any-generics
from typing import Generic, TypeVar
from file2 import T as T2

T = TypeVar('T', default=T2)
T = TypeVar("T", default=T2)

class ClassG1(Generic[T2, T]):
pass
Expand Down Expand Up @@ -587,3 +674,46 @@ def func_c4(
# reveal_type(b) # Revealed type is "Tuple[builtins.int, builtins.str]" # TODO
reveal_type(c) # N: Revealed type is "Tuple[builtins.int, builtins.float]"
[builtins fixtures/tuple.pyi]

[case testTypeVarDefaultsTypeAliasRecursive1]
# flags: --disallow-any-generics
from typing import Dict, List, TypeVar

T1 = TypeVar("T1")
T2 = TypeVar("T2", default=T1)

TD1 = Dict[T1, T2]

def func_d1(
a: TD1, # E: Missing type parameters for generic type "TD1"
b: TD1[int],
c: TD1[int, float],
) -> None:
reveal_type(a) # N: Revealed type is "builtins.dict[Any, Any]"
reveal_type(b) # N: Revealed type is "builtins.dict[builtins.int, builtins.int]"
reveal_type(c) # N: Revealed type is "builtins.dict[builtins.int, builtins.float]"
[builtins fixtures/dict.pyi]

[case testTypeVarDefaultsTypeAliasRecursive2]
from typing import Any, Dict, Generic, TypeVar

T1 = TypeVar("T1", default=str)
T2 = TypeVar("T2", default=T1)
Alias1 = Dict[T1, T2]
T3 = TypeVar("T3")
class A(Generic[T3]): ...

T4 = TypeVar("T4", default=A[Alias1])
class B(Generic[T4]): ...

def func_d3(
a: B,
b: B[A[Alias1[int]]],
c: B[A[Alias1[int, float]]],
d: B[int],
) -> None:
reveal_type(a) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.str, builtins.str]]]"
reveal_type(b) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.int]]]"
reveal_type(c) # N: Revealed type is "__main__.B[__main__.A[builtins.dict[builtins.int, builtins.float]]]"
reveal_type(d) # N: Revealed type is "__main__.B[builtins.int]"
[builtins fixtures/dict.pyi]

0 comments on commit 1df0dd2

Please sign in to comment.