From 7d0d1d9d505869d55ee71e370941a8a47122087e Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 16 Nov 2022 00:02:17 +0000 Subject: [PATCH] Fix crash on nested generic callable (#14093) Fixes #10244 Fixes #13515 This fixes only the crash part, I am going to fix also the embarrassing type variable clash in a separate PR, since it is completely unrelated issue. The crash happens because solver can call `is_suptype()` on the constraint bounds, and those can contain ``. Then if it is a generic callable type (e.g. `def [S] (S) -> T` when used as a context is erased to `def [S] (S) -> `), `is_subtype()` will try unifying them, causing the crash when applying unified arguments. My fix is to simply allow subtyping between callable types that contain ``, we anyway allow checking subtpying between all other types with `` components. And this technically can be useful, e.g. `[T <: DerivedGen1[], T <: DerivedGen2[]]` will be solved as `T <: NonGenBase`. Btw this crash technically has nothing to do with dataclasses, but it looks like there is no other way in mypy to define a callable with generic callable as argument type, if I try: ```python def foo(x: Callable[[S], T]) -> T: ... ``` to repro the crash, mypy instead interprets `foo` as `def [S, T] (x: Callable[[S], T]) -> T`, i.e. the argument type is not generic. I also tried callback protocols, but they also don't repro the crash (at least I can't find a repro), because protocols use variance for subtyping, before actually checking member types. --- mypy/applytype.py | 19 +++++++++++++----- mypy/expandtype.py | 29 ++++++++++++++++++++------- mypy/subtypes.py | 6 +++++- test-data/unit/check-dataclasses.test | 23 +++++++++++++++++++++ 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/mypy/applytype.py b/mypy/applytype.py index 1c401664568d..d7f31b36c244 100644 --- a/mypy/applytype.py +++ b/mypy/applytype.py @@ -73,6 +73,7 @@ def apply_generic_arguments( report_incompatible_typevar_value: Callable[[CallableType, Type, str, Context], None], context: Context, skip_unsatisfied: bool = False, + allow_erased_callables: bool = False, ) -> CallableType: """Apply generic type arguments to a callable type. @@ -130,18 +131,26 @@ def apply_generic_arguments( + callable.arg_names[star_index + 1 :] ) arg_types = ( - [expand_type(at, id_to_type) for at in callable.arg_types[:star_index]] + [ + expand_type(at, id_to_type, allow_erased_callables) + for at in callable.arg_types[:star_index] + ] + expanded - + [expand_type(at, id_to_type) for at in callable.arg_types[star_index + 1 :]] + + [ + expand_type(at, id_to_type, allow_erased_callables) + for at in callable.arg_types[star_index + 1 :] + ] ) else: - arg_types = [expand_type(at, id_to_type) for at in callable.arg_types] + arg_types = [ + expand_type(at, id_to_type, allow_erased_callables) for at in callable.arg_types + ] arg_kinds = callable.arg_kinds arg_names = callable.arg_names # Apply arguments to TypeGuard if any. if callable.type_guard is not None: - type_guard = expand_type(callable.type_guard, id_to_type) + type_guard = expand_type(callable.type_guard, id_to_type, allow_erased_callables) else: type_guard = None @@ -150,7 +159,7 @@ def apply_generic_arguments( return callable.copy_modified( arg_types=arg_types, - ret_type=expand_type(callable.ret_type, id_to_type), + ret_type=expand_type(callable.ret_type, id_to_type, allow_erased_callables), variables=remaining_tvars, type_guard=type_guard, arg_kinds=arg_kinds, diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 5a56857e1114..5bee9abc6dc8 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -39,20 +39,26 @@ @overload -def expand_type(typ: ProperType, env: Mapping[TypeVarId, Type]) -> ProperType: +def expand_type( + typ: ProperType, env: Mapping[TypeVarId, Type], allow_erased_callables: bool = ... +) -> ProperType: ... @overload -def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: +def expand_type( + typ: Type, env: Mapping[TypeVarId, Type], allow_erased_callables: bool = ... +) -> Type: ... -def expand_type(typ: Type, env: Mapping[TypeVarId, Type]) -> Type: +def expand_type( + typ: Type, env: Mapping[TypeVarId, Type], allow_erased_callables: bool = False +) -> Type: """Substitute any type variable references in a type given by a type environment. """ - return typ.accept(ExpandTypeVisitor(env)) + return typ.accept(ExpandTypeVisitor(env, allow_erased_callables)) @overload @@ -129,8 +135,11 @@ class ExpandTypeVisitor(TypeVisitor[Type]): variables: Mapping[TypeVarId, Type] # TypeVar id -> TypeVar value - def __init__(self, variables: Mapping[TypeVarId, Type]) -> None: + def __init__( + self, variables: Mapping[TypeVarId, Type], allow_erased_callables: bool = False + ) -> None: self.variables = variables + self.allow_erased_callables = allow_erased_callables def visit_unbound_type(self, t: UnboundType) -> Type: return t @@ -148,8 +157,14 @@ def visit_deleted_type(self, t: DeletedType) -> Type: return t def visit_erased_type(self, t: ErasedType) -> Type: - # Should not get here. - raise RuntimeError() + if not self.allow_erased_callables: + raise RuntimeError() + # This may happen during type inference if some function argument + # type is a generic callable, and its erased form will appear in inferred + # constraints, then solver may check subtyping between them, which will trigger + # unify_generic_callables(), this is why we can get here. In all other cases it + # is a sign of a bug, since should never appear in any stored types. + return t def visit_instance(self, t: Instance) -> Type: args = self.expand_types_with_unpack(list(t.args)) diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 7e49c19c42bb..ce91b08b2e53 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -1667,8 +1667,12 @@ def report(*args: Any) -> None: nonlocal had_errors had_errors = True + # This function may be called by the solver, so we need to allow erased types here. + # We anyway allow checking subtyping between other types containing + # (probably also because solver needs subtyping). See also comment in + # ExpandTypeVisitor.visit_erased_type(). applied = mypy.applytype.apply_generic_arguments( - type, non_none_inferred_vars, report, context=target + type, non_none_inferred_vars, report, context=target, allow_erased_callables=True ) if had_errors: return None diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 02abe8f1ddc4..d289ec5a8e58 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1958,3 +1958,26 @@ lst = SubLinkedList(1, LinkedList(2)) # E: Argument 2 to "SubLinkedList" has in reveal_type(lst.next) # N: Revealed type is "Union[__main__.SubLinkedList, None]" reveal_type(SubLinkedList) # N: Revealed type is "def (value: builtins.int, next: Union[__main__.SubLinkedList, None] =) -> __main__.SubLinkedList" [builtins fixtures/dataclasses.pyi] + +[case testNoCrashOnNestedGenericCallable] +from dataclasses import dataclass +from typing import Generic, TypeVar, Callable + +T = TypeVar('T') +R = TypeVar('R') +X = TypeVar('X') + +@dataclass +class Box(Generic[T]): + inner: T + +@dataclass +class Cont(Generic[R]): + run: Box[Callable[[X], R]] + +def const_two(x: T) -> str: + return "two" + +c = Cont(Box(const_two)) +reveal_type(c) # N: Revealed type is "__main__.Cont[builtins.str]" +[builtins fixtures/dataclasses.pyi]