From 9469ffceeb256f5fc6db81907e81b1e76cb79837 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Dec 2020 11:31:04 -0800 Subject: [PATCH 01/22] Tentative first steps for TypeGuard (PEP 647) --- mypy/test/testcheck.py | 1 + mypy/typeanal.py | 24 +++++++++++++++- mypy/types.py | 8 +++++- test-data/unit/check-typeguard.test | 28 +++++++++++++++++++ test-data/unit/lib-stub/typing_extensions.pyi | 2 ++ 5 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 test-data/unit/check-typeguard.test diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index eb1dbd9dcc30..eb61e66ddcf6 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -92,6 +92,7 @@ 'check-annotated.test', 'check-parameter-specification.test', 'check-generic-alias.test', + 'check-typeguard.test', ] # Tests that use Python 3.8-only AST features (like expression-scoped ignores): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 219be131c4af..04516a34638a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -345,6 +345,9 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Opt " and at least one annotation", t) return AnyType(TypeOfAny.from_error) return self.anal_type(t.args[0]) + elif self.anal_type_guard_arg(t, fullname) is not None: + # In most contexts, TypeGuard[...] acts as an alias for bool (ignoring its args) + return self.named_type('builtins.bool') return None def get_omitted_any(self, typ: Type, fullname: Optional[str] = None) -> AnyType: @@ -524,15 +527,34 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: variables = t.variables else: variables = self.bind_function_type_variables(t, t) + special = self.anal_type_guard(t.ret_type) ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=nested), ret_type=self.anal_type(t.ret_type, nested=nested), # If the fallback isn't filled in yet, # its type will be the falsey FakeInfo fallback=(t.fallback if t.fallback.type else self.named_type('builtins.function')), - variables=self.anal_var_defs(variables)) + variables=self.anal_var_defs(variables), + is_type_guard=(special is not None), + ) return ret + def anal_type_guard(self, t: Type) -> Optional[Type]: + if isinstance(t, UnboundType): + sym = self.lookup_qualified(t.name, t) + if sym is not None and sym.node is not None: + return self.anal_type_guard_arg(t, sym.node.fullname) + # TODO: What if it's an Instance? Then use t.type.fullname? + return None + + def anal_type_guard_arg(self, t: Type, fullname: str) -> Optional[Type]: + if fullname in ('typing_extensions.TypeGuard', 'typing.TypeGuard'): + if len(t.args) != 1: + self.fail("TypeGuard must have exactly one type argument", t) + return AnyType(TypeOfAny.from_error) + return self.anal_type(t.args[0]) + return None + def visit_overloaded(self, t: Overloaded) -> Type: # Overloaded types are manually constructed in semanal.py by analyzing the # AST and combining together the Callable types this visitor converts. diff --git a/mypy/types.py b/mypy/types.py index 10def3826120..f46e98b9b2aa 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1005,6 +1005,7 @@ class CallableType(FunctionLike): # tools that consume mypy ASTs 'def_extras', # Information about original definition we want to serialize. # This is used for more detailed error messages. + 'is_type_guard', # Of the form def (...) -> TypeGuard[...]. ) def __init__(self, @@ -1024,6 +1025,7 @@ def __init__(self, from_type_type: bool = False, bound_args: Sequence[Optional[Type]] = (), def_extras: Optional[Dict[str, Any]] = None, + is_type_guard: bool = False, ) -> None: super().__init__(line, column) assert len(arg_types) == len(arg_kinds) == len(arg_names) @@ -1058,6 +1060,7 @@ def __init__(self, not definition.is_static else None} else: self.def_extras = {} + self.is_type_guard = is_type_guard def copy_modified(self, arg_types: Bogus[Sequence[Type]] = _dummy, @@ -1075,7 +1078,9 @@ def copy_modified(self, special_sig: Bogus[Optional[str]] = _dummy, from_type_type: Bogus[bool] = _dummy, bound_args: Bogus[List[Optional[Type]]] = _dummy, - def_extras: Bogus[Dict[str, Any]] = _dummy) -> 'CallableType': + def_extras: Bogus[Dict[str, Any]] = _dummy, + is_type_guard: Bogus[bool] = _dummy, + ) -> 'CallableType': return CallableType( arg_types=arg_types if arg_types is not _dummy else self.arg_types, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, @@ -1094,6 +1099,7 @@ def copy_modified(self, from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, bound_args=bound_args if bound_args is not _dummy else self.bound_args, def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), + is_type_guard=is_type_guard if is_type_guard is not _dummy else self.is_type_guard, ) def var_arg(self) -> Optional[FormalArgument]: diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test new file mode 100644 index 000000000000..66b365e44315 --- /dev/null +++ b/test-data/unit/check-typeguard.test @@ -0,0 +1,28 @@ +[case testTypeGuardBasic] +from typing_extensions import TypeGuard +class Point: + x: int +def is_point(a: object) -> TypeGuard[Point]: + return not not a +def norm(a: object) -> None: + if is_point(a): + reveal_type(a) # N: Revealed type is '__main__.Point' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardArgsNone] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard: # E: TypeGuard must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardArgsTooMany] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[int, int]: # E: TypeGuard must have exactly one type argument + pass +[builtins fixtures/tuple.pyi] + +[case testTypeGuardArgType] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[42]: # E: Invalid type: try using Literal[42] instead? + pass +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/lib-stub/typing_extensions.pyi b/test-data/unit/lib-stub/typing_extensions.pyi index 946430d106a6..478e5dc1b283 100644 --- a/test-data/unit/lib-stub/typing_extensions.pyi +++ b/test-data/unit/lib-stub/typing_extensions.pyi @@ -24,6 +24,8 @@ Annotated: _SpecialForm = ... ParamSpec: _SpecialForm Concatenate: _SpecialForm +TypeGuard: _SpecialForm + # Fallback type for all typed dicts (does not exist at runtime). class _TypedDict(Mapping[str, object]): # Needed to make this class non-abstract. It is explicitly declared abstract in From 884d9af92ad7c46ca30d89ca9c8b190a6409f64b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Dec 2020 12:38:44 -0800 Subject: [PATCH 02/22] Rename is_type_guard to type_guard, fix repr and serialization --- mypy/fixup.py | 2 ++ mypy/typeanal.py | 2 +- mypy/types.py | 22 +++++++++++++++------- test-data/unit/check-serialize.test | 15 +++++++++++++++ test-data/unit/check-typeguard.test | 7 +++++++ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/mypy/fixup.py b/mypy/fixup.py index 30e1a0dae2b9..b90dba971e4f 100644 --- a/mypy/fixup.py +++ b/mypy/fixup.py @@ -192,6 +192,8 @@ def visit_callable_type(self, ct: CallableType) -> None: for arg in ct.bound_args: if arg: arg.accept(self) + if ct.type_guard is not None: + ct.type_guard.accept(self) def visit_overloaded(self, t: Overloaded) -> None: for ct in t.items(): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 04516a34638a..5756b651c27f 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -535,7 +535,7 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: fallback=(t.fallback if t.fallback.type else self.named_type('builtins.function')), variables=self.anal_var_defs(variables), - is_type_guard=(special is not None), + type_guard=special, ) return ret diff --git a/mypy/types.py b/mypy/types.py index f46e98b9b2aa..3785ffc36517 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1005,7 +1005,7 @@ class CallableType(FunctionLike): # tools that consume mypy ASTs 'def_extras', # Information about original definition we want to serialize. # This is used for more detailed error messages. - 'is_type_guard', # Of the form def (...) -> TypeGuard[...]. + 'type_guard', # T, if -> TypeGuard[T] (ret_type is bool in this case). ) def __init__(self, @@ -1025,7 +1025,7 @@ def __init__(self, from_type_type: bool = False, bound_args: Sequence[Optional[Type]] = (), def_extras: Optional[Dict[str, Any]] = None, - is_type_guard: bool = False, + type_guard: Optional[Type] = None, ) -> None: super().__init__(line, column) assert len(arg_types) == len(arg_kinds) == len(arg_names) @@ -1060,7 +1060,7 @@ def __init__(self, not definition.is_static else None} else: self.def_extras = {} - self.is_type_guard = is_type_guard + self.type_guard = type_guard def copy_modified(self, arg_types: Bogus[Sequence[Type]] = _dummy, @@ -1079,7 +1079,7 @@ def copy_modified(self, from_type_type: Bogus[bool] = _dummy, bound_args: Bogus[List[Optional[Type]]] = _dummy, def_extras: Bogus[Dict[str, Any]] = _dummy, - is_type_guard: Bogus[bool] = _dummy, + type_guard: Bogus[Optional[Type]] = _dummy, ) -> 'CallableType': return CallableType( arg_types=arg_types if arg_types is not _dummy else self.arg_types, @@ -1099,7 +1099,7 @@ def copy_modified(self, from_type_type=from_type_type if from_type_type is not _dummy else self.from_type_type, bound_args=bound_args if bound_args is not _dummy else self.bound_args, def_extras=def_extras if def_extras is not _dummy else dict(self.def_extras), - is_type_guard=is_type_guard if is_type_guard is not _dummy else self.is_type_guard, + type_guard=type_guard if type_guard is not _dummy else self.type_guard, ) def var_arg(self) -> Optional[FormalArgument]: @@ -1261,6 +1261,7 @@ def __eq__(self, other: object) -> bool: def serialize(self) -> JsonDict: # TODO: As an optimization, leave out everything related to # generic functions for non-generic functions. + assert self.type_guard is None or isinstance(self.type_guard, Instance), str(self.type_guard) return {'.class': 'CallableType', 'arg_types': [t.serialize() for t in self.arg_types], 'arg_kinds': self.arg_kinds, @@ -1275,6 +1276,8 @@ def serialize(self) -> JsonDict: 'bound_args': [(None if t is None else t.serialize()) for t in self.bound_args], 'def_extras': dict(self.def_extras), + 'type_guard': self.type_guard.serialize() + if isinstance(self.type_guard, Instance) else None, } @classmethod @@ -1292,7 +1295,9 @@ def deserialize(cls, data: JsonDict) -> 'CallableType': implicit=data['implicit'], bound_args=[(None if t is None else deserialize_type(t)) for t in data['bound_args']], - def_extras=data['def_extras'] + def_extras=data['def_extras'], + type_guard=(Instance.deserialize(data['type_guard']) + if data['type_guard'] is not None else None), ) @@ -2103,7 +2108,10 @@ def visit_callable_type(self, t: CallableType) -> str: s = '({})'.format(s) if not isinstance(get_proper_type(t.ret_type), NoneType): - s += ' -> {}'.format(t.ret_type.accept(self)) + if t.type_guard is not None: + s += ' -> TypeGuard[{}]'.format(t.type_guard.accept(self)) + else: + s += ' -> {}'.format(t.ret_type.accept(self)) if t.variables: vs = [] diff --git a/test-data/unit/check-serialize.test b/test-data/unit/check-serialize.test index 1aa9ac0662a2..b4982cc6f70a 100644 --- a/test-data/unit/check-serialize.test +++ b/test-data/unit/check-serialize.test @@ -224,6 +224,21 @@ def f(x: int) -> int: pass tmp/a.py:2: note: Revealed type is 'builtins.str' tmp/a.py:3: error: Unexpected keyword argument "x" for "f" +[case testSerializeTypeGuardFunction] +import a +[file a.py] +import b +[file a.py.2] +import b +reveal_type(b.guard('')) +reveal_type(b.guard) +[file b.py] +from typing_extensions import TypeGuard +def guard(a: object) -> TypeGuard[str]: pass +[builtins fixtures/tuple.pyi] +[out2] +tmp/a.py:2: note: Revealed type is 'builtins.bool' +tmp/a.py:3: note: Revealed type is 'def (a: builtins.object) -> TypeGuard[builtins.str]' -- -- Classes -- diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 66b365e44315..84893b93965d 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -26,3 +26,10 @@ from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard[42]: # E: Invalid type: try using Literal[42] instead? pass [builtins fixtures/tuple.pyi] + +[case testTypeGuardRepr] +from typing_extensions import TypeGuard +def foo(a: object) -> TypeGuard[int]: + pass +reveal_type(foo) # N: Revealed type is 'def (a: builtins.object) -> TypeGuard[builtins.int]' +[builtins fixtures/tuple.pyi] From f55b28407690ceff698080bdff498eb3ac8a47fb Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Dec 2020 17:13:06 -0800 Subject: [PATCH 03/22] Perhaps naive but passes the test --- mypy/checker.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mypy/checker.py b/mypy/checker.py index 17e894b9bc33..9c1c5b9df995 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -3957,6 +3957,7 @@ def find_isinstance_check(self, node: Expression ) -> Tuple[TypeMap, TypeMap]: """Find any isinstance checks (within a chain of ands). Includes implicit and explicit checks for None and calls to callable. + Also includes TypeGuard functions. Return value is a map of variables to their types if the condition is true and a map of variables to their types if the condition is false. @@ -4001,6 +4002,13 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM if literal(expr) == LITERAL_TYPE: vartype = type_map[expr] return self.conditional_callable_type_map(expr, vartype) + else: + type_guard = node.callee.node.type.type_guard + if type_guard is not None: + if len(node.args) < 1: # TODO: Is this an error? + return {}, {} + if literal(expr) == LITERAL_TYPE: + return {expr: type_guard}, {} elif isinstance(node, ComparisonExpr): # Step 1: Obtain the types of each operand and whether or not we can # narrow their types. (For example, we shouldn't try narrowing the From bdc97efee3899608c76bc39eae2904cbfbf42aad Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 29 Dec 2020 17:36:58 -0800 Subject: [PATCH 04/22] Add more tests, tweak tests (one new test fails) --- test-data/unit/check-typeguard.test | 84 ++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 84893b93965d..afecee4df9d6 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -1,27 +1,27 @@ [case testTypeGuardBasic] from typing_extensions import TypeGuard -class Point: - x: int -def is_point(a: object) -> TypeGuard[Point]: - return not not a -def norm(a: object) -> None: +class Point: pass +def is_point(a: object) -> TypeGuard[Point]: pass +def main(a: object) -> None: if is_point(a): reveal_type(a) # N: Revealed type is '__main__.Point' + else: + reveal_type(a) # N: Revealed type is 'builtins.object' [builtins fixtures/tuple.pyi] -[case testTypeGuardArgsNone] +[case testTypeGuardTypeArgsNone] from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard: # E: TypeGuard must have exactly one type argument pass [builtins fixtures/tuple.pyi] -[case testTypeGuardArgsTooMany] +[case testTypeGuardTypeArgsTooMany] from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard[int, int]: # E: TypeGuard must have exactly one type argument pass [builtins fixtures/tuple.pyi] -[case testTypeGuardArgType] +[case testTypeGuardTypeArgType] from typing_extensions import TypeGuard def foo(a: object) -> TypeGuard[42]: # E: Invalid type: try using Literal[42] instead? pass @@ -33,3 +33,71 @@ def foo(a: object) -> TypeGuard[int]: pass reveal_type(foo) # N: Revealed type is 'def (a: builtins.object) -> TypeGuard[builtins.int]' [builtins fixtures/tuple.pyi] + +[case testTypeGuardCallArgsNone] +from typing_extensions import TypeGuard +class Point: pass +# TODO: error on the 'def' line (insufficient args for type guard) +def is_point() -> TypeGuard[Point]: pass +def main(a: object) -> None: + if is_point(): + reveal_type(a) # N: Revealed type is 'builtins.object' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardCallArgsMultiple] +from typing_extensions import TypeGuard +class Point: pass +def is_point(a: object, b: object) -> TypeGuard[Point]: pass +def main(a: object, b: object) -> None: + if is_point(a, b): + reveal_type(a) # N: Revealed type is '__main__.Point' + reveal_type(b) # N: Revealed type is 'builtins.object' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardIsBool] +from typing_extensions import TypeGuard +def f(a: TypeGuard[int]) -> None: pass +reveal_type(f) # N: Revealed type is 'def (a: builtins.bool)' +a: TypeGuard[int] +reveal_type(a) # N: Revealed type is 'builtins.bool' +class C: + a: TypeGuard[int] +reveal_type(C().a) # N: Revealed type is 'builtins.bool' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithTypeVar] +from typing import TypeVar, Tuple +from typing_extensions import TypeGuard +T = TypeVar('T') +def is_two_element_tuple(a: Tuple[T, ...]) -> TypeGuard[Tuple[T, T]]: pass +def main(a: Tuple[T, ...]): + if is_two_element_tuple(a): + reveal_type(a) # N: Revealed type is 'Tuple[T`-1, T`-1]' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNonOverlapping] +from typing import List +from typing_extensions import TypeGuard +def is_str_list(a: List[object]) -> TypeGuard[List[str]]: pass +def main(a: List[object]): + if is_str_list(a): + reveal_type(a) # N: Revealed type is 'List[builtins.str]' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardUnionIn] +from typing import Union +from typing_extensions import TypeGuard +def is_foo(a: Union[int, str]) -> TypeGuard[str]: pass +def main(a: Union[str, int]) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is 'builtins.str' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardUnionOut] +from typing import Union +from typing_extensions import TypeGuard +def is_foo(a: object) -> TypeGuard[Union[int, str]]: pass +def main(a: object) -> None: + if is_foo(a): + reveal_type(a) # N: Revealed type is 'Union[builtins.int, builtins.str]' +[builtins fixtures/tuple.pyi] From 0ed54340e7d695ef50a465ea24f6ddca801d3c9b Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 10:02:17 -0800 Subject: [PATCH 05/22] Update typeshed --- mypy/typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeshed b/mypy/typeshed index fb753c4226ee..3b52d93ae309 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit fb753c4226ee9d1cfa2a84af4a94e6c32e939f47 +Subproject commit 3b52d93ae30997a34097b274bdbeb13e4cc054a3 From 47df17d21483513841b83f1ecb28ae836bddaa65 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 12:37:58 -0800 Subject: [PATCH 06/22] Make is_str_list() test work This is a bit ugly -- in find_isinstance_check() we wrap the type in a way that is only recognized in on specific other place, narrow_type_from_binder(). --- mypy/checker.py | 4 ++-- mypy/checkexpr.py | 5 ++++- mypy/types.py | 12 +++++++++++- test-data/unit/check-typeguard.test | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 9c1c5b9df995..08ac646f37a3 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -35,7 +35,7 @@ UnionType, TypeVarId, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef, is_named_instance, union_items, TypeQuery, LiteralType, is_optional, remove_optional, TypeTranslator, StarType, get_proper_type, ProperType, - get_proper_types, is_literal_type, TypeAliasType) + get_proper_types, is_literal_type, TypeAliasType, TypeGuardType) from mypy.sametypes import is_same_type from mypy.messages import ( MessageBuilder, make_inferred_type_note, append_invariance_notes, pretty_seq, @@ -4008,7 +4008,7 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM if len(node.args) < 1: # TODO: Is this an error? return {}, {} if literal(expr) == LITERAL_TYPE: - return {expr: type_guard}, {} + return {expr: TypeGuardType(type_guard)}, {} elif isinstance(node, ComparisonExpr): # Step 1: Obtain the types of each operand and whether or not we can # narrow their types. (For example, we shouldn't try narrowing the diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 40204e7c9ccf..ad758c7b993b 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -14,7 +14,7 @@ make_optional_type, ) from mypy.types import ( - Type, AnyType, CallableType, Overloaded, NoneType, TypeVarDef, + Type, AnyType, CallableType, Overloaded, NoneType, TypeGuardType, TypeVarDef, TupleType, TypedDictType, Instance, TypeVarType, ErasedType, UnionType, PartialType, DeletedType, UninhabitedType, TypeType, TypeOfAny, LiteralType, LiteralValue, is_named_instance, FunctionLike, @@ -4163,6 +4163,9 @@ def narrow_type_from_binder(self, expr: Expression, known_type: Type, """ if literal(expr) >= LITERAL_TYPE: restriction = self.chk.binder.get(expr) + if isinstance(restriction, TypeGuardType): + # A type guard forces the new type even if it doesn't overlap the old + return restriction.type_guard # If the current node is deferred, some variables may get Any types that they # otherwise wouldn't have. We don't want to narrow down these since it may # produce invalid inferred Optional[Any] types, at least. diff --git a/mypy/types.py b/mypy/types.py index 3785ffc36517..7b25f3c7367f 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -261,7 +261,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeAliasType': alias = TypeAliasType(None, args) alias.type_ref = data['type_ref'] return alias - + def copy_modified(self, *, args: Optional[List[Type]] = None) -> 'TypeAliasType': return TypeAliasType( @@ -270,6 +270,16 @@ def copy_modified(self, *, self.line, self.column) +class TypeGuardType(Type): + """Only used by find_instance_check() etc.""" + def __init__(self, type_guard: Type): + super().__init__(line=type_guard.line, column=type_guard.column) + self.type_guard = type_guard + + def __repr__(self) -> str: + return "TypeGuard({})".format(self.type_guard) + + class ProperType(Type): """Not a type alias. diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index afecee4df9d6..7e2764929dcc 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -81,7 +81,7 @@ from typing_extensions import TypeGuard def is_str_list(a: List[object]) -> TypeGuard[List[str]]: pass def main(a: List[object]): if is_str_list(a): - reveal_type(a) # N: Revealed type is 'List[builtins.str]' + reveal_type(a) # N: Revealed type is 'builtins.list[builtins.str]' [builtins fixtures/tuple.pyi] [case testTypeGuardUnionIn] From fbfeeb6dfe3a93036c2d0e9230c92865437191c6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 12:44:09 -0800 Subject: [PATCH 07/22] Add a test showing a controversial behavior --- test-data/unit/check-typeguard.test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 7e2764929dcc..533d1e981cfb 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -101,3 +101,12 @@ def main(a: object) -> None: if is_foo(a): reveal_type(a) # N: Revealed type is 'Union[builtins.int, builtins.str]' [builtins fixtures/tuple.pyi] + +[case TestTypeGuardNonzeroFloat] +from typing import Union +from typing_extensions import TypeGuard +def is_nonzero(a: object) -> TypeGuard[float]: pass +def main(a: int): + if is_nonzero(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/tuple.pyi] From 03e566fb64a0fdd98d4eab8054438e5bfaa29606 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 13:55:49 -0800 Subject: [PATCH 08/22] Fix mypy and lint --- mypy/checker.py | 10 ++++++---- mypy/checkexpr.py | 5 +++-- mypy/typeanal.py | 4 ++-- mypy/types.py | 12 ++++++------ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 08ac646f37a3..237f1cc2f9ea 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -22,7 +22,7 @@ Context, Decorator, PrintStmt, BreakStmt, PassStmt, ContinueStmt, ComparisonExpr, StarExpr, EllipsisExpr, RefExpr, PromoteExpr, Import, ImportFrom, ImportAll, ImportBase, TypeAlias, - ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, + ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, SYMBOL_FUNCBASE_TYPES, CONTRAVARIANT, COVARIANT, INVARIANT, TypeVarExpr, AssignmentExpr, is_final_node, ARG_NAMED) @@ -4003,12 +4003,14 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM vartype = type_map[expr] return self.conditional_callable_type_map(expr, vartype) else: - type_guard = node.callee.node.type.type_guard - if type_guard is not None: + if (isinstance(node.callee, RefExpr) + and isinstance(node.callee.node, SYMBOL_FUNCBASE_TYPES) + and isinstance(node.callee.node.type, CallableType) + and node.callee.node.type.type_guard is not None): if len(node.args) < 1: # TODO: Is this an error? return {}, {} if literal(expr) == LITERAL_TYPE: - return {expr: TypeGuardType(type_guard)}, {} + return {expr: TypeGuardType(node.callee.node.type.type_guard)}, {} elif isinstance(node, ComparisonExpr): # Step 1: Obtain the types of each operand and whether or not we can # narrow their types. (For example, we shouldn't try narrowing the diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ad758c7b993b..c66c705fb055 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -4163,8 +4163,9 @@ def narrow_type_from_binder(self, expr: Expression, known_type: Type, """ if literal(expr) >= LITERAL_TYPE: restriction = self.chk.binder.get(expr) - if isinstance(restriction, TypeGuardType): - # A type guard forces the new type even if it doesn't overlap the old + # Ignore the error about using get_proper_type(). + if isinstance(restriction, TypeGuardType): # type: ignore[misc] + # A type guard forces the new type even if it doesn't overlap the old. return restriction.type_guard # If the current node is deferred, some variables may get Any types that they # otherwise wouldn't have. We don't want to narrow down these since it may diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 5756b651c27f..3554f638d27c 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -536,7 +536,7 @@ def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: else self.named_type('builtins.function')), variables=self.anal_var_defs(variables), type_guard=special, - ) + ) return ret def anal_type_guard(self, t: Type) -> Optional[Type]: @@ -547,7 +547,7 @@ def anal_type_guard(self, t: Type) -> Optional[Type]: # TODO: What if it's an Instance? Then use t.type.fullname? return None - def anal_type_guard_arg(self, t: Type, fullname: str) -> Optional[Type]: + def anal_type_guard_arg(self, t: UnboundType, fullname: str) -> Optional[Type]: if fullname in ('typing_extensions.TypeGuard', 'typing.TypeGuard'): if len(t.args) != 1: self.fail("TypeGuard must have exactly one type argument", t) diff --git a/mypy/types.py b/mypy/types.py index 7b25f3c7367f..bf138f343b5a 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -261,7 +261,7 @@ def deserialize(cls, data: JsonDict) -> 'TypeAliasType': alias = TypeAliasType(None, args) alias.type_ref = data['type_ref'] return alias - + def copy_modified(self, *, args: Optional[List[Type]] = None) -> 'TypeAliasType': return TypeAliasType( @@ -1090,7 +1090,7 @@ def copy_modified(self, bound_args: Bogus[List[Optional[Type]]] = _dummy, def_extras: Bogus[Dict[str, Any]] = _dummy, type_guard: Bogus[Optional[Type]] = _dummy, - ) -> 'CallableType': + ) -> 'CallableType': return CallableType( arg_types=arg_types if arg_types is not _dummy else self.arg_types, arg_kinds=arg_kinds if arg_kinds is not _dummy else self.arg_kinds, @@ -1271,7 +1271,8 @@ def __eq__(self, other: object) -> bool: def serialize(self) -> JsonDict: # TODO: As an optimization, leave out everything related to # generic functions for non-generic functions. - assert self.type_guard is None or isinstance(self.type_guard, Instance), str(self.type_guard) + assert (self.type_guard is None + or isinstance(get_proper_type(self.type_guard), Instance)), str(self.type_guard) return {'.class': 'CallableType', 'arg_types': [t.serialize() for t in self.arg_types], 'arg_kinds': self.arg_kinds, @@ -1286,8 +1287,7 @@ def serialize(self) -> JsonDict: 'bound_args': [(None if t is None else t.serialize()) for t in self.bound_args], 'def_extras': dict(self.def_extras), - 'type_guard': self.type_guard.serialize() - if isinstance(self.type_guard, Instance) else None, + 'type_guard': self.type_guard.serialize() if self.type_guard is not None else None, } @classmethod @@ -1306,7 +1306,7 @@ def deserialize(cls, data: JsonDict) -> 'CallableType': bound_args=[(None if t is None else deserialize_type(t)) for t in data['bound_args']], def_extras=data['def_extras'], - type_guard=(Instance.deserialize(data['type_guard']) + type_guard=(deserialize_type(data['type_guard']) if data['type_guard'] is not None else None), ) From 93387bd00a0040ff1a0e4a396a4b998b983d0ff1 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 14:22:22 -0800 Subject: [PATCH 09/22] Update typeshed and fix test name typo --- mypy/typeshed | 2 +- test-data/unit/check-typeguard.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/typeshed b/mypy/typeshed index 3b52d93ae309..b3974b904fbe 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit 3b52d93ae30997a34097b274bdbeb13e4cc054a3 +Subproject commit b3974b904fbe76b2e42e32dbf4bc53b7ea5c5aab diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 533d1e981cfb..b61ba1158b87 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -102,7 +102,7 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is 'Union[builtins.int, builtins.str]' [builtins fixtures/tuple.pyi] -[case TestTypeGuardNonzeroFloat] +[case testTypeGuardNonzeroFloat] from typing import Union from typing_extensions import TypeGuard def is_nonzero(a: object) -> TypeGuard[float]: pass From 708f2e538ed33a9411de497dd3d95003a9cceca4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 14:33:04 -0800 Subject: [PATCH 10/22] Sync typeshed to the version that has TypeGuard --- mypy/typeshed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeshed b/mypy/typeshed index b3974b904fbe..dcaea8bf0619 160000 --- a/mypy/typeshed +++ b/mypy/typeshed @@ -1 +1 @@ -Subproject commit b3974b904fbe76b2e42e32dbf4bc53b7ea5c5aab +Subproject commit dcaea8bf06191d00c51a73fd932af09097cf208a From 0d2eb06de13bc27c11f710cca374f087d65b6f04 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 30 Dec 2020 16:23:34 -0800 Subject: [PATCH 11/22] Add two new tests - walrus - higher-order functions --- test-data/unit/check-typeguard.test | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index b61ba1158b87..98cf086c4682 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -103,10 +103,30 @@ def main(a: object) -> None: [builtins fixtures/tuple.pyi] [case testTypeGuardNonzeroFloat] -from typing import Union from typing_extensions import TypeGuard def is_nonzero(a: object) -> TypeGuard[float]: pass def main(a: int): if is_nonzero(a): reveal_type(a) # N: Revealed type is 'builtins.float' [builtins fixtures/tuple.pyi] + +[case testTypeGuardHigherOrder] +from typing import Callable, TypeVar, Iterable, List +from typing_extensions import TypeGuard +T = TypeVar('T') +R = TypeVar('R') +def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterable[R]: pass +def is_float(a: object) -> TypeGuard[float]: pass +a: List[object] = ["a", 0, 0.0] +# TODO: Make this pass +##reveal_type(filter(is_float, a)) ## N: Revealed type is 'typing.Iterable[float]' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWalrus] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: pass +def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # N: Revealed type is 'builtins.float' + reveal_type(a) # N: Revealed type is 'builtins.object' +[builtins fixtures/tuple.pyi] From 8264f8d2305835c7cceac7c4937604b6d5cfab48 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 31 Dec 2020 21:25:55 -0800 Subject: [PATCH 12/22] Minimal changes to make filter() test pass --- mypy/constraints.py | 7 ++++++- mypy/expandtype.py | 6 +++++- test-data/unit/check-typeguard.test | 4 ++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index 89b8e4527e24..70265285dadc 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -457,7 +457,12 @@ def visit_callable_type(self, template: CallableType) -> List[Constraint]: for t, a in zip(template.arg_types, cactual.arg_types): # Negate direction due to function argument type contravariance. res.extend(infer_constraints(t, a, neg_op(self.direction))) - res.extend(infer_constraints(template.ret_type, cactual.ret_type, + template_ret_type, cactual_ret_type = template.ret_type, cactual.ret_type + if template.type_guard is not None: + template_ret_type = template.type_guard + if cactual.type_guard is not None: + cactual_ret_type = cactual.type_guard + res.extend(infer_constraints(template_ret_type, cactual_ret_type, self.direction)) return res elif isinstance(self.actual, AnyType): diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 2e3db6b109a4..938ac287c448 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -96,8 +96,12 @@ def visit_type_var(self, t: TypeVarType) -> Type: return repl def visit_callable_type(self, t: CallableType) -> Type: + extra = {} + if t.type_guard is not None: + extra['type_guard'] = t.type_guard.accept(self) return t.copy_modified(arg_types=self.expand_types(t.arg_types), - ret_type=t.ret_type.accept(self)) + ret_type=t.ret_type.accept(self), + **extra) def visit_overloaded(self, t: Overloaded) -> Type: items = [] # type: List[CallableType] diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 98cf086c4682..b6ca7903f858 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -118,8 +118,8 @@ R = TypeVar('R') def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterable[R]: pass def is_float(a: object) -> TypeGuard[float]: pass a: List[object] = ["a", 0, 0.0] -# TODO: Make this pass -##reveal_type(filter(is_float, a)) ## N: Revealed type is 'typing.Iterable[float]' +b = filter(is_float, a) +reveal_type(b) # N: Revealed type is 'typing.Iterable[builtins.float*]' [builtins fixtures/tuple.pyi] [case testTypeGuardWalrus] From 43357852d65e0fafdff7f2f17064e3d8b6c364e3 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 1 Jan 2021 10:05:19 -0800 Subject: [PATCH 13/22] Fix mypy error in new code (corrected) --- mypy/expandtype.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 938ac287c448..9d6e80cdfbe7 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterable, List, TypeVar, Mapping, cast +from typing import Any, Dict, Iterable, List, TypeVar, Mapping, cast from mypy.types import ( Type, Instance, CallableType, TypeVisitor, UnboundType, AnyType, @@ -96,7 +96,7 @@ def visit_type_var(self, t: TypeVarType) -> Type: return repl def visit_callable_type(self, t: CallableType) -> Type: - extra = {} + extra = {} # type: Dict[str, Any] if t.type_guard is not None: extra['type_guard'] = t.type_guard.accept(self) return t.copy_modified(arg_types=self.expand_types(t.arg_types), From b34d2ac8d2e4cb46165e7e04651128b84a942d1e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 1 Jan 2021 16:30:18 -0800 Subject: [PATCH 14/22] Make methods work (adds a field to RefExpr) --- mypy/checker.py | 13 +++++-------- mypy/checkexpr.py | 5 +++++ mypy/nodes.py | 5 ++++- test-data/unit/check-typeguard.test | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 237f1cc2f9ea..43edabdfb6c8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -22,7 +22,7 @@ Context, Decorator, PrintStmt, BreakStmt, PassStmt, ContinueStmt, ComparisonExpr, StarExpr, EllipsisExpr, RefExpr, PromoteExpr, Import, ImportFrom, ImportAll, ImportBase, TypeAlias, - ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, SYMBOL_FUNCBASE_TYPES, + ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, CONTRAVARIANT, COVARIANT, INVARIANT, TypeVarExpr, AssignmentExpr, is_final_node, ARG_NAMED) @@ -4002,15 +4002,12 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM if literal(expr) == LITERAL_TYPE: vartype = type_map[expr] return self.conditional_callable_type_map(expr, vartype) - else: - if (isinstance(node.callee, RefExpr) - and isinstance(node.callee.node, SYMBOL_FUNCBASE_TYPES) - and isinstance(node.callee.node.type, CallableType) - and node.callee.node.type.type_guard is not None): - if len(node.args) < 1: # TODO: Is this an error? + elif isinstance(node.callee, RefExpr): + if node.callee.type_guard is not None: + if len(node.args) < 1: return {}, {} if literal(expr) == LITERAL_TYPE: - return {expr: TypeGuardType(node.callee.node.type.type_guard)}, {} + return {expr: TypeGuardType(node.callee.type_guard)}, {} elif isinstance(node, ComparisonExpr): # Step 1: Obtain the types of each operand and whether or not we can # narrow their types. (For example, we shouldn't try narrowing the diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index c66c705fb055..4a924d643676 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -317,6 +317,11 @@ def visit_call_expr_inner(self, e: CallExpr, allow_none_return: bool = False) -> ret_type=self.object_type(), fallback=self.named_type('builtins.function')) callee_type = get_proper_type(self.accept(e.callee, type_context, always_allow_any=True)) + if (isinstance(e.callee, RefExpr) + and isinstance(callee_type, CallableType) + and callee_type.type_guard is not None): + # Cache it for find_isinstance_check() + e.callee.type_guard = callee_type.type_guard if (self.chk.options.disallow_untyped_calls and self.chk.in_checked_function() and isinstance(callee_type, CallableType) diff --git a/mypy/nodes.py b/mypy/nodes.py index 0571788bf002..76521e8c2b38 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1448,7 +1448,8 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class RefExpr(Expression): """Abstract base class for name-like constructs""" - __slots__ = ('kind', 'node', 'fullname', 'is_new_def', 'is_inferred_def', 'is_alias_rvalue') + __slots__ = ('kind', 'node', 'fullname', 'is_new_def', 'is_inferred_def', 'is_alias_rvalue', + 'type_guard') def __init__(self) -> None: super().__init__() @@ -1467,6 +1468,8 @@ def __init__(self) -> None: self.is_inferred_def = False # Is this expression appears as an rvalue of a valid type alias definition? self.is_alias_rvalue = False + # Cache type guard from callable_type.type_guard + self.type_guard = None # type: Optional[mypy.types.Type] class NameExpr(RefExpr): diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index b6ca7903f858..26ea00dfe756 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -130,3 +130,17 @@ def main(a: object) -> None: reveal_type(x) # N: Revealed type is 'builtins.float' reveal_type(a) # N: Revealed type is 'builtins.object' [builtins fixtures/tuple.pyi] + +[case testTypeGuardMethod] +from typing_extensions import TypeGuard +class C: + def main(self, a: object) -> None: + if self.is_float(a): + reveal_type(self) # N: Revealed type is '__main__.C' + reveal_type(a) # N: Revealed type is 'builtins.float' + def is_float(self, a: object) -> TypeGuard[float]: pass +[builtins fixtures/tuple.pyi] + +# TODO: +# - can a type guard be a generator? or async? +# - all sorts of shenanigans with type variables From 9bdd7790c6a432727ba811879b74bb5836ea48ce Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 1 Jan 2021 16:32:18 -0800 Subject: [PATCH 15/22] Move walrus test to 3.8-only test file --- test-data/unit/check-python38.test | 9 +++++++++ test-data/unit/check-typeguard.test | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test-data/unit/check-python38.test b/test-data/unit/check-python38.test index dcbf96ac850f..7cb571cedc8d 100644 --- a/test-data/unit/check-python38.test +++ b/test-data/unit/check-python38.test @@ -392,3 +392,12 @@ def func() -> None: class Foo: def __init__(self) -> None: self.x = 123 + +[case testWalrusTypeGuard] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: pass +def main(a: object) -> None: + if is_float(x := a): + reveal_type(x) # N: Revealed type is 'builtins.float' + reveal_type(a) # N: Revealed type is 'builtins.object' +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 26ea00dfe756..a3e105e99782 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -122,15 +122,6 @@ b = filter(is_float, a) reveal_type(b) # N: Revealed type is 'typing.Iterable[builtins.float*]' [builtins fixtures/tuple.pyi] -[case testTypeGuardWalrus] -from typing_extensions import TypeGuard -def is_float(a: object) -> TypeGuard[float]: pass -def main(a: object) -> None: - if is_float(x := a): - reveal_type(x) # N: Revealed type is 'builtins.float' - reveal_type(a) # N: Revealed type is 'builtins.object' -[builtins fixtures/tuple.pyi] - [case testTypeGuardMethod] from typing_extensions import TypeGuard class C: From 639b0fca1f8082b7f0665458799d0eca16a76096 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 3 Jan 2021 20:19:41 -0800 Subject: [PATCH 16/22] Add cross-module test; remove test TODOs --- test-data/unit/check-typeguard.test | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index a3e105e99782..79df75cbc079 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -132,6 +132,16 @@ class C: def is_float(self, a: object) -> TypeGuard[float]: pass [builtins fixtures/tuple.pyi] -# TODO: -# - can a type guard be a generator? or async? -# - all sorts of shenanigans with type variables +[case testTypeGuardCrossModule] +import guard +from points import Point +def main(a: object) -> None: + if guard.is_point(a): + reveal_type(a) # N: Revealed type is 'points.Point' +[file guard.py] +from typing_extensions import TypeGuard +import points +def is_point(a: object) -> TypeGuard[points.Point]: pass +[file points.py] +class Point: pass +[builtins fixtures/tuple.pyi] From d96beea64a9f688e73a802f549d0fbdb78913202 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 8 Jan 2021 14:20:10 -0800 Subject: [PATCH 17/22] Add many new tests These mostly come from Jukka's suggestions. Most pass; 4 are currently being skipped: [case testTypeGuardWithKeywordArgsSwapped-skip] is_float(b=1, a=a) incorrectly asserts that b is a float [case testTypeGuardWithStarArgsTuple-skip] [case testTypeGuardWithStarArgsList-skip] Horrible things happen with *args [case testTypeGuardOverload-skip] A plain bool function is accepted by a higher-order function (overload) requiring a type guard [case testTypeGuardMethodOverride-skip] A plain bool function in a subclass incorrectly is allowed to override a type guard in the base class --- test-data/unit/check-typeguard.test | 166 ++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 79df75cbc079..133d4b5c4e60 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -145,3 +145,169 @@ def is_point(a: object) -> TypeGuard[points.Point]: pass [file points.py] class Point: pass [builtins fixtures/tuple.pyi] + +[case testTypeGuardBodyRequiresBool] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: + return "not a bool" # E: Incompatible return value type (got "str", expected "bool") +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNarrowToTypedDict] +from typing import Dict, TypedDict +from typing_extensions import TypeGuard +class User(TypedDict): + name: str + id: int +def is_user(a: Dict[str, object]) -> TypeGuard[User]: + return isinstance(a.get("name"), str) and isinstance(a.get("id"), int) +def main(a: Dict[str, object]) -> None: + if is_user(a): + reveal_type(a) # N: Revealed type is 'TypedDict('__main__.User', {'name': builtins.str, 'id': builtins.int})' +[builtins fixtures/dict.pyi] +[typing fixtures/typing-typeddict.pyi] + +[case testTypeGuardInAssert] +from typing_extensions import TypeGuard +def is_float(a: object) -> TypeGuard[float]: pass +def main(a: object) -> None: + assert is_float(a) + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardFromAny] +from typing import Any +from typing_extensions import TypeGuard +def is_objfloat(a: object) -> TypeGuard[float]: pass +def is_anyfloat(a: Any) -> TypeGuard[float]: pass +def objmain(a: object) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is 'builtins.float' + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +def anymain(a: Any) -> None: + if is_objfloat(a): + reveal_type(a) # N: Revealed type is 'builtins.float' + if is_anyfloat(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardNegatedAndElse] +from typing import Union +from typing_extensions import TypeGuard +def is_int(a: object) -> TypeGuard[int]: pass +def is_str(a: object) -> TypeGuard[str]: pass +def intmain(a: Union[int, str]) -> None: + if not is_int(a): + reveal_type(a) # N: Revealed type is 'Union[builtins.int, builtins.str]' + else: + reveal_type(a) # N: Revealed type is 'builtins.int' +def strmain(a: Union[int, str]) -> None: + if is_str(a): + reveal_type(a) # N: Revealed type is 'builtins.str' + else: + reveal_type(a) # N: Revealed type is 'Union[builtins.int, builtins.str]' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardClassMethod] +from typing_extensions import TypeGuard +class C: + @classmethod + def is_float(cls, a: object) -> TypeGuard[float]: pass + def method(self, a: object) -> None: + if self.is_float(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +def main(a: object) -> None: + if C.is_float(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/classmethod.pyi] + +[case testTypeGuardWithKeywordArgsEasy] +from typing_extensions import TypeGuard +def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass +def main(a: object) -> None: + if is_float(a=a, b=1): + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/tuple.pyi] + +# TODO: Make these work + +[case testTypeGuardWithKeywordArgsSwapped-skip] +from typing_extensions import TypeGuard +def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass +def main1(a: object) -> None: + if is_float(b=1, a=a): + reveal_type(a) # 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithStarArgsTuple-skip] +from typing_extensions import TypeGuard +def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass +def main(a: object) -> None: + aa = (a,) + if is_float(*aa): + reveal_type(aa) # 'Tuple[float]' + reveal_type(a) # 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardWithStarArgsList-skip] +from typing_extensions import TypeGuard +def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass +def main(a: object) -> None: + aa = [a] + if is_float(*aa): + reveal_type(aa) # 'List[float]' + reveal_type(a) # 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardOverload-skip] +from typing import overload, Any, Callable, Iterable, Iterator, List, Optional, TypeVar +from typing_extensions import TypeGuard + +T = TypeVar("T") +R = TypeVar("R") + +@overload +def filter(f: None, it: Iterable[Optional[T]]) -> Iterator[T]: ... +@overload +def filter(f: Callable[[T], Any], it: Iterable[T]) -> Iterator[T]: ... +@overload +def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterator[R]: ... +def filter(*args): pass + +def is_int(a: object) -> TypeGuard[int]: pass +def non_tg(a: object) -> bool: pass + +def main(a: List[Optional[int]]) -> None: + aa = filter(None, a) + reveal_type(aa) # N: Revealed type is 'typing.Iterator[builtins.int*]' + bb = filter(lambda x: x is not None, a) + reveal_type(bb) # N: Revealed type is 'typing.Iterator[Any]' + cc = filter(is_int, a) + reveal_type(cc) # N: Revealed type is 'typing.Iterator[builtins.int]' + # TODO: Make this work -- this matches the third overload instead of the second + dd = filter(non_tg, a) + reveal_type(dd) # N: Revealed type is 'typing.Iterator[Union[builtins.int, None]]' + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-full.pyi] + +[case testTypeGuardDecorated] +from typing import TypeVar +from typing_extensions import TypeGuard +T = TypeVar("T") +def decorator(f: T) -> T: pass +@decorator +def is_float(a: object) -> TypeGuard[float]: + pass +def main(a: object) -> None: + if is_float(a): + reveal_type(a) # N: Revealed type is 'builtins.float' +[builtins fixtures/tuple.pyi] + +[case testTypeGuardMethodOverride-skip] +from typing_extensions import TypeGuard +class C: + def is_float(self, a: object) -> TypeGuard[float]: pass +class D(C): + def is_float(self, a: object) -> bool: pass # E: Some error +[builtins fixtures/tuple.pyi] From 370818f8eb55b52951a0c3230c5f2fa5efbcc8eb Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 10 Jan 2021 17:41:43 -0800 Subject: [PATCH 18/22] Require that a type guard's first argument is positional --- mypy/checker.py | 4 ++- test-data/unit/check-typeguard.test | 46 ++++++++++------------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 43edabdfb6c8..83dd38130fa1 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4004,7 +4004,9 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM return self.conditional_callable_type_map(expr, vartype) elif isinstance(node.callee, RefExpr): if node.callee.type_guard is not None: - if len(node.args) < 1: + # TODO: Follow keyword args or *args, **kwargs + if node.arg_kinds[0] != nodes.ARG_POS: + self.fail("type guard requires positional argument", node) return {}, {} if literal(expr) == LITERAL_TYPE: return {expr: TypeGuardType(node.callee.type_guard)}, {} diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index 133d4b5c4e60..a8b4e2f9ab03 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -221,42 +221,28 @@ def main(a: object) -> None: reveal_type(a) # N: Revealed type is 'builtins.float' [builtins fixtures/classmethod.pyi] -[case testTypeGuardWithKeywordArgsEasy] +[case testTypeGuardRequiresPositionalArgs] from typing_extensions import TypeGuard def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass -def main(a: object) -> None: - if is_float(a=a, b=1): - reveal_type(a) # N: Revealed type is 'builtins.float' -[builtins fixtures/tuple.pyi] +def main1(a: object) -> None: + # This is debatable -- should we support these cases? -# TODO: Make these work + if is_float(a=a, b=1): # E: type guard requires positional argument + reveal_type(a) # N: Revealed type is 'builtins.object' -[case testTypeGuardWithKeywordArgsSwapped-skip] -from typing_extensions import TypeGuard -def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass -def main1(a: object) -> None: - if is_float(b=1, a=a): - reveal_type(a) # 'builtins.float' -[builtins fixtures/tuple.pyi] + if is_float(b=1, a=a): # E: type guard requires positional argument + reveal_type(a) # N: Revealed type is 'builtins.object' -[case testTypeGuardWithStarArgsTuple-skip] -from typing_extensions import TypeGuard -def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass -def main(a: object) -> None: - aa = (a,) - if is_float(*aa): - reveal_type(aa) # 'Tuple[float]' - reveal_type(a) # 'builtins.float' -[builtins fixtures/tuple.pyi] + ta = (a,) + if is_float(*ta): # E: type guard requires positional argument + reveal_type(ta) # N: Revealed type is 'Tuple[builtins.object]' + reveal_type(a) # N: Revealed type is 'builtins.object' + + la = [a] + if is_float(*la): # E: type guard requires positional argument + reveal_type(la) # N: Revealed type is 'builtins.list[builtins.object*]' + reveal_type(a) # N: Revealed type is 'builtins.object*' -[case testTypeGuardWithStarArgsList-skip] -from typing_extensions import TypeGuard -def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass -def main(a: object) -> None: - aa = [a] - if is_float(*aa): - reveal_type(aa) # 'List[float]' - reveal_type(a) # 'builtins.float' [builtins fixtures/tuple.pyi] [case testTypeGuardOverload-skip] From 9062adb65d92323894c4f1be7509ff29a2ceddd6 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 11 Jan 2021 13:40:42 -0800 Subject: [PATCH 19/22] Capitalize error message --- mypy/checker.py | 2 +- test-data/unit/check-typeguard.test | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 83dd38130fa1..5a69f502540a 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4006,7 +4006,7 @@ def find_isinstance_check_helper(self, node: Expression) -> Tuple[TypeMap, TypeM if node.callee.type_guard is not None: # TODO: Follow keyword args or *args, **kwargs if node.arg_kinds[0] != nodes.ARG_POS: - self.fail("type guard requires positional argument", node) + self.fail("Type guard requires positional argument", node) return {}, {} if literal(expr) == LITERAL_TYPE: return {expr: TypeGuardType(node.callee.type_guard)}, {} diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index a8b4e2f9ab03..d07ba86ddbb8 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -227,19 +227,19 @@ def is_float(a: object, b: object = 0) -> TypeGuard[float]: pass def main1(a: object) -> None: # This is debatable -- should we support these cases? - if is_float(a=a, b=1): # E: type guard requires positional argument + if is_float(a=a, b=1): # E: Type guard requires positional argument reveal_type(a) # N: Revealed type is 'builtins.object' - if is_float(b=1, a=a): # E: type guard requires positional argument + if is_float(b=1, a=a): # E: Type guard requires positional argument reveal_type(a) # N: Revealed type is 'builtins.object' ta = (a,) - if is_float(*ta): # E: type guard requires positional argument + if is_float(*ta): # E: Type guard requires positional argument reveal_type(ta) # N: Revealed type is 'Tuple[builtins.object]' reveal_type(a) # N: Revealed type is 'builtins.object' la = [a] - if is_float(*la): # E: type guard requires positional argument + if is_float(*la): # E: Type guard requires positional argument reveal_type(la) # N: Revealed type is 'builtins.list[builtins.object*]' reveal_type(a) # N: Revealed type is 'builtins.object*' From 5e76923604b1bf2742abf3e3680d78ec55a01b46 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 11 Jan 2021 13:59:50 -0800 Subject: [PATCH 20/22] Avoid using **extra if possible --- mypy/expandtype.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 9d6e80cdfbe7..bde5261a1f3c 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -96,12 +96,10 @@ def visit_type_var(self, t: TypeVarType) -> Type: return repl def visit_callable_type(self, t: CallableType) -> Type: - extra = {} # type: Dict[str, Any] - if t.type_guard is not None: - extra['type_guard'] = t.type_guard.accept(self) return t.copy_modified(arg_types=self.expand_types(t.arg_types), ret_type=t.ret_type.accept(self), - **extra) + type_guard=(t.type_guard.accept(self) + if t.type_guard is not None else None)) def visit_overloaded(self, t: Overloaded) -> Type: items = [] # type: List[CallableType] From 37d2a5ff7a245e58af18a84e04581ccddbfb8414 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 11 Jan 2021 17:21:56 -0800 Subject: [PATCH 21/22] Clean up testTypeGuardOverload -- it still fails, though --- test-data/unit/check-typeguard.test | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test-data/unit/check-typeguard.test b/test-data/unit/check-typeguard.test index d07ba86ddbb8..e4bf3dd5c931 100644 --- a/test-data/unit/check-typeguard.test +++ b/test-data/unit/check-typeguard.test @@ -246,32 +246,29 @@ def main1(a: object) -> None: [builtins fixtures/tuple.pyi] [case testTypeGuardOverload-skip] +# flags: --strict-optional from typing import overload, Any, Callable, Iterable, Iterator, List, Optional, TypeVar from typing_extensions import TypeGuard T = TypeVar("T") R = TypeVar("R") -@overload -def filter(f: None, it: Iterable[Optional[T]]) -> Iterator[T]: ... -@overload -def filter(f: Callable[[T], Any], it: Iterable[T]) -> Iterator[T]: ... @overload def filter(f: Callable[[T], TypeGuard[R]], it: Iterable[T]) -> Iterator[R]: ... +@overload +def filter(f: Callable[[T], bool], it: Iterable[T]) -> Iterator[T]: ... def filter(*args): pass -def is_int(a: object) -> TypeGuard[int]: pass -def non_tg(a: object) -> bool: pass +def is_int_typeguard(a: object) -> TypeGuard[int]: pass +def is_int_bool(a: object) -> bool: pass def main(a: List[Optional[int]]) -> None: - aa = filter(None, a) - reveal_type(aa) # N: Revealed type is 'typing.Iterator[builtins.int*]' bb = filter(lambda x: x is not None, a) - reveal_type(bb) # N: Revealed type is 'typing.Iterator[Any]' - cc = filter(is_int, a) - reveal_type(cc) # N: Revealed type is 'typing.Iterator[builtins.int]' - # TODO: Make this work -- this matches the third overload instead of the second - dd = filter(non_tg, a) + reveal_type(bb) # N: Revealed type is 'typing.Iterator[Union[builtins.int, None]]' + # Also, if you replace 'bool' with 'Any' in the second overload, bb is Iterator[Any] + cc = filter(is_int_typeguard, a) + reveal_type(cc) # N: Revealed type is 'typing.Iterator[builtins.int*]' + dd = filter(is_int_bool, a) reveal_type(dd) # N: Revealed type is 'typing.Iterator[Union[builtins.int, None]]' [builtins fixtures/tuple.pyi] From 896c90e1472021194d1de37470ada0db3a922140 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 18 Jan 2021 09:43:45 -0800 Subject: [PATCH 22/22] Fix lint --- mypy/expandtype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index bde5261a1f3c..f98e0750743b 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterable, List, TypeVar, Mapping, cast +from typing import Dict, Iterable, List, TypeVar, Mapping, cast from mypy.types import ( Type, Instance, CallableType, TypeVisitor, UnboundType, AnyType,