From 550423eddac904d99139a762077cd95bdda11daa Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Aug 2022 11:46:38 +0100 Subject: [PATCH 1/7] Enable definition of generic tuple types --- mypy/expandtype.py | 6 +++- mypy/nodes.py | 2 +- mypy/semanal.py | 40 ++++++++++++++++++------- mypy/semanal_namedtuple.py | 1 - mypy/typeanal.py | 6 +--- test-data/unit/check-classes.test | 10 +++++++ test-data/unit/check-namedtuple.test | 44 ++++++++++++++++++++++++++++ test-data/unit/check-tuples.test | 2 +- 8 files changed, 91 insertions(+), 20 deletions(-) diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 9a948ca2f115..418012c67095 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -296,7 +296,11 @@ def expand_types_with_unpack( def visit_tuple_type(self, t: TupleType) -> Type: items = self.expand_types_with_unpack(t.items) if isinstance(items, list): - return t.copy_modified(items=items) + fallback = t.partial_fallback.accept(self) + fallback = get_proper_type(fallback) + if not isinstance(fallback, Instance): + fallback = t.partial_fallback + return t.copy_modified(items=items, fallback=fallback) else: return items diff --git a/mypy/nodes.py b/mypy/nodes.py index b7b3a6ef87f3..c9f21895c387 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -3294,7 +3294,7 @@ def from_tuple_type(cls, info: TypeInfo) -> "TypeAlias": """Generate an alias to the tuple type described by a given TypeInfo.""" assert info.tuple_type return TypeAlias( - info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, [])), + info.tuple_type.copy_modified(fallback=mypy.types.Instance(info, info.defn.type_vars)), info.fullname, info.line, info.column, diff --git a/mypy/semanal.py b/mypy/semanal.py index 2a30783d5bdc..944520e932c4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1382,15 +1382,14 @@ def analyze_class(self, defn: ClassDef) -> None: return if self.analyze_namedtuple_classdef(defn): + if defn.info: + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) return # Create TypeInfo for class now that base classes and the MRO can be calculated. self.prepare_class_def(defn) - - defn.type_vars = tvar_defs - defn.info.type_vars = [] - # we want to make sure any additional logic in add_type_vars gets run - defn.info.add_type_vars() + self.setup_type_vars(defn, tvar_defs) if base_error: defn.info.fallback_to_any = True @@ -1403,6 +1402,19 @@ def analyze_class(self, defn: ClassDef) -> None: self.analyze_class_decorator(defn, decorator) self.analyze_class_body_common(defn) + def setup_type_vars(self, defn: ClassDef, tvar_defs: List[TypeVarLikeType]) -> None: + defn.type_vars = tvar_defs + defn.info.type_vars = [] + # we want to make sure any additional logic in add_type_vars gets run + defn.info.add_type_vars() + + def setup_alias_type_vars(self, defn: ClassDef) -> None: + assert defn.info.special_alias is not None + defn.info.special_alias.alias_tvars = list(defn.info.type_vars) + target = defn.info.special_alias.target + assert isinstance(target, ProperType) and isinstance(target, TupleType) + target.partial_fallback.args = tuple(defn.type_vars) + def is_core_builtin_class(self, defn: ClassDef) -> bool: return self.cur_mod_id == "builtins" and defn.name in CORE_BUILTIN_CLASSES @@ -1454,7 +1466,7 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: if info is None: self.mark_incomplete(defn.name, defn) else: - self.prepare_class_def(defn, info) + self.prepare_class_def(defn, info, custom_names=True) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): self.analyze_class_body_common(defn) @@ -1679,7 +1691,9 @@ def get_all_bases_tvars( tvars.extend(base_tvars) return remove_dups(tvars) - def prepare_class_def(self, defn: ClassDef, info: Optional[TypeInfo] = None) -> None: + def prepare_class_def( + self, defn: ClassDef, info: Optional[TypeInfo] = None, custom_names: bool = False + ) -> None: """Prepare for the analysis of a class definition. Create an empty TypeInfo and store it in a symbol table, or if the 'info' @@ -1691,10 +1705,13 @@ def prepare_class_def(self, defn: ClassDef, info: Optional[TypeInfo] = None) -> info = info or self.make_empty_type_info(defn) defn.info = info info.defn = defn - if not self.is_func_scope(): - info._fullname = self.qualified_name(defn.name) - else: - info._fullname = info.name + if not custom_names: + # Some special classes (in particular NamedTuples) use custom fullname logic. + # Don't override it here (also see comment below, this needs cleanup). + if not self.is_func_scope(): + info._fullname = self.qualified_name(defn.name) + else: + info._fullname = info.name local_name = defn.name if "@" in local_name: local_name = local_name.split("@")[0] @@ -1855,6 +1872,7 @@ def configure_tuple_base_class(self, defn: ClassDef, base: TupleType) -> Instanc if info.special_alias and has_placeholder(info.special_alias.target): self.defer(force_progress=True) info.update_tuple_type(base) + self.setup_alias_type_vars(defn) if base.partial_fallback.type.fullname == "builtins.tuple" and not has_placeholder(base): # Fallback can only be safely calculated after semantic analysis, since base diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 3903c52ab0e7..142c1c10132e 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -116,7 +116,6 @@ def analyze_namedtuple_classdef( info = self.build_namedtuple_typeinfo( defn.name, items, types, default_items, defn.line, existing_info ) - defn.info = info defn.analyzed = NamedTupleExpr(info, is_typed=True) defn.analyzed.line = defn.line defn.analyzed.column = defn.column diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d797c8306515..422572b67162 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -610,12 +610,8 @@ def analyze_type_with_type_info( if tup is not None: # The class has a Tuple[...] base class so it will be # represented as a tuple type. - if args: - self.fail("Generic tuple types not supported", ctx) - return AnyType(TypeOfAny.from_error) if info.special_alias: - # We don't support generic tuple types yet. - return TypeAliasType(info.special_alias, []) + return TypeAliasType(info.special_alias, self.anal_array(args)) return tup.copy_modified(items=self.anal_array(tup.items), fallback=instance) td = info.typeddict_type if td is not None: diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index a620c63ef58b..58281d434049 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -7329,3 +7329,13 @@ a: int = child.foo(1) b: str = child.bar("abc") c: float = child.baz(3.4) d: bool = child.foobar() + +[case testGenericTupleTypeCreation] +from typing import Generic, Tuple, TypeVar + +[builtins fixtures/tuple.pyi] + +[case testGenericTupleTypeSubclassing] +from typing import Generic, Tuple, TypeVar + +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index bfd6ea82d991..c086ff0010d5 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1162,3 +1162,47 @@ NT5 = NamedTuple(b'NT5', [('x', int), ('y', int)]) # E: "NamedTuple()" expects [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCreation] +from typing import Generic, NamedTuple, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]" +reveal_type(nts.value) # N: Revealed type is "builtins.str" + +nti = NT(key=0, value=0) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +reveal_type(nti.value) # N: Revealed type is "builtins.int" + +NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleMethods] +from typing import Generic, NamedTuple, TypeVar + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCustomMethods] +from typing import Generic, NamedTuple, TypeVar + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleSubtyping] +from typing import Generic, NamedTuple, TypeVar + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleJoin] +from typing import Generic, NamedTuple, TypeVar + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 0c43cff2fdb7..c6ae9e808f8a 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -883,9 +883,9 @@ from typing import TypeVar, Generic, Tuple T = TypeVar('T') class Test(Generic[T], Tuple[T]): pass x = Test() # type: Test[int] +reveal_type(x) # N: Revealed type is "Tuple[builtins.int, fallback=__main__.Test[builtins.int]]" [builtins fixtures/tuple.pyi] [out] -main:4: error: Generic tuple types not supported -- Variable-length tuples (Tuple[t, ...] with literal '...') From 83f78084197d038a466031b73a50bc228c43def9 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Aug 2022 15:23:08 +0100 Subject: [PATCH 2/7] Add support for mapping to supertype --- mypy/maptype.py | 27 ++++++++++++- mypy/semanal_namedtuple.py | 3 +- test-data/unit/check-namedtuple.test | 60 +++++++++++++++++++++++++++- test-data/unit/fixtures/tuple.pyi | 1 + 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/mypy/maptype.py b/mypy/maptype.py index 59d86d9f79b8..b52d7ff3403f 100644 --- a/mypy/maptype.py +++ b/mypy/maptype.py @@ -1,8 +1,19 @@ from typing import Dict, List +import mypy.typeops from mypy.expandtype import expand_type from mypy.nodes import TypeInfo -from mypy.types import AnyType, Instance, ProperType, Type, TypeOfAny, TypeVarId +from mypy.types import ( + AnyType, + Instance, + ProperType, + TupleType, + Type, + TypeOfAny, + TypeVarId, + get_proper_type, + has_type_vars, +) def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Instance: @@ -16,6 +27,20 @@ def map_instance_to_supertype(instance: Instance, superclass: TypeInfo) -> Insta # Fast path: `instance` already belongs to `superclass`. return instance + if superclass.fullname == "builtins.tuple" and instance.type.tuple_type: + if has_type_vars(instance.type.tuple_type): + # We special case mapping generic tuple types to tuple base, because for + # such tuples fallback can't be calculated before applying type arguments. + alias = instance.type.special_alias + assert alias is not None + if not alias._is_recursive: + # Unfortunately we can't support this for generic recursive tuples. + # If we skip this special casing we will fall back to tuple[Any, ...]. + env = instance_to_type_environment(instance) + tuple_type = get_proper_type(expand_type(instance.type.tuple_type, env)) + if isinstance(tuple_type, TupleType): + return mypy.typeops.tuple_fallback(tuple_type) + if not superclass.type_vars: # Fast path: `superclass` has no type variables to map to. return Instance(superclass, []) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 142c1c10132e..0efbea704a93 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -58,6 +58,7 @@ TypeType, TypeVarType, UnboundType, + has_type_vars, ) from mypy.util import get_unique_redefinition_name @@ -487,7 +488,7 @@ def build_namedtuple_typeinfo( # We can't calculate the complete fallback type until after semantic # analysis, since otherwise base classes might be incomplete. Postpone a # callback function that patches the fallback. - if not has_placeholder(tuple_base): + if not has_placeholder(tuple_base) and not has_type_vars(tuple_base): self.api.schedule_patch( PRIORITY_FALLBACKS, lambda: calculate_tuple_fallback(tuple_base) ) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index c086ff0010d5..5f93058d74b2 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1186,23 +1186,79 @@ NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "in [case testGenericNamedTupleMethods] from typing import Generic, NamedTuple, TypeVar +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T +x: int + +nti: NT[int] +reveal_type(nti * x) # N: Revealed type is "builtins.tuple[builtins.int, ...]" + +nts: NT[str] +reveal_type(nts * x) # N: Revealed type is "builtins.tuple[builtins.object, ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] [case testGenericNamedTupleCustomMethods] from typing import Generic, NamedTuple, TypeVar +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + def foo(self) -> T: ... + @classmethod + def from_value(cls, value: T) -> NT[T]: ... + +nts: NT[str] +reveal_type(nts.foo()) # N: Revealed type is "builtins.str" + +nti = NT.from_value(1) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +NT[str].from_value(1) # E: Argument 1 to "from_value" of "NT" has incompatible type "int"; expected "str" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] [case testGenericNamedTupleSubtyping] -from typing import Generic, NamedTuple, TypeVar +from typing import Generic, NamedTuple, TypeVar, Tuple +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +nti: NT[int] + +def foo(x: Tuple[int, ...]) -> None: ... +foo(nti) +foo(nts) # E: Argument 1 to "foo" has incompatible type "NT[str]"; expected "Tuple[int, ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] [case testGenericNamedTupleJoin] -from typing import Generic, NamedTuple, TypeVar +from typing import Generic, NamedTuple, TypeVar, Tuple + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +nts: NT[str] +nti: NT[int] +x: Tuple[int, ...] + +def foo(x: T, y: T) -> T: ... +reveal_type(foo(nti, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" + +# Here fallbacks are object because named tuples are invariant. +reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=builtins.object]" +reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=builtins.object]" +reveal_type(foo(nti, x)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" +reveal_type(foo(x, nti)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" +reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] diff --git a/test-data/unit/fixtures/tuple.pyi b/test-data/unit/fixtures/tuple.pyi index 42f178b5a459..6f40356bb5f0 100644 --- a/test-data/unit/fixtures/tuple.pyi +++ b/test-data/unit/fixtures/tuple.pyi @@ -25,6 +25,7 @@ class tuple(Sequence[Tco], Generic[Tco]): def count(self, obj: object) -> int: pass class function: pass class ellipsis: pass +class classmethod: pass # We need int and slice for indexing tuples. class int: From 417fc57906588f9e23c786f124850dcf39b18d43 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Aug 2022 17:08:58 +0100 Subject: [PATCH 3/7] Better logic for recursive generic named tuples --- mypy/constraints.py | 8 ++++++++ mypy/semanal.py | 11 ++++++----- test-data/unit/check-namedtuple.test | 10 +++++----- test-data/unit/check-recursive-types.test | 24 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 10 deletions(-) diff --git a/mypy/constraints.py b/mypy/constraints.py index d483fa1aeb40..e0c7d48e6662 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -880,6 +880,14 @@ def visit_tuple_type(self, template: TupleType) -> List[Constraint]: ] if isinstance(actual, TupleType) and len(actual.items) == len(template.items): + if ( + actual.partial_fallback.type.is_named_tuple + and template.partial_fallback.type.is_named_tuple + ): + # For named tuples using just the fallbacks usually gives better results. + return infer_constraints( + template.partial_fallback, actual.partial_fallback, self.direction + ) res: List[Constraint] = [] for i in range(len(template.items)): res.extend(infer_constraints(template.items[i], actual.items[i], self.direction)) diff --git a/mypy/semanal.py b/mypy/semanal.py index 944520e932c4..8589ef483193 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1381,10 +1381,7 @@ def analyze_class(self, defn: ClassDef) -> None: if self.analyze_typeddict_classdef(defn): return - if self.analyze_namedtuple_classdef(defn): - if defn.info: - self.setup_type_vars(defn, tvar_defs) - self.setup_alias_type_vars(defn) + if self.analyze_namedtuple_classdef(defn, tvar_defs): return # Create TypeInfo for class now that base classes and the MRO can be calculated. @@ -1447,7 +1444,9 @@ def analyze_typeddict_classdef(self, defn: ClassDef) -> bool: return True return False - def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: + def analyze_namedtuple_classdef( + self, defn: ClassDef, tvar_defs: List[TypeVarLikeType] + ) -> bool: """Check if this class can define a named tuple.""" if ( defn.info @@ -1467,6 +1466,8 @@ def analyze_namedtuple_classdef(self, defn: ClassDef) -> bool: self.mark_incomplete(defn.name, defn) else: self.prepare_class_def(defn, info, custom_names=True) + self.setup_type_vars(defn, tvar_defs) + self.setup_alias_type_vars(defn) with self.scope.class_scope(defn.info): with self.named_tuple_analyzer.save_namedtuple_body(info): self.analyze_class_body_common(defn) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 5f93058d74b2..5d0dcecc0b39 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1240,7 +1240,7 @@ foo(nts) # E: Argument 1 to "foo" has incompatible type "NT[str]"; expected "Tu [case testGenericNamedTupleJoin] from typing import Generic, NamedTuple, TypeVar, Tuple -T = TypeVar("T") +T = TypeVar("T", covariant=True) class NT(NamedTuple, Generic[T]): key: int value: T @@ -1249,12 +1249,12 @@ nts: NT[str] nti: NT[int] x: Tuple[int, ...] -def foo(x: T, y: T) -> T: ... +S = TypeVar("S") +def foo(x: S, y: S) -> S: ... reveal_type(foo(nti, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" -# Here fallbacks are object because named tuples are invariant. -reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=builtins.object]" -reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=builtins.object]" +reveal_type(foo(nti, nts)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" +reveal_type(foo(nts, nti)) # N: Revealed type is "Tuple[builtins.int, builtins.object, fallback=__main__.NT[builtins.object]]" reveal_type(foo(nti, x)) # N: Revealed type is "builtins.tuple[builtins.int, ...]" reveal_type(foo(nts, x)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" diff --git a/test-data/unit/check-recursive-types.test b/test-data/unit/check-recursive-types.test index b5a1fe6838b5..18e2d25cf7b3 100644 --- a/test-data/unit/check-recursive-types.test +++ b/test-data/unit/check-recursive-types.test @@ -611,6 +611,30 @@ def foo() -> None: reveal_type(b) # N: Revealed type is "Tuple[Any, builtins.int, fallback=__main__.B@4]" [builtins fixtures/tuple.pyi] +[case testBasicRecursiveGenericNamedTuple] +# flags: --enable-recursive-aliases +from typing import Generic, NamedTuple, TypeVar, Union + +T = TypeVar("T", covariant=True) +class NT(NamedTuple, Generic[T]): + key: int + value: Union[T, NT[T]] + +class A: ... +class B(A): ... + +nti: NT[int] = NT(key=0, value=NT(key=1, value=A())) # E: Argument "value" to "NT" has incompatible type "A"; expected "Union[int, NT[int]]" +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, Union[builtins.int, ...], fallback=__main__.NT[builtins.int]]" + +nta: NT[A] +ntb: NT[B] +nta = ntb # OK, covariance +ntb = nti # E: Incompatible types in assignment (expression has type "NT[int]", variable has type "NT[B]") + +def last(arg: NT[T]) -> T: ... +reveal_type(last(ntb)) # N: Revealed type is "__main__.B" +[builtins fixtures/tuple.pyi] + [case testBasicRecursiveTypedDictClass] # flags: --enable-recursive-aliases from typing import TypedDict From 12c4f02a7dde676c7d9a64c932fd9c5f749b850a Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 12 Aug 2022 17:19:54 +0100 Subject: [PATCH 4/7] More tests for regular classes --- test-data/unit/check-classes.test | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 58281d434049..e34bf24a3d79 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -7333,9 +7333,26 @@ d: bool = child.foobar() [case testGenericTupleTypeCreation] from typing import Generic, Tuple, TypeVar +T = TypeVar("T") +S = TypeVar("S") +class C(Tuple[T, S]): + def __init__(self, x: T, y: S) -> None: ... + def foo(self, arg: T) -> S: ... + +cis: C[int, str] +reveal_type(cis) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.C[builtins.int, builtins.str]]" +cii = C(0, 1) +reveal_type(cii) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.C[builtins.int, builtins.int]]" +reveal_type(cis.foo) # N: Revealed type is "def (arg: builtins.int) -> builtins.str" [builtins fixtures/tuple.pyi] [case testGenericTupleTypeSubclassing] -from typing import Generic, Tuple, TypeVar +from typing import Generic, Tuple, TypeVar, List + +T = TypeVar("T") +class C(Tuple[T, T]): ... +class D(C[List[T]]): ... +di: D[int] +reveal_type(di) # N: Revealed type is "Tuple[builtins.list[builtins.int], builtins.list[builtins.int], fallback=__main__.D[builtins.int]]" [builtins fixtures/tuple.pyi] From 41fab831f57c3da68ba15008493430487e5dc025 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 13 Aug 2022 13:56:44 +0100 Subject: [PATCH 5/7] Add generic alias test --- mypy/checkexpr.py | 3 +++ test-data/unit/check-namedtuple.test | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 565e20b9c243..65250effb40f 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3640,6 +3640,9 @@ def visit_type_application(self, tapp: TypeApplication) -> Type: if isinstance(item, Instance): tp = type_object_type(item.type, self.named_type) return self.apply_type_arguments_to_callable(tp, item.args, tapp) + elif isinstance(item, TupleType) and item.partial_fallback.type.is_named_tuple: + tp = type_object_type(item.partial_fallback.type, self.named_type) + return self.apply_type_arguments_to_callable(tp, item.partial_fallback.args, tapp) else: self.chk.fail(message_registry.ONLY_CLASS_APPLICATION, tapp) return AnyType(TypeOfAny.from_error) diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 5d0dcecc0b39..349ab0ab74f2 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1183,6 +1183,22 @@ NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "in [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] +[case testGenericNamedTupleAlias] +from typing import NamedTuple, Generic, TypeVar, List + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +Alias = NT[List[T]] + +an: Alias[str] +reveal_type(an) # N: Revealed type is "Tuple[builtins.int, builtins.list[builtins.str], fallback=__main__.NT[builtins.list[builtins.str]]]" +Alias[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "List[str]" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + [case testGenericNamedTupleMethods] from typing import Generic, NamedTuple, TypeVar From 98a637d8dfa9054d3f41c05458a4cd8ec2208501 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 13 Aug 2022 15:25:03 +0100 Subject: [PATCH 6/7] Add incremental tests --- test-data/unit/check-incremental.test | 23 ++++++++++++++++++++ test-data/unit/fine-grained.test | 30 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 0cf048bee959..2a6cea47b80a 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -5893,3 +5893,26 @@ reveal_type(a.n) tmp/c.py:4: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" tmp/c.py:5: error: Incompatible types in assignment (expression has type "Optional[N]", variable has type "int") tmp/c.py:7: note: Revealed type is "TypedDict('a.N', {'r': Union[TypedDict('b.M', {'r': Union[..., None], 'x': builtins.int}), None], 'x': builtins.int})" + +[case testGenericNamedTupleSerialization] +import b +[file a.py] +from typing import NamedTuple, Generic, TypeVar + +T = TypeVar("T") +class NT(NamedTuple, Generic[T]): + key: int + value: T + +[file b.py] +from a import NT +nt = NT(key=0, value="yes") +s: str = nt.value +[file b.py.2] +from a import NT +nt = NT(key=0, value=42) +s: str = nt.value +[builtins fixtures/tuple.pyi] +[out] +[out2] +tmp/b.py:3: error: Incompatible types in assignment (expression has type "int", variable has type "str") diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 2ce647f9cba1..d5a37d85d221 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -3472,6 +3472,36 @@ f(a.x) [out] == +[case testNamedTupleUpdateGeneric] +import b +[file a.py] +from typing import NamedTuple +class Point(NamedTuple): + x: int + y: int +[file a.py.2] +from typing import Generic, TypeVar, NamedTuple + +T = TypeVar("T") +class Point(NamedTuple, Generic[T]): + x: int + y: T +[file b.py] +from a import Point +def foo() -> None: + p = Point(x=0, y=1) + i: int = p.y +[file b.py.3] +from a import Point +def foo() -> None: + p = Point(x=0, y="no") + i: int = p.y +[builtins fixtures/tuple.pyi] +[out] +== +== +b.py:4: error: Incompatible types in assignment (expression has type "str", variable has type "int") + [case testNamedTupleUpdateNonRecursiveToRecursiveFine] # flags: --enable-recursive-aliases import c From 7905afbabeddf98303be3693d41f78da6179a03b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Sat, 13 Aug 2022 16:19:12 +0100 Subject: [PATCH 7/7] Add support for call syntax --- mypy/semanal.py | 35 +++++++++++++++++++++++-- mypy/semanal_namedtuple.py | 38 +++++++++++++++++++--------- mypy/semanal_shared.py | 7 +++++ mypy/tvar_scope.py | 5 ++++ test-data/unit/check-namedtuple.test | 28 ++++++++++++++++++++ 5 files changed, 99 insertions(+), 14 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8589ef483193..62200d48eb06 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1692,6 +1692,28 @@ def get_all_bases_tvars( tvars.extend(base_tvars) return remove_dups(tvars) + def get_and_bind_all_tvars(self, type_exprs: List[Expression]) -> List[TypeVarLikeType]: + """Return all type variable references in item type expressions. + This is a helper for generic TypedDicts and NamedTuples. Essentially it is + a simplified version of the logic we use for ClassDef bases. We duplicate + some amount of code, because it is hard to refactor common pieces. + """ + tvars = [] + for base_expr in type_exprs: + try: + base = self.expr_to_unanalyzed_type(base_expr) + except TypeTranslationError: + # This error will be caught later. + continue + base_tvars = base.accept(TypeVarLikeQuery(self.lookup_qualified, self.tvar_scope)) + tvars.extend(base_tvars) + tvars = remove_dups(tvars) # Variables are defined in order of textual appearance. + tvar_defs = [] + for name, tvar_expr in tvars: + tvar_def = self.tvar_scope.bind_new(name, tvar_expr) + tvar_defs.append(tvar_def) + return tvar_defs + def prepare_class_def( self, defn: ClassDef, info: Optional[TypeInfo] = None, custom_names: bool = False ) -> None: @@ -2666,7 +2688,7 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: return False lvalue = s.lvalues[0] name = lvalue.name - internal_name, info = self.named_tuple_analyzer.check_namedtuple( + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( s.rvalue, name, self.is_func_scope() ) if internal_name is None: @@ -2686,6 +2708,9 @@ def analyze_namedtuple_assign(self, s: AssignmentStmt) -> bool: # Yes, it's a valid namedtuple, but defer if it is not ready. if not info: self.mark_incomplete(name, lvalue, becomes_typeinfo=True) + else: + self.setup_type_vars(info.defn, tvar_defs) + self.setup_alias_type_vars(info.defn) return True def analyze_typeddict_assign(self, s: AssignmentStmt) -> bool: @@ -5868,10 +5893,16 @@ def expr_to_analyzed_type( self, expr: Expression, report_invalid_types: bool = True, allow_placeholder: bool = False ) -> Optional[Type]: if isinstance(expr, CallExpr): + # This is a legacy syntax intended mostly for Python 2, we keep it for + # backwards compatibility, but new features like generic named tuples + # and recursive named tuples will be not supported. expr.accept(self) - internal_name, info = self.named_tuple_analyzer.check_namedtuple( + internal_name, info, tvar_defs = self.named_tuple_analyzer.check_namedtuple( expr, None, self.is_func_scope() ) + if tvar_defs: + self.fail("Generic named tuples are not supported for legacy class syntax", expr) + self.note("Use either Python 3 class syntax, or the assignment syntax", expr) if internal_name is None: # Some form of namedtuple is the only valid type that looks like a call # expression. This isn't a valid type. diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 0efbea704a93..cf76d8c05d1e 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -56,6 +56,7 @@ Type, TypeOfAny, TypeType, + TypeVarLikeType, TypeVarType, UnboundType, has_type_vars, @@ -199,7 +200,7 @@ def check_namedtuple_classdef( def check_namedtuple( self, node: Expression, var_name: Optional[str], is_func_scope: bool - ) -> Tuple[Optional[str], Optional[TypeInfo]]: + ) -> Tuple[Optional[str], Optional[TypeInfo], List[TypeVarLikeType]]: """Check if a call defines a namedtuple. The optional var_name argument is the name of the variable to @@ -214,21 +215,21 @@ def check_namedtuple( report errors but return (some) TypeInfo. """ if not isinstance(node, CallExpr): - return None, None + return None, None, [] call = node callee = call.callee if not isinstance(callee, RefExpr): - return None, None + return None, None, [] fullname = callee.fullname if fullname == "collections.namedtuple": is_typed = False elif fullname in TYPED_NAMEDTUPLE_NAMES: is_typed = True else: - return None, None + return None, None, [] result = self.parse_namedtuple_args(call, fullname) if result: - items, types, defaults, typename, ok = result + items, types, defaults, typename, tvar_defs, ok = result else: # Error. Construct dummy return value. if var_name: @@ -242,10 +243,10 @@ def check_namedtuple( if name != var_name or is_func_scope: # NOTE: we skip local namespaces since they are not serialized. self.api.add_symbol_skip_local(name, info) - return var_name, info + return var_name, info, [] if not ok: # This is a valid named tuple but some types are not ready. - return typename, None + return typename, None, [] # We use the variable name as the class name if it exists. If # it doesn't, we use the name passed as an argument. We prefer @@ -304,7 +305,7 @@ def check_namedtuple( if name != var_name or is_func_scope: # NOTE: we skip local namespaces since they are not serialized. self.api.add_symbol_skip_local(name, info) - return typename, info + return typename, info, tvar_defs def store_namedtuple_info( self, info: TypeInfo, name: str, call: CallExpr, is_typed: bool @@ -315,7 +316,9 @@ def store_namedtuple_info( def parse_namedtuple_args( self, call: CallExpr, fullname: str - ) -> Optional[Tuple[List[str], List[Type], List[Expression], str, bool]]: + ) -> Optional[ + Tuple[List[str], List[Type], List[Expression], str, List[TypeVarLikeType], bool] + ]: """Parse a namedtuple() call into data needed to construct a type. Returns a 5-tuple: @@ -361,6 +364,7 @@ def parse_namedtuple_args( return None typename = cast(StrExpr, call.args[0]).value types: List[Type] = [] + tvar_defs = [] if not isinstance(args[1], (ListExpr, TupleExpr)): if fullname == "collections.namedtuple" and isinstance(args[1], StrExpr): str_expr = args[1] @@ -382,6 +386,12 @@ def parse_namedtuple_args( return None items = [cast(StrExpr, item).value for item in listexpr.items] else: + type_exprs = [ + t.items[1] + for t in listexpr.items + if isinstance(t, TupleExpr) and len(t.items) == 2 + ] + tvar_defs = self.api.get_and_bind_all_tvars(type_exprs) # The fields argument contains (name, type) tuples. result = self.parse_namedtuple_fields_with_types(listexpr.items, call) if result is None: @@ -389,7 +399,7 @@ def parse_namedtuple_args( return None items, types, _, ok = result if not ok: - return [], [], [], typename, False + return [], [], [], typename, [], False if not types: types = [AnyType(TypeOfAny.unannotated) for _ in items] underscore = [item for item in items if item.startswith("_")] @@ -402,7 +412,7 @@ def parse_namedtuple_args( if len(defaults) > len(items): self.fail(f'Too many defaults given in call to "{type_name}()"', call) defaults = defaults[: len(items)] - return items, types, defaults, typename, True + return items, types, defaults, typename, tvar_defs, True def parse_namedtuple_fields_with_types( self, nodes: List[Expression], context: Context @@ -523,7 +533,11 @@ def add_field( assert info.tuple_type is not None # Set by update_tuple_type() above. tvd = TypeVarType( - SELF_TVAR_NAME, info.fullname + "." + SELF_TVAR_NAME, -1, [], info.tuple_type + SELF_TVAR_NAME, + info.fullname + "." + SELF_TVAR_NAME, + self.api.tvar_scope.new_unique_func_id(), + [], + info.tuple_type, ) selftype = tvd diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index 2c1d843f4c7a..7fcfbe408dc1 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -32,6 +32,7 @@ TupleType, Type, TypeVarId, + TypeVarLikeType, get_proper_type, ) @@ -124,6 +125,8 @@ class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface): * Less need to pass around callback functions """ + tvar_scope: TypeVarLikeScope + @abstractmethod def lookup( self, name: str, ctx: Context, suppress_errors: bool = False @@ -158,6 +161,10 @@ def anal_type( ) -> Optional[Type]: raise NotImplementedError + @abstractmethod + def get_and_bind_all_tvars(self, type_exprs: List[Expression]) -> List[TypeVarLikeType]: + raise NotImplementedError + @abstractmethod def basic_new_typeinfo(self, name: str, basetype_or_fallback: Instance, line: int) -> TypeInfo: raise NotImplementedError diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 8464bb58b336..d8933b34663e 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -73,6 +73,11 @@ def class_frame(self, namespace: str) -> "TypeVarLikeScope": """A new scope frame for binding a class. Prohibits *this* class's tvars""" return TypeVarLikeScope(self.get_function_scope(), True, self, namespace=namespace) + def new_unique_func_id(self) -> int: + """Used by plugin-like code that needs to make synthetic generic functions.""" + self.func_id -= 1 + return self.func_id + def bind_new(self, name: str, tvar_expr: TypeVarLikeExpr) -> TypeVarLikeType: if self.is_class_scope: self.class_id += 1 diff --git a/test-data/unit/check-namedtuple.test b/test-data/unit/check-namedtuple.test index 349ab0ab74f2..e4f75f57280c 100644 --- a/test-data/unit/check-namedtuple.test +++ b/test-data/unit/check-namedtuple.test @@ -1278,3 +1278,31 @@ reveal_type(foo(x, nti)) # N: Revealed type is "builtins.tuple[builtins.int, .. reveal_type(foo(x, nts)) # N: Revealed type is "builtins.tuple[builtins.object, ...]" [builtins fixtures/tuple.pyi] [typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleCallSyntax] +from typing import NamedTuple, TypeVar + +T = TypeVar("T") +NT = NamedTuple("NT", [("key", int), ("value", T)]) +reveal_type(NT) # N: Revealed type is "def [T] (key: builtins.int, value: T`-1) -> Tuple[builtins.int, T`-1, fallback=__main__.NT[T`-1]]" + +nts: NT[str] +reveal_type(nts) # N: Revealed type is "Tuple[builtins.int, builtins.str, fallback=__main__.NT[builtins.str]]" + +nti = NT(key=0, value=0) +reveal_type(nti) # N: Revealed type is "Tuple[builtins.int, builtins.int, fallback=__main__.NT[builtins.int]]" +NT[str](key=0, value=0) # E: Argument "value" to "NT" has incompatible type "int"; expected "str" +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi] + +[case testGenericNamedTupleNoLegacySyntax] +from typing import TypeVar, NamedTuple + +T = TypeVar("T") +class C( + NamedTuple("_C", [("x", int), ("y", T)]) # E: Generic named tuples are not supported for legacy class syntax \ + # N: Use either Python 3 class syntax, or the assignment syntax +): ... + +[builtins fixtures/tuple.pyi] +[typing fixtures/typing-namedtuple.pyi]