From bb9f05132b0eb68600efb99ef67403fa3463fb9f Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 09:20:03 +0100 Subject: [PATCH 01/40] Implement support for returning TypedDict for dataclasses.asdict Relates to #5152 --- docs/source/additional_features.rst | 15 +- mypy/plugin.py | 1 + mypy/plugins/common.py | 27 ++- mypy/plugins/dataclasses.py | 126 +++++++++- mypy/plugins/default.py | 3 + test-data/unit/check-dataclasses.test | 298 ++++++++++++++++++++++++ test-data/unit/lib-stub/dataclasses.pyi | 7 +- 7 files changed, 460 insertions(+), 17 deletions(-) diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index fc151598cff0..1542c1a9d432 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -72,9 +72,22 @@ and :pep:`557`. Caveats/Known Issues ==================== -Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclasses.replace` and :py:func:`~dataclasses.asdict`, +Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclasses.replace`, have imprecise (too permissive) types. This will be fixed in future releases. +Calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the original dataclass +definition, transforming it recursively. There are, however, some limitations: + +* Subclasses of ``List``, ``Dict``, and ``Tuple`` appearing within dataclasses are transformed into reparameterized + versions of the respective base class, rather than a transformed version of the original subclass. + +* Recursion (e.g. dataclasses which reference each other) is not supported and results in an error. + +* ``NamedTuples`` appearing within dataclasses are transformed to ``Any`` + +* A more precise return type cannot be inferred for calls where ``dict_factory`` is set. + + Mypy does not yet recognize aliases of :py:func:`dataclasses.dataclass `, and will probably never recognize dynamically computed decorators. The following examples do **not** work: diff --git a/mypy/plugin.py b/mypy/plugin.py index ed2d80cfaf29..d6ccc3def1ee 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -209,6 +209,7 @@ class CheckerPluginInterface: docstrings in checker.py for more details. """ + modules = None # type: Dict[str, MypyFile] msg = None # type: MessageBuilder options = None # type: Options path = None # type: str diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 536022a1e09e..b57fd4d39527 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -1,14 +1,15 @@ -from typing import List, Optional, Union +from collections import OrderedDict +from typing import List, Optional, Union, Set from mypy.nodes import ( ARG_POS, MDEF, Argument, Block, CallExpr, ClassDef, Expression, SYMBOL_FUNCBASE_TYPES, FuncDef, PassStmt, RefExpr, SymbolTableNode, Var, JsonDict, ) -from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface +from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface, CheckerPluginInterface from mypy.semanal import set_callable_name from mypy.types import ( CallableType, Overloaded, Type, TypeVarDef, deserialize_type, get_proper_type, -) + TypedDictType, Instance, TPDICT_FB_NAMES) from mypy.typevars import fill_typevars from mypy.util import get_unique_redefinition_name from mypy.typeops import try_getting_str_literals # noqa: F401 # Part of public API @@ -155,8 +156,26 @@ def add_method_to_class( def deserialize_and_fixup_type( - data: Union[str, JsonDict], api: SemanticAnalyzerPluginInterface + data: Union[str, JsonDict], + api: Union[SemanticAnalyzerPluginInterface, CheckerPluginInterface] ) -> Type: typ = deserialize_type(data) typ.accept(TypeFixer(api.modules, allow_missing=False)) return typ + + +def get_anonymous_typeddict_type(api: CheckerPluginInterface) -> Instance: + for type_fullname in TPDICT_FB_NAMES: + try: + anonymous_typeddict_type = api.named_generic_type(type_fullname, []) + if anonymous_typeddict_type is not None: + return anonymous_typeddict_type + except KeyError: + continue + raise RuntimeError("No TypedDict fallback type found") + + +def make_anonymous_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, Type]', + required_keys: Set[str]) -> TypedDictType: + return TypedDictType(fields, required_keys=required_keys, + fallback=get_anonymous_typeddict_type(api)) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index b5c825394d13..f25bf18df273 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -1,19 +1,23 @@ """Plugin that provides support for dataclasses.""" -from typing import Dict, List, Set, Tuple, Optional +from collections import OrderedDict +from typing import Dict, List, Set, Tuple, Optional, FrozenSet, Callable, Union + from typing_extensions import Final +from mypy.maptype import map_instance_to_supertype from mypy.nodes import ( - ARG_OPT, ARG_POS, MDEF, Argument, AssignmentStmt, CallExpr, - Context, Expression, JsonDict, NameExpr, RefExpr, - SymbolTableNode, TempNode, TypeInfo, Var, TypeVarExpr, PlaceholderNode -) -from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface -from mypy.plugins.common import ( - add_method, _get_decorator_bool_argument, deserialize_and_fixup_type, + ARG_OPT, ARG_POS, MDEF, Argument, AssignmentStmt, CallExpr, Context, + Expression, JsonDict, NameExpr, RefExpr, SymbolTableNode, TempNode, + TypeInfo, Var, TypeVarExpr, PlaceholderNode ) -from mypy.types import Type, Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type +from mypy.plugin import ClassDefContext, FunctionContext, CheckerPluginInterface +from mypy.plugin import SemanticAnalyzerPluginInterface +from mypy.plugins.common import (add_method, _get_decorator_bool_argument, + make_anonymous_typeddict, deserialize_and_fixup_type) from mypy.server.trigger import make_wildcard_trigger +from mypy.types import (Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, + TupleType, UnionType, AnyType, TypeOfAny) # The set of decorators that generate dataclasses. dataclass_makers = { @@ -24,6 +28,10 @@ SELF_TVAR_NAME = '_DT' # type: Final +def is_type_dataclass(info: TypeInfo) -> bool: + return 'dataclass' in info.metadata + + class DataclassAttribute: def __init__( self, @@ -68,7 +76,8 @@ def serialize(self) -> JsonDict: @classmethod def deserialize( - cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface + cls, info: TypeInfo, data: JsonDict, + api: Union[SemanticAnalyzerPluginInterface, CheckerPluginInterface] ) -> 'DataclassAttribute': data = data.copy() typ = deserialize_and_fixup_type(data.pop('type'), api) @@ -276,7 +285,7 @@ def collect_attributes(self) -> Optional[List[DataclassAttribute]]: # we'll have unmodified attrs laying around. all_attrs = attrs.copy() for info in cls.info.mro[1:-1]: - if 'dataclass' not in info.metadata: + if not is_type_dataclass(info): continue super_attrs = [] @@ -365,3 +374,98 @@ def _collect_field_args(expr: Expression) -> Tuple[bool, Dict[str, Expression]]: args[name] = arg return True, args return False, {} + + +def asdict_callback(ctx: FunctionContext) -> Type: + positional_arg_types = ctx.arg_types[0] + + if positional_arg_types: + if len(ctx.arg_types) == 2: + # We can't infer a more precise for calls where dict_factory is set. + # At least for now, typeshed stubs for asdict don't allow you to pass in `dict` as + # dict_factory, so we can't special-case that. + return ctx.default_return_type + dataclass_instance = positional_arg_types[0] + dataclass_instance = get_proper_type(dataclass_instance) + if isinstance(dataclass_instance, Instance): + info = dataclass_instance.type + if not is_type_dataclass(info): + ctx.api.fail('asdict() should be called on dataclass instances', + dataclass_instance) + return _asdictify(ctx.api, ctx.context, dataclass_instance) + return ctx.default_return_type + + +def _transform_type_args(*, typ: Instance, transform: Callable[[Instance], Type]) -> List[Type]: + """For each type arg used in the Instance, call transform on it if the arg is an Instance.""" + new_args = [] + for arg in typ.args: + proper_arg = get_proper_type(arg) + if isinstance(proper_arg, Instance): + new_args.append(transform(proper_arg)) + else: + new_args.append(arg) + return new_args + + +def _asdictify(api: CheckerPluginInterface, context: Context, typ: Type) -> Type: + """Convert dataclasses into TypedDicts, recursively looking into built-in containers. + + It will look for dataclasses inside of tuples, lists, and dicts and convert them to TypedDicts. + """ + + def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: + typ = get_proper_type(typ) + if isinstance(typ, UnionType): + return UnionType([_asdictify_inner(item, seen_dataclasses) for item in typ.items]) + if isinstance(typ, Instance): + info = typ.type + if is_type_dataclass(info): + if info.fullname in seen_dataclasses: + api.fail( + "Recursive types are not supported in call to asdict, so falling back to " + "Dict[str, Any]", + context) + # Note: Would be nicer to fallback to default_return_type, but that is Any + # (due to overloads?) + return api.named_generic_type('builtins.dict', + [api.named_generic_type('builtins.str', []), + AnyType(TypeOfAny.implementation_artifact)]) + seen_dataclasses |= {info.fullname} + attrs = info.metadata['dataclass']['attributes'] + fields = OrderedDict() # type: OrderedDict[str, Type] + for data in attrs: + attr = DataclassAttribute.deserialize(info, data, api) + sym_node = info.names[attr.name] + attr_type = sym_node.type + assert attr_type is not None + fields[attr.name] = _asdictify_inner(attr_type, seen_dataclasses) + return make_anonymous_typeddict(api, fields=fields, + required_keys=set(fields.keys())) + elif info.has_base('builtins.list'): + supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( + 'builtins.list', []).type) + new_args = _transform_type_args( + typ=supertype_instance, + transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) + return api.named_generic_type('builtins.list', new_args) + elif info.has_base('builtins.dict'): + supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( + 'builtins.dict', []).type) + new_args = _transform_type_args( + typ=supertype_instance, + transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) + return api.named_generic_type('builtins.dict', new_args) + elif isinstance(typ, TupleType): + if typ.partial_fallback.type.is_named_tuple: + # For namedtuples, return Any. To properly support transforming namedtuples, + # we would have to generate a partial_fallback type for the TupleType and add it + # to the symbol table. It's not currently possibl to do this via the + # CheckerPluginInterface. Ideally it would use the same code as + # NamedTupleAnalyzer.build_namedtuple_typeinfo. + return AnyType(TypeOfAny.implementation_artifact) + return TupleType([_asdictify_inner(item, seen_dataclasses) for item in typ.items], + api.named_generic_type('builtins.tuple', []), implicit=typ.implicit) + return typ + + return _asdictify_inner(typ, seen_dataclasses=frozenset()) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 55a9a469e97b..3efc861787f6 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -23,6 +23,7 @@ class DefaultPlugin(Plugin): def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: from mypy.plugins import ctypes + from mypy.plugins import dataclasses if fullname == 'contextlib.contextmanager': return contextmanager_callback @@ -30,6 +31,8 @@ def get_function_hook(self, fullname: str return open_callback elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback + elif fullname == 'dataclasses.asdict': + return dataclasses.asdict_callback return None def get_method_signature_hook(self, fullname: str diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index f965ac54bff5..afe9dc65d078 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -990,3 +990,301 @@ class B(A): reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [builtins fixtures/property.pyi] + +[case testDataclassesAsdict] +from dataclasses import dataclass, asdict + +@dataclass +class Person: + name: str + age: int + +@dataclass +class NotQuiteAPerson: + name: str + other_field: str + + +reveal_type(Person) # N: Revealed type is 'def (name: builtins.str, age: builtins.int) -> __main__.Person' +reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' +Person(**asdict(Person('John', 32))) # Round-trip works +NotQuiteAPerson(**asdict(Person('John', 32))) # E: Extra argument "age" from **args for "NotQuiteAPerson" + +reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' + +[typing fixtures/typing-full.pyi] + +[case testDataclassesAsdict] +from dataclasses import dataclass, asdict + +@dataclass +class Person: + name: str + age: int + +reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' + +[typing fixtures/typing-full.pyi] + +[case testDataclassesAsdictRecursion] +from dataclasses import dataclass, asdict +from typing import Optional + +@dataclass +class C: + a: 'A' + +@dataclass +class B: + c: C + +@dataclass +class A: + b: Optional[B] = None + +# Recursion is not supported, so fall back +result = asdict(A(B(C(A())))) # E: Recursive types are not supported in call to asdict, so falling back to Dict[str, Any] +reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] + +[case testDataclassesAsdictUnions] +from dataclasses import dataclass, asdict +from typing import Union + +@dataclass +class Card: + last4: int + +@dataclass +class Customer: + card: Union[str, Card] + +reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' + +[typing fixtures/typing-full.pyi] + +[case testDataclassesAsdictDictFactory] +from dataclasses import dataclass, asdict +from typing import List, Tuple, Dict, Any + +@dataclass +class Person: + name: str + age: int + +def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: + pass + +# Passing in any a dict_factory falls back to default return value +reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] + +[case testDataclassesAsdictList] +from dataclasses import dataclass, asdict +from typing import List, Any + +@dataclass +class Person: + name: str + age: int + + +@dataclass +class Course: + participants: List[Person] + any_list: List[Any] + list_no_generic: list + +instance = Course( + participants=[Person("Joe", 32)], + any_list=[], + list_no_generic=[], +) +result = asdict(instance) +reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['any_list']) # N: Revealed type is 'builtins.list[Any]' +reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any]' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/list.pyi] + + +[case testDataclassesAsdictListSubclass] +from dataclasses import dataclass, asdict +from typing import List, Any, TypeVar, Generic + +@dataclass +class Person: + name: str + age: int + + +_T = TypeVar("_T") +class MyList(List[_T]): + pass + +_X = TypeVar("_X") +class MyListWith2TypeVars(List[_T], Generic[_T, _X]): + foo: _X + +_C = TypeVar("_C", Person, int) + +class MyListWithConstraint(List[_C], Generic[_C]): + pass + + +@dataclass +class Course: + list_subclass: MyList[Person] + list_subclass_2_typevars: MyListWith2TypeVars[Person, int] + list_subclass_with_constraint: MyListWithConstraint[Person] + +instance = Course( + list_subclass=MyList([]), + list_subclass_2_typevars=MyListWith2TypeVars[Person, int]([Person("John", 23)]), + list_subclass_with_constraint=MyListWithConstraint([Person("Tim", 29)]) +) +result = asdict(instance) + +# Supertypes (list) are returned, since there could be a constraint on the TypeVar +# used on the subclass such that when the type argument to the subclass is substituted with a TypedDict, +# it may not type-check. +reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/list.pyi] + + +[case testDataclassesAsdictDict] +from dataclasses import dataclass, asdict +from typing import Dict + +@dataclass +class Person: + name: str + age: int + +@dataclass +class Course: + participants_by_name: Dict[str, Person] + +instance = Course(participants_by_name={"Joe": Person("Joe", 32)}) +result = asdict(instance) +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name': builtins.str, 'age': builtins.int})]' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] + +[case testDataclassesAsdictDictSubclass] +from dataclasses import dataclass, asdict +from typing import Dict, Generic, TypeVar + +@dataclass +class Person: + name: str + age: int + +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_Other = TypeVar("_Other") +class MyDict(Dict[_KT, _VT], Generic[_Other, _KT, _VT]): + pass + +@dataclass +class Course: + participants_by_name: MyDict[int, str, Person] + +instance = Course(participants_by_name=MyDict[int, str, Person]([("Joe", Person("Joe", 32))])) +result = asdict(instance) +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name': builtins.str, 'age': builtins.int})]' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] + + +[case testDataclassesAsdictTuple] +from dataclasses import dataclass, asdict +from typing import Tuple + +@dataclass +class Person: + name: str + age: int + +@dataclass +class Course: + partners: Tuple[Person, Person] + +instance = Course(partners=(Person("Joe", 32), Person("John", 23))) +result = asdict(instance) +reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]' + + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + + + +[case testDataclassesAsdictTupleSubclass] +from dataclasses import dataclass, asdict +from typing import Tuple, Generic, TypeVar + +@dataclass +class Person: + name: str + age: int + +class MyTuple(Tuple[Person, str]): + pass + +@dataclass +class Course: + tuple_subclass: MyTuple + +instance = Course(tuple_subclass=MyTuple()) +result = asdict(instance) + +# For now, subclasses of Tuple are transformed to the Tuple base class +# This is because the subclass, if it itself contains dataclass fields, may be transformed in such a way that it +# is no longer compatible with the original Tuple class it is extending. +reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]' + + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testDataclassesAsdictNamedTuple] +from dataclasses import dataclass, asdict +from typing import NamedTuple + +@dataclass +class Person: + name: str + age: int + + +class Staff(NamedTuple): + teacher: Person + assistant: Person + + def staff_method(self): + pass + +@dataclass +class Course: + staff: Staff + +instance = Course(staff=Staff(teacher=Person("Joe", 32), assistant=Person("John", 23))) +result = asdict(instance) +# Due to implementation limitations, namedtuples are transformed to Any +reveal_type(result['staff']) # N: Revealed type is 'Any' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] diff --git a/test-data/unit/lib-stub/dataclasses.pyi b/test-data/unit/lib-stub/dataclasses.pyi index 160cfcd066ba..526a9831552d 100644 --- a/test-data/unit/lib-stub/dataclasses.pyi +++ b/test-data/unit/lib-stub/dataclasses.pyi @@ -1,4 +1,4 @@ -from typing import Any, Callable, Generic, Mapping, Optional, TypeVar, overload, Type +from typing import Any, Callable, Generic, Mapping, Optional, TypeVar, overload, Type, Dict, List, Tuple _T = TypeVar('_T') @@ -6,6 +6,11 @@ class InitVar(Generic[_T]): ... +@overload +def asdict(obj: Any) -> Dict[str, Any]: ... +@overload +def asdict(obj: Any, *, dict_factory: Callable[[List[Tuple[str, Any]]], _T]) -> _T: ... + @overload def dataclass(_cls: Type[_T]) -> Type[_T]: ... From e2f9f068c8a272f28e8206032478bbe3e4602229 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 09:40:13 +0100 Subject: [PATCH 02/40] Remove redundant test. Fix comment typo. --- test-data/unit/check-dataclasses.test | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index afe9dc65d078..5a94c9d6a148 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -994,29 +994,6 @@ reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [case testDataclassesAsdict] from dataclasses import dataclass, asdict -@dataclass -class Person: - name: str - age: int - -@dataclass -class NotQuiteAPerson: - name: str - other_field: str - - -reveal_type(Person) # N: Revealed type is 'def (name: builtins.str, age: builtins.int) -> __main__.Person' -reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' -Person(**asdict(Person('John', 32))) # Round-trip works -NotQuiteAPerson(**asdict(Person('John', 32))) # E: Extra argument "age" from **args for "NotQuiteAPerson" - -reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' - -[typing fixtures/typing-full.pyi] - -[case testDataclassesAsdict] -from dataclasses import dataclass, asdict - @dataclass class Person: name: str @@ -1078,7 +1055,7 @@ class Person: def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: pass -# Passing in any a dict_factory falls back to default return value +# Passing in a dict_factory should not return a TypedDict reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' [typing fixtures/typing-full.pyi] From 2f6ec2d4642bb05eab4382119a451161d28c3a00 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 09:46:51 +0100 Subject: [PATCH 03/40] Test for cases where dataclasses.asdict is called on non-dataclass instance. --- mypy/plugins/dataclasses.py | 9 +++++---- test-data/unit/check-dataclasses.test | 6 ++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index f25bf18df273..52c43d4202ce 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -389,10 +389,11 @@ def asdict_callback(ctx: FunctionContext) -> Type: dataclass_instance = get_proper_type(dataclass_instance) if isinstance(dataclass_instance, Instance): info = dataclass_instance.type - if not is_type_dataclass(info): - ctx.api.fail('asdict() should be called on dataclass instances', - dataclass_instance) - return _asdictify(ctx.api, ctx.context, dataclass_instance) + if is_type_dataclass(info): + return _asdictify(ctx.api, ctx.context, dataclass_instance) + + ctx.api.fail("'dataclasses.asdict' should be called on dataclass instances", + ctx.context) return ctx.default_return_type diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 5a94c9d6a148..8a6f78fed46e 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -999,7 +999,13 @@ class Person: name: str age: int +class NonDataclass: + pass + reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' +asdict(Person) # E: 'dataclasses.asdict' should be called on dataclass instances +asdict(NonDataclass()) # E: 'dataclasses.asdict' should be called on dataclass instances + [typing fixtures/typing-full.pyi] From a9779e2dc1a8aeefd61867c9d0037c8cc0815d1b Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 09:54:54 +0100 Subject: [PATCH 04/40] Clean up tests, and test more edge-cases. --- mypy/plugins/dataclasses.py | 13 +++++++------ test-data/unit/check-dataclasses.test | 27 +++++++++------------------ 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 52c43d4202ce..0724913ac093 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -380,17 +380,18 @@ def asdict_callback(ctx: FunctionContext) -> Type: positional_arg_types = ctx.arg_types[0] if positional_arg_types: - if len(ctx.arg_types) == 2: - # We can't infer a more precise for calls where dict_factory is set. - # At least for now, typeshed stubs for asdict don't allow you to pass in `dict` as - # dict_factory, so we can't special-case that. - return ctx.default_return_type dataclass_instance = positional_arg_types[0] dataclass_instance = get_proper_type(dataclass_instance) if isinstance(dataclass_instance, Instance): info = dataclass_instance.type if is_type_dataclass(info): - return _asdictify(ctx.api, ctx.context, dataclass_instance) + if len(ctx.arg_types) == 1: + return _asdictify(ctx.api, ctx.context, dataclass_instance) + else: + # We can't infer a more precise for calls where dict_factory is set. + # At least for now, typeshed stubs for asdict don't allow you to pass in + # `dict` as dict_factory, so we can't special-case that. + return ctx.default_return_type ctx.api.fail("'dataclasses.asdict' should be called on dataclass instances", ctx.context) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 8a6f78fed46e..c2a99d9dc955 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -993,6 +993,7 @@ reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [case testDataclassesAsdict] from dataclasses import dataclass, asdict +from typing import List, Tuple, Dict, Any @dataclass class Person: @@ -1006,8 +1007,16 @@ reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name asdict(Person) # E: 'dataclasses.asdict' should be called on dataclass instances asdict(NonDataclass()) # E: 'dataclasses.asdict' should be called on dataclass instances +def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: + pass + +asdict(NonDataclass(), dict_factory=my_dict_factory) # E: 'dataclasses.asdict' should be called on dataclass instances + +# Passing in a dict_factory should not return a TypedDict +reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' [typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictRecursion] from dataclasses import dataclass, asdict @@ -1049,24 +1058,6 @@ reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': [typing fixtures/typing-full.pyi] -[case testDataclassesAsdictDictFactory] -from dataclasses import dataclass, asdict -from typing import List, Tuple, Dict, Any - -@dataclass -class Person: - name: str - age: int - -def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: - pass - -# Passing in a dict_factory should not return a TypedDict -reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' - -[typing fixtures/typing-full.pyi] -[builtins fixtures/dict.pyi] - [case testDataclassesAsdictList] from dataclasses import dataclass, asdict from typing import List, Any From c5d0a155a227a8a778bcdc668b24ce2d2b56515a Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 09:56:26 +0100 Subject: [PATCH 05/40] Remove no-longer-needed type: ignore on CheckerPluginInterface.modules in proper_plugin --- misc/proper_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/proper_plugin.py b/misc/proper_plugin.py index c30999448387..4f1f6d115356 100644 --- a/misc/proper_plugin.py +++ b/misc/proper_plugin.py @@ -127,7 +127,7 @@ def proper_types_hook(ctx: FunctionContext) -> Type: def get_proper_type_instance(ctx: FunctionContext) -> Instance: - types = ctx.api.modules['mypy.types'] # type: ignore + types = ctx.api.modules['mypy.types'] proper_type_info = types.names['ProperType'] assert isinstance(proper_type_info.node, TypeInfo) return Instance(proper_type_info.node, []) From 454431d2354f77f5b0fc07d9214c39b8ad3e96cf Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 26 Mar 2020 10:05:02 +0100 Subject: [PATCH 06/40] Make typeddicts non-total. --- mypy/plugins/dataclasses.py | 2 +- test-data/unit/check-dataclasses.test | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 0724913ac093..c1ea6f468e92 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -443,7 +443,7 @@ def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: assert attr_type is not None fields[attr.name] = _asdictify_inner(attr_type, seen_dataclasses) return make_anonymous_typeddict(api, fields=fields, - required_keys=set(fields.keys())) + required_keys=set()) elif info.has_base('builtins.list'): supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( 'builtins.list', []).type) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index c2a99d9dc955..ccdf371ecf54 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1003,7 +1003,7 @@ class Person: class NonDataclass: pass -reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' +reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name'?: builtins.str, 'age'?: builtins.int})' asdict(Person) # E: 'dataclasses.asdict' should be called on dataclass instances asdict(NonDataclass()) # E: 'dataclasses.asdict' should be called on dataclass instances @@ -1036,7 +1036,7 @@ class A: # Recursion is not supported, so fall back result = asdict(A(B(C(A())))) # E: Recursive types are not supported in call to asdict, so falling back to Dict[str, Any] -reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})' +reveal_type(result) # N: Revealed type is 'TypedDict({'b'?: Union[TypedDict({'c'?: TypedDict({'a'?: builtins.dict[builtins.str, Any]})}), None]})' [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] @@ -1054,7 +1054,7 @@ class Card: class Customer: card: Union[str, Card] -reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' +reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card'?: Union[builtins.str, TypedDict({'last4'?: builtins.int})]})' [typing fixtures/typing-full.pyi] @@ -1080,7 +1080,7 @@ instance = Course( list_no_generic=[], ) result = asdict(instance) -reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' reveal_type(result['any_list']) # N: Revealed type is 'builtins.list[Any]' reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any]' @@ -1128,9 +1128,9 @@ result = asdict(instance) # Supertypes (list) are returned, since there could be a constraint on the TypeVar # used on the subclass such that when the type argument to the subclass is substituted with a TypedDict, # it may not type-check. -reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' -reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' -reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] @@ -1151,7 +1151,7 @@ class Course: instance = Course(participants_by_name={"Joe": Person("Joe", 32)}) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1177,7 +1177,7 @@ class Course: instance = Course(participants_by_name=MyDict[int, str, Person]([("Joe", Person("Joe", 32))])) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1198,7 +1198,7 @@ class Course: instance = Course(partners=(Person("Joe", 32), Person("John", 23))) result = asdict(instance) -reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name'?: builtins.str, 'age'?: builtins.int}), TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' [typing fixtures/typing-full.pyi] @@ -1228,7 +1228,7 @@ result = asdict(instance) # For now, subclasses of Tuple are transformed to the Tuple base class # This is because the subclass, if it itself contains dataclass fields, may be transformed in such a way that it # is no longer compatible with the original Tuple class it is extending. -reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]' +reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name'?: builtins.str, 'age'?: builtins.int}), builtins.str]' [typing fixtures/typing-full.pyi] From d809e8b6bb44c105ca0576503b696b4583f603ba Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Mon, 30 Mar 2020 09:41:59 +0200 Subject: [PATCH 07/40] Address some of review comments (formatting, docs, code nitpicks, remove recursion error). --- docs/source/additional_features.rst | 13 ++----------- mypy/plugins/common.py | 3 ++- mypy/plugins/dataclasses.py | 27 +++++++++++++-------------- test-data/unit/check-dataclasses.test | 2 +- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index 1542c1a9d432..fae1a09245f3 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -76,17 +76,8 @@ Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclass have imprecise (too permissive) types. This will be fixed in future releases. Calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the original dataclass -definition, transforming it recursively. There are, however, some limitations: - -* Subclasses of ``List``, ``Dict``, and ``Tuple`` appearing within dataclasses are transformed into reparameterized - versions of the respective base class, rather than a transformed version of the original subclass. - -* Recursion (e.g. dataclasses which reference each other) is not supported and results in an error. - -* ``NamedTuples`` appearing within dataclasses are transformed to ``Any`` - -* A more precise return type cannot be inferred for calls where ``dict_factory`` is set. - +definition, transforming it recursively. There are, however, some limitations. In particular, a precise return type +cannot be inferred for recursive dataclasses, and for calls where ``dict_factory`` is set. Mypy does not yet recognize aliases of :py:func:`dataclasses.dataclass `, and will probably never recognize dynamically computed decorators. The following examples diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index b57fd4d39527..a8ebf853c787 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -9,7 +9,8 @@ from mypy.semanal import set_callable_name from mypy.types import ( CallableType, Overloaded, Type, TypeVarDef, deserialize_type, get_proper_type, - TypedDictType, Instance, TPDICT_FB_NAMES) + TypedDictType, Instance, TPDICT_FB_NAMES +) from mypy.typevars import fill_typevars from mypy.util import get_unique_redefinition_name from mypy.typeops import try_getting_str_literals # noqa: F401 # Part of public API diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index c1ea6f468e92..361e5f054ac4 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -13,11 +13,15 @@ ) from mypy.plugin import ClassDefContext, FunctionContext, CheckerPluginInterface from mypy.plugin import SemanticAnalyzerPluginInterface -from mypy.plugins.common import (add_method, _get_decorator_bool_argument, - make_anonymous_typeddict, deserialize_and_fixup_type) +from mypy.plugins.common import ( + add_method, _get_decorator_bool_argument, make_anonymous_typeddict, + deserialize_and_fixup_type +) from mypy.server.trigger import make_wildcard_trigger -from mypy.types import (Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, - TupleType, UnionType, AnyType, TypeOfAny) +from mypy.types import ( + Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, UnionType, + AnyType, TypeOfAny +) # The set of decorators that generate dataclasses. dataclass_makers = { @@ -380,11 +384,9 @@ def asdict_callback(ctx: FunctionContext) -> Type: positional_arg_types = ctx.arg_types[0] if positional_arg_types: - dataclass_instance = positional_arg_types[0] - dataclass_instance = get_proper_type(dataclass_instance) + dataclass_instance = get_proper_type(positional_arg_types[0]) if isinstance(dataclass_instance, Instance): - info = dataclass_instance.type - if is_type_dataclass(info): + if is_type_dataclass(dataclass_instance.type): if len(ctx.arg_types) == 1: return _asdictify(ctx.api, ctx.context, dataclass_instance) else: @@ -424,10 +426,7 @@ def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: info = typ.type if is_type_dataclass(info): if info.fullname in seen_dataclasses: - api.fail( - "Recursive types are not supported in call to asdict, so falling back to " - "Dict[str, Any]", - context) + # Recursive types not supported, so fall back to Dict[str, Any] # Note: Would be nicer to fallback to default_return_type, but that is Any # (due to overloads?) return api.named_generic_type('builtins.dict', @@ -446,14 +445,14 @@ def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: required_keys=set()) elif info.has_base('builtins.list'): supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( - 'builtins.list', []).type) + 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) new_args = _transform_type_args( typ=supertype_instance, transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) return api.named_generic_type('builtins.list', new_args) elif info.has_base('builtins.dict'): supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( - 'builtins.dict', []).type) + 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), AnyType(TypeOfAny.implementation_artifact)]).type) new_args = _transform_type_args( typ=supertype_instance, transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index ccdf371ecf54..86f1e4af473f 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1035,7 +1035,7 @@ class A: b: Optional[B] = None # Recursion is not supported, so fall back -result = asdict(A(B(C(A())))) # E: Recursive types are not supported in call to asdict, so falling back to Dict[str, Any] +result = asdict(A(B(C(A())))) reveal_type(result) # N: Revealed type is 'TypedDict({'b'?: Union[TypedDict({'c'?: TypedDict({'a'?: builtins.dict[builtins.str, Any]})}), None]})' [typing fixtures/typing-full.pyi] From 4d195ccde18fe3894529cb79f16cd066967acb8c Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Mon, 30 Mar 2020 09:54:54 +0200 Subject: [PATCH 08/40] Simplify: Remove _transform_type_args and remove unneeded None check in get_anonymous_typeddict_type. --- mypy/plugins/common.py | 4 +--- mypy/plugins/dataclasses.py | 30 +++++++++--------------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index a8ebf853c787..91b910dfe441 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -168,9 +168,7 @@ def deserialize_and_fixup_type( def get_anonymous_typeddict_type(api: CheckerPluginInterface) -> Instance: for type_fullname in TPDICT_FB_NAMES: try: - anonymous_typeddict_type = api.named_generic_type(type_fullname, []) - if anonymous_typeddict_type is not None: - return anonymous_typeddict_type + return api.named_generic_type(type_fullname, []) except KeyError: continue raise RuntimeError("No TypedDict fallback type found") diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 361e5f054ac4..6a8457578a9c 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -400,18 +400,6 @@ def asdict_callback(ctx: FunctionContext) -> Type: return ctx.default_return_type -def _transform_type_args(*, typ: Instance, transform: Callable[[Instance], Type]) -> List[Type]: - """For each type arg used in the Instance, call transform on it if the arg is an Instance.""" - new_args = [] - for arg in typ.args: - proper_arg = get_proper_type(arg) - if isinstance(proper_arg, Instance): - new_args.append(transform(proper_arg)) - else: - new_args.append(arg) - return new_args - - def _asdictify(api: CheckerPluginInterface, context: Context, typ: Type) -> Type: """Convert dataclasses into TypedDicts, recursively looking into built-in containers. @@ -446,17 +434,17 @@ def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: elif info.has_base('builtins.list'): supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) - new_args = _transform_type_args( - typ=supertype_instance, - transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) - return api.named_generic_type('builtins.list', new_args) + return api.named_generic_type('builtins.list', [ + _asdictify_inner(supertype_instance.args[0], seen_dataclasses) + ]) elif info.has_base('builtins.dict'): supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( - 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), AnyType(TypeOfAny.implementation_artifact)]).type) - new_args = _transform_type_args( - typ=supertype_instance, - transform=lambda arg: _asdictify_inner(arg, seen_dataclasses)) - return api.named_generic_type('builtins.dict', new_args) + 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), + AnyType(TypeOfAny.implementation_artifact)]).type) + return api.named_generic_type('builtins.dict', [ + _asdictify_inner(supertype_instance.args[0], seen_dataclasses), + _asdictify_inner(supertype_instance.args[1], seen_dataclasses) + ]) elif isinstance(typ, TupleType): if typ.partial_fallback.type.is_named_tuple: # For namedtuples, return Any. To properly support transforming namedtuples, From b33798c09027bdc9e28f8c56c550d5b528e833eb Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Mon, 30 Mar 2020 10:25:10 +0200 Subject: [PATCH 09/40] Fix unused import --- mypy/plugins/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 6a8457578a9c..051f502f24fa 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -1,7 +1,7 @@ """Plugin that provides support for dataclasses.""" from collections import OrderedDict -from typing import Dict, List, Set, Tuple, Optional, FrozenSet, Callable, Union +from typing import Dict, List, Set, Tuple, Optional, FrozenSet, Union from typing_extensions import Final From fe19bc9d83b8471bbe876beee8b81f512e95c7fc Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Mon, 30 Mar 2020 11:09:33 +0200 Subject: [PATCH 10/40] Add fine-grained test for dataclasses.asdict. --- mypy/plugins/default.py | 10 +++++++-- mypy/test/testfinegrained.py | 1 + test-data/unit/check-dataclasses.test | 32 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 3efc861787f6..f258362a218b 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -1,8 +1,8 @@ from functools import partial -from typing import Callable, Optional, List +from typing import Callable, Optional, List, Tuple from mypy import message_registry -from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr +from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, CheckerPluginInterface, @@ -20,6 +20,12 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" + def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: + # Add modules for TypedDict fallback types as a plugin dependency + # (used by dataclasses plugin). + return [(10, ".".join(typeddict_fallback_name.split('.')[:-1]), -1) + for typeddict_fallback_name in TPDICT_FB_NAMES] + def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: from mypy.plugins import ctypes diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index d4ed18cab095..b48e354e09f3 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -49,6 +49,7 @@ class FineGrainedSuite(DataSuite): 'fine-grained-modules.test', 'fine-grained-follow-imports.test', 'fine-grained-suggest.test', + 'fine-grained-dataclasses.test', ] # Whether to use the fine-grained cache in the testing. This is overridden diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 86f1e4af473f..aeeabf665b95 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1262,3 +1262,35 @@ reveal_type(result['staff']) # N: Revealed type is 'Any' [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] + + + +[case testDataclassesAsdictNamedTuple] +from dataclasses import dataclass, asdict +from typing import NamedTuple + +@dataclass +class Person: + name: str + age: int + + +class Staff(NamedTuple): + teacher: Person + assistant: Person + + def staff_method(self): + pass + +@dataclass +class Course: + staff: Staff + +instance = Course(staff=Staff(teacher=Person("Joe", 32), assistant=Person("John", 23))) +result = asdict(instance) +# Due to implementation limitations, namedtuples are transformed to Any +reveal_type(result['staff']) # N: Revealed type is 'Any' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + From 6328a7cd6141aeca4b4d6b61549fbec626a5b163 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Mon, 30 Mar 2020 11:19:25 +0200 Subject: [PATCH 11/40] Oops, add forgotten fine-grained dataclasses test. And remove redundant test. --- test-data/unit/check-dataclasses.test | 31 ---------------- test-data/unit/fine-grained-dataclasses.test | 37 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 31 deletions(-) create mode 100644 test-data/unit/fine-grained-dataclasses.test diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index aeeabf665b95..5dd1bd8a2680 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1263,34 +1263,3 @@ reveal_type(result['staff']) # N: Revealed type is 'Any' [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] - - -[case testDataclassesAsdictNamedTuple] -from dataclasses import dataclass, asdict -from typing import NamedTuple - -@dataclass -class Person: - name: str - age: int - - -class Staff(NamedTuple): - teacher: Person - assistant: Person - - def staff_method(self): - pass - -@dataclass -class Course: - staff: Staff - -instance = Course(staff=Staff(teacher=Person("Joe", 32), assistant=Person("John", 23))) -result = asdict(instance) -# Due to implementation limitations, namedtuples are transformed to Any -reveal_type(result['staff']) # N: Revealed type is 'Any' - -[typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] - diff --git a/test-data/unit/fine-grained-dataclasses.test b/test-data/unit/fine-grained-dataclasses.test new file mode 100644 index 000000000000..fff511f3939a --- /dev/null +++ b/test-data/unit/fine-grained-dataclasses.test @@ -0,0 +1,37 @@ +-- Test cases for fine-grained incremental checking of dataclasses +-- +-- The comment at the top of fine-grained.test explains how these tests +-- work. + +[case testDataclassesAsdictFineGrained] +[file a.py] +from dataclasses import dataclass +from b import AttributeInOtherModule + +@dataclass +class MyDataclass: + attr: AttributeInOtherModule + +[file b.py] +AttributeInOtherModule = str +[file c.py] +from dataclasses import asdict +from a import MyDataclass +reveal_type(asdict(MyDataclass('John'))) + +[file b.py.2] +from typing import List +class MyList(List[int]): + pass +AttributeInOtherModule = MyList +[file c.py.2] +from dataclasses import asdict +from a import MyDataclass +from b import MyList +reveal_type(asdict(MyDataclass(MyList()))) + +[out] +c.py:3: note: Revealed type is 'TypedDict({'attr'?: builtins.str})' +== +c.py:4: note: Revealed type is 'TypedDict({'attr'?: builtins.list[builtins.int]})' +[builtins fixtures/tuple.pyi] \ No newline at end of file From 4694299e5be712b4c14c8dbc90dabb8c54278654 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 1 Apr 2020 23:01:36 +0200 Subject: [PATCH 12/40] Only import the module containing TypedDict fallback if dataclasses is imported. Reduces a lot of failing tests. Unfortunately, still some tests require tuple to be defined (used by mypy_extensions). --- mypy/plugins/default.py | 30 ++++++++++++++++---- test-data/unit/check-dataclasses.test | 40 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index f258362a218b..ff13ee1a680a 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -2,7 +2,9 @@ from typing import Callable, Optional, List, Tuple from mypy import message_registry -from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile +from mypy.nodes import ( + Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile, ImportFrom, Import, ImportAll +) from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, CheckerPluginInterface, @@ -21,10 +23,28 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: - # Add modules for TypedDict fallback types as a plugin dependency - # (used by dataclasses plugin). - return [(10, ".".join(typeddict_fallback_name.split('.')[:-1]), -1) - for typeddict_fallback_name in TPDICT_FB_NAMES] + def is_dataclasses_imported(file: MypyFile): + for i in file.imports: + if isinstance(i, Import): + for id, as_id in i.ids: + if 'dataclasses' in id: + return True + elif isinstance(i, ImportFrom) or isinstance(i, ImportAll): + if 'dataclasses' in i.id: + return True + return False + + # Add modules for TypedDict fallback types as a plugin dependency only if dataclasses + # module is imported (it is used to support dataclasses.asdict). + if is_dataclasses_imported(file): + if self.python_version == (3, 6): + return [(10, "mypy_extensions", -1)] + elif self.python_version == (3, 7): + return [(10, "typing_extensions", -1)] + elif self.python_version == (3, 8): + return [(10, "typing", -1)] + else: + return [] def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 5dd1bd8a2680..07506767a511 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1018,6 +1018,46 @@ reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Reve [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] + +[case testDataclassesAsdictPython36] +# flags: --python-version 3.6 +from dataclasses import dataclass, asdict + +@dataclass +class Foo: + bar: str + +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + + +[case testDataclassesAsdictPython37] +# flags: --python-version 3.7 +from dataclasses import dataclass, asdict + +@dataclass +class Foo: + bar: str + +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' + +[typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + +[case testDataclassesAsdictPython38] +# flags: --python-version 3.8 +from dataclasses import dataclass, asdict + +@dataclass +class Foo: + bar: str + +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' + +[typing fixtures/typing-full.pyi] + [case testDataclassesAsdictRecursion] from dataclasses import dataclass, asdict from typing import Optional From 9c29081719e145958fed59d38a86cddc16e84b99 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 15:16:30 +0200 Subject: [PATCH 13/40] Only enable TypedDict for Python >= 3.8. This is because TypedDict is only available (from typing) without installing external packages from that version. It also fixes issues where I would have to change a lot of unit tests if mypy_extensions were depended on always by the default plugin. Because apparently, mypy_extensions depended on tuple being defined. --- docs/source/additional_features.rst | 6 ++-- mypy/plugins/dataclasses.py | 11 ++++--- mypy/plugins/default.py | 33 +++++--------------- test-data/unit/check-dataclasses.test | 30 +++++++----------- test-data/unit/fine-grained-dataclasses.test | 4 ++- 5 files changed, 31 insertions(+), 53 deletions(-) diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index fae1a09245f3..e90d06604e34 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -75,9 +75,9 @@ Caveats/Known Issues Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclasses.replace`, have imprecise (too permissive) types. This will be fixed in future releases. -Calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the original dataclass -definition, transforming it recursively. There are, however, some limitations. In particular, a precise return type -cannot be inferred for recursive dataclasses, and for calls where ``dict_factory`` is set. +In Python version 3.8 and above, calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the +original dataclass definition, transforming it recursively. There are, however, some limitations. In particular, a +precise return type cannot be inferred for recursive dataclasses, and for calls where ``dict_factory`` is set. Mypy does not yet recognize aliases of :py:func:`dataclasses.dataclass `, and will probably never recognize dynamically computed decorators. The following examples diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 051f502f24fa..cb7a39bd690d 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -380,17 +380,18 @@ def _collect_field_args(expr: Expression) -> Tuple[bool, Dict[str, Expression]]: return False, {} -def asdict_callback(ctx: FunctionContext) -> Type: +def asdict_callback(ctx: FunctionContext, return_typeddicts: bool = False) -> Type: + """Check calls to asdict pass in a dataclass. Optionally, return TypedDicts.""" positional_arg_types = ctx.arg_types[0] if positional_arg_types: dataclass_instance = get_proper_type(positional_arg_types[0]) if isinstance(dataclass_instance, Instance): if is_type_dataclass(dataclass_instance.type): - if len(ctx.arg_types) == 1: - return _asdictify(ctx.api, ctx.context, dataclass_instance) + if len(ctx.arg_types) == 1 and return_typeddicts: + return _asdictify(ctx.api, dataclass_instance) else: - # We can't infer a more precise for calls where dict_factory is set. + # We can't infer a more precise type for calls where dict_factory is set. # At least for now, typeshed stubs for asdict don't allow you to pass in # `dict` as dict_factory, so we can't special-case that. return ctx.default_return_type @@ -400,7 +401,7 @@ def asdict_callback(ctx: FunctionContext) -> Type: return ctx.default_return_type -def _asdictify(api: CheckerPluginInterface, context: Context, typ: Type) -> Type: +def _asdictify(api: CheckerPluginInterface, typ: Type) -> Type: """Convert dataclasses into TypedDicts, recursively looking into built-in containers. It will look for dataclasses inside of tuples, lists, and dicts and convert them to TypedDicts. diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index ff13ee1a680a..bb1e8b9afff8 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -2,9 +2,7 @@ from typing import Callable, Optional, List, Tuple from mypy import message_registry -from mypy.nodes import ( - Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile, ImportFrom, Import, ImportAll -) +from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, CheckerPluginInterface, @@ -23,28 +21,10 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: - def is_dataclasses_imported(file: MypyFile): - for i in file.imports: - if isinstance(i, Import): - for id, as_id in i.ids: - if 'dataclasses' in id: - return True - elif isinstance(i, ImportFrom) or isinstance(i, ImportAll): - if 'dataclasses' in i.id: - return True - return False - - # Add modules for TypedDict fallback types as a plugin dependency only if dataclasses - # module is imported (it is used to support dataclasses.asdict). - if is_dataclasses_imported(file): - if self.python_version == (3, 6): - return [(10, "mypy_extensions", -1)] - elif self.python_version == (3, 7): - return [(10, "typing_extensions", -1)] - elif self.python_version == (3, 8): - return [(10, "typing", -1)] - else: - return [] + if self.python_version >= (3, 8): + # Add module needed for anonymous TypedDict (used to support dataclasses.asdict) + return [(10, "typing", -1)] + return [] def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: @@ -58,7 +38,8 @@ def get_function_hook(self, fullname: str elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback elif fullname == 'dataclasses.asdict': - return dataclasses.asdict_callback + return partial(dataclasses.asdict_callback, + return_typeddicts=self.python_version >= (3, 8)) return None def get_method_signature_hook(self, fullname: str diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 07506767a511..e45bfb6542bf 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -992,6 +992,7 @@ reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [builtins fixtures/property.pyi] [case testDataclassesAsdict] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Tuple, Dict, Any @@ -1027,11 +1028,9 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'Any' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] - [case testDataclassesAsdictPython37] # flags: --python-version 3.7 @@ -1041,24 +1040,12 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' - -[typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] - -[case testDataclassesAsdictPython38] -# flags: --python-version 3.8 -from dataclasses import dataclass, asdict - -@dataclass -class Foo: - bar: str - -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar'?: builtins.str})' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'Any' [typing fixtures/typing-full.pyi] [case testDataclassesAsdictRecursion] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Optional @@ -1079,10 +1066,10 @@ result = asdict(A(B(C(A())))) reveal_type(result) # N: Revealed type is 'TypedDict({'b'?: Union[TypedDict({'c'?: TypedDict({'a'?: builtins.dict[builtins.str, Any]})}), None]})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] [builtins fixtures/dict.pyi] [case testDataclassesAsdictUnions] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Union @@ -1099,6 +1086,7 @@ reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card'?: [typing fixtures/typing-full.pyi] [case testDataclassesAsdictList] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Any @@ -1129,6 +1117,7 @@ reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any] [case testDataclassesAsdictListSubclass] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Any, TypeVar, Generic @@ -1177,6 +1166,7 @@ reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'buil [case testDataclassesAsdictDict] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Dict @@ -1197,6 +1187,7 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [builtins fixtures/dict.pyi] [case testDataclassesAsdictDictSubclass] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Dict, Generic, TypeVar @@ -1224,6 +1215,7 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [case testDataclassesAsdictTuple] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Tuple @@ -1247,6 +1239,7 @@ reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name'?: [case testDataclassesAsdictTupleSubclass] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Tuple, Generic, TypeVar @@ -1275,6 +1268,7 @@ reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'n [builtins fixtures/tuple.pyi] [case testDataclassesAsdictNamedTuple] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import NamedTuple diff --git a/test-data/unit/fine-grained-dataclasses.test b/test-data/unit/fine-grained-dataclasses.test index fff511f3939a..ea8a8920bbcb 100644 --- a/test-data/unit/fine-grained-dataclasses.test +++ b/test-data/unit/fine-grained-dataclasses.test @@ -4,6 +4,7 @@ -- work. [case testDataclassesAsdictFineGrained] +# flags: --python-version 3.8 [file a.py] from dataclasses import dataclass from b import AttributeInOtherModule @@ -34,4 +35,5 @@ reveal_type(asdict(MyDataclass(MyList()))) c.py:3: note: Revealed type is 'TypedDict({'attr'?: builtins.str})' == c.py:4: note: Revealed type is 'TypedDict({'attr'?: builtins.list[builtins.int]})' -[builtins fixtures/tuple.pyi] \ No newline at end of file +[typing fixtures/typing-full.pyi] +[builtins fixtures/list.pyi] From d7df77a089345ae657fc060159d5162ad99b5302 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 15:45:48 +0200 Subject: [PATCH 14/40] Refactor asdict implementation to use TypeTranslator instead of recursive functions. --- mypy/plugins/dataclasses.py | 124 ++++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index cb7a39bd690d..6c52dc92b9d0 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -1,7 +1,7 @@ """Plugin that provides support for dataclasses.""" from collections import OrderedDict -from typing import Dict, List, Set, Tuple, Optional, FrozenSet, Union +from typing import Dict, List, Set, Tuple, Optional, Union from typing_extensions import Final @@ -18,9 +18,10 @@ deserialize_and_fixup_type ) from mypy.server.trigger import make_wildcard_trigger +from mypy.type_visitor import TypeTranslator from mypy.types import ( Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, UnionType, - AnyType, TypeOfAny + AnyType, TypeOfAny, TypeAliasType ) # The set of decorators that generate dataclasses. @@ -401,61 +402,72 @@ def asdict_callback(ctx: FunctionContext, return_typeddicts: bool = False) -> Ty return ctx.default_return_type +class AsDictVisitor(TypeTranslator): + def __init__(self, api: CheckerPluginInterface) -> None: + self.api = api + self.seen_dataclasses = set() # type: Set[str] + + def visit_type_alias_type(self, t: TypeAliasType) -> Type: + return t.copy_modified(args=[a.accept(self) for a in t.args]) + + def visit_instance(self, t: Instance) -> Type: + info = t.type + if is_type_dataclass(info): + if info.fullname in self.seen_dataclasses: + # Recursive types not supported, so fall back to Dict[str, Any] + # Note: Would be nicer to fallback to default_return_type, but that is Any + # (due to overloads?) + return self.api.named_generic_type( + 'builtins.dict', [self.api.named_generic_type('builtins.str', []), + AnyType(TypeOfAny.implementation_artifact)]) + attrs = info.metadata['dataclass']['attributes'] + fields = OrderedDict() # type: OrderedDict[str, Type] + self.seen_dataclasses.add(info.fullname) + for data in attrs: + attr = DataclassAttribute.deserialize(info, data, self.api) + sym_node = info.names[attr.name] + attr_type = sym_node.type + assert attr_type is not None + fields[attr.name] = attr_type.accept(self) + self.seen_dataclasses.remove(info.fullname) + return make_anonymous_typeddict(self.api, fields=fields, + required_keys=set()) + elif info.has_base('builtins.list'): + supertype_instance = map_instance_to_supertype(t, self.api.named_generic_type( + 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) + return self.api.named_generic_type('builtins.list', [ + supertype_instance.args[0].accept(self) + ]) + elif info.has_base('builtins.dict'): + supertype_instance = map_instance_to_supertype(t, self.api.named_generic_type( + 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), + AnyType(TypeOfAny.implementation_artifact)]).type) + return self.api.named_generic_type('builtins.dict', [ + supertype_instance.args[0].accept(self), + supertype_instance.args[1].accept(self) + ]) + return t + + def visit_union_type(self, t: UnionType) -> Type: + return UnionType([item.accept(self) for item in t.items]) + + def visit_tuple_type(self, t: TupleType) -> Type: + if t.partial_fallback.type.is_named_tuple: + # For namedtuples, return Any. To properly support transforming namedtuples, + # we would have to generate a partial_fallback type for the TupleType and add it + # to the symbol table. It's not currently possibl to do this via the + # CheckerPluginInterface. Ideally it would use the same code as + # NamedTupleAnalyzer.build_namedtuple_typeinfo. + return AnyType(TypeOfAny.implementation_artifact) + # Note: Tuple subclasses not supported, hence overriding the fallback + return t.copy_modified(items=[item.accept(self) for item in t.items], + fallback=self.api.named_generic_type('builtins.tuple', [])) + + def _asdictify(api: CheckerPluginInterface, typ: Type) -> Type: """Convert dataclasses into TypedDicts, recursively looking into built-in containers. - It will look for dataclasses inside of tuples, lists, and dicts and convert them to TypedDicts. + It will look for dataclasses inside of tuples, lists, and dicts and convert them to + TypedDicts. """ - - def _asdictify_inner(typ: Type, seen_dataclasses: FrozenSet[str]) -> Type: - typ = get_proper_type(typ) - if isinstance(typ, UnionType): - return UnionType([_asdictify_inner(item, seen_dataclasses) for item in typ.items]) - if isinstance(typ, Instance): - info = typ.type - if is_type_dataclass(info): - if info.fullname in seen_dataclasses: - # Recursive types not supported, so fall back to Dict[str, Any] - # Note: Would be nicer to fallback to default_return_type, but that is Any - # (due to overloads?) - return api.named_generic_type('builtins.dict', - [api.named_generic_type('builtins.str', []), - AnyType(TypeOfAny.implementation_artifact)]) - seen_dataclasses |= {info.fullname} - attrs = info.metadata['dataclass']['attributes'] - fields = OrderedDict() # type: OrderedDict[str, Type] - for data in attrs: - attr = DataclassAttribute.deserialize(info, data, api) - sym_node = info.names[attr.name] - attr_type = sym_node.type - assert attr_type is not None - fields[attr.name] = _asdictify_inner(attr_type, seen_dataclasses) - return make_anonymous_typeddict(api, fields=fields, - required_keys=set()) - elif info.has_base('builtins.list'): - supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( - 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) - return api.named_generic_type('builtins.list', [ - _asdictify_inner(supertype_instance.args[0], seen_dataclasses) - ]) - elif info.has_base('builtins.dict'): - supertype_instance = map_instance_to_supertype(typ, api.named_generic_type( - 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), - AnyType(TypeOfAny.implementation_artifact)]).type) - return api.named_generic_type('builtins.dict', [ - _asdictify_inner(supertype_instance.args[0], seen_dataclasses), - _asdictify_inner(supertype_instance.args[1], seen_dataclasses) - ]) - elif isinstance(typ, TupleType): - if typ.partial_fallback.type.is_named_tuple: - # For namedtuples, return Any. To properly support transforming namedtuples, - # we would have to generate a partial_fallback type for the TupleType and add it - # to the symbol table. It's not currently possibl to do this via the - # CheckerPluginInterface. Ideally it would use the same code as - # NamedTupleAnalyzer.build_namedtuple_typeinfo. - return AnyType(TypeOfAny.implementation_artifact) - return TupleType([_asdictify_inner(item, seen_dataclasses) for item in typ.items], - api.named_generic_type('builtins.tuple', []), implicit=typ.implicit) - return typ - - return _asdictify_inner(typ, seen_dataclasses=frozenset()) + return typ.accept(AsDictVisitor(api)) From e9a56ba41949caecf430dcc9704a18adbbe13a2a Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 15:52:40 +0200 Subject: [PATCH 15/40] Made TypedDicts returned by asdict total again. --- mypy/plugins/dataclasses.py | 2 +- test-data/unit/check-dataclasses.test | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 6c52dc92b9d0..38bd1bff1b7f 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -431,7 +431,7 @@ def visit_instance(self, t: Instance) -> Type: fields[attr.name] = attr_type.accept(self) self.seen_dataclasses.remove(info.fullname) return make_anonymous_typeddict(self.api, fields=fields, - required_keys=set()) + required_keys=set(fields.keys())) elif info.has_base('builtins.list'): supertype_instance = map_instance_to_supertype(t, self.api.named_generic_type( 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index e45bfb6542bf..2b45a63d48d4 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1004,7 +1004,7 @@ class Person: class NonDataclass: pass -reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name'?: builtins.str, 'age'?: builtins.int})' +reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' asdict(Person) # E: 'dataclasses.asdict' should be called on dataclass instances asdict(NonDataclass()) # E: 'dataclasses.asdict' should be called on dataclass instances @@ -1063,7 +1063,7 @@ class A: # Recursion is not supported, so fall back result = asdict(A(B(C(A())))) -reveal_type(result) # N: Revealed type is 'TypedDict({'b'?: Union[TypedDict({'c'?: TypedDict({'a'?: builtins.dict[builtins.str, Any]})}), None]})' +reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})' [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1081,7 +1081,7 @@ class Card: class Customer: card: Union[str, Card] -reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card'?: Union[builtins.str, TypedDict({'last4'?: builtins.int})]})' +reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' [typing fixtures/typing-full.pyi] @@ -1108,7 +1108,7 @@ instance = Course( list_no_generic=[], ) result = asdict(instance) -reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' reveal_type(result['any_list']) # N: Revealed type is 'builtins.list[Any]' reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any]' @@ -1157,9 +1157,9 @@ result = asdict(instance) # Supertypes (list) are returned, since there could be a constraint on the TypeVar # used on the subclass such that when the type argument to the subclass is substituted with a TypedDict, # it may not type-check. -reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' -reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' -reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] @@ -1181,7 +1181,7 @@ class Course: instance = Course(participants_by_name={"Joe": Person("Joe", 32)}) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name': builtins.str, 'age': builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1208,7 +1208,7 @@ class Course: instance = Course(participants_by_name=MyDict[int, str, Person]([("Joe", Person("Joe", 32))])) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name': builtins.str, 'age': builtins.int})]' [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1230,7 +1230,7 @@ class Course: instance = Course(partners=(Person("Joe", 32), Person("John", 23))) result = asdict(instance) -reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name'?: builtins.str, 'age'?: builtins.int}), TypedDict({'name'?: builtins.str, 'age'?: builtins.int})]' +reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]' [typing fixtures/typing-full.pyi] @@ -1261,7 +1261,7 @@ result = asdict(instance) # For now, subclasses of Tuple are transformed to the Tuple base class # This is because the subclass, if it itself contains dataclass fields, may be transformed in such a way that it # is no longer compatible with the original Tuple class it is extending. -reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name'?: builtins.str, 'age'?: builtins.int}), builtins.str]' +reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]' [typing fixtures/typing-full.pyi] From 2e5240ebc169da1d49a8cac9b8f7331461e2232f Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 15:53:53 +0200 Subject: [PATCH 16/40] Fixed test after total change. --- test-data/unit/fine-grained-dataclasses.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-data/unit/fine-grained-dataclasses.test b/test-data/unit/fine-grained-dataclasses.test index ea8a8920bbcb..c5498e5f829c 100644 --- a/test-data/unit/fine-grained-dataclasses.test +++ b/test-data/unit/fine-grained-dataclasses.test @@ -32,8 +32,8 @@ from b import MyList reveal_type(asdict(MyDataclass(MyList()))) [out] -c.py:3: note: Revealed type is 'TypedDict({'attr'?: builtins.str})' +c.py:3: note: Revealed type is 'TypedDict({'attr': builtins.str})' == -c.py:4: note: Revealed type is 'TypedDict({'attr'?: builtins.list[builtins.int]})' +c.py:4: note: Revealed type is 'TypedDict({'attr': builtins.list[builtins.int]})' [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] From 52a1c2737903474fad80342fab99f480d0fb4e3c Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 20:19:08 +0200 Subject: [PATCH 17/40] Make code a bit more readable, and a bit more robust. - Iterate through type args to list/dict rather than assuming there are 1 or 2 type args, respectively. - Also remove unnecessary overriding of visit_union_type. --- mypy/plugins/dataclasses.py | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 38bd1bff1b7f..e8423b54d42c 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -20,8 +20,8 @@ from mypy.server.trigger import make_wildcard_trigger from mypy.type_visitor import TypeTranslator from mypy.types import ( - Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, UnionType, - AnyType, TypeOfAny, TypeAliasType + Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, AnyType, + TypeOfAny, TypeAliasType ) # The set of decorators that generate dataclasses. @@ -382,7 +382,7 @@ def _collect_field_args(expr: Expression) -> Tuple[bool, Dict[str, Expression]]: def asdict_callback(ctx: FunctionContext, return_typeddicts: bool = False) -> Type: - """Check calls to asdict pass in a dataclass. Optionally, return TypedDicts.""" + """Check that calls to asdict pass in a dataclass. Optionally, return TypedDicts.""" positional_arg_types = ctx.arg_types[0] if positional_arg_types: @@ -412,14 +412,14 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: def visit_instance(self, t: Instance) -> Type: info = t.type + any_type = AnyType(TypeOfAny.implementation_artifact) if is_type_dataclass(info): if info.fullname in self.seen_dataclasses: # Recursive types not supported, so fall back to Dict[str, Any] # Note: Would be nicer to fallback to default_return_type, but that is Any # (due to overloads?) return self.api.named_generic_type( - 'builtins.dict', [self.api.named_generic_type('builtins.str', []), - AnyType(TypeOfAny.implementation_artifact)]) + 'builtins.dict', [self.api.named_generic_type('builtins.str', []), any_type]) attrs = info.metadata['dataclass']['attributes'] fields = OrderedDict() # type: OrderedDict[str, Type] self.seen_dataclasses.add(info.fullname) @@ -433,24 +433,17 @@ def visit_instance(self, t: Instance) -> Type: return make_anonymous_typeddict(self.api, fields=fields, required_keys=set(fields.keys())) elif info.has_base('builtins.list'): - supertype_instance = map_instance_to_supertype(t, self.api.named_generic_type( - 'builtins.list', [AnyType(TypeOfAny.implementation_artifact)]).type) - return self.api.named_generic_type('builtins.list', [ - supertype_instance.args[0].accept(self) - ]) + supertype = map_instance_to_supertype(t, self.api.named_generic_type( + 'builtins.list', [any_type]).type) + return self.api.named_generic_type('builtins.list', + self.translate_types(supertype.args)) elif info.has_base('builtins.dict'): - supertype_instance = map_instance_to_supertype(t, self.api.named_generic_type( - 'builtins.dict', [AnyType(TypeOfAny.implementation_artifact), - AnyType(TypeOfAny.implementation_artifact)]).type) - return self.api.named_generic_type('builtins.dict', [ - supertype_instance.args[0].accept(self), - supertype_instance.args[1].accept(self) - ]) + supertype = map_instance_to_supertype(t, self.api.named_generic_type( + 'builtins.dict', [any_type, any_type]).type) + return self.api.named_generic_type('builtins.dict', + self.translate_types(supertype.args)) return t - def visit_union_type(self, t: UnionType) -> Type: - return UnionType([item.accept(self) for item in t.items]) - def visit_tuple_type(self, t: TupleType) -> Type: if t.partial_fallback.type.is_named_tuple: # For namedtuples, return Any. To properly support transforming namedtuples, @@ -460,7 +453,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: # NamedTupleAnalyzer.build_namedtuple_typeinfo. return AnyType(TypeOfAny.implementation_artifact) # Note: Tuple subclasses not supported, hence overriding the fallback - return t.copy_modified(items=[item.accept(self) for item in t.items], + return t.copy_modified(items=self.translate_types(t.items), fallback=self.api.named_generic_type('builtins.tuple', [])) From 43f174c94ed0e22d88420dd5dc76b91e2c54b95d Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 20:22:50 +0200 Subject: [PATCH 18/40] Fix typo --- mypy/plugins/dataclasses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index e8423b54d42c..9763b871c04d 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -448,7 +448,7 @@ def visit_tuple_type(self, t: TupleType) -> Type: if t.partial_fallback.type.is_named_tuple: # For namedtuples, return Any. To properly support transforming namedtuples, # we would have to generate a partial_fallback type for the TupleType and add it - # to the symbol table. It's not currently possibl to do this via the + # to the symbol table. It's not currently possible to do this via the # CheckerPluginInterface. Ideally it would use the same code as # NamedTupleAnalyzer.build_namedtuple_typeinfo. return AnyType(TypeOfAny.implementation_artifact) From 227ba90751dfc7db46a2d4853da3fffb327a4405 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 2 Apr 2020 20:50:56 +0200 Subject: [PATCH 19/40] After refactoring to use TypeTranslator, ensure Callable and Type[..] are not transformed. --- mypy/plugins/dataclasses.py | 12 ++++++-- test-data/unit/check-dataclasses.test | 40 +++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 9763b871c04d..2df04f885a26 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -21,8 +21,8 @@ from mypy.type_visitor import TypeTranslator from mypy.types import ( Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, AnyType, - TypeOfAny, TypeAliasType -) + TypeOfAny, TypeAliasType, + CallableType, TypeType) # The set of decorators that generate dataclasses. dataclass_makers = { @@ -456,6 +456,14 @@ def visit_tuple_type(self, t: TupleType) -> Type: return t.copy_modified(items=self.translate_types(t.items), fallback=self.api.named_generic_type('builtins.tuple', [])) + def visit_callable_type(self, t: CallableType) -> Type: + # Leave e.g. Callable[[SomeDataclass], SomeDataclass] alone + return t + + def visit_type_type(self, t: TypeType) -> Type: + # Leave e.g. Type[SomeDataclass] alone + return t + def _asdictify(api: CheckerPluginInterface, typ: Type) -> Type: """Convert dataclasses into TypedDicts, recursively looking into built-in containers. diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 2b45a63d48d4..89d43d2616e0 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1085,6 +1085,46 @@ reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': [typing fixtures/typing-full.pyi] +[case testDataclassesAsdictType] +# flags: --python-version 3.8 +from dataclasses import dataclass, asdict +from typing import Type + +@dataclass +class Card: + last4: int + +@dataclass +class Customer: + card: Type[Card] + +# Type[...] hould be left alone +reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': Type[__main__.Card]})' + +[typing fixtures/typing-full.pyi] + + +[case testDataclassesAsdictCallable] +# flags: --python-version 3.8 +from dataclasses import dataclass, asdict +from typing import Callable + +@dataclass +class Card: + last4: int + +@dataclass +class Customer: + card: Callable[[Card], Card] + +def func(_: Card) -> Card: pass + +# Type[...] hould be left alone +reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': def (__main__.Card) -> __main__.Card})' + +[typing fixtures/typing-full.pyi] + + [case testDataclassesAsdictList] # flags: --python-version 3.8 from dataclasses import dataclass, asdict From d12c665197a8e880ab25ed5619b37294552e3a52 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 8 Apr 2020 23:21:54 +0200 Subject: [PATCH 20/40] Address second review comments. - Remove limitation that Python must be 3.8+. - Move fine-grained test of asdict to main fine-grained test file. - Correct fine-grained test of asdict so it actually tests it properly. This doesn't pass currently... - Try to lookup anonymous TypedDict fallback using named_type_or_none, - Add additional dependency only on typing_extensions. - copied from SemanticAnalyzer to TypeChecker. --- docs/source/additional_features.rst | 6 +-- mypy/checker.py | 19 +++++++++- mypy/plugin.py | 4 ++ mypy/plugins/common.py | 10 +---- mypy/plugins/dataclasses.py | 6 +-- mypy/plugins/default.py | 12 +++--- mypy/semanal_typeddict.py | 18 ++++++--- mypy/test/testfinegrained.py | 1 - test-data/unit/check-dataclasses.test | 32 +++++++++------- test-data/unit/fine-grained-dataclasses.test | 39 -------------------- test-data/unit/fine-grained.test | 29 +++++++++++++++ 11 files changed, 92 insertions(+), 84 deletions(-) delete mode 100644 test-data/unit/fine-grained-dataclasses.test diff --git a/docs/source/additional_features.rst b/docs/source/additional_features.rst index e90d06604e34..fae1a09245f3 100644 --- a/docs/source/additional_features.rst +++ b/docs/source/additional_features.rst @@ -75,9 +75,9 @@ Caveats/Known Issues Some functions in the :py:mod:`dataclasses` module, such as :py:func:`~dataclasses.replace`, have imprecise (too permissive) types. This will be fixed in future releases. -In Python version 3.8 and above, calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the -original dataclass definition, transforming it recursively. There are, however, some limitations. In particular, a -precise return type cannot be inferred for recursive dataclasses, and for calls where ``dict_factory`` is set. +Calls to :py:func:`~dataclasses.asdict` will return a ``TypedDict`` based on the original dataclass +definition, transforming it recursively. There are, however, some limitations. In particular, a precise return type +cannot be inferred for recursive dataclasses, and for calls where ``dict_factory`` is set. Mypy does not yet recognize aliases of :py:func:`dataclasses.dataclass `, and will probably never recognize dynamically computed decorators. The following examples diff --git a/mypy/checker.py b/mypy/checker.py index c012251dad9f..375f3c96b150 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -24,8 +24,8 @@ Import, ImportFrom, ImportAll, ImportBase, TypeAlias, ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, CONTRAVARIANT, COVARIANT, INVARIANT, TypeVarExpr, AssignmentExpr, - is_final_node, - ARG_NAMED) + is_final_node, ARG_NAMED, PlaceholderNode +) from mypy import nodes from mypy.literals import literal, literal_hash, Key from mypy.typeanal import has_any_from_unimported_type, check_for_explicit_any @@ -4593,6 +4593,21 @@ def named_type(self, name: str) -> Instance: any_type = AnyType(TypeOfAny.from_omitted_generics) return Instance(node, [any_type] * len(node.defn.type_vars)) + def named_type_or_none(self, qualified_name: str, + args: Optional[List[Type]] = None) -> Optional[Instance]: + sym = self.lookup_qualified(qualified_name) + if not sym or isinstance(sym.node, PlaceholderNode): + return None + node = sym.node + if isinstance(node, TypeAlias): + assert isinstance(node.target, Instance) # type: ignore + node = node.target.type + assert isinstance(node, TypeInfo), node + if args is not None: + # TODO: assert len(args) == len(node.defn.type_vars) + return Instance(node, args) + return Instance(node, [AnyType(TypeOfAny.unannotated)] * len(node.defn.type_vars)) + def named_generic_type(self, name: str, args: List[Type]) -> Instance: """Return an instance with the given name and type arguments. diff --git a/mypy/plugin.py b/mypy/plugin.py index d6ccc3def1ee..2550189d0a13 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -231,6 +231,10 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: """Construct an instance of a builtin type with given type arguments.""" raise NotImplementedError + def named_type_or_none(self, qualified_name: str, + args: Optional[List[Type]] = None) -> Optional[Instance]: + raise NotImplementedError + @trait class SemanticAnalyzerPluginInterface: diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 91b910dfe441..143037b95ecb 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -7,6 +7,7 @@ ) from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface, CheckerPluginInterface from mypy.semanal import set_callable_name +from mypy.semanal_typeddict import get_anonymous_typeddict_type from mypy.types import ( CallableType, Overloaded, Type, TypeVarDef, deserialize_type, get_proper_type, TypedDictType, Instance, TPDICT_FB_NAMES @@ -165,15 +166,6 @@ def deserialize_and_fixup_type( return typ -def get_anonymous_typeddict_type(api: CheckerPluginInterface) -> Instance: - for type_fullname in TPDICT_FB_NAMES: - try: - return api.named_generic_type(type_fullname, []) - except KeyError: - continue - raise RuntimeError("No TypedDict fallback type found") - - def make_anonymous_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, Type]', required_keys: Set[str]) -> TypedDictType: return TypedDictType(fields, required_keys=required_keys, diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 2df04f885a26..fd8b34eed66a 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -381,15 +381,15 @@ def _collect_field_args(expr: Expression) -> Tuple[bool, Dict[str, Expression]]: return False, {} -def asdict_callback(ctx: FunctionContext, return_typeddicts: bool = False) -> Type: - """Check that calls to asdict pass in a dataclass. Optionally, return TypedDicts.""" +def asdict_callback(ctx: FunctionContext) -> Type: + """Check that calls to asdict pass in a dataclass. If possible, return TypedDicts.""" positional_arg_types = ctx.arg_types[0] if positional_arg_types: dataclass_instance = get_proper_type(positional_arg_types[0]) if isinstance(dataclass_instance, Instance): if is_type_dataclass(dataclass_instance.type): - if len(ctx.arg_types) == 1 and return_typeddicts: + if len(ctx.arg_types) == 1: return _asdictify(ctx.api, dataclass_instance) else: # We can't infer a more precise type for calls where dict_factory is set. diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index bb1e8b9afff8..b4fbba16048f 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -2,7 +2,9 @@ from typing import Callable, Optional, List, Tuple from mypy import message_registry -from mypy.nodes import Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile +from mypy.nodes import ( + Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile, ImportFrom, Import, ImportAll +) from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, CheckerPluginInterface, @@ -21,10 +23,7 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: - if self.python_version >= (3, 8): - # Add module needed for anonymous TypedDict (used to support dataclasses.asdict) - return [(10, "typing", -1)] - return [] + return [(10, "typing_extensions", -1)] def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: @@ -38,8 +37,7 @@ def get_function_hook(self, fullname: str elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback elif fullname == 'dataclasses.asdict': - return partial(dataclasses.asdict_callback, - return_typeddicts=self.python_version >= (3, 8)) + return dataclasses.asdict_callback return None def get_method_signature_hook(self, fullname: str diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 99a1e1395379..95dbff473e0f 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -1,10 +1,12 @@ """Semantic analysis of TypedDict definitions.""" from mypy.ordered_dict import OrderedDict -from typing import Optional, List, Set, Tuple +from collections import OrderedDict +from typing import Optional, List, Set, Tuple, Union from typing_extensions import Final -from mypy.types import Type, AnyType, TypeOfAny, TypedDictType, TPDICT_NAMES +from mypy.plugin import CheckerPluginInterface +from mypy.types import Type, AnyType, TypeOfAny, TypedDictType, TPDICT_NAMES, Instance from mypy.nodes import ( CallExpr, TypedDictExpr, Expression, NameExpr, Context, StrExpr, BytesExpr, UnicodeExpr, ClassDef, RefExpr, TypeInfo, AssignmentStmt, PassStmt, ExpressionStmt, EllipsisExpr, TempNode, @@ -304,10 +306,7 @@ def fail_typeddict_arg(self, message: str, def build_typeddict_typeinfo(self, name: str, items: List[str], types: List[Type], required_keys: Set[str]) -> TypeInfo: - # Prefer typing then typing_extensions if available. - fallback = (self.api.named_type_or_none('typing._TypedDict', []) or - self.api.named_type_or_none('typing_extensions._TypedDict', []) or - self.api.named_type_or_none('mypy_extensions._TypedDict', [])) + fallback = get_anonymous_typeddict_type(self.api) assert fallback is not None info = self.api.basic_new_typeinfo(name, fallback) info.typeddict_type = TypedDictType(OrderedDict(zip(items, types)), required_keys, @@ -322,3 +321,10 @@ def is_typeddict(self, expr: Expression) -> bool: def fail(self, msg: str, ctx: Context) -> None: self.api.fail(msg, ctx) + +def get_anonymous_typeddict_type( + api: Union[SemanticAnalyzerInterface, CheckerPluginInterface]) -> Instance: + # Prefer typing then typing_extensions if available. + return (api.named_type_or_none('typing._TypedDict', []) or + api.named_type_or_none('typing_extensions._TypedDict', []) or + api.named_type_or_none('mypy_extensions._TypedDict', [])) \ No newline at end of file diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index b48e354e09f3..d4ed18cab095 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -49,7 +49,6 @@ class FineGrainedSuite(DataSuite): 'fine-grained-modules.test', 'fine-grained-follow-imports.test', 'fine-grained-suggest.test', - 'fine-grained-dataclasses.test', ] # Whether to use the fine-grained cache in the testing. This is overridden diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 89d43d2616e0..8b867e3ac243 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -992,7 +992,6 @@ reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [builtins fixtures/property.pyi] [case testDataclassesAsdict] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Tuple, Dict, Any @@ -1028,9 +1027,11 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'Any' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' [typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] + [case testDataclassesAsdictPython37] # flags: --python-version 3.7 @@ -1040,13 +1041,25 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'Any' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' [typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] -[case testDataclassesAsdictRecursion] +[case testDataclassesAsdictPython38] # flags: --python-version 3.8 from dataclasses import dataclass, asdict + +@dataclass +class Foo: + bar: str + +reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' + +[typing fixtures/typing-full.pyi] + +[case testDataclassesAsdictRecursion] +from dataclasses import dataclass, asdict from typing import Optional @dataclass @@ -1066,10 +1079,10 @@ result = asdict(A(B(C(A())))) reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})' [typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] [builtins fixtures/dict.pyi] [case testDataclassesAsdictUnions] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Union @@ -1086,7 +1099,6 @@ reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': [typing fixtures/typing-full.pyi] [case testDataclassesAsdictType] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Type @@ -1105,7 +1117,6 @@ reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': T [case testDataclassesAsdictCallable] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Callable @@ -1126,7 +1137,6 @@ reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': d [case testDataclassesAsdictList] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Any @@ -1157,7 +1167,6 @@ reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any] [case testDataclassesAsdictListSubclass] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import List, Any, TypeVar, Generic @@ -1206,7 +1215,6 @@ reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'buil [case testDataclassesAsdictDict] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Dict @@ -1227,7 +1235,6 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [builtins fixtures/dict.pyi] [case testDataclassesAsdictDictSubclass] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Dict, Generic, TypeVar @@ -1255,7 +1262,6 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [case testDataclassesAsdictTuple] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Tuple @@ -1279,7 +1285,6 @@ reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': [case testDataclassesAsdictTupleSubclass] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import Tuple, Generic, TypeVar @@ -1308,7 +1313,6 @@ reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'n [builtins fixtures/tuple.pyi] [case testDataclassesAsdictNamedTuple] -# flags: --python-version 3.8 from dataclasses import dataclass, asdict from typing import NamedTuple diff --git a/test-data/unit/fine-grained-dataclasses.test b/test-data/unit/fine-grained-dataclasses.test deleted file mode 100644 index c5498e5f829c..000000000000 --- a/test-data/unit/fine-grained-dataclasses.test +++ /dev/null @@ -1,39 +0,0 @@ --- Test cases for fine-grained incremental checking of dataclasses --- --- The comment at the top of fine-grained.test explains how these tests --- work. - -[case testDataclassesAsdictFineGrained] -# flags: --python-version 3.8 -[file a.py] -from dataclasses import dataclass -from b import AttributeInOtherModule - -@dataclass -class MyDataclass: - attr: AttributeInOtherModule - -[file b.py] -AttributeInOtherModule = str -[file c.py] -from dataclasses import asdict -from a import MyDataclass -reveal_type(asdict(MyDataclass('John'))) - -[file b.py.2] -from typing import List -class MyList(List[int]): - pass -AttributeInOtherModule = MyList -[file c.py.2] -from dataclasses import asdict -from a import MyDataclass -from b import MyList -reveal_type(asdict(MyDataclass(MyList()))) - -[out] -c.py:3: note: Revealed type is 'TypedDict({'attr': builtins.str})' -== -c.py:4: note: Revealed type is 'TypedDict({'attr': builtins.list[builtins.int]})' -[typing fixtures/typing-full.pyi] -[builtins fixtures/list.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 5761f6cb337c..e605f02bf9dc 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -910,6 +910,35 @@ class A: main:3: error: Too few arguments for "C" == +[case testDataclassesAsdictFineGrained] +# flags: --python-version 3.8 +[file a.py] +from dataclasses import dataclass +from b import AttributeInOtherModule + +@dataclass +class MyDataclass: + attr: AttributeInOtherModule + +my_var: MyDataclass + +[file b.py] +AttributeInOtherModule = str +[file c.py] +from dataclasses import asdict +from a import my_var +asdict(my_var)['attr'] + "foo" + +[file b.py.2] +AttributeInOtherModule = int + +[out] +== +c.py:3 error: Unsupported left operand type for + ("int") + +[typing fixtures/typing-typeddict.pyi] +[builtins fixtures/fine_grained.pyi] + [case testAttrsUpdate1] [file a.py] import attr From 45e72d7720c140bc23b4f8627431767a3fd958d8 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 8 Apr 2020 23:39:24 +0200 Subject: [PATCH 21/40] Fix return type --- mypy/plugins/common.py | 4 +++- mypy/semanal_typeddict.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 143037b95ecb..1279d774c9af 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -168,5 +168,7 @@ def deserialize_and_fixup_type( def make_anonymous_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, Type]', required_keys: Set[str]) -> TypedDictType: + fallback = get_anonymous_typeddict_type(api) + assert fallback is not None return TypedDictType(fields, required_keys=required_keys, - fallback=get_anonymous_typeddict_type(api)) + fallback=fallback) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 95dbff473e0f..aaebd0e26fb8 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -323,7 +323,7 @@ def fail(self, msg: str, ctx: Context) -> None: self.api.fail(msg, ctx) def get_anonymous_typeddict_type( - api: Union[SemanticAnalyzerInterface, CheckerPluginInterface]) -> Instance: + api: Union[SemanticAnalyzerInterface, CheckerPluginInterface]) -> Optional[Instance]: # Prefer typing then typing_extensions if available. return (api.named_type_or_none('typing._TypedDict', []) or api.named_type_or_none('typing_extensions._TypedDict', []) or From a03f0333b8da10f85bcd6cd26df02a63ff016516 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 3 Jun 2020 23:02:37 +0200 Subject: [PATCH 22/40] Try to address more review comments and fix flake8 --- mypy/checker.py | 28 +++++++++++++++++++++++++--- mypy/plugins/common.py | 2 +- mypy/plugins/default.py | 2 +- mypy/semanal_typeddict.py | 3 ++- test-data/unit/deps.test | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 6 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 375f3c96b150..103fee008968 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -24,7 +24,7 @@ Import, ImportFrom, ImportAll, ImportBase, TypeAlias, ARG_POS, ARG_STAR, LITERAL_TYPE, MDEF, GDEF, CONTRAVARIANT, COVARIANT, INVARIANT, TypeVarExpr, AssignmentExpr, - is_final_node, ARG_NAMED, PlaceholderNode + is_final_node, ARG_NAMED ) from mypy import nodes from mypy.literals import literal, literal_hash, Key @@ -4595,8 +4595,8 @@ def named_type(self, name: str) -> Instance: def named_type_or_none(self, qualified_name: str, args: Optional[List[Type]] = None) -> Optional[Instance]: - sym = self.lookup_qualified(qualified_name) - if not sym or isinstance(sym.node, PlaceholderNode): + sym = self.lookup_fully_qualified_or_none(qualified_name) + if not sym: return None node = sym.node if isinstance(node, TypeAlias): @@ -4691,6 +4691,28 @@ def lookup_qualified(self, name: str) -> SymbolTableNode: msg = "Failed qualified lookup: '{}' (fullname = '{}')." raise KeyError(msg.format(last, name)) + def lookup_fully_qualified_or_none(self, fullname: str) -> Optional[SymbolTableNode]: + """Lookup a fully qualified name that refers to a module-level definition. + + Don't assume that the name is defined. This happens in the global namespace -- + the local module namespace is ignored. This does not dereference indirect + refs. + + Note that this can't be used for names nested in class namespaces. + """ + # TODO: unify/clean-up/simplify lookup methods, see #4157. + # TODO: support nested classes (but consider performance impact, + # we might keep the module level only lookup for thing like 'builtins.int'). + assert '.' in fullname + module, name = fullname.rsplit('.', maxsplit=1) + if module not in self.modules: + return None + filenode = self.modules[module] + result = filenode.names.get(name) + if result is None: + raise KeyError("Failed fully qualified lookup: (fullname = '{}').".format(fullname)) + return result + @contextmanager def enter_partial_types(self, *, is_function: bool = False, is_class: bool = False) -> Iterator[None]: diff --git a/mypy/plugins/common.py b/mypy/plugins/common.py index 1279d774c9af..3dc24e3a90a4 100644 --- a/mypy/plugins/common.py +++ b/mypy/plugins/common.py @@ -10,7 +10,7 @@ from mypy.semanal_typeddict import get_anonymous_typeddict_type from mypy.types import ( CallableType, Overloaded, Type, TypeVarDef, deserialize_type, get_proper_type, - TypedDictType, Instance, TPDICT_FB_NAMES + TypedDictType ) from mypy.typevars import fill_typevars from mypy.util import get_unique_redefinition_name diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index b4fbba16048f..a7c75ee4e3eb 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -3,7 +3,7 @@ from mypy import message_registry from mypy.nodes import ( - Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile, ImportFrom, Import, ImportAll + Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile ) from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index aaebd0e26fb8..8c0cd8dece09 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -322,9 +322,10 @@ def is_typeddict(self, expr: Expression) -> bool: def fail(self, msg: str, ctx: Context) -> None: self.api.fail(msg, ctx) + def get_anonymous_typeddict_type( api: Union[SemanticAnalyzerInterface, CheckerPluginInterface]) -> Optional[Instance]: # Prefer typing then typing_extensions if available. return (api.named_type_or_none('typing._TypedDict', []) or api.named_type_or_none('typing_extensions._TypedDict', []) or - api.named_type_or_none('mypy_extensions._TypedDict', [])) \ No newline at end of file + api.named_type_or_none('mypy_extensions._TypedDict', [])) diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 8c074abc83a2..950cbfc4392a 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1446,3 +1446,36 @@ class B(A): -> m -> m -> m + + +[case testDataclassDeps] +# flags: --python-version 3.7 +from dataclasses import dataclass + +Z = int + +@dataclass +class A: + x: Z + +@dataclass +class B(A): + y: int +[builtins fixtures/list.pyi] + +[out] + -> , m + -> + -> + -> , m.B.__init__ + -> + -> + -> + -> + -> m, m.A, m.B + -> m + -> m + -> m.B + -> m + -> m + -> m From b4d7e154b45fbc622ee87c5cc7986be5b75b3965 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 3 Jun 2020 23:34:15 +0200 Subject: [PATCH 23/40] Add fine grained deps test to help debug asdict dependencies. --- test-data/unit/deps.test | 37 +++++++++++--------------------- test-data/unit/fine-grained.test | 5 ++--- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 950cbfc4392a..2ef978495972 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1448,34 +1448,21 @@ class B(A): -> m -[case testDataclassDeps] -# flags: --python-version 3.7 +[case testDataclassAsdictDeps] +# flags: --python-version 3.8 from dataclasses import dataclass - -Z = int +from b import AttributeInOtherModule @dataclass -class A: - x: Z +class MyDataclass: + attr: AttributeInOtherModule -@dataclass -class B(A): - y: int -[builtins fixtures/list.pyi] +my_var: MyDataclass + +[file b.py] +AttributeInOtherModule = str + +[typing fixtures/typing-typeddict.pyi] +[builtins fixtures/fine_grained.pyi] [out] - -> , m - -> - -> - -> , m.B.__init__ - -> - -> - -> - -> - -> m, m.A, m.B - -> m - -> m - -> m.B - -> m - -> m - -> m diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index e605f02bf9dc..0fdb5c34a526 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -932,13 +932,12 @@ asdict(my_var)['attr'] + "foo" [file b.py.2] AttributeInOtherModule = int +[typing fixtures/typing-typeddict.pyi] +[builtins fixtures/fine_grained.pyi] [out] == c.py:3 error: Unsupported left operand type for + ("int") -[typing fixtures/typing-typeddict.pyi] -[builtins fixtures/fine_grained.pyi] - [case testAttrsUpdate1] [file a.py] import attr From d96d977522de08ba048bf14c516cd4f6739ecccd Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 3 Jun 2020 23:38:43 +0200 Subject: [PATCH 24/40] Fix some asdict tests missing tuple dependency --- test-data/unit/check-dataclasses.test | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 8b867e3ac243..196be7e97da6 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1057,6 +1057,7 @@ class Foo: reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' [typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] [case testDataclassesAsdictRecursion] from dataclasses import dataclass, asdict @@ -1097,6 +1098,7 @@ class Customer: reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' [typing fixtures/typing-full.pyi] +[builtins fixtures/tuple.pyi] [case testDataclassesAsdictType] from dataclasses import dataclass, asdict @@ -1114,7 +1116,7 @@ class Customer: reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': Type[__main__.Card]})' [typing fixtures/typing-full.pyi] - +[builtins fixtures/tuple.pyi] [case testDataclassesAsdictCallable] from dataclasses import dataclass, asdict @@ -1134,7 +1136,7 @@ def func(_: Card) -> Card: pass reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': def (__main__.Card) -> __main__.Card})' [typing fixtures/typing-full.pyi] - +[builtins fixtures/tuple.pyi] [case testDataclassesAsdictList] from dataclasses import dataclass, asdict From 441b665b29242f7770e00cf0f7292dc4936a4861 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 4 Jun 2020 19:28:45 +0200 Subject: [PATCH 25/40] Revert "Fix some asdict tests missing tuple dependency" This reverts commit 3252ab69756af0dd981cbdcede48219fbee9e61d. --- test-data/unit/check-dataclasses.test | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 196be7e97da6..8b867e3ac243 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1057,7 +1057,6 @@ class Foo: reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] [case testDataclassesAsdictRecursion] from dataclasses import dataclass, asdict @@ -1098,7 +1097,6 @@ class Customer: reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] [case testDataclassesAsdictType] from dataclasses import dataclass, asdict @@ -1116,7 +1114,7 @@ class Customer: reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': Type[__main__.Card]})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] + [case testDataclassesAsdictCallable] from dataclasses import dataclass, asdict @@ -1136,7 +1134,7 @@ def func(_: Card) -> Card: pass reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': def (__main__.Card) -> __main__.Card})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] + [case testDataclassesAsdictList] from dataclasses import dataclass, asdict From 344ca6af176a962bf1167731e1860fb216837c3f Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 4 Jun 2020 19:29:33 +0200 Subject: [PATCH 26/40] Don't need dep on typing_extensions --- mypy/plugins/default.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index a7c75ee4e3eb..64c0e2ecddad 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -22,9 +22,6 @@ class DefaultPlugin(Plugin): """Type checker plugin that is enabled by default.""" - def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]: - return [(10, "typing_extensions", -1)] - def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: from mypy.plugins import ctypes From c8858fa4884fa7bfc8e738c2630402cb558e7259 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Tue, 9 Jun 2020 20:13:47 +0200 Subject: [PATCH 27/40] Checker lookup_fully_qualified_or_none: Don't raise KeyError, return None instead --- mypy/checker.py | 2 -- test-data/unit/deps.test | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mypy/checker.py b/mypy/checker.py index 103fee008968..fce649f52bf8 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4709,8 +4709,6 @@ def lookup_fully_qualified_or_none(self, fullname: str) -> Optional[SymbolTableN return None filenode = self.modules[module] result = filenode.names.get(name) - if result is None: - raise KeyError("Failed fully qualified lookup: (fullname = '{}').".format(fullname)) return result @contextmanager diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 2ef978495972..9175039c32bf 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1448,7 +1448,7 @@ class B(A): -> m -[case testDataclassAsdictDeps] +[case testDataclassesAsdictDeps] # flags: --python-version 3.8 from dataclasses import dataclass from b import AttributeInOtherModule From 20d7716e9d25210fafa5abb5753266414c20e329 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Tue, 9 Jun 2020 21:23:54 +0200 Subject: [PATCH 28/40] Add dependencies for asdict on the referenced dataclasses and its attributes. It makes one of the fine-grained tests pass, but not the other? --- mypy/checker.py | 7 +++++++ mypy/plugin.py | 9 +++++++++ mypy/plugins/dataclasses.py | 3 +++ test-data/unit/deps.test | 12 ++++++++++++ test-data/unit/fine-grained.test | 2 +- 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mypy/checker.py b/mypy/checker.py index fce649f52bf8..611681b98623 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -4619,6 +4619,13 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: # TODO: assert len(args) == len(info.defn.type_vars) return Instance(info, args) + def add_plugin_dependency(self, trigger: str, target: Optional[str] = None) -> None: + if target is None: + target = self.tscope.current_target() + + cur_module_node = self.modules[self.tscope.current_module_id()] + cur_module_node.plugin_deps.setdefault(trigger, set()).add(target) + def lookup_typeinfo(self, fullname: str) -> TypeInfo: # Assume that the name refers to a class. sym = self.lookup_qualified(fullname) diff --git a/mypy/plugin.py b/mypy/plugin.py index 2550189d0a13..c4a3426c1208 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -231,10 +231,19 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: """Construct an instance of a builtin type with given type arguments.""" raise NotImplementedError + @abstractmethod def named_type_or_none(self, qualified_name: str, args: Optional[List[Type]] = None) -> Optional[Instance]: raise NotImplementedError + @abstractmethod + def add_plugin_dependency(self, trigger: str, target: Optional[str] = None) -> None: + """Specify semantic dependencies for generated methods/variables. + + See the same function on SemanticAnalyzerPluginInterface for more details. + """ + raise NotImplementedError + @trait class SemanticAnalyzerPluginInterface: diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index fd8b34eed66a..09fb55d34177 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -414,6 +414,9 @@ def visit_instance(self, t: Instance) -> Type: info = t.type any_type = AnyType(TypeOfAny.implementation_artifact) if is_type_dataclass(info): + # The resultant type from the asdict call depends on the set of attributes in the + # referenced dataclass and all dataclasses that are referenced by it + self.api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) if info.fullname in self.seen_dataclasses: # Recursive types not supported, so fall back to Dict[str, Any] # Note: Would be nicer to fallback to default_return_type, but that is Any diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 9175039c32bf..b99631fcccdd 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1450,6 +1450,12 @@ class B(A): [case testDataclassesAsdictDeps] # flags: --python-version 3.8 +from dataclasses import asdict +from a import my_var +x = asdict(my_var) +x['attr'] + "foo" + +[file a.py] from dataclasses import dataclass from b import AttributeInOtherModule @@ -1466,3 +1472,9 @@ AttributeInOtherModule = str [builtins fixtures/fine_grained.pyi] [out] + -> m + -> m + -> m + -> m + -> m + -> m \ No newline at end of file diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index 0fdb5c34a526..0c4180f37e0f 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -936,7 +936,7 @@ AttributeInOtherModule = int [builtins fixtures/fine_grained.pyi] [out] == -c.py:3 error: Unsupported left operand type for + ("int") +c.py:3: error: Unsupported operand types for + ("int" and "str") [case testAttrsUpdate1] [file a.py] From 5fec41b9b03d1741ae79f6718be2a114d7234a69 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Tue, 18 Aug 2020 23:09:11 +0200 Subject: [PATCH 29/40] Fix fine-grained no-cache test by adding correct dep on dataclass attrs. --- mypy/plugins/dataclasses.py | 6 ++---- test-data/unit/deps.test | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 09fb55d34177..20320718a010 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -17,7 +17,7 @@ add_method, _get_decorator_bool_argument, make_anonymous_typeddict, deserialize_and_fixup_type ) -from mypy.server.trigger import make_wildcard_trigger +from mypy.server.trigger import make_wildcard_trigger, make_trigger from mypy.type_visitor import TypeTranslator from mypy.types import ( Instance, NoneType, TypeVarDef, TypeVarType, get_proper_type, Type, TupleType, AnyType, @@ -414,9 +414,6 @@ def visit_instance(self, t: Instance) -> Type: info = t.type any_type = AnyType(TypeOfAny.implementation_artifact) if is_type_dataclass(info): - # The resultant type from the asdict call depends on the set of attributes in the - # referenced dataclass and all dataclasses that are referenced by it - self.api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) if info.fullname in self.seen_dataclasses: # Recursive types not supported, so fall back to Dict[str, Any] # Note: Would be nicer to fallback to default_return_type, but that is Any @@ -428,6 +425,7 @@ def visit_instance(self, t: Instance) -> Type: self.seen_dataclasses.add(info.fullname) for data in attrs: attr = DataclassAttribute.deserialize(info, data, self.api) + self.api.add_plugin_dependency(make_trigger(info.fullname + "." + attr.name)) sym_node = info.names[attr.name] attr_type = sym_node.type assert attr_type is not None diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index b99631fcccdd..6c1f6e51473f 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1473,7 +1473,7 @@ AttributeInOtherModule = str [out] -> m - -> m + -> m -> m -> m -> m From 5862c163684a2c60393f04b36d0e747e4f4e1a2c Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Tue, 18 Aug 2020 23:21:11 +0200 Subject: [PATCH 30/40] remove unused imports --- mypy/plugins/default.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 64c0e2ecddad..fd045ff47eb2 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -1,9 +1,9 @@ from functools import partial -from typing import Callable, Optional, List, Tuple +from typing import Callable, Optional, List from mypy import message_registry from mypy.nodes import ( - Expression, StrExpr, IntExpr, DictExpr, UnaryExpr, MypyFile + Expression, StrExpr, IntExpr, DictExpr, UnaryExpr ) from mypy.plugin import ( Plugin, FunctionContext, MethodContext, MethodSigContext, AttributeContext, ClassDefContext, From d7e03101a5b9bbf49cfcb5c0edd9b35cf5ab1fdb Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 17 Feb 2021 23:34:26 +0100 Subject: [PATCH 31/40] Remove error when passing a "non-dataclass" to asdict to reduce false-positives when type-checking code that calls asdict on "non-specific" or "generic" dataclasses. Small cleanup of control flow and comment for clarity. --- mypy/plugins/dataclasses.py | 8 +------- test-data/unit/check-dataclasses.test | 7 ++++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 1e49db86918c..0602a08a0ca7 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -403,15 +403,9 @@ def asdict_callback(ctx: FunctionContext) -> Type: if isinstance(dataclass_instance, Instance): if is_type_dataclass(dataclass_instance.type): if len(ctx.arg_types) == 1: + # Can only infer a more precise type for calls where dict_factory is not set. return _asdictify(ctx.api, dataclass_instance) - else: - # We can't infer a more precise type for calls where dict_factory is set. - # At least for now, typeshed stubs for asdict don't allow you to pass in - # `dict` as dict_factory, so we can't special-case that. - return ctx.default_return_type - ctx.api.fail("'dataclasses.asdict' should be called on dataclass instances", - ctx.context) return ctx.default_return_type diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index dfd19b20201b..dfb230a10f09 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1122,13 +1122,14 @@ class NonDataclass: pass reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' -asdict(Person) # E: 'dataclasses.asdict' should be called on dataclass instances -asdict(NonDataclass()) # E: 'dataclasses.asdict' should be called on dataclass instances +# It's OK to call on a non-dataclass, to reduce false-positives. +reveal_type(asdict(Person)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' +reveal_type(asdict(NonDataclass())) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: pass -asdict(NonDataclass(), dict_factory=my_dict_factory) # E: 'dataclasses.asdict' should be called on dataclass instances +reveal_type(asdict(NonDataclass(), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' # Passing in a dict_factory should not return a TypedDict reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' From 080c00c2477b5283a7fe8fc4bedebaad4ebd76df Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Wed, 17 Feb 2021 23:37:44 +0100 Subject: [PATCH 32/40] Fix flake8 --- mypy/semanal_typeddict.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index 8315576c591d..9addcae72e99 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -324,9 +324,10 @@ def is_typeddict(self, expr: Expression) -> bool: def fail(self, msg: str, ctx: Context, *, code: Optional[ErrorCode] = None) -> None: self.api.fail(msg, ctx, code=code) + def get_anonymous_typeddict_type( api: Union[SemanticAnalyzerInterface, CheckerPluginInterface]) -> Optional[Instance]: # Prefer typing then typing_extensions if available. return (api.named_type_or_none('typing._TypedDict', []) or api.named_type_or_none('typing_extensions._TypedDict', []) or - api.named_type_or_none('mypy_extensions._TypedDict', [])) \ No newline at end of file + api.named_type_or_none('mypy_extensions._TypedDict', [])) From 9e45f8fc0cb278ff93614d285dec1939cc702365 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 18 Feb 2021 00:23:55 +0100 Subject: [PATCH 33/40] Fix asdict tests (require using python version 3.7 minimum). --- test-data/unit/check-dataclasses.test | 46 +++++++++++---------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index dfb230a10f09..3ba8554f5e05 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1110,6 +1110,7 @@ reveal_type(B) # N: Revealed type is 'def (foo: builtins.int) -> __main__.B' [builtins fixtures/property.pyi] [case testDataclassesAsdict] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import List, Tuple, Dict, Any @@ -1123,8 +1124,8 @@ class NonDataclass: reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' # It's OK to call on a non-dataclass, to reduce false-positives. -reveal_type(asdict(Person)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' -reveal_type(asdict(NonDataclass())) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' +reveal_type(asdict(Person)) # N: Revealed type is 'builtins.dict[builtins.str, Any]' +reveal_type(asdict(NonDataclass())) # N: Revealed type is 'builtins.dict[builtins.str, Any]' def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: pass @@ -1137,9 +1138,8 @@ reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Reve [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] - -[case testDataclassesAsdictPython36] -# flags: --python-version 3.6 +[case testDataclassesAsdictPython38] +# flags: --python-version 3.8 from dataclasses import dataclass, asdict @dataclass @@ -1149,24 +1149,9 @@ class Foo: reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' [typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] - -[case testDataclassesAsdictPython37] -# flags: --python-version 3.7 -from dataclasses import dataclass, asdict - -@dataclass -class Foo: - bar: str - -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' - -[typing fixtures/typing-full.pyi] -[builtins fixtures/tuple.pyi] - -[case testDataclassesAsdictPython38] -# flags: --python-version 3.8 +[case testDataclassesAsdictPython39] +# flags: --python-version 3.9 from dataclasses import dataclass, asdict @dataclass @@ -1178,6 +1163,7 @@ reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builti [typing fixtures/typing-full.pyi] [case testDataclassesAsdictRecursion] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Optional @@ -1202,6 +1188,7 @@ reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c' [builtins fixtures/dict.pyi] [case testDataclassesAsdictUnions] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Union @@ -1218,6 +1205,7 @@ reveal_type(asdict(Customer("foo"))) # N: Revealed type is 'TypedDict({'card': [typing fixtures/typing-full.pyi] [case testDataclassesAsdictType] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Type @@ -1236,6 +1224,7 @@ reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': T [case testDataclassesAsdictCallable] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Callable @@ -1256,6 +1245,7 @@ reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': d [case testDataclassesAsdictList] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import List, Any @@ -1286,6 +1276,7 @@ reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any] [case testDataclassesAsdictListSubclass] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import List, Any, TypeVar, Generic @@ -1332,8 +1323,8 @@ reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'buil [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] - [case testDataclassesAsdictDict] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Dict @@ -1354,6 +1345,7 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [builtins fixtures/dict.pyi] [case testDataclassesAsdictDictSubclass] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Dict, Generic, TypeVar @@ -1379,8 +1371,8 @@ reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] - [case testDataclassesAsdictTuple] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Tuple @@ -1397,13 +1389,11 @@ instance = Course(partners=(Person("Joe", 32), Person("John", 23))) result = asdict(instance) reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]' - [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] - - [case testDataclassesAsdictTupleSubclass] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import Tuple, Generic, TypeVar @@ -1427,11 +1417,11 @@ result = asdict(instance) # is no longer compatible with the original Tuple class it is extending. reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]' - [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] [case testDataclassesAsdictNamedTuple] +# flags: --python-version 3.7 from dataclasses import dataclass, asdict from typing import NamedTuple From 74ebc6f1c88937bfac609fa69da35919108f20b4 Mon Sep 17 00:00:00 2001 From: Seth Yastrov Date: Thu, 19 Aug 2021 23:27:18 +0200 Subject: [PATCH 34/40] Fix tests for quoting changes --- test-data/unit/check-dataclasses.test | 70 +++++++++++++-------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 724c0bd940d4..dffc92a33b60 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1290,18 +1290,18 @@ class Person: class NonDataclass: pass -reveal_type(asdict(Person('John', 32))) # N: Revealed type is 'TypedDict({'name': builtins.str, 'age': builtins.int})' +reveal_type(asdict(Person('John', 32))) # N: Revealed type is "TypedDict({'name': builtins.str, 'age': builtins.int})" # It's OK to call on a non-dataclass, to reduce false-positives. -reveal_type(asdict(Person)) # N: Revealed type is 'builtins.dict[builtins.str, Any]' -reveal_type(asdict(NonDataclass())) # N: Revealed type is 'builtins.dict[builtins.str, Any]' +reveal_type(asdict(Person)) # N: Revealed type is "builtins.dict[builtins.str, Any]" +reveal_type(asdict(NonDataclass())) # N: Revealed type is "builtins.dict[builtins.str, Any]" def my_dict_factory(seq: List[Tuple[str, Any]]) -> Dict[str, Any]: pass -reveal_type(asdict(NonDataclass(), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' +reveal_type(asdict(NonDataclass(), dict_factory=my_dict_factory)) # N: Revealed type is "builtins.dict*[builtins.str, Any]" # Passing in a dict_factory should not return a TypedDict -reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is 'builtins.dict*[builtins.str, Any]' +reveal_type(asdict(Person('John', 32), dict_factory=my_dict_factory)) # N: Revealed type is "builtins.dict*[builtins.str, Any]" [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1314,7 +1314,7 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is "TypedDict({'bar': builtins.str})" [typing fixtures/typing-full.pyi] @@ -1326,7 +1326,7 @@ from dataclasses import dataclass, asdict class Foo: bar: str -reveal_type(asdict(Foo('bar'))) # N: Revealed type is 'TypedDict({'bar': builtins.str})' +reveal_type(asdict(Foo('bar'))) # N: Revealed type is "TypedDict({'bar': builtins.str})" [typing fixtures/typing-full.pyi] @@ -1349,7 +1349,7 @@ class A: # Recursion is not supported, so fall back result = asdict(A(B(C(A())))) -reveal_type(result) # N: Revealed type is 'TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})' +reveal_type(result) # N: Revealed type is "TypedDict({'b': Union[TypedDict({'c': TypedDict({'a': builtins.dict[builtins.str, Any]})}), None]})" [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] @@ -1368,7 +1368,7 @@ class Card: class Customer: card: Union[str, Card] -reveal_type(asdict(Customer('foo'))) # N: Revealed type is 'TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})' +reveal_type(asdict(Customer("foo"))) # N: Revealed type is "TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})" [typing fixtures/typing-full.pyi] @@ -1386,7 +1386,7 @@ class Customer: card: Type[Card] # Type[...] hould be left alone -reveal_type(asdict(Customer(Card))) # N: Revealed type is 'TypedDict({'card': Type[__main__.Card]})' +reveal_type(asdict(Customer(Card))) # N: Revealed type is "TypedDict({'card': Type[__main__.Card]})" [typing fixtures/typing-full.pyi] @@ -1407,7 +1407,7 @@ class Customer: def func(_: Card) -> Card: pass # Type[...] hould be left alone -reveal_type(asdict(Customer(func))) # N: Revealed type is 'TypedDict({'card': def (__main__.Card) -> __main__.Card})' +reveal_type(asdict(Customer(func))) # N: Revealed type is "TypedDict({'card': def (__main__.Card) -> __main__.Card})" [typing fixtures/typing-full.pyi] @@ -1430,14 +1430,14 @@ class Course: list_no_generic: list instance = Course( - participants=[Person('Joe', 32)], + participants=[Person("Joe", 32)], any_list=[], list_no_generic=[], ) result = asdict(instance) -reveal_type(result['participants']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' -reveal_type(result['any_list']) # N: Revealed type is 'builtins.list[Any]' -reveal_type(result['list_no_generic']) # N: Revealed type is 'builtins.list[Any]' +reveal_type(result['participants']) # N: Revealed type is "builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]" +reveal_type(result['any_list']) # N: Revealed type is "builtins.list[Any]" +reveal_type(result['list_no_generic']) # N: Revealed type is "builtins.list[Any]" [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] @@ -1454,15 +1454,15 @@ class Person: age: int -_T = TypeVar('_T') +_T = TypeVar("_T") class MyList(List[_T]): pass -_X = TypeVar('_X') +_X = TypeVar("_X") class MyListWith2TypeVars(List[_T], Generic[_T, _X]): foo: _X -_C = TypeVar('_C', Person, int) +_C = TypeVar("_C", Person, int) class MyListWithConstraint(List[_C], Generic[_C]): pass @@ -1476,17 +1476,17 @@ class Course: instance = Course( list_subclass=MyList([]), - list_subclass_2_typevars=MyListWith2TypeVars[Person, int]([Person('John', 23)]), - list_subclass_with_constraint=MyListWithConstraint([Person('Tim', 29)]) + list_subclass_2_typevars=MyListWith2TypeVars[Person, int]([Person("John", 23)]), + list_subclass_with_constraint=MyListWithConstraint([Person("Tim", 29)]) ) result = asdict(instance) # Supertypes (list) are returned, since there could be a constraint on the TypeVar # used on the subclass such that when the type argument to the subclass is substituted with a TypedDict, # it may not type-check. -reveal_type(result['list_subclass']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' -reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' -reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is 'builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['list_subclass']) # N: Revealed type is "builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]" +reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is "builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]" +reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is "builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]" [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] @@ -1505,9 +1505,9 @@ class Person: class Course: participants_by_name: Dict[str, Person] -instance = Course(participants_by_name={'Joe': Person('Joe', 32)}) +instance = Course(participants_by_name={"Joe": Person("Joe", 32)}) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str, TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is "builtins.dict[builtins.str, TypedDict({'name': builtins.str, 'age': builtins.int})]" [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1522,9 +1522,9 @@ class Person: name: str age: int -_KT = TypeVar('_KT') -_VT = TypeVar('_VT') -_Other = TypeVar('_Other') +_KT = TypeVar("_KT") +_VT = TypeVar("_VT") +_Other = TypeVar("_Other") class MyDict(Dict[_KT, _VT], Generic[_Other, _KT, _VT]): pass @@ -1532,9 +1532,9 @@ class MyDict(Dict[_KT, _VT], Generic[_Other, _KT, _VT]): class Course: participants_by_name: MyDict[int, str, Person] -instance = Course(participants_by_name=MyDict[int, str, Person]([('Joe', Person('Joe', 32))])) +instance = Course(participants_by_name=MyDict[int, str, Person]([("Joe", Person("Joe", 32))])) result = asdict(instance) -reveal_type(result['participants_by_name']) # N: Revealed type is 'builtins.dict[builtins.str*, TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['participants_by_name']) # N: Revealed type is "builtins.dict[builtins.str*, TypedDict({'name': builtins.str, 'age': builtins.int})]" [typing fixtures/typing-full.pyi] [builtins fixtures/dict.pyi] @@ -1553,9 +1553,9 @@ class Person: class Course: partners: Tuple[Person, Person] -instance = Course(partners=(Person('Joe', 32), Person('John', 23))) +instance = Course(partners=(Person("Joe", 32), Person("John", 23))) result = asdict(instance) -reveal_type(result['partners']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]' +reveal_type(result['partners']) # N: Revealed type is "Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), TypedDict({'name': builtins.str, 'age': builtins.int})]" [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] @@ -1583,7 +1583,7 @@ result = asdict(instance) # For now, subclasses of Tuple are transformed to the Tuple base class # This is because the subclass, if it itself contains dataclass fields, may be transformed in such a way that it # is no longer compatible with the original Tuple class it is extending. -reveal_type(result['tuple_subclass']) # N: Revealed type is 'Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]' +reveal_type(result['tuple_subclass']) # N: Revealed type is "Tuple[TypedDict({'name': builtins.str, 'age': builtins.int}), builtins.str]" [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] @@ -1610,10 +1610,10 @@ class Staff(NamedTuple): class Course: staff: Staff -instance = Course(staff=Staff(teacher=Person('Joe', 32), assistant=Person('John', 23))) +instance = Course(staff=Staff(teacher=Person("Joe", 32), assistant=Person("John", 23))) result = asdict(instance) # Due to implementation limitations, namedtuples are transformed to Any -reveal_type(result['staff']) # N: Revealed type is 'Any' +reveal_type(result['staff']) # N: Revealed type is "Any" [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] From f54e503a5c2bf4647a9e2f1edc1e61fe54255f53 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Tue, 18 Jan 2022 14:59:43 +0800 Subject: [PATCH 35/40] Fix --- mypy/plugins/dataclasses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 01e8b5937e9a..107f6291c9d8 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -497,10 +497,10 @@ def _add_dataclass_fields_magic_attribute(self) -> None: attr_name = '__dataclass_fields__' any_type = AnyType(TypeOfAny.explicit) field_type = self._ctx.api.named_type_or_none('dataclasses.Field', [any_type]) or any_type - attr_type = self._ctx.api.named_type('builtins.dict', [ + attr_type = self._ctx.api.named_type_or_none('builtins.dict', [ self._ctx.api.named_type('builtins.str'), field_type, - ]) + ]) or any_type var = Var(name=attr_name, type=attr_type) var.info = self._ctx.cls.info var._fullname = self._ctx.cls.info.fullname + '.' + attr_name From 9820cfc4d45ecdce3430adde80ae729749da2b63 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Tue, 18 Jan 2022 16:21:22 +0800 Subject: [PATCH 36/40] Add fixture for tests --- mypy/plugins/dataclasses.py | 4 ++-- test-data/unit/check-dataclasses.test | 13 ++++++++++--- test-data/unit/deps.test | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 107f6291c9d8..01e8b5937e9a 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -497,10 +497,10 @@ def _add_dataclass_fields_magic_attribute(self) -> None: attr_name = '__dataclass_fields__' any_type = AnyType(TypeOfAny.explicit) field_type = self._ctx.api.named_type_or_none('dataclasses.Field', [any_type]) or any_type - attr_type = self._ctx.api.named_type_or_none('builtins.dict', [ + attr_type = self._ctx.api.named_type('builtins.dict', [ self._ctx.api.named_type('builtins.str'), field_type, - ]) or any_type + ]) var = Var(name=attr_name, type=attr_type) var.info = self._ctx.cls.info var._fullname = self._ctx.cls.info.fullname + '.' + attr_name diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index d8b3638f5c99..11e37d60e521 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1379,6 +1379,7 @@ class Foo: reveal_type(asdict(Foo('bar'))) # N: Revealed type is "TypedDict({'bar': builtins.str})" [typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictPython39] # flags: --python-version 3.9 @@ -1391,6 +1392,7 @@ class Foo: reveal_type(asdict(Foo('bar'))) # N: Revealed type is "TypedDict({'bar': builtins.str})" [typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictRecursion] # flags: --python-version 3.7 @@ -1433,6 +1435,7 @@ class Customer: reveal_type(asdict(Customer("foo"))) # N: Revealed type is "TypedDict({'card': Union[builtins.str, TypedDict({'last4': builtins.int})]})" [typing fixtures/typing-full.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictType] # flags: --python-version 3.7 @@ -1451,7 +1454,7 @@ class Customer: reveal_type(asdict(Customer(Card))) # N: Revealed type is "TypedDict({'card': Type[__main__.Card]})" [typing fixtures/typing-full.pyi] - +[builtins fixtures/dict.pyi] [case testDataclassesAsdictCallable] # flags: --python-version 3.7 @@ -1472,7 +1475,7 @@ def func(_: Card) -> Card: pass reveal_type(asdict(Customer(func))) # N: Revealed type is "TypedDict({'card': def (__main__.Card) -> __main__.Card})" [typing fixtures/typing-full.pyi] - +[builtins fixtures/dict.pyi] [case testDataclassesAsdictList] # flags: --python-version 3.7 @@ -1503,7 +1506,7 @@ reveal_type(result['list_no_generic']) # N: Revealed type is "builtins.list[Any] [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] - +[builtins fixtures/dict.pyi] [case testDataclassesAsdictListSubclass] # flags: --python-version 3.7 @@ -1552,6 +1555,7 @@ reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is "buil [typing fixtures/typing-full.pyi] [builtins fixtures/list.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictDict] # flags: --python-version 3.7 @@ -1621,6 +1625,7 @@ reveal_type(result['partners']) # N: Revealed type is "Tuple[TypedDict({'name': [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictTupleSubclass] # flags: --python-version 3.7 @@ -1649,6 +1654,7 @@ reveal_type(result['tuple_subclass']) # N: Revealed type is "Tuple[TypedDict({'n [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] [case testDataclassesAsdictNamedTuple] # flags: --python-version 3.7 @@ -1679,6 +1685,7 @@ reveal_type(result['staff']) # N: Revealed type is "Any" [typing fixtures/typing-full.pyi] [builtins fixtures/tuple.pyi] +[builtins fixtures/dict.pyi] [case testDataclassFieldDoesNotFailOnKwargsUnpacking] # flags: --python-version 3.7 diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index 530b6857dd63..dcd4e3df1b2a 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1470,6 +1470,7 @@ AttributeInOtherModule = str [typing fixtures/typing-typeddict.pyi] [builtins fixtures/fine_grained.pyi] +[builtins fixtures/dict.pyi] [out] -> m From 9f49cac3be8d58d5a480f2fe9eeec41455013e60 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Tue, 18 Jan 2022 18:15:06 +0800 Subject: [PATCH 37/40] Add fixture for tests --- test-data/unit/check-dataclasses.test | 1 - test-data/unit/deps.test | 2 +- test-data/unit/fine-grained.test | 1 + test-data/unit/fixtures/dict.pyi | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test-data/unit/check-dataclasses.test b/test-data/unit/check-dataclasses.test index 11e37d60e521..4aab98611478 100644 --- a/test-data/unit/check-dataclasses.test +++ b/test-data/unit/check-dataclasses.test @@ -1554,7 +1554,6 @@ reveal_type(result['list_subclass_2_typevars']) # N: Revealed type is "builtins. reveal_type(result['list_subclass_with_constraint']) # N: Revealed type is "builtins.list[TypedDict({'name': builtins.str, 'age': builtins.int})]" [typing fixtures/typing-full.pyi] -[builtins fixtures/list.pyi] [builtins fixtures/dict.pyi] [case testDataclassesAsdictDict] diff --git a/test-data/unit/deps.test b/test-data/unit/deps.test index dcd4e3df1b2a..ad37de853404 100644 --- a/test-data/unit/deps.test +++ b/test-data/unit/deps.test @@ -1470,7 +1470,7 @@ AttributeInOtherModule = str [typing fixtures/typing-typeddict.pyi] [builtins fixtures/fine_grained.pyi] -[builtins fixtures/dict.pyi] +[builtins fixtures/primitives.pyi] [out] -> m diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index f68f9a448771..ef11d8adc9b3 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -934,6 +934,7 @@ AttributeInOtherModule = int [typing fixtures/typing-typeddict.pyi] [builtins fixtures/fine_grained.pyi] +[builtins fixtures/primitives.pyi] [out] == c.py:3: error: Unsupported operand types for + ("int" and "str") diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index fd509de8a6c2..ef5891c8ed4b 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -39,6 +39,7 @@ class unicode: pass # needed for py2 docstrings class bytes: pass class list(Sequence[T]): # needed by some test cases + def __init__(self, x: Iterable[T]) -> None: pass def __getitem__(self, x: int) -> T: pass def __iter__(self) -> Iterator[T]: pass def __mul__(self, x: int) -> list[T]: pass From fcd1ff5ddf3fb8fd46c450ec8aefe9445bdc3812 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Tue, 18 Jan 2022 18:42:21 +0800 Subject: [PATCH 38/40] Add fixture for tests --- test-data/unit/fixtures/dict.pyi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index ef5891c8ed4b..3f484a11d17e 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -39,6 +39,9 @@ class unicode: pass # needed for py2 docstrings class bytes: pass class list(Sequence[T]): # needed by some test cases + @overload + def __init__(self) -> None: pass + @overload def __init__(self, x: Iterable[T]) -> None: pass def __getitem__(self, x: int) -> T: pass def __iter__(self) -> Iterator[T]: pass From 6e1585dc0618aca24df85d92eee604809df1f6c3 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Tue, 18 Jan 2022 19:10:35 +0800 Subject: [PATCH 39/40] Fix --- test-data/unit/check-functions.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index bdf75b2dc58c..8d6a649a12f9 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1777,7 +1777,7 @@ P = Callable[[mypy_extensions.VarArg(int)], int] # ok Q = Callable[[Arg(int, type=int)], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "type" R = Callable[[Arg(int, 'x', name='y')], int] # E: Invalid type alias: expression is not a valid type # E: Value of type "int" is not indexable # E: "Arg" gets multiple values for keyword argument "name" -[builtins fixtures/dict.pyi] +[builtins fixtures/primitives.pyi] [case testCallableParsing] from typing import Callable From e22656202f90442be1aecc422d10dfeb3f4adae1 Mon Sep 17 00:00:00 2001 From: 97littleleaf11 <97littleleaf11@gmail.com> Date: Wed, 19 Jan 2022 16:36:17 +0800 Subject: [PATCH 40/40] Test for a workaround --- mypy/plugins/dataclasses.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/plugins/dataclasses.py b/mypy/plugins/dataclasses.py index 6268f200c004..bf53b41aee1d 100644 --- a/mypy/plugins/dataclasses.py +++ b/mypy/plugins/dataclasses.py @@ -577,7 +577,7 @@ def asdict_callback(ctx: FunctionContext) -> Type: class AsDictVisitor(TypeTranslator): def __init__(self, api: CheckerPluginInterface) -> None: self.api = api - self.seen_dataclasses = set() # type: Set[str] + self.seen_dataclasses: Set[str] = set() def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) @@ -593,12 +593,15 @@ def visit_instance(self, t: Instance) -> Type: return self.api.named_generic_type( 'builtins.dict', [self.api.named_generic_type('builtins.str', []), any_type]) attrs = info.metadata['dataclass']['attributes'] - fields = OrderedDict() # type: OrderedDict[str, Type] + fields: OrderedDict[str, Type] = OrderedDict() self.seen_dataclasses.add(info.fullname) for data in attrs: attr = DataclassAttribute.deserialize(info, data, self.api) self.api.add_plugin_dependency(make_trigger(info.fullname + "." + attr.name)) - sym_node = info.names[attr.name] + # TODO: attr.name should be available + sym_node = info.names.get(attr.name, None) + if sym_node is None: + continue attr_type = sym_node.type assert attr_type is not None fields[attr.name] = attr_type.accept(self)