Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add signature for dataclasses.replace #14849

Merged
merged 49 commits into from
Jun 17, 2023
Merged
Show file tree
Hide file tree
Changes from 38 commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
6dbaf9c
Add signature for dataclasses.replace
ikonst Mar 7, 2023
4dcbe44
add ClassVar
ikonst Mar 7, 2023
7ed3741
prevent misleading note
ikonst Mar 7, 2023
89257b5
stash in a secret class-private
ikonst Mar 8, 2023
32b1d47
fix typing
ikonst Mar 8, 2023
1f08816
docs and naming
ikonst Mar 8, 2023
c456a5f
inst -> obj
ikonst Mar 8, 2023
8118d29
add the secret symbol to deps.test
ikonst Mar 8, 2023
789cb2b
language
ikonst Mar 8, 2023
9f0974c
nit
ikonst Mar 8, 2023
a37e406
nit
ikonst Mar 8, 2023
367c0e9
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst Mar 9, 2023
0e84c4f
make obj positional-only
ikonst Mar 14, 2023
9cfc081
Merge branch '2023-03-06-dataclasses-replace' of https://github.com/i…
ikonst Mar 14, 2023
7b907cf
add pythoneval test
ikonst Mar 14, 2023
985db60
use py3.7 syntax
ikonst Mar 14, 2023
15dbb7b
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst Mar 17, 2023
3227fde
Fix lint
ikonst Mar 17, 2023
2dbf249
use syntactically invalid name for symbol
ikonst Mar 30, 2023
b32881c
Merge branch '2023-03-06-dataclasses-replace' of https://github.com/i…
ikonst Mar 30, 2023
26056a4
Merge remote-tracking branch 'origin/master' into 2023-03-06-dataclas…
ikonst Mar 30, 2023
40315b7
mypy-replace must use name mangling prefix
ikonst Mar 30, 2023
c005895
Generic dataclass support
ikonst Mar 30, 2023
d71bc21
add fine-grained test
ikonst Mar 30, 2023
d914b94
disable for arbitrary transforms
ikonst Mar 31, 2023
2cb6dee
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst Apr 6, 2023
5735726
fix testDataclassTransformReplace
ikonst Apr 7, 2023
d38897e
better error message for generics
ikonst Apr 7, 2023
04f0ee3
add support for typevars
ikonst Apr 8, 2023
f402b86
streamline
ikonst Apr 8, 2023
306c3f3
self-check
ikonst Apr 8, 2023
9b491f5
Merge remote-tracking branch 'origin/master' into 2023-03-06-dataclas…
ikonst Apr 21, 2023
283fe3d
Add improved union support from #15050
ikonst Apr 21, 2023
29780e9
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst May 1, 2023
9c43ab6
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst May 3, 2023
94024ba
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst May 9, 2023
70240c4
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst May 16, 2023
99bf973
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst May 17, 2023
8fe75a7
add to testReplaceUnion
ikonst May 22, 2023
bea50e8
testReplaceUnion: fix B.y to be int
ikonst May 22, 2023
cd35951
rename testcase replace to testReplace
ikonst May 22, 2023
d71a7c0
Merge remote-tracking branch 'origin/master' into 2023-03-06-dataclas…
ikonst May 22, 2023
957744e
Merge branch 'master' into 2023-03-06-dataclasses-replace
ikonst Jun 2, 2023
6abf6ff
Merge remote-tracking branch 'origin/master' into pr/ikonst/14849
ikonst Jun 12, 2023
4c4fc94
remove get_expression_type hack
ikonst Jun 12, 2023
be4a290
assert isinstance(replace_sig, ProperType)
ikonst Jun 17, 2023
ee0ae21
add a TODO for _meet_replace_sigs
ikonst Jun 17, 2023
3f656f8
Merge remote-tracking branch 'origin/master' into 2023-03-06-dataclas…
ikonst Jun 17, 2023
65d6e89
Merge remote-tracking branch 'origin/master' into 2023-03-06-dataclas…
ikonst Jun 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 166 additions & 1 deletion mypy/plugins/dataclasses.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

from mypy import errorcodes, message_registry
from mypy.expandtype import expand_type, expand_type_by_instance
from mypy.meet import meet_types
from mypy.messages import format_type_bare
from mypy.nodes import (
ARG_NAMED,
ARG_NAMED_OPT,
Expand Down Expand Up @@ -38,7 +40,7 @@
TypeVarExpr,
Var,
)
from mypy.plugin import ClassDefContext, SemanticAnalyzerPluginInterface
from mypy.plugin import ClassDefContext, FunctionSigContext, SemanticAnalyzerPluginInterface
from mypy.plugins.common import (
_get_callee_type,
_get_decorator_bool_argument,
Expand All @@ -56,10 +58,13 @@
Instance,
LiteralType,
NoneType,
ProperType,
TupleType,
Type,
TypeOfAny,
TypeVarType,
UninhabitedType,
UnionType,
get_proper_type,
)
from mypy.typevars import fill_typevars
Expand All @@ -76,6 +81,7 @@
frozen_default=False,
field_specifiers=("dataclasses.Field", "dataclasses.field"),
)
_INTERNAL_REPLACE_SYM_NAME = "__mypy-replace"


class DataclassAttribute:
Expand Down Expand Up @@ -335,13 +341,47 @@ def transform(self) -> bool:

self._add_dataclass_fields_magic_attribute()

if self._spec is _TRANSFORM_SPEC_FOR_DATACLASSES:
self._add_internal_replace_method(attributes)

info.metadata["dataclass"] = {
"attributes": [attr.serialize() for attr in attributes],
"frozen": decorator_arguments["frozen"],
}

return True

def _add_internal_replace_method(self, attributes: list[DataclassAttribute]) -> None:
"""
Stashes the signature of 'dataclasses.replace(...)' for this specific dataclass
to be used later whenever 'dataclasses.replace' is called for this dataclass.
"""
arg_types: list[Type] = []
arg_kinds = []
arg_names: list[str | None] = []

info = self._cls.info
for attr in attributes:
attr_type = attr.expand_type(info)
assert attr_type is not None
arg_types.append(attr_type)
arg_kinds.append(
ARG_NAMED if attr.is_init_var and not attr.has_default else ARG_NAMED_OPT
)
arg_names.append(attr.name)

signature = CallableType(
arg_types=arg_types,
arg_kinds=arg_kinds,
arg_names=arg_names,
ret_type=NoneType(),
fallback=self._api.named_type("builtins.function"),
)

self._cls.info.names[_INTERNAL_REPLACE_SYM_NAME] = SymbolTableNode(
kind=MDEF, node=FuncDef(typ=signature), plugin_generated=True
)

def add_slots(
self, info: TypeInfo, attributes: list[DataclassAttribute], *, correct_version: bool
) -> None:
Expand Down Expand Up @@ -884,3 +924,128 @@ def _has_direct_dataclass_transform_metaclass(info: TypeInfo) -> bool:
info.declared_metaclass is not None
and info.declared_metaclass.type.dataclass_transform_spec is not None
)


def _fail_not_dataclass(ctx: FunctionSigContext, t: Type, parent_t: Type) -> None:
t_name = format_type_bare(t, ctx.api.options)
if parent_t is t:
msg = (
f'Argument 1 to "replace" has a variable type "{t_name}" not bound to a dataclass'
if isinstance(t, TypeVarType)
else f'Argument 1 to "replace" has incompatible type "{t_name}"; expected a dataclass'
)
else:
pt_name = format_type_bare(parent_t, ctx.api.options)
msg = (
f'Argument 1 to "replace" has type "{pt_name}" whose item "{t_name}" is not bound to a dataclass'
if isinstance(t, TypeVarType)
else f'Argument 1 to "replace" has incompatible type "{pt_name}" whose item "{t_name}" is not a dataclass'
)

ctx.api.fail(msg, ctx.context)


def _get_expanded_dataclasses_fields(
ctx: FunctionSigContext, typ: ProperType, display_typ: ProperType, parent_typ: ProperType
) -> list[CallableType] | None:
"""
For a given type, determine what dataclasses it can be: for each class, return the field types.
For generic classes, the field types are expanded.
If the type contains Any or a non-dataclass, returns None; in the latter case, also reports an error.
"""
if isinstance(typ, AnyType):
return None
elif isinstance(typ, UnionType):
ret: list[CallableType] | None = []
for item in typ.relevant_items():
item = get_proper_type(item)
item_types = _get_expanded_dataclasses_fields(ctx, item, item, parent_typ)
if ret is not None and item_types is not None:
ret += item_types
else:
ret = None # but keep iterating to emit all errors
return ret
elif isinstance(typ, TypeVarType):
return _get_expanded_dataclasses_fields(
ctx, get_proper_type(typ.upper_bound), display_typ, parent_typ
)
elif isinstance(typ, Instance):
replace_sym = typ.type.get_method(_INTERNAL_REPLACE_SYM_NAME)
if replace_sym is None:
_fail_not_dataclass(ctx, display_typ, parent_typ)
return None
replace_sig = get_proper_type(replace_sym.type)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to call get+proper_type() here, since we know we added it, you can just add assert isinstance(..., ProperType) instead.

Copy link
Contributor Author

@ikonst ikonst May 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It ends up looking awkward like this:

-replace_sig = get_proper_type(replace_sym.type)
+replace_sig = replace_sym.type
+assert isinstance(replace_sig, ProperType)
 assert isinstance(replace_sig, CallableType)

Isn't it simpler to do what I did before, even if taking a few more cycles?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it actually looks fine to have two asserts. It has two additional benefits:

  • It clearly indicates the intent
  • It reduces number of get_proper_type() in the code, so people we hopefully cargo cult them less (we had a bunch of weird crashes caused by blindly copying some get_proper_type() calls).

assert isinstance(replace_sig, CallableType)
return [expand_type_by_instance(replace_sig, typ)]
else:
_fail_not_dataclass(ctx, display_typ, parent_typ)
return None


def _meet_replace_sigs(sigs: list[CallableType]) -> CallableType:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like hack. Is this because the plugin systems expects a CallableType returned from the plugin? Maybe we can instead relax this expectation and allow returning say a union of callable types?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ooh, that's an interesting idea. It didn't cross my mind I could return a union of callables and let mypy handle it in check_union_call.

Copy link
Contributor Author

@ikonst ikonst May 22, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

p.s. I'll be happy to simplify this code, but do you reckon we should first change the plugin API and then rework this PR, or merge as-is, change plugin API and then refactor?

(A bit worried the devil's in the details and that maybe it's not as easy as it sounds, or not equivalent.)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is better to do it in a follow up PR, but leave a TODO or open an issue (or both).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"""
Produces the lowest bound of the 'replace' signatures of multiple dataclasses.
"""
args = {
name: (typ, kind)
for name, typ, kind in zip(sigs[0].arg_names, sigs[0].arg_types, sigs[0].arg_kinds)
}

for sig in sigs[1:]:
sig_args = {
name: (typ, kind)
for name, typ, kind in zip(sig.arg_names, sig.arg_types, sig.arg_kinds)
}
for name in (*args.keys(), *sig_args.keys()):
sig_typ, sig_kind = args.get(name, (UninhabitedType(), ARG_NAMED_OPT))
sig2_typ, sig2_kind = sig_args.get(name, (UninhabitedType(), ARG_NAMED_OPT))
args[name] = (
meet_types(sig_typ, sig2_typ),
ARG_NAMED_OPT if sig_kind == sig2_kind == ARG_NAMED_OPT else ARG_NAMED,
)

return sigs[0].copy_modified(
arg_names=list(args.keys()),
arg_types=[typ for typ, _ in args.values()],
arg_kinds=[kind for _, kind in args.values()],
)


def replace_function_sig_callback(ctx: FunctionSigContext) -> CallableType:
"""
Returns a signature for the 'dataclasses.replace' function that's dependent on the type
of the first positional argument.
"""
if len(ctx.args) != 2:
# Ideally the name and context should be callee's, but we don't have it in FunctionSigContext.
ctx.api.fail(f'"{ctx.default_signature.name}" has unexpected type annotation', ctx.context)
return ctx.default_signature

if len(ctx.args[0]) != 1:
return ctx.default_signature # leave it to the type checker to complain

obj_arg = ctx.args[0][0]

# <hack>
from mypy.checker import TypeChecker

assert isinstance(ctx.api, TypeChecker)
obj_type = ctx.api.expr_checker.accept(obj_arg)
# </hack>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW this kind of hack is quite common (I myself used it several times, for old sqlalchemy plugin, and internally). I would propose to instead add another plugin hook, e.g. get_function_signature_hook_2 (similar to what we have for classes), that would be called after infer_arg_types_in_context() with a context that includes inferred argument types.

cc @JukkaL what do you think?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This proposal still stands. Do we have an issue for this?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's #14845 and #10216, though not one spelling out the "get_function_signature_hook_2" solution.


obj_type = get_proper_type(obj_type)
inst_type_str = format_type_bare(obj_type, ctx.api.options)

replace_sigs = _get_expanded_dataclasses_fields(ctx, obj_type, obj_type, obj_type)
if replace_sigs is None:
return ctx.default_signature
replace_sig = _meet_replace_sigs(replace_sigs)

return replace_sig.copy_modified(
arg_names=[None, *replace_sig.arg_names],
arg_kinds=[ARG_POS, *replace_sig.arg_kinds],
arg_types=[obj_type, *replace_sig.arg_types],
ret_type=obj_type,
fallback=ctx.default_signature.fallback,
name=f"{ctx.default_signature.name} of {inst_type_str}",
)
4 changes: 3 additions & 1 deletion mypy/plugins/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,12 @@ def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type]
def get_function_signature_hook(
self, fullname: str
) -> Callable[[FunctionSigContext], FunctionLike] | None:
from mypy.plugins import attrs
from mypy.plugins import attrs, dataclasses

if fullname in ("attr.evolve", "attrs.evolve", "attr.assoc", "attrs.assoc"):
return attrs.evolve_function_sig_callback
elif fullname == "dataclasses.replace":
return dataclasses.replace_function_sig_callback
return None

def get_method_signature_hook(
Expand Down
19 changes: 19 additions & 0 deletions test-data/unit/check-dataclass-transform.test
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,24 @@ reveal_type(bar.base) # N: Revealed type is "builtins.int"
[typing fixtures/typing-full.pyi]
[builtins fixtures/dataclasses.pyi]

[case testDataclassTransformReplace]
from dataclasses import replace
from typing import dataclass_transform, Type

@dataclass_transform()
def my_dataclass(cls: Type) -> Type:
return cls

@my_dataclass
class Person:
name: str

p = Person('John')
y = replace(p, name='Bob') # E: Argument 1 to "replace" has incompatible type "Person"; expected a dataclass

[typing fixtures/typing-full.pyi]
[builtins fixtures/dataclasses.pyi]

[case testDataclassTransformSimpleDescriptor]
# flags: --python-version 3.11

Expand Down Expand Up @@ -1051,5 +1069,6 @@ class Desc2:
class C:
x: Desc # E: Unsupported signature for "__set__" in "Desc"
y: Desc2 # E: Unsupported "__set__" in "Desc2"

[typing fixtures/typing-full.pyi]
[builtins fixtures/dataclasses.pyi]
Loading