-
-
Notifications
You must be signed in to change notification settings - Fork 636
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add initial support for a parametrize
builtin to generate multiple copies of a target
#14408
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,6 +6,7 @@ | |
These are always activated and cannot be disabled. | ||
""" | ||
from pants.bsp.rules import rules as bsp_rules | ||
from pants.build_graph.build_file_aliases import BuildFileAliases | ||
from pants.core.goals import ( | ||
check, | ||
export, | ||
|
@@ -39,6 +40,7 @@ | |
stripped_source_files, | ||
subprocess_environment, | ||
) | ||
from pants.engine.internals.parametrize import Parametrize | ||
from pants.goal import anonymous_telemetry, stats_aggregator | ||
from pants.python import binaries as python_binaries | ||
from pants.source import source_root | ||
|
@@ -86,3 +88,9 @@ def target_types(): | |
ResourcesGeneratorTarget, | ||
RelocatedFiles, | ||
] | ||
|
||
|
||
def build_file_aliases(): | ||
return BuildFileAliases( | ||
objects={"parametrize": Parametrize}, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. V happy this is not a CAOF, I'm eagerly awaiting dropping that infrastructure. |
||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -39,6 +39,7 @@ | |
SpecsSnapshot, | ||
) | ||
from pants.engine.internals import native_engine | ||
from pants.engine.internals.parametrize import Parametrize | ||
from pants.engine.internals.target_adaptor import TargetAdaptor | ||
from pants.engine.rules import Get, MultiGet, collect_rules, rule | ||
from pants.engine.target import ( | ||
|
@@ -96,7 +97,7 @@ | |
from pants.util.frozendict import FrozenDict | ||
from pants.util.logging import LogLevel | ||
from pants.util.ordered_set import FrozenOrderedSet, OrderedSet | ||
from pants.util.strutil import bullet_list | ||
from pants.util.strutil import bullet_list, pluralize | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
@@ -177,17 +178,27 @@ def warn_deprecated_field_type(request: _WarnDeprecatedFieldRequest) -> _WarnDep | |
|
||
@dataclass(frozen=True) | ||
class _TargetParametrizations: | ||
"""All parametrizations and generated targets for a single input Address.""" | ||
"""All parametrizations and generated targets for a single input Address. | ||
|
||
original_target: Target | ||
If a Target has been parametrized, it might _not_ be present in this output, due to it not being | ||
addressable using its un-parameterized Address. | ||
""" | ||
|
||
original_target: Target | None | ||
parametrizations: FrozenDict[Address, Target] | ||
|
||
def get(self, address: Address) -> Target | None: | ||
if self.original_target.address == address: | ||
if self.original_target and self.original_target.address == address: | ||
return self.original_target | ||
return self.parametrizations.get(address) | ||
|
||
def generated_or_generator(self, maybe_generator: Address) -> Iterator[Target]: | ||
if not self.original_target: | ||
raise ValueError( | ||
"A `parametrized` target cannot be consumed without its parameters specified.\n" | ||
f"Target `{maybe_generator}` can be addressed as:\n" | ||
f"{bullet_list(addr.spec for addr in self.parametrizations)}" | ||
) | ||
if self.parametrizations: | ||
# Generated Targets. | ||
yield from self.parametrizations.values() | ||
|
@@ -219,7 +230,7 @@ async def resolve_target_parametrizations( | |
): | ||
await Get(_WarnDeprecatedTarget, _WarnDeprecatedTargetRequest(target_type)) | ||
|
||
target: Target | ||
target: Target | None | ||
generate_request = target_types_to_generate_requests.request_for(target_type) | ||
if generate_request: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a huge if block... possible/worth to split out (or perhaps await Joshuas rule parsing that supports following function calls) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sorry, I rushed this one in without doing this. I'm going to open a ticket for followup (since I'm out of town for the next few days), but will clean this up before |
||
# Split out the `propagated_fields` before construction. | ||
|
@@ -240,6 +251,17 @@ async def resolve_target_parametrizations( | |
if field_value is not None: | ||
template_fields[field_type.alias] = field_value | ||
|
||
generator_fields_parametrized = { | ||
name for name, field in generator_fields.items() if isinstance(field, Parametrize) | ||
} | ||
if generator_fields_parametrized: | ||
noun = pluralize(len(generator_fields_parametrized), "field", include_count=False) | ||
raise ValueError( | ||
f"Only fields which will be moved to generated targets may be parametrized, " | ||
f"so target generator {address} (with type {target_type.alias}) cannot " | ||
f"parametrize the {generator_fields_parametrized} {noun}." | ||
) | ||
|
||
target = target_type(generator_fields, address, union_membership) | ||
|
||
overrides = {} | ||
|
@@ -260,21 +282,44 @@ async def resolve_target_parametrizations( | |
else: | ||
overrides = overrides_field.flatten() | ||
|
||
generated = await Get( | ||
GeneratedTargets, | ||
GenerateTargetsRequest, | ||
generate_request( | ||
target, | ||
template_address=address, | ||
template=template_fields, | ||
overrides=overrides, | ||
), | ||
all_generated = await MultiGet( | ||
Get( | ||
GeneratedTargets, | ||
GenerateTargetsRequest, | ||
generate_request( | ||
target, | ||
template_address=address, | ||
template=template, | ||
overrides={ | ||
name: dict(Parametrize.expand(address, override)) | ||
for name, override in overrides.items() | ||
}, | ||
), | ||
) | ||
for address, template in Parametrize.expand(address, template_fields) | ||
) | ||
generated = FrozenDict( | ||
GeneratedTargets( | ||
target, (t for generated_batch in all_generated for t in generated_batch.values()) | ||
) | ||
) | ||
else: | ||
target = target_type(target_adaptor.kwargs, address, union_membership) | ||
generated = GeneratedTargets(target, ()) | ||
first, *rest = Parametrize.expand(address, target_adaptor.kwargs) | ||
if rest: | ||
target = None | ||
generated = FrozenDict( | ||
( | ||
parameterized_address, | ||
target_type(parameterized_fields, parameterized_address, union_membership), | ||
) | ||
for parameterized_address, parameterized_fields in (first, *rest) | ||
) | ||
else: | ||
target = target_type(target_adaptor.kwargs, address, union_membership) | ||
generated = FrozenDict() | ||
|
||
for field_type in target.field_types: | ||
# TODO: Move to Target constructor. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nack, we need to use the engine for singleton so that we don't warn hundreds of times. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yea... but we could use the |
||
for field_type in target.field_types if target else (): | ||
if ( | ||
field_type.deprecated_alias is not None | ||
and field_type.deprecated_alias in target_adaptor.kwargs | ||
|
@@ -291,8 +336,10 @@ async def resolve_target( | |
) -> WrappedTarget: | ||
base_address = address.maybe_convert_to_target_generator() | ||
parametrizations = await Get(_TargetParametrizations, Address, base_address) | ||
if address.is_generated_target and not target_types_to_generate_requests.is_generator( | ||
parametrizations.original_target | ||
if ( | ||
address.is_generated_target | ||
and parametrizations.original_target | ||
and not target_types_to_generate_requests.is_generator(parametrizations.original_target) | ||
): | ||
# TODO: This is an accommodation to allow using file/generator Addresses for non-generator | ||
# atom targets. See https://github.com/pantsbuild/pants/issues/14419. | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is
target_generator
a misnomer? Maybebase_target
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yea, it probably is. It used to be
build_target
, although I likebase
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Probably best as a followup though?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Fine with me.
Maybe open an issue tracking all the little fixes like this? I know you have some dedicated tickets for bigger changes.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Will do.