diff --git a/src/python/pants/backend/codegen/avro/target_types.py b/src/python/pants/backend/codegen/avro/target_types.py index ef4aeb0c4da..39d5e4fb33e 100644 --- a/src/python/pants/backend/codegen/avro/target_types.py +++ b/src/python/pants/backend/codegen/avro/target_types.py @@ -1,26 +1,19 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.engine.fs import PathGlobs, Paths -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, AllTargets, Dependencies, - GeneratedTargets, - GenerateTargetsRequest, MultipleSourcesField, OverridesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, Target, + TargetFilesGenerator, Targets, generate_file_based_overrides_field_help_message, - generate_file_level_targets, ) -from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior from pants.util.docutil import doc_url from pants.util.logging import LogLevel @@ -81,7 +74,7 @@ class AvroSourcesOverridesField(OverridesField): ) -class AvroSourcesGeneratorTarget(Target): +class AvroSourcesGeneratorTarget(TargetFilesGenerator): alias = "avro_sources" core_fields = ( *COMMON_TARGET_FIELDS, @@ -89,47 +82,14 @@ class AvroSourcesGeneratorTarget(Target): AvroSourcesGeneratingSourcesField, AvroSourcesOverridesField, ) - help = "Generate a `avro_source` target for each file in the `sources` field." - - -class GenerateTargetsFromAvroSources(GenerateTargetsRequest): - generate_from = AvroSourcesGeneratorTarget - - -@rule -async def generate_targets_from_avro_sources( - request: GenerateTargetsFromAvroSources, - files_not_found_behavior: FilesNotFoundBehavior, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[AvroSourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - AvroSourceTarget, - request.generator, - sources_paths.files, - union_membership, - # Note: Avro files cannot import from other Avro files, so do not add dependencies. - add_dependencies_on_all_siblings=False, - overrides=all_overrides, + generated_target_cls = AvroSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + AvroDependenciesField, ) + moved_fields = () + help = "Generate a `avro_source` target for each file in the `sources` field." def rules(): - return ( - *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromAvroSources), - ) + return collect_rules() diff --git a/src/python/pants/backend/codegen/avro/target_types_test.py b/src/python/pants/backend/codegen/avro/target_types_test.py index 7c83bb45488..bd0e2323a89 100644 --- a/src/python/pants/backend/codegen/avro/target_types_test.py +++ b/src/python/pants/backend/codegen/avro/target_types_test.py @@ -7,13 +7,10 @@ from textwrap import dedent from pants.backend.codegen.avro import target_types -from pants.backend.codegen.avro.target_types import ( - AvroSourcesGeneratorTarget, - AvroSourceTarget, - GenerateTargetsFromAvroSources, -) +from pants.backend.codegen.avro.target_types import AvroSourcesGeneratorTarget, AvroSourceTarget from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import SingleSourceField, Tags from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -21,7 +18,7 @@ def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), - QueryRule(GeneratedTargets, [GenerateTargetsFromAvroSources]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[AvroSourcesGeneratorTarget], ) @@ -42,8 +39,6 @@ def test_generate_source_targets() -> None: } ) - generator = rule_runner.get_target(Address("src/avro", target_name="lib")) - def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> AvroSourceTarget: return AvroSourceTarget( {SingleSourceField.alias: rel_fp, Tags.alias: tags}, @@ -51,12 +46,11 @@ def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> AvroSourceTarget: residence_dir=os.path.dirname(os.path.join("src/avro", rel_fp)), ) - generated = rule_runner.request(GeneratedTargets, [GenerateTargetsFromAvroSources(generator)]) - assert generated == GeneratedTargets( - generator, - { - gen_tgt("f1.avsc", tags=["overridden"]), - gen_tgt("f2.avpr"), - gen_tgt("subdir/f.avsc"), - }, - ) + generated = rule_runner.request( + _TargetParametrizations, [Address("src/avro", target_name="lib")] + ).parametrizations + assert set(generated.values()) == { + gen_tgt("f1.avsc", tags=["overridden"]), + gen_tgt("f2.avpr"), + gen_tgt("subdir/f.avsc"), + } diff --git a/src/python/pants/backend/codegen/protobuf/target_types.py b/src/python/pants/backend/codegen/protobuf/target_types.py index ffd6ae16811..b4f75e08815 100644 --- a/src/python/pants/backend/codegen/protobuf/target_types.py +++ b/src/python/pants/backend/codegen/protobuf/target_types.py @@ -2,27 +2,23 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.codegen.protobuf.protoc import Protoc -from pants.engine.fs import PathGlobs, Paths -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, AllTargets, BoolField, Dependencies, - GeneratedTargets, - GenerateTargetsRequest, MultipleSourcesField, OverridesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, Target, + TargetFilesGenerator, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, Targets, generate_file_based_overrides_field_help_message, - generate_file_level_targets, ) -from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior +from pants.engine.unions import UnionRule from pants.util.docutil import doc_url from pants.util.logging import LogLevel @@ -76,6 +72,20 @@ class ProtobufSourceTarget(Target): # ----------------------------------------------------------------------------------------------- +class GeneratorSettingsRequest(TargetFilesGeneratorSettingsRequest): + pass + + +@rule +def generator_settings( + _: GeneratorSettingsRequest, + protoc: Protoc, +) -> TargetFilesGeneratorSettings: + return TargetFilesGeneratorSettings( + add_dependencies_on_all_siblings=not protoc.dependency_inference + ) + + class ProtobufSourcesGeneratingSourcesField(MultipleSourcesField): default = ("*.proto",) expected_file_extensions = (".proto",) @@ -94,56 +104,26 @@ class ProtobufSourcesOverridesField(OverridesField): ) -class ProtobufSourcesGeneratorTarget(Target): +class ProtobufSourcesGeneratorTarget(TargetFilesGenerator): alias = "protobuf_sources" core_fields = ( *COMMON_TARGET_FIELDS, ProtobufDependenciesField, ProtobufSourcesGeneratingSourcesField, - ProtobufGrpcToggleField, ProtobufSourcesOverridesField, ) - help = "Generate a `protobuf_source` target for each file in the `sources` field." - - -class GenerateTargetsFromProtobufSources(GenerateTargetsRequest): - generate_from = ProtobufSourcesGeneratorTarget - - -@rule -async def generate_targets_from_protobuf_sources( - request: GenerateTargetsFromProtobufSources, - files_not_found_behavior: FilesNotFoundBehavior, - protoc: Protoc, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ProtobufSourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - ProtobufSourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not protoc.dependency_inference, - overrides=all_overrides, + generated_target_cls = ProtobufSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + ProtobufDependenciesField, ) + moved_fields = (ProtobufGrpcToggleField,) + settings_request_cls = GeneratorSettingsRequest + help = "Generate a `protobuf_source` target for each file in the `sources` field." def rules(): - return ( + return [ *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromProtobufSources), - ) + UnionRule(TargetFilesGeneratorSettingsRequest, GeneratorSettingsRequest), + ] diff --git a/src/python/pants/backend/codegen/protobuf/target_types_test.py b/src/python/pants/backend/codegen/protobuf/target_types_test.py index 374866dd1c5..7f2057ef3db 100644 --- a/src/python/pants/backend/codegen/protobuf/target_types_test.py +++ b/src/python/pants/backend/codegen/protobuf/target_types_test.py @@ -8,12 +8,12 @@ from pants.backend.codegen.protobuf import target_types from pants.backend.codegen.protobuf.target_types import ( - GenerateTargetsFromProtobufSources, ProtobufSourcesGeneratorTarget, ProtobufSourceTarget, ) from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import SingleSourceField, Tags from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -21,7 +21,7 @@ def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), - QueryRule(GeneratedTargets, [GenerateTargetsFromProtobufSources]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[ProtobufSourcesGeneratorTarget], ) @@ -42,8 +42,6 @@ def test_generate_source_targets() -> None: } ) - generator = rule_runner.get_target(Address("src/proto", target_name="lib")) - def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ProtobufSourceTarget: return ProtobufSourceTarget( {SingleSourceField.alias: rel_fp, Tags.alias: tags}, @@ -52,13 +50,10 @@ def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ProtobufSourceTarget: ) generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromProtobufSources(generator)] - ) - assert generated == GeneratedTargets( - generator, - { - gen_tgt("f1.proto", tags=["overridden"]), - gen_tgt("f2.proto"), - gen_tgt("subdir/f.proto"), - }, - ) + _TargetParametrizations, [Address("src/proto", target_name="lib")] + ).parametrizations + assert set(generated.values()) == { + gen_tgt("f1.proto", tags=["overridden"]), + gen_tgt("f2.proto"), + gen_tgt("subdir/f.proto"), + } diff --git a/src/python/pants/backend/codegen/thrift/target_types.py b/src/python/pants/backend/codegen/thrift/target_types.py index 41fb8a4c123..dd9524ef0b3 100644 --- a/src/python/pants/backend/codegen/thrift/target_types.py +++ b/src/python/pants/backend/codegen/thrift/target_types.py @@ -2,26 +2,22 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.codegen.thrift.subsystem import ThriftSubsystem -from pants.engine.fs import PathGlobs, Paths -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, AllTargets, Dependencies, - GeneratedTargets, - GenerateTargetsRequest, MultipleSourcesField, OverridesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, Target, + TargetFilesGenerator, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, Targets, generate_file_based_overrides_field_help_message, - generate_file_level_targets, ) -from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior +from pants.engine.unions import UnionRule from pants.util.docutil import doc_url from pants.util.logging import LogLevel @@ -41,6 +37,20 @@ def find_all_thrift_targets(targets: AllTargets) -> AllThriftTargets: return AllThriftTargets(tgt for tgt in targets if tgt.has_field(ThriftSourceField)) +class GeneratorSettingsRequest(TargetFilesGeneratorSettingsRequest): + pass + + +@rule +def generator_settings( + _: GeneratorSettingsRequest, + thrift: ThriftSubsystem, +) -> TargetFilesGeneratorSettings: + return TargetFilesGeneratorSettings( + add_dependencies_on_all_siblings=not thrift.dependency_inference + ) + + # ----------------------------------------------------------------------------------------------- # `thrift_source` target # ----------------------------------------------------------------------------------------------- @@ -82,7 +92,7 @@ class ThriftSourcesOverridesField(OverridesField): ) -class ThriftSourcesGeneratorTarget(Target): +class ThriftSourcesGeneratorTarget(TargetFilesGenerator): alias = "thrift_sources" core_fields = ( *COMMON_TARGET_FIELDS, @@ -90,47 +100,18 @@ class ThriftSourcesGeneratorTarget(Target): ThriftSourcesGeneratingSourcesField, ThriftSourcesOverridesField, ) - help = "Generate a `thrift_source` target for each file in the `sources` field." - - -class GenerateTargetsFromThriftSources(GenerateTargetsRequest): - generate_from = ThriftSourcesGeneratorTarget - - -@rule -async def generate_targets_from_thrift_sources( - request: GenerateTargetsFromThriftSources, - files_not_found_behavior: FilesNotFoundBehavior, - thrift: ThriftSubsystem, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ThriftSourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - ThriftSourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not thrift.dependency_inference, - overrides=all_overrides, + generated_target_cls = ThriftSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + ThriftDependenciesField, ) + moved_fields = () + settings_request_cls = GeneratorSettingsRequest + help = "Generate a `thrift_source` target for each file in the `sources` field." def rules(): - return ( + return [ *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromThriftSources), - ) + UnionRule(TargetFilesGeneratorSettingsRequest, GeneratorSettingsRequest), + ] diff --git a/src/python/pants/backend/codegen/thrift/target_types_test.py b/src/python/pants/backend/codegen/thrift/target_types_test.py index d0c0013a809..a88f62b34b2 100644 --- a/src/python/pants/backend/codegen/thrift/target_types_test.py +++ b/src/python/pants/backend/codegen/thrift/target_types_test.py @@ -8,12 +8,12 @@ from pants.backend.codegen.thrift import target_types from pants.backend.codegen.thrift.target_types import ( - GenerateTargetsFromThriftSources, ThriftSourcesGeneratorTarget, ThriftSourceTarget, ) from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import SingleSourceField, Tags from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -21,7 +21,7 @@ def test_generate_source_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), - QueryRule(GeneratedTargets, [GenerateTargetsFromThriftSources]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[ThriftSourcesGeneratorTarget], ) @@ -42,8 +42,6 @@ def test_generate_source_targets() -> None: } ) - generator = rule_runner.get_target(Address("src/thrift", target_name="lib")) - def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ThriftSourceTarget: return ThriftSourceTarget( {SingleSourceField.alias: rel_fp, Tags.alias: tags}, @@ -51,12 +49,11 @@ def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ThriftSourceTarget: residence_dir=os.path.dirname(os.path.join("src/thrift", rel_fp)), ) - generated = rule_runner.request(GeneratedTargets, [GenerateTargetsFromThriftSources(generator)]) - assert generated == GeneratedTargets( - generator, - { - gen_tgt("f1.thrift", tags=["overridden"]), - gen_tgt("f2.thrift"), - gen_tgt("subdir/f.thrift"), - }, - ) + generated = rule_runner.request( + _TargetParametrizations, [Address("src/thrift", target_name="lib")] + ).parametrizations + assert set(generated.values()) == { + gen_tgt("f1.thrift", tags=["overridden"]), + gen_tgt("f2.thrift"), + gen_tgt("subdir/f.thrift"), + } diff --git a/src/python/pants/backend/go/target_type_rules.py b/src/python/pants/backend/go/target_type_rules.py index 224e23b4ff1..e3b63417b6c 100644 --- a/src/python/pants/backend/go/target_type_rules.py +++ b/src/python/pants/backend/go/target_type_rules.py @@ -13,6 +13,7 @@ GoBinaryMainPackageField, GoBinaryMainPackageRequest, GoImportPathField, + GoModSourcesField, GoModTarget, GoPackageSourcesField, GoThirdPartyPackageDependenciesField, @@ -206,7 +207,7 @@ async def generate_targets_from_go_mod( union_membership: UnionMembership, ) -> GeneratedTargets: generator_addr = request.generator.address - go_mod_info = await Get(GoModInfo, GoModInfoRequest(generator_addr)) + go_mod_info = await Get(GoModInfo, GoModInfoRequest(request.generator[GoModSourcesField])) all_packages = await Get( AllThirdPartyPackages, AllThirdPartyPackagesRequest(go_mod_info.digest, go_mod_info.mod_path), diff --git a/src/python/pants/backend/go/target_type_rules_test.py b/src/python/pants/backend/go/target_type_rules_test.py index e5f0c9c99dc..6653da97513 100644 --- a/src/python/pants/backend/go/target_type_rules_test.py +++ b/src/python/pants/backend/go/target_type_rules_test.py @@ -8,10 +8,7 @@ import pytest from pants.backend.go import target_type_rules -from pants.backend.go.target_type_rules import ( - GenerateTargetsFromGoModRequest, - InjectGoBinaryMainDependencyRequest, -) +from pants.backend.go.target_type_rules import InjectGoBinaryMainDependencyRequest from pants.backend.go.target_types import ( GoBinaryMainPackage, GoBinaryMainPackageField, @@ -36,11 +33,11 @@ from pants.build_graph.address import Address from pants.core.target_types import GenericTarget from pants.engine.addresses import Addresses +from pants.engine.internals.graph import _TargetParametrizations from pants.engine.rules import QueryRule from pants.engine.target import ( Dependencies, DependenciesRequest, - GeneratedTargets, InjectedDependencies, InvalidFieldException, InvalidTargetException, @@ -64,6 +61,7 @@ def rule_runner() -> RuleRunner: *build_pkg.rules(), *link.rules(), *assembly.rules(), + QueryRule(_TargetParametrizations, [Address]), QueryRule(Addresses, [DependenciesRequest]), QueryRule(GoBinaryMainPackage, [GoBinaryMainPackageRequest]), QueryRule(InjectedDependencies, [InjectGoBinaryMainDependencyRequest]), @@ -196,8 +194,7 @@ def test_generate_package_targets(rule_runner: RuleRunner) -> None: "src/go/another_dir/subdir/f.go": "", } ) - generator = rule_runner.get_target(Address("src/go")) - generated = rule_runner.request(GeneratedTargets, [GenerateTargetsFromGoModRequest(generator)]) + generated = rule_runner.request(_TargetParametrizations, [Address("src/go")]).parametrizations def gen_third_party_tgt(import_path: str) -> GoThirdPartyPackageTarget: return GoThirdPartyPackageTarget( @@ -205,28 +202,22 @@ def gen_third_party_tgt(import_path: str) -> GoThirdPartyPackageTarget: Address("src/go", generated_name=import_path), ) - expected = GeneratedTargets( - generator, - { - gen_third_party_tgt(pkg) - for pkg in ( - "github.com/google/uuid", - "github.com/google/go-cmp/cmp", - "github.com/google/go-cmp/cmp/cmpopts", - "github.com/google/go-cmp/cmp/internal/diff", - "github.com/google/go-cmp/cmp/internal/flags", - "github.com/google/go-cmp/cmp/internal/function", - "github.com/google/go-cmp/cmp/internal/testprotos", - "github.com/google/go-cmp/cmp/internal/teststructs", - "github.com/google/go-cmp/cmp/internal/value", - "golang.org/x/xerrors", - "golang.org/x/xerrors/internal", - ) - }, - ) - assert list(generated.keys()) == list(expected.keys()) - for addr, tgt in generated.items(): - assert tgt == expected[addr] + assert set(generated.values()) == { + gen_third_party_tgt(pkg) + for pkg in ( + "github.com/google/uuid", + "github.com/google/go-cmp/cmp", + "github.com/google/go-cmp/cmp/cmpopts", + "github.com/google/go-cmp/cmp/internal/diff", + "github.com/google/go-cmp/cmp/internal/flags", + "github.com/google/go-cmp/cmp/internal/function", + "github.com/google/go-cmp/cmp/internal/testprotos", + "github.com/google/go-cmp/cmp/internal/teststructs", + "github.com/google/go-cmp/cmp/internal/value", + "golang.org/x/xerrors", + "golang.org/x/xerrors/internal", + ) + } def test_third_party_package_targets_cannot_be_manually_created() -> None: diff --git a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py index a8b575b3448..9440fc3d60f 100644 --- a/src/python/pants/backend/go/util_rules/first_party_pkg_test.py +++ b/src/python/pants/backend/go/util_rules/first_party_pkg_test.py @@ -28,16 +28,10 @@ FirstPartyPkgImportPath, FirstPartyPkgImportPathRequest, ) -from pants.core.target_types import ( - GenerateTargetsFromResources, - ResourcesGeneratorTarget, - generate_targets_from_resources, -) +from pants.core.target_types import ResourcesGeneratorTarget from pants.engine.addresses import Address from pants.engine.fs import PathGlobs, Snapshot from pants.engine.rules import QueryRule -from pants.engine.target import GenerateTargetsRequest -from pants.engine.unions import UnionRule from pants.testutil.rule_runner import RuleRunner, engine_error @@ -53,8 +47,6 @@ def rule_runner() -> RuleRunner: *build_pkg.rules(), *link.rules(), *assembly.rules(), - generate_targets_from_resources, - UnionRule(GenerateTargetsRequest, GenerateTargetsFromResources), QueryRule(FallibleFirstPartyPkgAnalysis, [FirstPartyPkgAnalysisRequest]), QueryRule(FallibleFirstPartyPkgDigest, [FirstPartyPkgDigestRequest]), QueryRule(FirstPartyPkgImportPath, [FirstPartyPkgImportPathRequest]), diff --git a/src/python/pants/backend/go/util_rules/go_mod.py b/src/python/pants/backend/go/util_rules/go_mod.py index 11f984c5955..b9a9af9f98b 100644 --- a/src/python/pants/backend/go/util_rules/go_mod.py +++ b/src/python/pants/backend/go/util_rules/go_mod.py @@ -74,18 +74,24 @@ class GoModInfo: @dataclass(frozen=True) class GoModInfoRequest(EngineAwareParameter): - address: Address + source: Address | GoModSourcesField def debug_hint(self) -> str: - return self.address.spec + if isinstance(self.source, Address): + return self.source.spec + else: + return self.source.address.spec @rule async def determine_go_mod_info( request: GoModInfoRequest, ) -> GoModInfo: - wrapped_target = await Get(WrappedTarget, Address, request.address) - sources_field = wrapped_target.target[GoModSourcesField] + if isinstance(request.source, Address): + wrapped_target = await Get(WrappedTarget, Address, request.source) + sources_field = wrapped_target.target[GoModSourcesField] + else: + sources_field = request.source go_mod_path = sources_field.go_mod_path go_mod_dir = os.path.dirname(go_mod_path) diff --git a/src/python/pants/backend/java/target_types.py b/src/python/pants/backend/java/target_types.py index 582eb5bf356..738bd4b1813 100644 --- a/src/python/pants/backend/java/target_types.py +++ b/src/python/pants/backend/java/target_types.py @@ -5,21 +5,16 @@ from dataclasses import dataclass -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import collect_rules from pants.engine.target import ( COMMON_TARGET_FIELDS, Dependencies, FieldSet, - GeneratedTargets, - GenerateTargetsRequest, MultipleSourcesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, Target, - generate_file_level_targets, + TargetFilesGenerator, ) -from pants.engine.unions import UnionMembership, UnionRule from pants.jvm.target_types import ( JunitTestSourceField, JvmCompatibleResolvesField, @@ -75,38 +70,25 @@ class JavaTestsGeneratorSourcesField(JavaGeneratorSourcesField): default = ("*Test.java",) -class JunitTestsGeneratorTarget(Target): +class JunitTestsGeneratorTarget(TargetFilesGenerator): alias = "junit_tests" core_fields = ( *COMMON_TARGET_FIELDS, + Dependencies, JavaTestsGeneratorSourcesField, + ) + generated_target_cls = JunitTestTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, Dependencies, + ) + moved_fields = ( JvmResolveField, JvmProvidesTypesField, ) help = "Generate a `junit_test` target for each file in the `sources` field." -class GenerateTargetsFromJunitTests(GenerateTargetsRequest): - generate_from = JunitTestsGeneratorTarget - - -@rule -async def generate_targets_from_junit_tests( - request: GenerateTargetsFromJunitTests, union_membership: UnionMembership -) -> GeneratedTargets: - paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[JavaTestsGeneratorSourcesField]) - ) - return generate_file_level_targets( - JunitTestTarget, - request.generator, - paths.files, - union_membership, - add_dependencies_on_all_siblings=False, - ) - - # ----------------------------------------------------------------------------------------------- # `java_source` and `java_sources` targets # ----------------------------------------------------------------------------------------------- @@ -128,42 +110,23 @@ class JavaSourcesGeneratorSourcesField(JavaGeneratorSourcesField): default = ("*.java",) + tuple(f"!{pat}" for pat in JavaTestsGeneratorSourcesField.default) -class JavaSourcesGeneratorTarget(Target): +class JavaSourcesGeneratorTarget(TargetFilesGenerator): alias = "java_sources" core_fields = ( *COMMON_TARGET_FIELDS, Dependencies, JavaSourcesGeneratorSourcesField, JvmCompatibleResolvesField, - JvmProvidesTypesField, ) - help = "Generate a `java_source` target for each file in the `sources` field." - - -class GenerateTargetsFromJavaSources(GenerateTargetsRequest): - generate_from = JavaSourcesGeneratorTarget - - -@rule -async def generate_targets_from_java_sources( - request: GenerateTargetsFromJavaSources, union_membership: UnionMembership -) -> GeneratedTargets: - paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[JavaSourcesGeneratorSourcesField]) - ) - return generate_file_level_targets( - JavaSourceTarget, - request.generator, - paths.files, - union_membership, - add_dependencies_on_all_siblings=False, - use_source_field=True, + generated_target_cls = JavaSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + JvmCompatibleResolvesField, ) + moved_fields = (JvmProvidesTypesField,) + help = "Generate a `java_source` target for each file in the `sources` field." def rules(): - return ( - *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromJunitTests), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromJavaSources), - ) + return collect_rules() diff --git a/src/python/pants/backend/plugin_development/pants_requirements_test.py b/src/python/pants/backend/plugin_development/pants_requirements_test.py index ebdd07af945..289129982c3 100644 --- a/src/python/pants/backend/plugin_development/pants_requirements_test.py +++ b/src/python/pants/backend/plugin_development/pants_requirements_test.py @@ -6,7 +6,6 @@ from pants.backend.plugin_development import pants_requirements from pants.backend.plugin_development.pants_requirements import ( - GenerateFromPantsRequirementsRequest, PantsRequirementsTargetGenerator, determine_version, ) @@ -17,7 +16,7 @@ PythonRequirementsField, ) from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets +from pants.engine.internals.graph import _TargetParametrizations from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -38,7 +37,7 @@ def test_target_generator() -> None: rule_runner = RuleRunner( rules=( *pants_requirements.rules(), - QueryRule(GeneratedTargets, [GenerateFromPantsRequirementsRequest]), + QueryRule(_TargetParametrizations, [Address]), ), target_types=[PantsRequirementsTargetGenerator], ) @@ -54,10 +53,9 @@ def test_target_generator() -> None: } ) - generator = rule_runner.get_target(Address("", target_name="default")) result = rule_runner.request( - GeneratedTargets, [GenerateFromPantsRequirementsRequest(generator)] - ) + _TargetParametrizations, [Address("", target_name="default")] + ).parametrizations assert len(result) == 2 pants_req = next(t for t in result.values() if t.address.generated_name == "pantsbuild.pants") testutil_req = next( @@ -74,10 +72,9 @@ def test_target_generator() -> None: for t in (pants_req, testutil_req): assert not t[PythonRequirementResolveField].value - generator = rule_runner.get_target(Address("", target_name="no_testutil")) result = rule_runner.request( - GeneratedTargets, [GenerateFromPantsRequirementsRequest(generator)] - ) + _TargetParametrizations, [Address("", target_name="no_testutil")] + ).parametrizations assert len(result) == 1 assert next(iter(result.keys())).generated_name == "pantsbuild.pants" pants_req = next(iter(result.values())) diff --git a/src/python/pants/backend/project_info/count_loc_test.py b/src/python/pants/backend/project_info/count_loc_test.py index e5c2672c3fb..12f842dc709 100644 --- a/src/python/pants/backend/project_info/count_loc_test.py +++ b/src/python/pants/backend/project_info/count_loc_test.py @@ -5,6 +5,7 @@ from pants.backend.project_info import count_loc from pants.backend.project_info.count_loc import CountLinesOfCode +from pants.backend.python import target_types_rules from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.util_rules import external_tool from pants.engine.target import MultipleSourcesField, Target @@ -23,7 +24,11 @@ class ElixirTarget(Target): @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - rules=[*count_loc.rules(), *external_tool.rules()], + rules=[ + *count_loc.rules(), + *external_tool.rules(), + *target_types_rules.rules(), + ], target_types=[PythonSourcesGeneratorTarget, ElixirTarget], ) diff --git a/src/python/pants/backend/project_info/dependencies_test.py b/src/python/pants/backend/project_info/dependencies_test.py index a4cd0da7d8a..8e75716923f 100644 --- a/src/python/pants/backend/project_info/dependencies_test.py +++ b/src/python/pants/backend/project_info/dependencies_test.py @@ -8,6 +8,7 @@ import pytest from pants.backend.project_info.dependencies import Dependencies, rules +from pants.backend.python import target_types_rules from pants.backend.python.target_types import PythonRequirementTarget, PythonSourcesGeneratorTarget from pants.engine.target import SpecialCasedDependencies, Target from pants.testutil.rule_runner import RuleRunner @@ -27,7 +28,10 @@ class SpecialDepsTarget(Target): @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - rules=rules(), + rules=[ + *rules(), + *target_types_rules.rules(), + ], target_types=[PythonSourcesGeneratorTarget, PythonRequirementTarget, SpecialDepsTarget], ) diff --git a/src/python/pants/backend/project_info/peek_test.py b/src/python/pants/backend/project_info/peek_test.py index 4909e0a6430..48b332b670a 100644 --- a/src/python/pants/backend/project_info/peek_test.py +++ b/src/python/pants/backend/project_info/peek_test.py @@ -8,7 +8,7 @@ from pants.backend.project_info import peek from pants.backend.project_info.peek import Peek, TargetData, TargetDatas from pants.base.specs import AddressSpecs, DescendantAddresses -from pants.core.target_types import ArchiveTarget, FilesGeneratorTarget, GenericTarget +from pants.core.target_types import ArchiveTarget, FilesGeneratorTarget, FileTarget, GenericTarget from pants.engine.addresses import Address from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -193,17 +193,29 @@ def test_get_target_data(rule_runner: RuleRunner) -> None: } ) tds = rule_runner.request(TargetDatas, [AddressSpecs([DescendantAddresses("foo")])]) - assert tds == TargetDatas( - [ - TargetData( - GenericTarget({"dependencies": [":baz"]}, Address("foo", target_name="bar")), - None, - ("foo:baz",), + assert list(tds) == [ + TargetData( + GenericTarget({"dependencies": [":baz"]}, Address("foo", target_name="bar")), + None, + ("foo/a.txt:baz", "foo/b.txt:baz"), + ), + TargetData( + FilesGeneratorTarget({"sources": ["*.txt"]}, Address("foo", target_name="baz")), + ("foo/a.txt", "foo/b.txt"), + ("foo/a.txt:baz", "foo/b.txt:baz"), + ), + TargetData( + FileTarget( + {"source": "a.txt"}, Address("foo", relative_file_path="a.txt", target_name="baz") ), - TargetData( - FilesGeneratorTarget({"sources": ["*.txt"]}, Address("foo", target_name="baz")), - ("foo/a.txt", "foo/b.txt"), - tuple(), + ("foo/a.txt",), + (), + ), + TargetData( + FileTarget( + {"source": "b.txt"}, Address("foo", relative_file_path="b.txt", target_name="baz") ), - ] - ) + ("foo/b.txt",), + (), + ), + ] diff --git a/src/python/pants/backend/python/goals/repl_integration_test.py b/src/python/pants/backend/python/goals/repl_integration_test.py index c3130cb692c..55ac85b031e 100644 --- a/src/python/pants/backend/python/goals/repl_integration_test.py +++ b/src/python/pants/backend/python/goals/repl_integration_test.py @@ -16,6 +16,7 @@ PythonSourcesGeneratorTarget, PythonSourceTarget, ) +from pants.backend.python.target_types_rules import rules as target_types_rules from pants.backend.python.util_rules import local_dists, pex_from_targets from pants.backend.python.util_rules.pex import PexProcess from pants.backend.python.util_rules.pex_from_targets import NoCompatibleResolveException @@ -41,6 +42,7 @@ def rule_runner() -> RuleRunner: *python_repl.rules(), *pex_from_targets.rules(), *local_dists.rules(), + *target_types_rules(), QueryRule(Process, (PexProcess,)), ], target_types=[ diff --git a/src/python/pants/backend/python/goals/setup_py_test.py b/src/python/pants/backend/python/goals/setup_py_test.py index 28f43eb720c..1a105022aa7 100644 --- a/src/python/pants/backend/python/goals/setup_py_test.py +++ b/src/python/pants/backend/python/goals/setup_py_test.py @@ -738,6 +738,7 @@ def test_get_requirements() -> None: get_requirements, get_owned_dependencies, get_exporting_owner, + *target_types_rules.rules(), SubsystemRule(SetupPyGeneration), QueryRule(ExportedTargetRequirements, (DependencyOwner,)), ] @@ -817,6 +818,7 @@ def test_get_requirements_with_exclude() -> None: get_requirements, get_owned_dependencies, get_exporting_owner, + *target_types_rules.rules(), SubsystemRule(SetupPyGeneration), QueryRule(ExportedTargetRequirements, (DependencyOwner,)), ] @@ -878,6 +880,7 @@ def test_owned_dependencies() -> None: rules=[ get_owned_dependencies, get_exporting_owner, + *target_types_rules.rules(), QueryRule(OwnedDependencies, (DependencyOwner,)), ] ) @@ -960,6 +963,7 @@ def exporting_owner_rule_runner() -> RuleRunner: return create_setup_py_rule_runner( rules=[ get_exporting_owner, + *target_types_rules.rules(), QueryRule(ExportedTarget, (OwnedDependency,)), ] ) diff --git a/src/python/pants/backend/python/lint/black/subsystem_test.py b/src/python/pants/backend/python/lint/black/subsystem_test.py index 59eabfb91f4..d3b3ad97f70 100644 --- a/src/python/pants/backend/python/lint/black/subsystem_test.py +++ b/src/python/pants/backend/python/lint/black/subsystem_test.py @@ -5,6 +5,7 @@ from textwrap import dedent +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import GeneratePythonLockfile from pants.backend.python.lint.black import skip_field from pants.backend.python.lint.black.subsystem import Black, BlackLockfileSentinel @@ -20,6 +21,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: rules=[ *subsystem_rules(), *skip_field.rules(), + *target_types_rules.rules(), QueryRule(GeneratePythonLockfile, [BlackLockfileSentinel]), ], target_types=[PythonSourcesGeneratorTarget, GenericTarget], diff --git a/src/python/pants/backend/python/macros/common_fields.py b/src/python/pants/backend/python/macros/common_fields.py index 92219aed66c..92e83f3414c 100644 --- a/src/python/pants/backend/python/macros/common_fields.py +++ b/src/python/pants/backend/python/macros/common_fields.py @@ -3,9 +3,7 @@ from __future__ import annotations -from typing import Any, ClassVar, Dict, Iterable, Tuple - -from packaging.utils import canonicalize_name as canonicalize_project_name +from typing import ClassVar, Dict, Iterable, Tuple from pants.backend.python.target_types import ( PythonRequirementModulesField, @@ -77,6 +75,3 @@ class RequirementsOverrideField(OverridesField): "You can specify the same requirement in multiple keys, so long as you don't " "override the same field more than one time for the requirement." ) - - def flatten_and_normalize(self) -> dict[str, dict[str, Any]]: - return {canonicalize_project_name(req): v for req, v in super().flatten().items()} diff --git a/src/python/pants/backend/python/macros/pipenv_requirements.py b/src/python/pants/backend/python/macros/pipenv_requirements.py index 14b853b8a0f..a9845303844 100644 --- a/src/python/pants/backend/python/macros/pipenv_requirements.py +++ b/src/python/pants/backend/python/macros/pipenv_requirements.py @@ -96,7 +96,7 @@ async def generate_from_pipenv_requirement( module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value - overrides = generator[RequirementsOverrideField].flatten_and_normalize() + overrides = {canonicalize_project_name(k): v for k, v in request.overrides.items()} inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() diff --git a/src/python/pants/backend/python/macros/pipenv_requirements_test.py b/src/python/pants/backend/python/macros/pipenv_requirements_test.py index b7c601ced4b..5fa96b92085 100644 --- a/src/python/pants/backend/python/macros/pipenv_requirements_test.py +++ b/src/python/pants/backend/python/macros/pipenv_requirements_test.py @@ -8,13 +8,11 @@ import pytest from pants.backend.python.macros import pipenv_requirements -from pants.backend.python.macros.pipenv_requirements import ( - GenerateFromPipenvRequirementsRequest, - PipenvRequirementsTargetGenerator, -) +from pants.backend.python.macros.pipenv_requirements import PipenvRequirementsTargetGenerator from pants.backend.python.target_types import PythonRequirementsFileTarget, PythonRequirementTarget from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, Target +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import Target from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -23,7 +21,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=( *pipenv_requirements.rules(), - QueryRule(GeneratedTargets, [GenerateFromPipenvRequirementsRequest]), + QueryRule(_TargetParametrizations, [Address]), ), target_types=[PipenvRequirementsTargetGenerator], ) @@ -37,11 +35,8 @@ def assert_pipenv_requirements( expected_targets: set[Target], ) -> None: rule_runner.write_files({"BUILD": build_file_entry, "Pipfile.lock": dumps(pipfile_lock)}) - generator = rule_runner.get_target(Address("", target_name="reqs")) - result = rule_runner.request( - GeneratedTargets, [GenerateFromPipenvRequirementsRequest(generator)] - ) - assert set(result.values()) == expected_targets + result = rule_runner.request(_TargetParametrizations, [Address("", target_name="reqs")]) + assert set(result.parametrizations.values()) == expected_targets def test_pipfile_lock(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py index 3f2ae51516f..a97d3f970e6 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements.py +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -453,7 +453,7 @@ async def generate_from_python_requirement( module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value - overrides = generator[RequirementsOverrideField].flatten_and_normalize() + overrides = {canonicalize_project_name(k): v for k, v in request.overrides.items()} inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py index c8408e9bb47..cc1c9cbb635 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -11,7 +11,6 @@ from pants.backend.python.macros import poetry_requirements from pants.backend.python.macros.poetry_requirements import ( - GenerateFromPoetryRequirementsRequest, PoetryRequirementsTargetGenerator, PyprojectAttr, PyProjectToml, @@ -26,7 +25,8 @@ from pants.backend.python.pip_requirement import PipRequirement from pants.backend.python.target_types import PythonRequirementsFileTarget, PythonRequirementTarget from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, Target +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import Target from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error # --------------------------------------------------------------------------------- @@ -402,7 +402,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *poetry_requirements.rules(), - QueryRule(GeneratedTargets, [GenerateFromPoetryRequirementsRequest]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[PoetryRequirementsTargetGenerator], ) @@ -417,11 +417,8 @@ def assert_poetry_requirements( pyproject_toml_relpath: str = "pyproject.toml", ) -> None: rule_runner.write_files({"BUILD": build_file_entry, pyproject_toml_relpath: pyproject_toml}) - generator = rule_runner.get_target(Address("", target_name="reqs")) - result = rule_runner.request( - GeneratedTargets, [GenerateFromPoetryRequirementsRequest(generator)] - ) - assert set(result.values()) == expected_targets + result = rule_runner.request(_TargetParametrizations, [Address("", target_name="reqs")]) + assert set(result.parametrizations.values()) == expected_targets def test_pyproject_toml(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/python/macros/python_requirements.py b/src/python/pants/backend/python/macros/python_requirements.py index 7f9f13c0710..1251b0e826e 100644 --- a/src/python/pants/backend/python/macros/python_requirements.py +++ b/src/python/pants/backend/python/macros/python_requirements.py @@ -108,7 +108,7 @@ async def generate_from_python_requirement( module_mapping = generator[ModuleMappingField].value stubs_mapping = generator[TypeStubsModuleMappingField].value - overrides = generator[RequirementsOverrideField].flatten_and_normalize() + overrides = {canonicalize_project_name(k): v for k, v in request.overrides.items()} inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() diff --git a/src/python/pants/backend/python/macros/python_requirements_test.py b/src/python/pants/backend/python/macros/python_requirements_test.py index 533320f1d8b..c4f6a5937f0 100644 --- a/src/python/pants/backend/python/macros/python_requirements_test.py +++ b/src/python/pants/backend/python/macros/python_requirements_test.py @@ -8,13 +8,11 @@ import pytest from pants.backend.python.macros import python_requirements -from pants.backend.python.macros.python_requirements import ( - GenerateFromPythonRequirementsRequest, - PythonRequirementsTargetGenerator, -) +from pants.backend.python.macros.python_requirements import PythonRequirementsTargetGenerator from pants.backend.python.target_types import PythonRequirementsFileTarget, PythonRequirementTarget from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, Target +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import Target from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error @@ -23,7 +21,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *python_requirements.rules(), - QueryRule(GeneratedTargets, [GenerateFromPythonRequirementsRequest]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[PythonRequirementsTargetGenerator], ) @@ -38,11 +36,8 @@ def assert_python_requirements( requirements_txt_relpath: str = "requirements.txt", ) -> None: rule_runner.write_files({"BUILD": build_file_entry, requirements_txt_relpath: requirements_txt}) - generator = rule_runner.get_target(Address("", target_name="reqs")) - result = rule_runner.request( - GeneratedTargets, [GenerateFromPythonRequirementsRequest(generator)] - ) - assert set(result.values()) == expected_targets + result = rule_runner.request(_TargetParametrizations, [Address("", target_name="reqs")]) + assert set(result.parametrizations.values()) == expected_targets def test_requirements_txt(rule_runner: RuleRunner) -> None: diff --git a/src/python/pants/backend/python/subsystems/ipython_test.py b/src/python/pants/backend/python/subsystems/ipython_test.py index d9b1ed8a8cb..3abc032fc34 100644 --- a/src/python/pants/backend/python/subsystems/ipython_test.py +++ b/src/python/pants/backend/python/subsystems/ipython_test.py @@ -5,6 +5,7 @@ from textwrap import dedent +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import GeneratePythonLockfile from pants.backend.python.subsystems.ipython import IPythonLockfileSentinel from pants.backend.python.subsystems.ipython import rules as subsystem_rules @@ -16,7 +17,11 @@ def test_setup_lockfile_interpreter_constraints() -> None: rule_runner = RuleRunner( - rules=[*subsystem_rules(), QueryRule(GeneratePythonLockfile, [IPythonLockfileSentinel])], + rules=[ + *subsystem_rules(), + *target_types_rules.rules(), + QueryRule(GeneratePythonLockfile, [IPythonLockfileSentinel]), + ], target_types=[PythonSourcesGeneratorTarget, GenericTarget], ) diff --git a/src/python/pants/backend/python/subsystems/setuptools_test.py b/src/python/pants/backend/python/subsystems/setuptools_test.py index 4a20c9bccb4..4fcfe51e74f 100644 --- a/src/python/pants/backend/python/subsystems/setuptools_test.py +++ b/src/python/pants/backend/python/subsystems/setuptools_test.py @@ -5,6 +5,7 @@ from textwrap import dedent +from pants.backend.python import target_types_rules from pants.backend.python.goals.lockfile import GeneratePythonLockfile from pants.backend.python.macros.python_artifact import PythonArtifact from pants.backend.python.subsystems import setuptools @@ -18,6 +19,7 @@ def test_setup_lockfile_interpreter_constraints() -> None: rule_runner = RuleRunner( rules=[ *setuptools.rules(), + *target_types_rules.rules(), QueryRule(GeneratePythonLockfile, [SetuptoolsLockfileSentinel]), ], target_types=[PythonSourcesGeneratorTarget, PythonDistribution], diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 210e9c5d026..99fc13e68de 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -53,6 +53,8 @@ StringField, StringSequenceField, Target, + TargetFilesGenerator, + TargetFilesGeneratorSettingsRequest, TriBoolField, ValidNumbers, generate_file_based_overrides_field_help_message, @@ -127,6 +129,15 @@ def normalized_value(self, python_setup: PythonSetup) -> str: return resolve +# ----------------------------------------------------------------------------------------------- +# Target generation support +# ----------------------------------------------------------------------------------------------- + + +class PythonFilesGeneratorSettingsRequest(TargetFilesGeneratorSettingsRequest): + pass + + # ----------------------------------------------------------------------------------------------- # `pex_binary` and `pex_binaries` target # ----------------------------------------------------------------------------------------------- @@ -647,11 +658,9 @@ class SkipPythonTestsField(BoolField): help = "If true, don't run this target's tests." -_PYTHON_TEST_COMMON_FIELDS = ( +_PYTHON_TEST_MOVED_FIELDS = ( *COMMON_TARGET_FIELDS, - InterpreterConstraintsField, PythonResolveField, - PythonTestsDependenciesField, PythonTestsTimeoutField, RuntimePackageDependenciesField, PythonTestsExtraEnvVarsField, @@ -661,7 +670,12 @@ class SkipPythonTestsField(BoolField): class PythonTestTarget(Target): alias = "python_test" - core_fields = (*_PYTHON_TEST_COMMON_FIELDS, PythonTestSourceField) + core_fields = ( + *_PYTHON_TEST_MOVED_FIELDS, + PythonTestsDependenciesField, + PythonTestSourceField, + InterpreterConstraintsField, + ) help = ( "A single Python test file, written in either Pytest style or unittest style.\n\n" "All test util code, including `conftest.py`, should go into a dedicated `python_source` " @@ -704,13 +718,21 @@ class PythonTestsOverrideField(OverridesField): ) -class PythonTestsGeneratorTarget(Target): +class PythonTestsGeneratorTarget(TargetFilesGenerator): alias = "python_tests" core_fields = ( - *_PYTHON_TEST_COMMON_FIELDS, + PythonTestsDependenciesField, PythonTestsGeneratingSourcesField, + InterpreterConstraintsField, PythonTestsOverrideField, ) + generated_target_cls = PythonTestTarget + copied_fields = ( + PythonTestsDependenciesField, + InterpreterConstraintsField, + ) + moved_fields = _PYTHON_TEST_MOVED_FIELDS + settings_request_cls = PythonFilesGeneratorSettingsRequest help = "Generate a `python_test` target for each file in the `sources` field." @@ -756,17 +778,23 @@ class PythonSourcesGeneratingSourcesField(PythonGeneratingSourcesBase): ) -class PythonTestUtilsGeneratorTarget(Target): +class PythonTestUtilsGeneratorTarget(TargetFilesGenerator): alias = "python_test_utils" # Keep in sync with `PythonSourcesGeneratorTarget`, outside of the `sources` field. core_fields = ( *COMMON_TARGET_FIELDS, - InterpreterConstraintsField, Dependencies, - PythonResolveField, PythonTestUtilsGeneratingSourcesField, PythonSourcesOverridesField, ) + generated_target_cls = PythonSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + InterpreterConstraintsField, + ) + moved_fields = (PythonResolveField,) + settings_request_cls = PythonFilesGeneratorSettingsRequest help = ( "Generate a `python_source` target for each file in the `sources` field.\n\n" "This target generator is intended for test utility files like `conftest.py`, although it " @@ -776,17 +804,24 @@ class PythonTestUtilsGeneratorTarget(Target): ) -class PythonSourcesGeneratorTarget(Target): +class PythonSourcesGeneratorTarget(TargetFilesGenerator): alias = "python_sources" # Keep in sync with `PythonTestUtilsGeneratorTarget`, outside of the `sources` field. core_fields = ( *COMMON_TARGET_FIELDS, - InterpreterConstraintsField, Dependencies, - PythonResolveField, PythonSourcesGeneratingSourcesField, + InterpreterConstraintsField, PythonSourcesOverridesField, ) + generated_target_cls = PythonSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + InterpreterConstraintsField, + ) + moved_fields = (PythonResolveField,) + settings_request_cls = PythonFilesGeneratorSettingsRequest help = ( "Generate a `python_source` target for each file in the `sources` field.\n\n" "You can either use this target generator or `python_test_utils` for test utility files " diff --git a/src/python/pants/backend/python/target_types_rules.py b/src/python/pants/backend/python/target_types_rules.py index 013d838abdc..21832a0ad4e 100644 --- a/src/python/pants/backend/python/target_types_rules.py +++ b/src/python/pants/backend/python/target_types_rules.py @@ -32,16 +32,9 @@ PythonDistributionDependenciesField, PythonDistributionEntryPoint, PythonDistributionEntryPointsField, + PythonFilesGeneratorSettingsRequest, PythonProvidesField, PythonResolveField, - PythonSourcesGeneratingSourcesField, - PythonSourcesGeneratorTarget, - PythonSourceTarget, - PythonTestsGeneratingSourcesField, - PythonTestsGeneratorTarget, - PythonTestTarget, - PythonTestUtilsGeneratingSourcesField, - PythonTestUtilsGeneratorTarget, ResolvedPexEntryPoint, ResolvedPythonDistributionEntryPoints, ResolvePexEntryPointRequest, @@ -60,14 +53,12 @@ InjectedDependencies, InvalidFieldException, OverridesField, - SourcesPaths, - SourcesPathsRequest, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, Targets, WrappedTarget, - generate_file_level_targets, ) from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior from pants.source.source_root import SourceRoot, SourceRootRequest from pants.util.docutil import doc_url from pants.util.frozendict import FrozenDict @@ -77,120 +68,21 @@ logger = logging.getLogger(__name__) -# ----------------------------------------------------------------------------------------------- -# `python_sources`, `python_tests`, and `python_test_utils` target generation rules -# ----------------------------------------------------------------------------------------------- - - -class GenerateTargetsFromPythonTests(GenerateTargetsRequest): - generate_from = PythonTestsGeneratorTarget - - -@rule -async def generate_targets_from_python_tests( - request: GenerateTargetsFromPythonTests, - files_not_found_behavior: FilesNotFoundBehavior, - python_infer: PythonInferSubsystem, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[PythonTestsGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - PythonTestTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not python_infer.imports, - overrides=all_overrides, - ) - - -class GenerateTargetsFromPythonSources(GenerateTargetsRequest): - generate_from = PythonSourcesGeneratorTarget - - -@rule -async def generate_targets_from_python_sources( - request: GenerateTargetsFromPythonSources, - python_infer: PythonInferSubsystem, - files_not_found_behavior: FilesNotFoundBehavior, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[PythonSourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - PythonSourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not python_infer.imports, - overrides=all_overrides, - ) - - -class GenerateTargetsFromPythonTestUtils(GenerateTargetsRequest): - generate_from = PythonTestUtilsGeneratorTarget - - @rule -async def generate_targets_from_python_test_utils( - request: GenerateTargetsFromPythonTestUtils, +def python_files_generator_settings( + _: PythonFilesGeneratorSettingsRequest, python_infer: PythonInferSubsystem, - files_not_found_behavior: FilesNotFoundBehavior, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[PythonTestUtilsGeneratingSourcesField]) - ) +) -> TargetFilesGeneratorSettings: + return TargetFilesGeneratorSettings(add_dependencies_on_all_siblings=not python_infer.imports) - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - return generate_file_level_targets( - PythonSourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not python_infer.imports, - overrides=all_overrides, - ) +# ----------------------------------------------------------------------------------------------- +# `pex_binary` target generation rules +# ----------------------------------------------------------------------------------------------- class GenerateTargetsFromPexBinaries(GenerateTargetsRequest): + # TODO: This can be deprecated in favor of `parametrize`. generate_from = PexBinariesGeneratorTarget @@ -201,7 +93,7 @@ async def generate_targets_from_pex_binaries( ) -> GeneratedTargets: generator_addr = request.generator.address entry_points_field = request.generator[PexEntryPointsField].value or [] - overrides = request.generator[OverridesField].flatten() + overrides = request.overrides inherited_fields = { field.alias: field.value for field in request.generator.field_values.values() @@ -552,9 +444,7 @@ def rules(): return ( *collect_rules(), *import_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonTests), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonSources), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromPythonTestUtils), + UnionRule(TargetFilesGeneratorSettingsRequest, PythonFilesGeneratorSettingsRequest), UnionRule(GenerateTargetsRequest, GenerateTargetsFromPexBinaries), UnionRule(InjectDependenciesRequest, InjectPexBinaryEntryPointDependency), UnionRule(InjectDependenciesRequest, InjectPythonDistributionDependencies), diff --git a/src/python/pants/backend/python/target_types_test.py b/src/python/pants/backend/python/target_types_test.py index f05b0f849e8..8d0f355369e 100644 --- a/src/python/pants/backend/python/target_types_test.py +++ b/src/python/pants/backend/python/target_types_test.py @@ -40,10 +40,6 @@ parse_requirements_file, ) from pants.backend.python.target_types_rules import ( - GenerateTargetsFromPexBinaries, - GenerateTargetsFromPythonSources, - GenerateTargetsFromPythonTests, - GenerateTargetsFromPythonTestUtils, InjectPexBinaryEntryPointDependency, InjectPythonDistributionDependencies, resolve_pex_entry_point, @@ -51,9 +47,9 @@ from pants.backend.python.util_rules import python_sources from pants.core.goals.generate_lockfiles import UnrecognizedResolveNamesError from pants.engine.addresses import Address +from pants.engine.internals.graph import _TargetParametrizations from pants.engine.internals.scheduler import ExecutionError from pants.engine.target import ( - GeneratedTargets, InjectedDependencies, InvalidFieldException, InvalidFieldTypeException, @@ -563,10 +559,7 @@ def test_generate_source_and_test_targets() -> None: *target_types_rules.rules(), *import_rules(), *python_sources.rules(), - QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTests]), - QueryRule(GeneratedTargets, [GenerateTargetsFromPythonSources]), - QueryRule(GeneratedTargets, [GenerateTargetsFromPythonTestUtils]), - QueryRule(GeneratedTargets, [GenerateTargetsFromPexBinaries]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[ PythonTestsGeneratorTarget, @@ -623,11 +616,6 @@ def test_generate_source_and_test_targets() -> None: } ) - sources_generator = rule_runner.get_target(Address("src/py", target_name="lib")) - tests_generator = rule_runner.get_target(Address("src/py", target_name="tests")) - test_utils_generator = rule_runner.get_target(Address("src/py", target_name="test_utils")) - pex_binaries_generator = rule_runner.get_target(Address("src/py", target_name="pexes")) - def gen_source_tgt( rel_fp: str, tags: list[str] | None = None, *, tgt_name: str ) -> PythonSourceTarget: @@ -652,49 +640,36 @@ def gen_pex_binary_tgt(entry_point: str, tags: list[str] | None = None) -> PexBi ) sources_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromPythonSources(sources_generator)] - ) + _TargetParametrizations, [Address("src/py", target_name="lib")] + ).parametrizations tests_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromPythonTests(tests_generator)] - ) + _TargetParametrizations, [Address("src/py", target_name="tests")] + ).parametrizations test_utils_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromPythonTestUtils(test_utils_generator)] - ) + _TargetParametrizations, [Address("src/py", target_name="test_utils")] + ).parametrizations pex_binaries_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromPexBinaries(pex_binaries_generator)] - ) - - assert sources_generated == GeneratedTargets( - sources_generator, - { - gen_source_tgt("f1.py", tags=["overridden"], tgt_name="lib"), - gen_source_tgt("f2.py", tgt_name="lib"), - gen_source_tgt("subdir/f.py", tgt_name="lib"), - }, - ) - assert tests_generated == GeneratedTargets( - tests_generator, - { - gen_test_tgt("f1_test.py", tags=["overridden"]), - gen_test_tgt("f2_test.py"), - gen_test_tgt("subdir/f_test.py"), - }, - ) + _TargetParametrizations, [Address("src/py", target_name="pexes")] + ).parametrizations - assert test_utils_generated == GeneratedTargets( - test_utils_generator, - { - gen_source_tgt("conftest.py", tags=["overridden"], tgt_name="test_utils"), - gen_source_tgt("subdir/conftest.py", tgt_name="test_utils"), - }, - ) + assert set(sources_generated.values()) == { + gen_source_tgt("f1.py", tags=["overridden"], tgt_name="lib"), + gen_source_tgt("f2.py", tgt_name="lib"), + gen_source_tgt("subdir/f.py", tgt_name="lib"), + } + assert set(tests_generated.values()) == { + gen_test_tgt("f1_test.py", tags=["overridden"]), + gen_test_tgt("f2_test.py"), + gen_test_tgt("subdir/f_test.py"), + } - assert pex_binaries_generated == GeneratedTargets( - pex_binaries_generator, - { - gen_pex_binary_tgt("f1.py"), - gen_pex_binary_tgt("f2:foo", tags=["overridden"]), - gen_pex_binary_tgt("subdir.f.py", tags=["overridden"]), - gen_pex_binary_tgt("subdir.f:main"), - }, - ) + assert set(test_utils_generated.values()) == { + gen_source_tgt("conftest.py", tags=["overridden"], tgt_name="test_utils"), + gen_source_tgt("subdir/conftest.py", tgt_name="test_utils"), + } + assert set(pex_binaries_generated.values()) == { + gen_pex_binary_tgt("f1.py"), + gen_pex_binary_tgt("f2:foo", tags=["overridden"]), + gen_pex_binary_tgt("subdir.f.py", tags=["overridden"]), + gen_pex_binary_tgt("subdir.f:main"), + } diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py index aa3ad002a7e..3c303a66a72 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets_test.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets_test.py @@ -13,6 +13,7 @@ import pytest +from pants.backend.python import target_types_rules from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import ( PythonRequirementsField, @@ -46,6 +47,7 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *pex_from_targets.rules(), + *target_types_rules.rules(), QueryRule(PexRequest, (PexFromTargetsRequest,)), QueryRule(GlobalRequirementConstraints, ()), QueryRule(ChosenPythonResolve, [ChosenPythonResolveRequest]), @@ -92,28 +94,41 @@ def test_no_compatible_resolve_error() -> None: def test_choose_compatible_resolve(rule_runner: RuleRunner) -> None: - def create_build(*, req_resolve: str, source_resolve: str, test_resolve: str) -> str: - return dedent( - f"""\ - python_source(name="dep", source="dep.py", resolve="{source_resolve}") - python_requirement( - name="req", requirements=[], resolve="{req_resolve}" - ) - python_test( - name="test", - source="tests.py", - dependencies=[":dep", ":req"], - resolve="{test_resolve}", - ) - """ - ) + def create_target_files( + directory: str, *, req_resolve: str, source_resolve: str, test_resolve: str + ) -> dict[str | PurePath, str | bytes]: + return { + f"{directory}/BUILD": dedent( + f"""\ + python_source(name="dep", source="dep.py", resolve="{source_resolve}") + python_requirement( + name="req", requirements=[], resolve="{req_resolve}" + ) + python_test( + name="test", + source="tests.py", + dependencies=[":dep", ":req"], + resolve="{test_resolve}", + ) + """ + ), + f"{directory}/tests.py": "", + f"{directory}/dep.py": "", + } - rule_runner.set_options(["--python-resolves={'a': '', 'b': ''}", "--python-enable-resolves"]) + rule_runner.set_options( + ["--python-resolves={'a': '', 'b': ''}", "--python-enable-resolves"], env_inherit={"PATH"} + ) rule_runner.write_files( { # Note that each of these BUILD files are entirely self-contained. - "valid/BUILD": create_build(req_resolve="a", source_resolve="a", test_resolve="a"), - "invalid/BUILD": create_build(req_resolve="a", source_resolve="b", test_resolve="b"), + **create_target_files("valid", req_resolve="a", source_resolve="a", test_resolve="a"), + **create_target_files( + "invalid", + req_resolve="a", + source_resolve="a", + test_resolve="b", + ), } ) diff --git a/src/python/pants/backend/scala/compile/scalac_test.py b/src/python/pants/backend/scala/compile/scalac_test.py index 19cb45f8901..9a3e0834304 100644 --- a/src/python/pants/backend/scala/compile/scalac_test.py +++ b/src/python/pants/backend/scala/compile/scalac_test.py @@ -33,7 +33,7 @@ maybe_skip_jdk_test, ) from pants.jvm.util_rules import rules as util_rules -from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner, logging +from pants.testutil.rule_runner import PYTHON_BOOTSTRAP_ENV, QueryRule, RuleRunner @pytest.fixture @@ -88,7 +88,6 @@ def main(args: Array[String]): Unit = { ) -@logging @maybe_skip_jdk_test def test_compile_no_deps(rule_runner: RuleRunner) -> None: rule_runner.write_files( @@ -135,7 +134,6 @@ def test_compile_no_deps(rule_runner: RuleRunner) -> None: assert check_result.exit_code == 0 -@logging @maybe_skip_jdk_test def test_compile_with_deps(rule_runner: RuleRunner) -> None: rule_runner.write_files( @@ -183,7 +181,6 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None: @maybe_skip_jdk_test -@logging def test_compile_with_missing_dep_fails(rule_runner: RuleRunner) -> None: rule_runner.write_files( { diff --git a/src/python/pants/backend/scala/target_types.py b/src/python/pants/backend/scala/target_types.py index 4b8745ca3a6..97a6e32c3f0 100644 --- a/src/python/pants/backend/scala/target_types.py +++ b/src/python/pants/backend/scala/target_types.py @@ -5,22 +5,20 @@ from dataclasses import dataclass -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, Dependencies, FieldSet, - GeneratedTargets, - GenerateTargetsRequest, MultipleSourcesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, StringField, Target, - generate_file_level_targets, + TargetFilesGenerator, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, ) -from pants.engine.unions import UnionMembership, UnionRule +from pants.engine.unions import UnionRule from pants.jvm.target_types import ( JunitTestSourceField, JvmCompatibleResolvesField, @@ -29,6 +27,16 @@ ) +class ScalaSettingsRequest(TargetFilesGeneratorSettingsRequest): + pass + + +@rule +def scala_settings_request(_: ScalaSettingsRequest) -> TargetFilesGeneratorSettings: + # TODO: See https://github.com/pantsbuild/pants/issues/14382. + return TargetFilesGeneratorSettings(add_dependencies_on_all_siblings=True) + + class ScalaSourceField(SingleSourceField): expected_file_extensions = (".scala",) @@ -76,42 +84,29 @@ class ScalatestTestsGeneratorSourcesField(ScalaGeneratorSourcesField): default = ("*Spec.scala", "*Suite.scala") -class ScalatestTestsGeneratorTarget(Target): +class ScalatestTestsGeneratorTarget(TargetFilesGenerator): alias = "scalatest_tests" core_fields = ( *COMMON_TARGET_FIELDS, ScalatestTestsGeneratorSourcesField, Dependencies, + ) + generated_target_cls = ScalatestTestTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + ) + moved_fields = ( JvmResolveField, JvmProvidesTypesField, ) + settings_request_cls = ScalaSettingsRequest help = ( "Generate a `scalatest_test` target for each file in the `sources` field (defaults to " f"all files in the directory matching {ScalatestTestsGeneratorSourcesField.default})." ) -class GenerateTargetsFromScalatestTests(GenerateTargetsRequest): - generate_from = ScalatestTestsGeneratorTarget - - -@rule -async def generate_targets_from_scala_scalatest_tests( - request: GenerateTargetsFromScalatestTests, union_membership: UnionMembership -) -> GeneratedTargets: - paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ScalatestTestsGeneratorSourcesField]) - ) - return generate_file_level_targets( - ScalatestTestTarget, - request.generator, - paths.files, - union_membership, - add_dependencies_on_all_siblings=True, - use_source_field=True, - ) - - # ----------------------------------------------------------------------------------------------- # `scala_junit_tests` # ----------------------------------------------------------------------------------------------- @@ -137,39 +132,26 @@ class ScalaJunitTestsGeneratorSourcesField(ScalaGeneratorSourcesField): default = ("*Test.scala",) -class ScalaJunitTestsGeneratorTarget(Target): +class ScalaJunitTestsGeneratorTarget(TargetFilesGenerator): alias = "scala_junit_tests" core_fields = ( *COMMON_TARGET_FIELDS, ScalaJunitTestsGeneratorSourcesField, Dependencies, + ) + generated_target_cls = ScalaJunitTestTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + ) + moved_fields = ( JvmResolveField, JvmProvidesTypesField, ) + settings_request_cls = ScalaSettingsRequest help = "Generate a `scala_junit_test` target for each file in the `sources` field." -class GenerateTargetsFromScalaJunitTests(GenerateTargetsRequest): - generate_from = ScalaJunitTestsGeneratorTarget - - -@rule -async def generate_targets_from_scala_junit_tests( - request: GenerateTargetsFromScalaJunitTests, union_membership: UnionMembership -) -> GeneratedTargets: - paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ScalaJunitTestsGeneratorSourcesField]) - ) - return generate_file_level_targets( - ScalaJunitTestTarget, - request.generator, - paths.files, - union_membership, - add_dependencies_on_all_siblings=True, - use_source_field=True, - ) - - # ----------------------------------------------------------------------------------------------- # `scala_source` target # ----------------------------------------------------------------------------------------------- @@ -202,39 +184,25 @@ class ScalaSourcesGeneratorSourcesField(ScalaGeneratorSourcesField): ) -class ScalaSourcesGeneratorTarget(Target): +class ScalaSourcesGeneratorTarget(TargetFilesGenerator): alias = "scala_sources" core_fields = ( *COMMON_TARGET_FIELDS, Dependencies, + JvmCompatibleResolvesField, ScalaSourcesGeneratorSourcesField, + ) + generated_target_cls = ScalaSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, JvmCompatibleResolvesField, - JvmProvidesTypesField, ) + moved_fields = (JvmProvidesTypesField,) + settings_request_cls = ScalaSettingsRequest help = "Generate a `scala_source` target for each file in the `sources` field." -class GenerateTargetsFromScalaSources(GenerateTargetsRequest): - generate_from = ScalaSourcesGeneratorTarget - - -@rule -async def generate_targets_from_scala_sources( - request: GenerateTargetsFromScalaSources, union_membership: UnionMembership -) -> GeneratedTargets: - paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ScalaSourcesGeneratorSourcesField]) - ) - return generate_file_level_targets( - ScalaSourceTarget, - request.generator, - paths.files, - union_membership, - add_dependencies_on_all_siblings=True, - use_source_field=True, - ) - - # ----------------------------------------------------------------------------------------------- # `scalac_plugin` target # ----------------------------------------------------------------------------------------------- @@ -274,7 +242,5 @@ class ScalacPluginTarget(Target): def rules(): return ( *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromScalaJunitTests), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromScalaSources), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromScalatestTests), + UnionRule(TargetFilesGeneratorSettingsRequest, ScalaSettingsRequest), ) diff --git a/src/python/pants/backend/shell/shell_command.py b/src/python/pants/backend/shell/shell_command.py index 896bd40416f..18b57f7fcfc 100644 --- a/src/python/pants/backend/shell/shell_command.py +++ b/src/python/pants/backend/shell/shell_command.py @@ -51,6 +51,7 @@ GeneratedSources, GenerateSourcesRequest, SourcesField, + Target, TransitiveTargets, TransitiveTargetsRequest, WrappedTarget, @@ -67,8 +68,8 @@ class GenerateFilesFromShellCommandRequest(GenerateSourcesRequest): @dataclass(frozen=True) -class ShellCommandProcessRequest(WrappedTarget): - pass +class ShellCommandProcessRequest: + target: Target class RunShellCommand(RunFieldSet): diff --git a/src/python/pants/backend/shell/target_types.py b/src/python/pants/backend/shell/target_types.py index 453e561c854..ecad7d269f6 100644 --- a/src/python/pants/backend/shell/target_types.py +++ b/src/python/pants/backend/shell/target_types.py @@ -9,30 +9,26 @@ from pants.backend.shell.shell_setup import ShellSetup from pants.core.goals.test import RuntimePackageDependenciesField -from pants.engine.fs import PathGlobs, Paths from pants.engine.process import BinaryPathTest -from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.rules import collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, BoolField, Dependencies, - GeneratedTargets, - GenerateTargetsRequest, IntField, MultipleSourcesField, OverridesField, SingleSourceField, - SourcesPaths, - SourcesPathsRequest, StringField, StringSequenceField, Target, + TargetFilesGenerator, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, ValidNumbers, generate_file_based_overrides_field_help_message, - generate_file_level_targets, ) -from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior +from pants.engine.unions import UnionRule from pants.util.enums import match @@ -46,6 +42,20 @@ class ShellGeneratingSourcesBase(MultipleSourcesField): uses_source_roots = False +class ShellGeneratorSettingsRequest(TargetFilesGeneratorSettingsRequest): + pass + + +@rule +def generator_settings( + _: ShellGeneratorSettingsRequest, + shell_setup: ShellSetup, +) -> TargetFilesGeneratorSettings: + return TargetFilesGeneratorSettings( + add_dependencies_on_all_siblings=not shell_setup.dependency_inference + ) + + # ----------------------------------------------------------------------------------------------- # `shunit2_test` target # ----------------------------------------------------------------------------------------------- @@ -167,57 +177,28 @@ class Shunit2TestsOverrideField(OverridesField): ) -class Shunit2TestsGeneratorTarget(Target): +class Shunit2TestsGeneratorTarget(TargetFilesGenerator): alias = "shunit2_tests" core_fields = ( *COMMON_TARGET_FIELDS, Shunit2TestsGeneratorSourcesField, Shunit2TestDependenciesField, + Shunit2TestsOverrideField, + ) + generated_target_cls = Shunit2TestTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Shunit2TestDependenciesField, + ) + moved_fields = ( Shunit2TestTimeoutField, SkipShunit2TestsField, Shunit2ShellField, RuntimePackageDependenciesField, - Shunit2TestsOverrideField, ) help = "Generate a `shunit2_test` target for each file in the `sources` field." -class GenerateTargetsFromShunit2Tests(GenerateTargetsRequest): - generate_from = Shunit2TestsGeneratorTarget - - -@rule -async def generate_targets_from_shunit2_tests( - request: GenerateTargetsFromShunit2Tests, - files_not_found_behavior: FilesNotFoundBehavior, - shell_setup: ShellSetup, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[Shunit2TestsGeneratorSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - Shunit2TestTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not shell_setup.dependency_inference, - overrides=all_overrides, - ) - - # ----------------------------------------------------------------------------------------------- # `shell_source` and `shell_sources` targets # ----------------------------------------------------------------------------------------------- @@ -246,7 +227,7 @@ class ShellSourcesOverridesField(OverridesField): ) -class ShellSourcesGeneratorTarget(Target): +class ShellSourcesGeneratorTarget(TargetFilesGenerator): alias = "shell_sources" core_fields = ( *COMMON_TARGET_FIELDS, @@ -254,43 +235,13 @@ class ShellSourcesGeneratorTarget(Target): ShellSourcesGeneratingSourcesField, ShellSourcesOverridesField, ) - help = "Generate a `shell_source` target for each file in the `sources` field." - - -class GenerateTargetsFromShellSources(GenerateTargetsRequest): - generate_from = ShellSourcesGeneratorTarget - - -@rule -async def generate_targets_from_shell_sources( - request: GenerateTargetsFromShellSources, - files_not_found_behavior: FilesNotFoundBehavior, - shell_setup: ShellSetup, - union_membership: UnionMembership, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ShellSourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - ShellSourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=not shell_setup.dependency_inference, - overrides=all_overrides, + generated_target_cls = ShellSourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, ) + moved_fields = () + help = "Generate a `shell_source` target for each file in the `sources` field." # ----------------------------------------------------------------------------------------------- @@ -418,8 +369,7 @@ class ShellCommandRunTarget(Target): def rules(): - return ( + return [ *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromShunit2Tests), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromShellSources), - ) + UnionRule(TargetFilesGeneratorSettingsRequest, ShellGeneratorSettingsRequest), + ] diff --git a/src/python/pants/backend/shell/target_types_test.py b/src/python/pants/backend/shell/target_types_test.py index 3e8e966057f..367b59f7520 100644 --- a/src/python/pants/backend/shell/target_types_test.py +++ b/src/python/pants/backend/shell/target_types_test.py @@ -10,8 +10,6 @@ from pants.backend.shell import target_types from pants.backend.shell.target_types import ( - GenerateTargetsFromShellSources, - GenerateTargetsFromShunit2Tests, ShellSourcesGeneratorTarget, ShellSourceTarget, Shunit2Shell, @@ -19,7 +17,8 @@ Shunit2TestTarget, ) from pants.engine.addresses import Address -from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.engine.internals.graph import _TargetParametrizations +from pants.engine.target import SingleSourceField, Tags from pants.testutil.rule_runner import QueryRule, RuleRunner @@ -69,8 +68,7 @@ def test_generate_source_and_test_targets() -> None: rule_runner = RuleRunner( rules=[ *target_types.rules(), - QueryRule(GeneratedTargets, [GenerateTargetsFromShunit2Tests]), - QueryRule(GeneratedTargets, [GenerateTargetsFromShellSources]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[Shunit2TestsGeneratorTarget, ShellSourcesGeneratorTarget], ) @@ -100,9 +98,6 @@ def test_generate_source_and_test_targets() -> None: } ) - sources_generator = rule_runner.get_target(Address("src/sh", target_name="lib")) - tests_generator = rule_runner.get_target(Address("src/sh", target_name="tests")) - def gen_source_tgt(rel_fp: str, tags: list[str] | None = None) -> ShellSourceTarget: return ShellSourceTarget( {SingleSourceField.alias: rel_fp, Tags.alias: tags}, @@ -118,25 +113,19 @@ def gen_test_tgt(rel_fp: str, tags: list[str] | None = None) -> Shunit2TestTarge ) sources_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromShellSources(sources_generator)] - ) + _TargetParametrizations, [Address("src/sh", target_name="lib")] + ).parametrizations tests_generated = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromShunit2Tests(tests_generator)] - ) + _TargetParametrizations, [Address("src/sh", target_name="tests")] + ).parametrizations - assert sources_generated == GeneratedTargets( - sources_generator, - { - gen_source_tgt("f1.sh", tags=["overridden"]), - gen_source_tgt("f2.sh"), - gen_source_tgt("subdir/f.sh"), - }, - ) - assert tests_generated == GeneratedTargets( - tests_generator, - { - gen_test_tgt("f1_test.sh", tags=["overridden"]), - gen_test_tgt("f2_test.sh"), - gen_test_tgt("subdir/f_test.sh"), - }, - ) + assert set(sources_generated.values()) == { + gen_source_tgt("f1.sh", tags=["overridden"]), + gen_source_tgt("f2.sh"), + gen_source_tgt("subdir/f.sh"), + } + assert set(tests_generated.values()) == { + gen_test_tgt("f1_test.sh", tags=["overridden"]), + gen_test_tgt("f2_test.sh"), + gen_test_tgt("subdir/f_test.sh"), + } diff --git a/src/python/pants/backend/terraform/target_gen_test.py b/src/python/pants/backend/terraform/target_gen_test.py index e9b12998e81..21b0edc91f5 100644 --- a/src/python/pants/backend/terraform/target_gen_test.py +++ b/src/python/pants/backend/terraform/target_gen_test.py @@ -6,7 +6,6 @@ import pytest from pants.backend.terraform import target_gen -from pants.backend.terraform.target_gen import GenerateTerraformModuleTargetsRequest from pants.backend.terraform.target_types import ( TerraformModulesGeneratorTarget, TerraformModuleSourcesField, @@ -14,8 +13,8 @@ ) from pants.core.util_rules import external_tool from pants.engine.addresses import Address +from pants.engine.internals.graph import _TargetParametrizations from pants.engine.rules import QueryRule -from pants.engine.target import GeneratedTargets from pants.testutil.rule_runner import RuleRunner @@ -26,7 +25,7 @@ def rule_runner() -> RuleRunner: rules=[ *external_tool.rules(), *target_gen.rules(), - QueryRule(GeneratedTargets, [GenerateTerraformModuleTargetsRequest]), + QueryRule(_TargetParametrizations, [Address]), ], ) @@ -44,24 +43,19 @@ def test_target_generation_at_build_root(rule_runner: RuleRunner) -> None: generator_addr = Address("", target_name="tf_mods") generator = rule_runner.get_target(generator_addr) - targets = rule_runner.request( - GeneratedTargets, [GenerateTerraformModuleTargetsRequest(generator)] - ) - assert targets == GeneratedTargets( - generator, - [ - TerraformModuleTarget( - {TerraformModuleSourcesField.alias: ("src/tf/foo/versions.tf",)}, - generator_addr.create_generated("src/tf/foo"), - residence_dir="src/tf/foo", - ), - TerraformModuleTarget( - {TerraformModuleSourcesField.alias: ("src/tf/outputs.tf", "src/tf/versions.tf")}, - generator_addr.create_generated("src/tf"), - residence_dir="src/tf", - ), - ], - ) + targets = rule_runner.request(_TargetParametrizations, [generator.address]) + assert set(targets.parametrizations.values()) == { + TerraformModuleTarget( + {TerraformModuleSourcesField.alias: ("src/tf/foo/versions.tf",)}, + generator_addr.create_generated("src/tf/foo"), + residence_dir="src/tf/foo", + ), + TerraformModuleTarget( + {TerraformModuleSourcesField.alias: ("src/tf/outputs.tf", "src/tf/versions.tf")}, + generator_addr.create_generated("src/tf"), + residence_dir="src/tf", + ), + } def test_target_generation_at_subdir(rule_runner: RuleRunner) -> None: @@ -75,21 +69,16 @@ def test_target_generation_at_subdir(rule_runner: RuleRunner) -> None: generator_addr = Address("src/tf") generator = rule_runner.get_target(generator_addr) - targets = rule_runner.request( - GeneratedTargets, [GenerateTerraformModuleTargetsRequest(generator)] - ) - assert targets == GeneratedTargets( - generator, - [ - TerraformModuleTarget( - {TerraformModuleSourcesField.alias: ("foo/versions.tf",)}, - generator_addr.create_generated("foo"), - residence_dir="src/tf/foo", - ), - TerraformModuleTarget( - {TerraformModuleSourcesField.alias: ("versions.tf",)}, - generator_addr.create_generated("."), - residence_dir="src/tf", - ), - ], - ) + targets = rule_runner.request(_TargetParametrizations, [generator.address]) + assert set(targets.parametrizations.values()) == { + TerraformModuleTarget( + {TerraformModuleSourcesField.alias: ("foo/versions.tf",)}, + generator_addr.create_generated("foo"), + residence_dir="src/tf/foo", + ), + TerraformModuleTarget( + {TerraformModuleSourcesField.alias: ("versions.tf",)}, + generator_addr.create_generated("."), + residence_dir="src/tf", + ), + } diff --git a/src/python/pants/build_graph/address.py b/src/python/pants/build_graph/address.py index 2e3e84d4900..8d472cc63b8 100644 --- a/src/python/pants/build_graph/address.py +++ b/src/python/pants/build_graph/address.py @@ -302,6 +302,10 @@ def is_default_target(self) -> bool: """ return self._target_name is None + @property + def is_parametrized(self) -> bool: + return bool(self.parameters) + @property def filename(self) -> str: if self._relative_file_path is None: @@ -387,7 +391,7 @@ def sanitize(s: str) -> str: def maybe_convert_to_target_generator(self) -> Address: """If this address is generated, convert it to its generator target. - Otherwise, return itself unmodified. + Otherwise, return self unmodified. """ if self.is_generated_target: return self.__class__( @@ -412,7 +416,7 @@ def maybe_convert_to_generated_target(self) -> Address: def create_generated(self, generated_name: str) -> Address: if self.is_generated_target: - raise AssertionError("Cannot call `create_generated` on `{self}`.") + raise AssertionError(f"Cannot call `create_generated` on `{self}`.") return self.__class__( self.spec_path, target_name=self._target_name, @@ -420,6 +424,16 @@ def create_generated(self, generated_name: str) -> Address: generated_name=generated_name, ) + def create_file(self, relative_file_path: str) -> Address: + if self.is_generated_target: + raise AssertionError(f"Cannot call `create_file` on `{self}`.") + return self.__class__( + self.spec_path, + target_name=self._target_name, + parameters=self.parameters, + relative_file_path=relative_file_path, + ) + def __eq__(self, other): if not isinstance(other, Address): return False diff --git a/src/python/pants/core/target_types.py b/src/python/pants/core/target_types.py index d4b6bd552f9..f98b426f6f9 100644 --- a/src/python/pants/core/target_types.py +++ b/src/python/pants/core/target_types.py @@ -12,15 +12,7 @@ ) from pants.core.util_rules.archive import ArchiveFormat, CreateArchive from pants.engine.addresses import UnparsedAddressInputs -from pants.engine.fs import ( - AddPrefix, - Digest, - MergeDigests, - PathGlobs, - Paths, - RemovePrefix, - Snapshot, -) +from pants.engine.fs import AddPrefix, Digest, MergeDigests, RemovePrefix, Snapshot from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, @@ -29,26 +21,21 @@ FieldSetsPerTarget, FieldSetsPerTargetRequest, GeneratedSources, - GeneratedTargets, GenerateSourcesRequest, - GenerateTargetsRequest, HydratedSources, HydrateSourcesRequest, MultipleSourcesField, OverridesField, SingleSourceField, SourcesField, - SourcesPaths, - SourcesPathsRequest, SpecialCasedDependencies, StringField, Target, + TargetFilesGenerator, Targets, generate_file_based_overrides_field_help_message, - generate_file_level_targets, ) -from pants.engine.unions import UnionMembership, UnionRule -from pants.option.global_options import FilesNotFoundBehavior +from pants.engine.unions import UnionRule from pants.util.docutil import bin_name from pants.util.logging import LogLevel @@ -90,7 +77,7 @@ class FilesOverridesField(OverridesField): ) -class FilesGeneratorTarget(Target): +class FilesGeneratorTarget(TargetFilesGenerator): alias = "files" core_fields = ( *COMMON_TARGET_FIELDS, @@ -98,42 +85,13 @@ class FilesGeneratorTarget(Target): FilesGeneratingSourcesField, FilesOverridesField, ) - help = "Generate a `file` target for each file in the `sources` field." - - -class GenerateTargetsFromFiles(GenerateTargetsRequest): - generate_from = FilesGeneratorTarget - - -@rule -async def generate_targets_from_files( - request: GenerateTargetsFromFiles, - union_membership: UnionMembership, - files_not_found_behavior: FilesNotFoundBehavior, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[FilesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - FileTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=False, - overrides=all_overrides, + generated_target_cls = FileTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, ) + moved_fields = () + help = "Generate a `file` target for each file in the `sources` field." # ----------------------------------------------------------------------------------------------- @@ -303,7 +261,7 @@ class ResourcesOverridesField(OverridesField): ) -class ResourcesGeneratorTarget(Target): +class ResourcesGeneratorTarget(TargetFilesGenerator): alias = "resources" core_fields = ( *COMMON_TARGET_FIELDS, @@ -311,13 +269,15 @@ class ResourcesGeneratorTarget(Target): ResourcesGeneratingSourcesField, ResourcesOverridesField, ) + generated_target_cls = ResourceTarget + copied_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + ) + moved_fields = () help = "Generate a `resource` target for each file in the `sources` field." -class GenerateTargetsFromResources(GenerateTargetsRequest): - generate_from = ResourcesGeneratorTarget - - @dataclass(frozen=True) class ResourcesFieldSet(FieldSet): required_fields = (ResourceSourceField,) @@ -332,37 +292,6 @@ class ResourcesGeneratorFieldSet(FieldSet): sources: ResourcesGeneratingSourcesField -@rule -async def generate_targets_from_resources( - request: GenerateTargetsFromResources, - union_membership: UnionMembership, - files_not_found_behavior: FilesNotFoundBehavior, -) -> GeneratedTargets: - sources_paths = await Get( - SourcesPaths, SourcesPathsRequest(request.generator[ResourcesGeneratingSourcesField]) - ) - - all_overrides = {} - overrides_field = request.generator[OverridesField] - if overrides_field.value: - _all_override_paths = await MultiGet( - Get(Paths, PathGlobs, path_globs) - for path_globs in overrides_field.to_path_globs(files_not_found_behavior) - ) - all_overrides = overrides_field.flatten_paths( - dict(zip(_all_override_paths, overrides_field.value.values())) - ) - - return generate_file_level_targets( - ResourceTarget, - request.generator, - sources_paths.files, - union_membership, - add_dependencies_on_all_siblings=False, - overrides=all_overrides, - ) - - # ----------------------------------------------------------------------------------------------- # `target` generic target # ----------------------------------------------------------------------------------------------- @@ -498,8 +427,6 @@ async def package_archive_target(field_set: ArchiveFieldSet) -> BuiltPackage: def rules(): return ( *collect_rules(), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromFiles), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromResources), UnionRule(GenerateSourcesRequest, RelocateFilesViaCodegenRequest), UnionRule(PackageFieldSet, ArchiveFieldSet), ) diff --git a/src/python/pants/core/target_types_test.py b/src/python/pants/core/target_types_test.py index 06c61cebb38..153acbcbc54 100644 --- a/src/python/pants/core/target_types_test.py +++ b/src/python/pants/core/target_types_test.py @@ -13,7 +13,6 @@ from pants.backend.python.goals import package_pex_binary from pants.backend.python.target_types import PexBinary from pants.backend.python.util_rules import pex_from_targets -from pants.core import target_types as core_target_types from pants.core.goals.package import BuiltPackage from pants.core.target_types import ( ArchiveFieldSet, @@ -21,8 +20,6 @@ FilesGeneratorTarget, FileSourceField, FileTarget, - GenerateTargetsFromFiles, - GenerateTargetsFromResources, RelocatedFiles, RelocateFilesViaCodegenRequest, ResourcesGeneratorTarget, @@ -34,9 +31,9 @@ from pants.core.util_rules.source_files import rules as source_files_rules from pants.engine.addresses import Address from pants.engine.fs import EMPTY_SNAPSHOT, DigestContents, FileContent +from pants.engine.internals.graph import _TargetParametrizations from pants.engine.target import ( GeneratedSources, - GeneratedTargets, SingleSourceField, SourcesField, Tags, @@ -266,10 +263,7 @@ def get_file(fp: str) -> bytes: def test_generate_file_and_resource_targets() -> None: rule_runner = RuleRunner( rules=[ - core_target_types.generate_targets_from_files, - core_target_types.generate_targets_from_resources, - QueryRule(GeneratedTargets, [GenerateTargetsFromFiles]), - QueryRule(GeneratedTargets, [GenerateTargetsFromResources]), + QueryRule(_TargetParametrizations, [Address]), ], target_types=[FilesGeneratorTarget, ResourcesGeneratorTarget], ) @@ -296,9 +290,6 @@ def test_generate_file_and_resource_targets() -> None: } ) - files_generator = rule_runner.get_target(Address("assets", target_name="files")) - resources_generator = rule_runner.get_target(Address("assets", target_name="resources")) - def gen_file_tgt(rel_fp: str, tags: list[str] | None = None) -> FileTarget: return FileTarget( {SingleSourceField.alias: rel_fp, Tags.alias: tags}, @@ -314,25 +305,19 @@ def gen_resource_tgt(rel_fp: str, tags: list[str] | None = None) -> ResourceTarg ) generated_files = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromFiles(files_generator)] - ) + _TargetParametrizations, [Address("assets", target_name="files")] + ).parametrizations generated_resources = rule_runner.request( - GeneratedTargets, [GenerateTargetsFromResources(resources_generator)] - ) - - assert generated_files == GeneratedTargets( - files_generator, - { - gen_file_tgt("f1.ext", tags=["overridden"]), - gen_file_tgt("f2.ext"), - gen_file_tgt("subdir/f.ext"), - }, - ) - assert generated_resources == GeneratedTargets( - resources_generator, - { - gen_resource_tgt("f1.ext", tags=["overridden"]), - gen_resource_tgt("f2.ext"), - gen_resource_tgt("subdir/f.ext"), - }, - ) + _TargetParametrizations, [Address("assets", target_name="resources")] + ).parametrizations + + assert set(generated_files.values()) == { + gen_file_tgt("f1.ext", tags=["overridden"]), + gen_file_tgt("f2.ext"), + gen_file_tgt("subdir/f.ext"), + } + assert set(generated_resources.values()) == { + gen_resource_tgt("f1.ext", tags=["overridden"]), + gen_resource_tgt("f2.ext"), + gen_resource_tgt("subdir/f.ext"), + } diff --git a/src/python/pants/engine/internals/build_files_test.py b/src/python/pants/engine/internals/build_files_test.py index 31d1e2feb9c..843e24764b2 100644 --- a/src/python/pants/engine/internals/build_files_test.py +++ b/src/python/pants/engine/internals/build_files_test.py @@ -37,13 +37,15 @@ GeneratedTargets, GenerateTargetsRequest, MultipleSourcesField, + SingleSourceField, SourcesPaths, SourcesPathsRequest, Tags, Target, - generate_file_level_targets, + TargetGenerator, + _generate_file_level_targets, ) -from pants.engine.unions import UnionRule +from pants.engine.unions import UnionMembership, UnionRule from pants.testutil.rule_runner import ( MockGet, QueryRule, @@ -126,42 +128,55 @@ class MockTgt(Target): class MockGeneratedTarget(Target): alias = "generated" - core_fields = (Dependencies, MultipleSourcesField, Tags) + core_fields = (Dependencies, SingleSourceField, Tags) -class MockTargetGenerator(Target): +class MockTargetGenerator(TargetGenerator): alias = "generator" core_fields = (Dependencies, MultipleSourcesField, Tags) + copied_fields = (Tags,) + moved_fields = () class MockGenerateTargetsRequest(GenerateTargetsRequest): generate_from = MockTargetGenerator +# TODO: This method duplicates the builtin `generate_file_targets`, with the exception that it +# intentionally generates both using file addresses and generated addresses. When we remove +# `use_generated_address_syntax=True`, we should remove this implementation and have +# `MockTargetGenerator` subclass `TargetFilesGenerator` instead. @rule -async def generate_mock_generated_target(request: MockGenerateTargetsRequest) -> GeneratedTargets: +async def generate_mock_generated_target( + request: MockGenerateTargetsRequest, + union_membership: UnionMembership, +) -> GeneratedTargets: paths = await Get(SourcesPaths, SourcesPathsRequest(request.generator[MultipleSourcesField])) # Generate using both "file address" and "generated target" syntax. return GeneratedTargets( request.generator, [ - *generate_file_level_targets( + *_generate_file_level_targets( MockGeneratedTarget, request.generator, paths.files, - None, + request.template_address, + request.template, + request.overrides, + union_membership, add_dependencies_on_all_siblings=True, use_generated_address_syntax=False, - use_source_field=False, ).values(), - *generate_file_level_targets( + *_generate_file_level_targets( MockGeneratedTarget, request.generator, paths.files, - None, + request.template_address, + request.template, + request.overrides, + union_membership, add_dependencies_on_all_siblings=True, use_generated_address_syntax=True, - use_source_field=False, ).values(), ], ) @@ -549,6 +564,5 @@ def test_address_specs_generated_target_does_not_belong_to_generator( address_specs_rule_runner, [AddressLiteralSpec("demo/f.txt", "not_owner")] ) assert ( - f"The address `demo/f.txt:not_owner` is not generated by the `{MockTargetGenerator.alias}` " - f"target `demo:not_owner`" + "The address `demo/f.txt:not_owner` was not generated by the target `demo:not_owner`" ) in str(exc.value) diff --git a/src/python/pants/engine/internals/graph.py b/src/python/pants/engine/internals/graph.py index 5c8d7f43873..7a8ab5848c9 100644 --- a/src/python/pants/engine/internals/graph.py +++ b/src/python/pants/engine/internals/graph.py @@ -9,7 +9,7 @@ import os.path from dataclasses import dataclass from pathlib import PurePath -from typing import Iterable, NamedTuple, Sequence, cast +from typing import Iterable, Iterator, NamedTuple, Sequence, cast from pants.base.deprecated import warn_or_error from pants.base.exceptions import ResolveError @@ -64,7 +64,9 @@ InferredDependencies, InjectDependenciesRequest, InjectedDependencies, + MultipleSourcesField, NoApplicableTargetsBehavior, + OverridesField, RegisteredTargetTypes, SecondaryOwnerMixin, SourcesField, @@ -72,6 +74,10 @@ SourcesPathsRequest, SpecialCasedDependencies, Target, + TargetFilesGenerator, + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, + TargetGenerator, TargetRootsToFieldSets, TargetRootsToFieldSetsRequest, Targets, @@ -81,8 +87,9 @@ UnexpandedTargets, UnrecognizedTargetTypeException, WrappedTarget, + _generate_file_level_targets, ) -from pants.engine.unions import UnionMembership +from pants.engine.unions import UnionMembership, UnionRule from pants.option.global_options import FilesNotFoundBehavior, GlobalOptions, OwnersNotFoundBehavior from pants.source.filespec import matches_filespec from pants.util.docutil import bin_name, doc_url @@ -168,54 +175,137 @@ def warn_deprecated_field_type(request: _WarnDeprecatedFieldRequest) -> _WarnDep return _WarnDeprecatedField() +@dataclass(frozen=True) +class _TargetParametrizations: + """All parametrizations and generated targets for a single input Address.""" + + original_target: Target + parametrizations: FrozenDict[Address, Target] + + def get(self, address: Address) -> Target | None: + if self.original_target.address == address: + return self.original_target + return self.parametrizations.get(address) + + def generated_or_generator(self, maybe_generator: Address) -> Iterator[Target]: + if self.parametrizations: + # Generated Targets. + yield from self.parametrizations.values() + else: + # Did not generate targets. + yield self.original_target + + @rule -async def resolve_target( +async def resolve_target_parametrizations( address: Address, registered_target_types: RegisteredTargetTypes, union_membership: UnionMembership, target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests, -) -> WrappedTarget: - if not address.is_generated_target: - target_adaptor = await Get(TargetAdaptor, Address, address) - target_type = registered_target_types.aliases_to_types.get(target_adaptor.type_alias, None) - if target_type is None: - raise UnrecognizedTargetTypeException( - target_adaptor.type_alias, registered_target_types, address + files_not_found_behavior: FilesNotFoundBehavior, +) -> _TargetParametrizations: + assert not address.is_generated_target and not address.is_parametrized + + target_adaptor = await Get(TargetAdaptor, Address, address) + target_type = registered_target_types.aliases_to_types.get(target_adaptor.type_alias, None) + if target_type is None: + raise UnrecognizedTargetTypeException( + target_adaptor.type_alias, registered_target_types, address + ) + if ( + target_type.deprecated_alias is not None + and target_type.deprecated_alias == target_adaptor.type_alias + and not address.is_generated_target + ): + await Get(_WarnDeprecatedTarget, _WarnDeprecatedTargetRequest(target_type)) + + target: Target + generate_request = target_types_to_generate_requests.request_for(target_type) + if generate_request: + # Split out the `propagated_fields` before construction. + generator_fields = dict(target_adaptor.kwargs) + template_fields = {} + # TODO: Require for all instances before landing. + if issubclass(target_type, TargetGenerator): + copied_fields = ( + *target_type.copied_fields, + *target_type._find_plugin_fields(union_membership), ) + for field_type in copied_fields: + field_value = generator_fields.get(field_type.alias, None) + if field_value is not None: + template_fields[field_type.alias] = field_value + for field_type in target_type.moved_fields: + field_value = generator_fields.pop(field_type.alias, None) + if field_value is not None: + template_fields[field_type.alias] = field_value + + target = target_type(generator_fields, address, union_membership) + + overrides = {} + if target.has_field(OverridesField): + overrides_field = target[OverridesField] + overrides_flattened = overrides_field.flatten() + if issubclass(target_type, TargetFilesGenerator): + override_globs = OverridesField.to_path_globs( + target.address, overrides_flattened, files_not_found_behavior + ) + override_paths = await MultiGet( + Get(Paths, PathGlobs, path_globs) for path_globs in override_globs + ) + overrides = OverridesField.flatten_paths( + target.address, + zip(override_paths, override_globs, overrides_flattened.values()), + ) + else: + overrides = overrides_field.flatten() + + generated = await Get( + GeneratedTargets, + GenerateTargetsRequest, + generate_request( + target, + template_address=address, + template=template_fields, + overrides=overrides, + ), + ) + else: + target = target_type(target_adaptor.kwargs, address, union_membership) + generated = GeneratedTargets(target, ()) + + for field_type in target.field_types: if ( - target_type.deprecated_alias is not None - and target_type.deprecated_alias == target_adaptor.type_alias - and not address.is_generated_target + field_type.deprecated_alias is not None + and field_type.deprecated_alias in target_adaptor.kwargs ): - await Get(_WarnDeprecatedTarget, _WarnDeprecatedTargetRequest(target_type)) - target = target_type(target_adaptor.kwargs, address, union_membership) - for field_type in target.field_types: - if ( - field_type.deprecated_alias is not None - and field_type.deprecated_alias in target_adaptor.kwargs - ): - await Get(_WarnDeprecatedField, _WarnDeprecatedFieldRequest(field_type)) - return WrappedTarget(target) - - wrapped_generator_tgt = await Get( - WrappedTarget, Address, address.maybe_convert_to_target_generator() - ) - generator_tgt = wrapped_generator_tgt.target - if not target_types_to_generate_requests.is_generator(generator_tgt): - # TODO: Error in this case. You should not use a generator address (or file address) if - # the generator does not actually generate. - return wrapped_generator_tgt - - generate_request = target_types_to_generate_requests[type(generator_tgt)] - generated = await Get(GeneratedTargets, GenerateTargetsRequest, generate_request(generator_tgt)) - if address not in generated: + await Get(_WarnDeprecatedField, _WarnDeprecatedFieldRequest(field_type)) + + return _TargetParametrizations(target, generated) + + +@rule +async def resolve_target( + address: Address, + target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests, +) -> WrappedTarget: + base_address = address.maybe_convert_to_target_generator() + parametrizations = await Get(_TargetParametrizations, Address, base_address) + if address.is_generated_target and not target_types_to_generate_requests.is_generator( + parametrizations.original_target + ): + # TODO: This is an accommodation to allow using file/generator Addresses for non-generator + # atom targets. See https://github.com/pantsbuild/pants/issues/14419. + return WrappedTarget(parametrizations.original_target) + target = parametrizations.get(address) + if target is None: raise ValueError( - f"The address `{address}` is not generated by the `{generator_tgt.alias}` target " - f"`{generator_tgt.address}`, which only generates these addresses:\n\n" - f"{bullet_list(addr.spec for addr in generated)}\n\n" + f"The address `{address}` was not generated by the target `{base_address}`, which " + "only generated these addresses:\n\n" + f"{bullet_list(addr.spec for addr in parametrizations.parametrizations)}\n\n" "Did you mean to use one of those addresses?" ) - return WrappedTarget(generated[address]) + return WrappedTarget(target) @rule @@ -223,30 +313,27 @@ async def resolve_targets( targets: UnexpandedTargets, target_types_to_generate_requests: TargetTypesToGenerateTargetsRequests, ) -> Targets: - # Replace all generating targets with what it generates. Otherwise, keep it. If a target + # Replace all generating targets with what they generate. Otherwise, keep them. If a target # generator does not generate any targets, keep the target generator. # TODO: This method does not preserve the order of inputs. expanded_targets: OrderedSet[Target] = OrderedSet() generator_targets = [] - generate_gets = [] + parametrizations_gets = [] for tgt in targets: if ( target_types_to_generate_requests.is_generator(tgt) and not tgt.address.is_generated_target ): generator_targets.append(tgt) - generate_request = target_types_to_generate_requests[type(tgt)] - generate_gets.append( - Get(GeneratedTargets, GenerateTargetsRequest, generate_request(tgt)) - ) + parametrizations_gets.append(Get(_TargetParametrizations, Address, tgt.address)) else: expanded_targets.add(tgt) - all_generated_targets = await MultiGet(generate_gets) + all_generated_targets = await MultiGet(parametrizations_gets) expanded_targets.update( tgt - for generator, generated_targets in zip(generator_targets, all_generated_targets) - for tgt in (generated_targets.values() if generated_targets else {generator}) + for generator, parametrizations in zip(generator_targets, all_generated_targets) + for tgt in parametrizations.generated_or_generator(generator.address) ) return Targets(expanded_targets) @@ -989,11 +1076,8 @@ async def resolve_dependencies( # If it's a target generator, inject dependencies on all of its generated targets. generated_addresses: tuple[Address, ...] = () if target_types_to_generate_requests.is_generator(tgt) and not tgt.address.is_generated_target: - generate_request = target_types_to_generate_requests[type(tgt)] - generated_targets = await Get( - GeneratedTargets, GenerateTargetsRequest, generate_request(tgt) - ) - generated_addresses = tuple(generated_targets.keys()) + parametrizations = await Get(_TargetParametrizations, Address, tgt.address) + generated_addresses = tuple(parametrizations.parametrizations.keys()) # If the target has `SpecialCasedDependencies`, such as the `archive` target having # `files` and `packages` fields, then we possibly include those too. We don't want to always @@ -1245,5 +1329,44 @@ def find_valid_field_sets( ) +class GenerateFileTargets(GenerateTargetsRequest): + generate_from = TargetFilesGenerator + + +@rule +async def generate_file_targets( + request: GenerateFileTargets, + files_not_found_behavior: FilesNotFoundBehavior, + union_membership: UnionMembership, +) -> GeneratedTargets: + sources_paths = await Get( + SourcesPaths, SourcesPathsRequest(request.generator[MultipleSourcesField]) + ) + + add_dependencies_on_all_siblings = False + if request.generator.settings_request_cls: + generator_settings = await Get( + TargetFilesGeneratorSettings, + TargetFilesGeneratorSettingsRequest, + request.generator.settings_request_cls(), + ) + add_dependencies_on_all_siblings = generator_settings.add_dependencies_on_all_siblings + + return _generate_file_level_targets( + type(request.generator).generated_target_cls, + request.generator, + sources_paths.files, + request.template_address, + request.template, + request.overrides, + union_membership, + add_dependencies_on_all_siblings=add_dependencies_on_all_siblings, + ) + + def rules(): - return collect_rules() + + return [ + *collect_rules(), + UnionRule(GenerateTargetsRequest, GenerateFileTargets), + ] diff --git a/src/python/pants/engine/internals/graph_test.py b/src/python/pants/engine/internals/graph_test.py index a16f90983ce..86ce24de30c 100644 --- a/src/python/pants/engine/internals/graph_test.py +++ b/src/python/pants/engine/internals/graph_test.py @@ -1,6 +1,8 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + import itertools import os.path from dataclasses import dataclass @@ -38,6 +40,7 @@ OwnersRequest, TooManyTargetsException, TransitiveExcludesNotSupportedError, + _TargetParametrizations, ) from pants.engine.internals.scheduler import ExecutionError from pants.engine.rules import Get, MultiGet, rule @@ -52,9 +55,7 @@ ExplicitlyProvidedDependencies, FieldSet, GeneratedSources, - GeneratedTargets, GenerateSourcesRequest, - GenerateTargetsRequest, HydratedSources, HydrateSourcesRequest, InferDependenciesRequest, @@ -63,6 +64,7 @@ InjectedDependencies, MultipleSourcesField, NoApplicableTargetsBehavior, + OverridesField, SecondaryOwnerMixin, SingleSourceField, SourcesPaths, @@ -71,12 +73,12 @@ StringField, Tags, Target, + TargetFilesGenerator, TargetRootsToFieldSets, TargetRootsToFieldSetsRequest, Targets, TransitiveTargets, TransitiveTargetsRequest, - generate_file_level_targets, ) from pants.engine.unions import UnionMembership, UnionRule, union from pants.source.filespec import Filespec @@ -107,36 +109,21 @@ class MockTarget(Target): class MockGeneratedTarget(Target): alias = "generated" - core_fields = (MockDependencies, SingleSourceField) + core_fields = (MockDependencies, Tags, SingleSourceField) -class MockTargetGenerator(Target): +class MockTargetGenerator(TargetFilesGenerator): alias = "generator" - core_fields = (Dependencies, MultipleSourcesField) - - -class MockGenerateTargetsRequest(GenerateTargetsRequest): - generate_from = MockTargetGenerator - - -@rule -async def generate_mock_generated_target(request: MockGenerateTargetsRequest) -> GeneratedTargets: - paths = await Get(SourcesPaths, SourcesPathsRequest(request.generator[MultipleSourcesField])) - return generate_file_level_targets( - MockGeneratedTarget, - request.generator, - paths.files, - None, - add_dependencies_on_all_siblings=False, - ) + core_fields = (Dependencies, MultipleSourcesField, OverridesField) + generated_target_cls = MockGeneratedTarget + copied_fields = (Dependencies,) + moved_fields = (Tags,) @pytest.fixture def transitive_targets_rule_runner() -> RuleRunner: return RuleRunner( rules=[ - generate_mock_generated_target, - UnionRule(GenerateTargetsRequest, MockGenerateTargetsRequest), QueryRule(AllTargets, [AllTargetsRequest]), QueryRule(AllUnexpandedTargets, [AllTargetsRequest]), QueryRule(CoarsenedTargets, [Addresses]), @@ -582,8 +569,8 @@ def test_resolve_generated_target(transitive_targets_rule_runner: RuleRunner) -> Address("", target_name="generator", relative_file_path="no_owner.txt") ) - # Using a "file address" on a target that does not generate file-level targets will fall back - # to the target generator. This is temporary until we remove file address syntax. + # TODO: Using a "file address" on a target that does not generate file-level targets will fall + # back to the target generator. See https://github.com/pantsbuild/pants/issues/14419. non_generator_file_address = Address( "", target_name="non-generator", relative_file_path="f1.txt" ) @@ -662,8 +649,6 @@ class MockSecondaryOwnerTarget(Target): def owners_rule_runner() -> RuleRunner: return RuleRunner( rules=[ - generate_mock_generated_target, - UnionRule(GenerateTargetsRequest, MockGenerateTargetsRequest), QueryRule(Owners, [OwnersRequest]), ], target_types=[ @@ -806,8 +791,6 @@ def test_owners_build_file(owners_rule_runner: RuleRunner) -> None: def specs_rule_runner() -> RuleRunner: return RuleRunner( rules=[ - generate_mock_generated_target, - UnionRule(GenerateTargetsRequest, MockGenerateTargetsRequest), QueryRule(Addresses, [FilesystemSpecs]), QueryRule(Addresses, [Specs]), ], @@ -923,6 +906,106 @@ def test_resolve_addresses_from_specs(specs_rule_runner: RuleRunner) -> None: } +# ----------------------------------------------------------------------------------------------- +# Test file-level target generation and parameterization. +# ----------------------------------------------------------------------------------------------- + + +@pytest.fixture +def generated_targets_rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + QueryRule(_TargetParametrizations, [Address]), + ], + target_types=[MockTargetGenerator, MockGeneratedTarget], + ) + + +def assert_generated( + rule_runner: RuleRunner, + address: Address, + build_content: str, + files: list[str], + expected: set[Target], +) -> None: + rule_runner.write_files( + { + f"{address.spec_path}/BUILD": build_content, + **{os.path.join(address.spec_path, f): "" for f in files}, + } + ) + parametrizations = rule_runner.request(_TargetParametrizations, [address]) + assert expected == set(t for t in parametrizations.parametrizations.values()) + + +def test_generate_multiple(generated_targets_rule_runner: RuleRunner) -> None: + assert_generated( + generated_targets_rule_runner, + Address("demo"), + "generator(tags=['tag'], sources=['*.ext'])", + ["f1.ext", "f2.ext"], + { + MockGeneratedTarget( + {SingleSourceField.alias: "f1.ext", Tags.alias: ["tag"]}, + Address("demo", relative_file_path="f1.ext"), + residence_dir="demo", + ), + MockGeneratedTarget( + {SingleSourceField.alias: "f2.ext", Tags.alias: ["tag"]}, + Address("demo", relative_file_path="f2.ext"), + residence_dir="demo", + ), + }, + ) + + +def test_generate_subdir(generated_targets_rule_runner: RuleRunner) -> None: + assert_generated( + generated_targets_rule_runner, + Address("src/fortran", target_name="demo"), + "generator(name='demo', sources=['**/*.f95'])", + ["subdir/demo.f95"], + { + MockGeneratedTarget( + {SingleSourceField.alias: "subdir/demo.f95"}, + Address("src/fortran", target_name="demo", relative_file_path="subdir/demo.f95"), + residence_dir="src/fortran/subdir", + ) + }, + ) + + +def test_generate_overrides(generated_targets_rule_runner: RuleRunner) -> None: + assert_generated( + generated_targets_rule_runner, + Address("example"), + "generator(sources=['*.ext'], tags=['override_me'], overrides={'f1.ext': {'tags': ['overridden']}})", + ["f1.ext"], + { + MockGeneratedTarget( + { + SingleSourceField.alias: "f1.ext", + Tags.alias: ["overridden"], + }, + Address("example", relative_file_path="f1.ext"), + ), + }, + ) + + +def test_generate_overrides_unused(generated_targets_rule_runner: RuleRunner) -> None: + with engine_error( + contains="Unused file paths in the `overrides` field for demo:demo: ['fake.ext']" + ): + assert_generated( + generated_targets_rule_runner, + Address("demo"), + "generator(sources=['*.ext'], overrides={'fake.ext': {'tags': ['irrelevant']}})", + ["f1.ext"], + set(), + ) + + # ----------------------------------------------------------------------------------------------- # Test FieldSets. Also see `engine/target_test.py`. # ----------------------------------------------------------------------------------------------- @@ -1331,7 +1414,7 @@ def hydrate(sources_cls: Type[MultipleSourcesField], sources: Iterable[str]) -> # ----------------------------------------------------------------------------------------------- -class SmalltalkSources(MultipleSourcesField): +class SmalltalkSource(SingleSourceField): pass @@ -1346,7 +1429,7 @@ class AvroLibrary(Target): class GenerateSmalltalkFromAvroRequest(GenerateSourcesRequest): input = AvroSources - output = SmalltalkSources + output = SmalltalkSource @rule @@ -1392,8 +1475,7 @@ def test_codegen_generates_sources(codegen_rule_runner: RuleRunner) -> None: addr = setup_codegen_protocol_tgt(codegen_rule_runner) protocol_sources = AvroSources(["*.avro"], addr) assert ( - protocol_sources.can_generate(SmalltalkSources, codegen_rule_runner.union_membership) - is True + protocol_sources.can_generate(SmalltalkSource, codegen_rule_runner.union_membership) is True ) # First, get the original protocol sources. @@ -1415,12 +1497,12 @@ def test_codegen_generates_sources(codegen_rule_runner: RuleRunner) -> None: HydratedSources, [ HydrateSourcesRequest( - protocol_sources, for_sources_types=[SmalltalkSources], enable_codegen=True + protocol_sources, for_sources_types=[SmalltalkSource], enable_codegen=True ) ], ) assert generated_via_hydrate_sources.snapshot.files == ("src/smalltalk/f.st",) - assert generated_via_hydrate_sources.sources_type == SmalltalkSources + assert generated_via_hydrate_sources.sources_type == SmalltalkSource def test_codegen_works_with_subclass_fields(codegen_rule_runner: RuleRunner) -> None: @@ -1431,14 +1513,13 @@ class CustomAvroSources(AvroSources): protocol_sources = CustomAvroSources(["*.avro"], addr) assert ( - protocol_sources.can_generate(SmalltalkSources, codegen_rule_runner.union_membership) - is True + protocol_sources.can_generate(SmalltalkSource, codegen_rule_runner.union_membership) is True ) generated = codegen_rule_runner.request( HydratedSources, [ HydrateSourcesRequest( - protocol_sources, for_sources_types=[SmalltalkSources], enable_codegen=True + protocol_sources, for_sources_types=[SmalltalkSource], enable_codegen=True ) ], ) @@ -1469,11 +1550,11 @@ def test_ambiguous_codegen_implementations_exception() -> None: # This error message is quite complex. We test that it correctly generates the message. class SmalltalkGenerator1(GenerateSourcesRequest): input = AvroSources - output = SmalltalkSources + output = SmalltalkSource class SmalltalkGenerator2(GenerateSourcesRequest): input = AvroSources - output = SmalltalkSources + output = SmalltalkSource class AdaSources(MultipleSourcesField): pass @@ -1487,9 +1568,9 @@ class IrrelevantSources(MultipleSourcesField): # Test when all generators have the same input and output. exc = AmbiguousCodegenImplementationsException( - [SmalltalkGenerator1, SmalltalkGenerator2], for_sources_types=[SmalltalkSources] + [SmalltalkGenerator1, SmalltalkGenerator2], for_sources_types=[SmalltalkSource] ) - assert "can generate SmalltalkSources from AvroSources" in str(exc) + assert "can generate SmalltalkSource from AvroSources" in str(exc) assert "* SmalltalkGenerator1" in str(exc) assert "* SmalltalkGenerator2" in str(exc) @@ -1497,11 +1578,11 @@ class IrrelevantSources(MultipleSourcesField): # the call site used too expansive of a `for_sources_types` argument. exc = AmbiguousCodegenImplementationsException( [SmalltalkGenerator1, AdaGenerator], - for_sources_types=[SmalltalkSources, AdaSources, IrrelevantSources], + for_sources_types=[SmalltalkSource, AdaSources, IrrelevantSources], ) - assert "can generate one of ['AdaSources', 'SmalltalkSources'] from AvroSources" in str(exc) + assert "can generate one of ['AdaSources', 'SmalltalkSource'] from AvroSources" in str(exc) assert "IrrelevantSources" not in str(exc) - assert "* SmalltalkGenerator1 -> SmalltalkSources" in str(exc) + assert "* SmalltalkGenerator1 -> SmalltalkSource" in str(exc) assert "* AdaGenerator -> AdaSources" in str(exc) @@ -1561,18 +1642,27 @@ def inject_custom_smalltalk_deps(_: InjectCustomSmalltalkDependencies) -> Inject return InjectedDependencies([Address("", target_name="custom_injected")]) -class SmalltalkLibrarySources(SmalltalkSources): +class SmalltalkLibrarySource(SmalltalkSource): pass class SmalltalkLibrary(Target): - alias = "smalltalk" + alias = "smalltalk_library" # Note that we use MockDependencies so that we support transitive excludes (`!!`). - core_fields = (MockDependencies, SmalltalkLibrarySources) + core_fields = (MockDependencies, SmalltalkLibrarySource) + + +class SmalltalkLibraryGenerator(TargetFilesGenerator): + alias = "smalltalk_libraries" + # Note that we use MockDependencies so that we support transitive excludes (`!!`). + core_fields = (MockDependencies, MultipleSourcesField) + generated_target_cls = SmalltalkLibrary + copied_fields = (MockDependencies,) + moved_fields = () class InferSmalltalkDependencies(InferDependenciesRequest): - infer_from = SmalltalkSources + infer_from = SmalltalkLibrarySource @rule @@ -1590,25 +1680,6 @@ async def infer_smalltalk_dependencies(request: InferSmalltalkDependencies) -> I return InferredDependencies(resolved) -class GenerateTargetsFromSmallTalkLibraryRequest(GenerateTargetsRequest): - generate_from = SmalltalkLibrary - - -@rule -async def generate_targets_from_smalltalk_library( - request: GenerateTargetsFromSmallTalkLibraryRequest, -) -> GeneratedTargets: - paths = await Get(SourcesPaths, SourcesPathsRequest(request.generator[SmalltalkSources])) - return generate_file_level_targets( - SmalltalkLibrary, - request.generator, - paths.files, - None, - add_dependencies_on_all_siblings=False, - use_source_field=False, - ) - - @pytest.fixture def dependencies_rule_runner() -> RuleRunner: return RuleRunner( @@ -1616,15 +1687,13 @@ def dependencies_rule_runner() -> RuleRunner: inject_smalltalk_deps, inject_custom_smalltalk_deps, infer_smalltalk_dependencies, - generate_targets_from_smalltalk_library, QueryRule(Addresses, [DependenciesRequest]), QueryRule(ExplicitlyProvidedDependencies, [DependenciesRequest]), UnionRule(InjectDependenciesRequest, InjectSmalltalkDependencies), UnionRule(InjectDependenciesRequest, InjectCustomSmalltalkDependencies), UnionRule(InferDependenciesRequest, InferSmalltalkDependencies), - UnionRule(GenerateTargetsRequest, GenerateTargetsFromSmallTalkLibraryRequest), ], - target_types=[SmalltalkLibrary], + target_types=[SmalltalkLibraryGenerator], ) @@ -1648,11 +1717,11 @@ def test_explicitly_provided_dependencies(dependencies_rule_runner: RuleRunner) { "files/f.txt": "", "files/transitive_exclude.txt": "", - "files/BUILD": "smalltalk(sources=['*.txt'])", - "a/b/c/BUILD": "smalltalk()", + "files/BUILD": "smalltalk_libraries(sources=['*.txt'])", + "a/b/c/BUILD": "smalltalk_libraries()", "demo/subdir/BUILD": dedent( """\ - smalltalk( + smalltalk_libraries( dependencies=[ 'a/b/c', '!a/b/c', @@ -1681,11 +1750,11 @@ def test_explicitly_provided_dependencies(dependencies_rule_runner: RuleRunner) def test_normal_resolution(dependencies_rule_runner: RuleRunner) -> None: dependencies_rule_runner.write_files( { - "src/smalltalk/BUILD": "smalltalk(dependencies=['//:dep1', '//:dep2', ':sibling'])", - "no_deps/BUILD": "smalltalk()", + "src/smalltalk/BUILD": "smalltalk_libraries(dependencies=['//:dep1', '//:dep2', ':sibling'])", + "no_deps/BUILD": "smalltalk_libraries()", # An ignore should override an include. "ignore/BUILD": ( - "smalltalk(dependencies=['//:dep1', '!//:dep1', '//:dep2', '!!//:dep2'])" + "smalltalk_libraries(dependencies=['//:dep1', '!//:dep1', '//:dep2', '!!//:dep2'])" ), } ) @@ -1709,10 +1778,10 @@ def test_explicit_file_dependencies(dependencies_rule_runner: RuleRunner) -> Non "src/smalltalk/util/f2.st": "", "src/smalltalk/util/f3.st": "", "src/smalltalk/util/f4.st": "", - "src/smalltalk/util/BUILD": "smalltalk(sources=['*.st'])", + "src/smalltalk/util/BUILD": "smalltalk_libraries(sources=['*.st'])", "src/smalltalk/BUILD": dedent( """\ - smalltalk( + smalltalk_libraries( dependencies=[ './util/f1.st', 'src/smalltalk/util/f2.st', @@ -1737,7 +1806,7 @@ def test_explicit_file_dependencies(dependencies_rule_runner: RuleRunner) -> Non def test_dependency_injection(dependencies_rule_runner: RuleRunner) -> None: - dependencies_rule_runner.write_files({"BUILD": "smalltalk(name='target')"}) + dependencies_rule_runner.write_files({"BUILD": "smalltalk_libraries(name='target')"}) def assert_injected(deps_cls: Type[Dependencies], *, injected: List[Address]) -> None: provided_deps = ["//:provided"] @@ -1775,12 +1844,12 @@ def test_dependency_inference(dependencies_rule_runner: RuleRunner) -> None: "inferred_and_provided2.st": "", "BUILD": dedent( """\ - smalltalk(name='inferred1') - smalltalk(name='inferred2') - smalltalk(name='inferred_but_ignored1', sources=['inferred_but_ignored1.st']) - smalltalk(name='inferred_but_ignored2', sources=['inferred_but_ignored2.st']) - smalltalk(name='inferred_and_provided1') - smalltalk(name='inferred_and_provided2') + smalltalk_libraries(name='inferred1') + smalltalk_libraries(name='inferred2') + smalltalk_libraries(name='inferred_but_ignored1', sources=['inferred_but_ignored1.st']) + smalltalk_libraries(name='inferred_but_ignored2', sources=['inferred_but_ignored2.st']) + smalltalk_libraries(name='inferred_and_provided1') + smalltalk_libraries(name='inferred_and_provided2') """ ), "demo/f1.st": dedent( @@ -1799,7 +1868,7 @@ def test_dependency_inference(dependencies_rule_runner: RuleRunner) -> None: ), "demo/BUILD": dedent( """\ - smalltalk( + smalltalk_libraries( sources=['*.st'], dependencies=[ '//:inferred_and_provided1', @@ -1817,15 +1886,8 @@ def test_dependency_inference(dependencies_rule_runner: RuleRunner) -> None: dependencies_rule_runner, Address("demo"), expected=[ - Address("", target_name="inferred1"), - Address("", relative_file_path="inferred2.st", target_name="inferred2"), Address("", target_name="inferred_and_provided1"), Address("", target_name="inferred_and_provided2"), - Address( - "", - relative_file_path="inferred_and_provided2.st", - target_name="inferred_and_provided2", - ), Address("demo", relative_file_path="f1.st"), Address("demo", relative_file_path="f2.st"), ], @@ -1863,8 +1925,8 @@ def test_depends_on_generated_targets(dependencies_rule_runner: RuleRunner) -> N { "src/smalltalk/f1.st": "", "src/smalltalk/f2.st": "", - "src/smalltalk/BUILD": "smalltalk(sources=['*.st'])", - "src/smalltalk/util/BUILD": "smalltalk()", + "src/smalltalk/BUILD": "smalltalk_libraries(sources=['*.st'])", + "src/smalltalk/util/BUILD": "smalltalk_libraries()", } ) assert_dependencies_resolved( diff --git a/src/python/pants/engine/target.py b/src/python/pants/engine/target.py index 4e5be13e11b..7555b73eca2 100644 --- a/src/python/pants/engine/target.py +++ b/src/python/pants/engine/target.py @@ -4,6 +4,7 @@ from __future__ import annotations import collections.abc +import dataclasses import enum import itertools import logging @@ -46,7 +47,7 @@ ) from pants.engine.unions import UnionMembership, UnionRule, union from pants.option.global_options import FilesNotFoundBehavior -from pants.source.filespec import Filespec, matches_filespec +from pants.source.filespec import Filespec from pants.util.collections import ensure_list, ensure_str_list from pants.util.dirutil import fast_relpath from pants.util.docutil import bin_name, doc_url @@ -808,14 +809,91 @@ class AllTargetsRequest: # Target generation # ----------------------------------------------------------------------------------------------- -_Tgt = TypeVar("_Tgt", bound=Target) + +class TargetGenerator(Target): + """A Target type which generates other Targets via installed `@rule` logic. + + To act as a generator, a Target type should subclass this base class and install generation + `@rule`s which consume a corresponding GenerateTargetsRequest subclass to produce + GeneratedTargets. + """ + + # The generated Target class. + generated_target_cls: ClassVar[type[Target]] + + # Fields which have their values copied from the generator Target to the generated Target. + # + # Must be a subset of `core_fields`. + # + # Fields should be copied from the generator to the generated when their semantic meaning is + # the same for both Target types, and when it is valuable for them to be introspected on + # either the generator or generated target (such as by `peek`, or in `filter`). + copied_fields: ClassVar[Tuple[Type[Field], ...]] + + # Fields which are specified to instances of the generator Target, but which are propagated + # to generated Targets rather than being stored on the generator Target. + # + # Must be disjoint from `core_fields`. + # + # Only Fields which are moved to the generated Target are allowed to be `parametrize`d. But + # it can also be the case that a Field only makes sense semantically when it is applied to + # the generated Target (for example, for an individual file), and the generator Target is just + # acting as a convenient place for them to be specified. + moved_fields: ClassVar[Tuple[Type[Field], ...]] + + +class TargetFilesGenerator(TargetGenerator): + """A TargetGenerator which generates a Target per file matched by the generator. + + Unlike TargetGenerator, no additional `@rules` are required to be installed, because generation + is implemented declaratively. But an optional `settings_request_cls` can be declared to + dynamically control some settings of generation. + """ + + settings_request_cls: ClassVar[type[TargetFilesGeneratorSettingsRequest] | None] = None + + +@union +class TargetFilesGeneratorSettingsRequest: + """An optional union to provide dynamic settings for a `TargetFilesGenerator`. + + See `TargetFilesGenerator`. + """ + + +@dataclass +class TargetFilesGeneratorSettings: + # Set `add_dependencies_on_all_siblings` to True so that each file-level target depends on all + # other generated targets from the target generator. This is useful if both are true: + # + # a) file-level targets usually need their siblings to be present to work. Most target types + # (Python, Java, Shell, etc) meet this, except for `files` and `resources` which have no + # concept of "imports" + # b) dependency inference cannot infer dependencies on sibling files. + # + # Otherwise, set `add_dependencies_on_all_siblings` to `False` so that dependencies are + # finer-grained. + add_dependencies_on_all_siblings: bool = False + + +_TargetGenerator = TypeVar("_TargetGenerator", bound=TargetGenerator) @union @dataclass(frozen=True) -class GenerateTargetsRequest(Generic[_Tgt]): - generate_from: ClassVar[type[_Tgt]] - generator: _Tgt +class GenerateTargetsRequest(Generic[_TargetGenerator]): + generate_from: ClassVar[type[_TargetGenerator]] + + # The TargetGenerator instance to generate targets for. + generator: _TargetGenerator + # The base Address to generate for. Note that due to parametrization, this may not + # always be the Address of the underlying target. + template_address: Address + # The `TargetGenerator.moved_field/copied_field` Field values that the generator + # should generate targets with. + template: dict[str, Any] = dataclasses.field(hash=False) + # Per-generated-Target overrides. + overrides: dict[str, dict[str, Any]] = dataclasses.field(hash=False) class GeneratedTargets(FrozenDict[Address, Target]): @@ -855,20 +933,27 @@ def __init__(self, generator: Target, generated_targets: Iterable[Target]) -> No class TargetTypesToGenerateTargetsRequests(FrozenDict[Type[Target], Type[GenerateTargetsRequest]]): def is_generator(self, tgt: Target) -> bool: """Does this target type generate other targets?""" - return type(tgt) in self + return bool(self.request_for(type(tgt))) + def request_for(self, tgt_cls: type[Target]) -> type[GenerateTargetsRequest] | None: + """Return the request type for the given Target, or None.""" + if issubclass(tgt_cls, TargetFilesGenerator): + return self.get(TargetFilesGenerator) + return self.get(tgt_cls) -def generate_file_level_targets( + +def _generate_file_level_targets( generated_target_cls: type[Target], generator: Target, paths: Sequence[str], + template_address: Address, + template: dict[str, Any], + overrides: dict[str, dict[str, Any]], # NB: Should only ever be set to `None` in tests. union_membership: UnionMembership | None, *, add_dependencies_on_all_siblings: bool, use_generated_address_syntax: bool = False, - use_source_field: bool = True, - overrides: dict[str, dict[str, Any]] | None = None, ) -> GeneratedTargets: """Generate one new target for each path, using the same fields as the generator target except for the `sources` field only referring to the path and using a new address. @@ -887,24 +972,14 @@ def generate_file_level_targets( `overrides` allows changing the fields for particular targets. It expects the full file path as the key. """ - if not generator.has_field(Dependencies) or not generator.has_field(SourcesField): - raise AssertionError( - f"The `{generator.alias}` target {generator.address.spec} does " - "not have both a `dependencies` and `sources` field, and thus cannot generate a " - f"`{generated_target_cls.alias}` target." - ) all_generated_addresses = [] for fp in paths: - relativized_fp = fast_relpath(fp, generator.address.spec_path) + relativized_fp = fast_relpath(fp, template_address.spec_path) all_generated_addresses.append( - generator.address.create_generated(relativized_fp) + template_address.create_generated(relativized_fp) if use_generated_address_syntax - else Address( - generator.address.spec_path, - target_name=generator.address.target_name, - relative_file_path=relativized_fp, - ) + else template_address.create_file(relativized_fp) ) all_generated_address_specs = ( @@ -913,37 +988,29 @@ def generate_file_level_targets( else FrozenOrderedSet() ) - used_overrides = set() - normalized_overrides = overrides or {} + normalized_overrides = dict(overrides or {}) def gen_tgt(full_fp: str, address: Address) -> Target: - generated_target_fields: dict[str, ImmutableValue] = {} - for field in generator.field_values.values(): - value: ImmutableValue - if isinstance(field, MultipleSourcesField): - if not bool(matches_filespec(field.filespec, paths=[full_fp])): - raise AssertionError( - f"Target {generator.address.spec}'s `sources` field does not match a file " - f"{full_fp}." - ) - value = address._relative_file_path or address.generated_name - if use_source_field: - generated_target_fields[SingleSourceField.alias] = value - else: - generated_target_fields[MultipleSourcesField.alias] = (value,) - elif add_dependencies_on_all_siblings and isinstance(field, Dependencies): - generated_target_fields[Dependencies.alias] = (field.value or ()) + tuple( - all_generated_address_specs - {address.spec} - ) - elif isinstance(field, OverridesField): - continue - elif field.value != field.default: - generated_target_fields[field.alias] = field.value + generated_target_fields = dict(template) if full_fp in normalized_overrides: - used_overrides.add(full_fp) - generated_target_fields.update(normalized_overrides[full_fp]) + generated_target_fields.update(normalized_overrides.pop(full_fp)) + + if add_dependencies_on_all_siblings: + if not generator.has_field(Dependencies): + raise AssertionError( + f"The `{generator.alias}` target {template_address.spec} does " + "not have a `dependencies` field, and thus cannot " + "`add_dependencies_on_all_siblings`." + ) + original_deps = generated_target_fields.get(Dependencies.alias, ()) + generated_target_fields[Dependencies.alias] = tuple(original_deps) + tuple( + all_generated_address_specs - {address.spec} + ) + generated_target_fields[SingleSourceField.alias] = ( + address._relative_file_path or address.generated_name + ) return generated_target_cls( generated_target_fields, address, @@ -953,21 +1020,19 @@ def gen_tgt(full_fp: str, address: Address) -> Target: result = tuple(gen_tgt(fp, address) for fp, address in zip(paths, all_generated_addresses)) - unused_overrides = set(normalized_overrides.keys()) - used_overrides - if unused_overrides: + if normalized_overrides: unused_relative_paths = sorted( - fast_relpath(fp, generator.address.spec_path) for fp in unused_overrides + fast_relpath(fp, template_address.spec_path) for fp in normalized_overrides ) all_valid_relative_paths = sorted( cast(str, tgt.address._relative_file_path or tgt.address.generated_name) for tgt in result ) raise InvalidFieldException( - f"Unused file paths in the `overrides` field for {generator.address}: " + f"Unused file paths in the `overrides` field for {template_address}: " f"{sorted(unused_relative_paths)}" f"\n\nDid you mean one of these valid paths?\n\n" - f"{all_valid_relative_paths}\n\n" - f"Tip: if you want to override a value for all generated targets, set the ..." + f"{all_valid_relative_paths}" ) return GeneratedTargets(generator, result) @@ -2361,30 +2426,32 @@ def __hash__(self) -> int: # The value might have unhashable elements like `list`, so we stringify it. return hash((self.__class__, repr(self.value))) - def _relativize_globs(self, globs: tuple[str, ...]) -> tuple[str, ...]: - return tuple( - f"!{os.path.join(self.address.spec_path, glob[1:])}" - if glob.startswith("!") - else os.path.join(self.address.spec_path, glob) - for glob in globs - ) - + @classmethod def to_path_globs( - self, files_not_found_behavior: FilesNotFoundBehavior + cls, + address: Address, + overrides_keys: Iterable[str], + files_not_found_behavior: FilesNotFoundBehavior, ) -> tuple[PathGlobs, ...]: """Create a `PathGlobs` for each key. This should only be used if the keys are file globs. """ - if not self.value: - return () + + def relativize_glob(glob: str) -> str: + return ( + f"!{os.path.join(address.spec_path, glob[1:])}" + if glob.startswith("!") + else os.path.join(address.spec_path, glob) + ) + return tuple( PathGlobs( - self._relativize_globs(globs), + [relativize_glob(glob)], glob_match_error_behavior=files_not_found_behavior.to_glob_match_error_behavior(), - description_of_origin=f"the `overrides` field for {self.address}", + description_of_origin=f"the `overrides` field for {address}", ) - for globs in self.value + for glob in overrides_keys ) def flatten(self) -> dict[str, dict[str, Any]]: @@ -2409,13 +2476,18 @@ def flatten(self) -> dict[str, dict[str, Any]]: ) return result + @classmethod def flatten_paths( - self, paths_to_overrides: Mapping[Paths, dict[str, Any]] + cls, + address: Address, + paths_and_overrides: Iterable[tuple[Paths, PathGlobs, dict[str, Any]]], ) -> dict[str, dict[str, Any]]: """Combine all overrides for each file into a single dictionary.""" result: dict[str, dict[str, Any]] = {} - for paths, override in paths_to_overrides.items(): - for path in paths.files: + for paths, globs, override in paths_and_overrides: + # NB: If some globs did not result in any Paths, we preserve them to ensure that + # unconsumed overrides trigger errors during generation. + for path in paths.files or globs.globs: for field, value in override.items(): if path not in result: result[path] = {field: value} @@ -2423,12 +2495,11 @@ def flatten_paths( if field not in result[path]: result[path][field] = value continue - relpath = fast_relpath(path, self.address.spec_path) + relpath = fast_relpath(path, address.spec_path) raise InvalidFieldException( - f"Conflicting overrides in the `{self.alias}` field of " - f"`{self.address}` for the relative path `{relpath}` for " - f"the field `{field}`. You cannot specify the same field name " - "multiple times for the same path.\n\n" + f"Conflicting overrides for `{address}` for the relative path " + f"`{relpath}` for the field `{field}`. You cannot specify the same field " + f"name multiple times for the same path.\n\n" f"(One override sets the field to `{repr(result[path][field])}` " f"but another sets to `{repr(value)}`.)" ) diff --git a/src/python/pants/engine/target_test.py b/src/python/pants/engine/target_test.py index ff599f0a6d7..a2cfa9862df 100644 --- a/src/python/pants/engine/target_test.py +++ b/src/python/pants/engine/target_test.py @@ -9,11 +9,10 @@ import pytest from pants.engine.addresses import Address -from pants.engine.fs import GlobExpansionConjunction, GlobMatchErrorBehavior, Paths +from pants.engine.fs import GlobExpansionConjunction, GlobMatchErrorBehavior, PathGlobs, Paths from pants.engine.target import ( AsyncFieldMixin, BoolField, - Dependencies, DictStringToStringField, DictStringToStringSequenceField, ExplicitlyProvidedDependencies, @@ -38,10 +37,8 @@ SingleSourceField, StringField, StringSequenceField, - Tags, Target, ValidNumbers, - generate_file_level_targets, targets_with_sources_types, ) from pants.engine.unions import UnionMembership @@ -450,138 +447,6 @@ def test_target_residence_dir() -> None: # ----------------------------------------------------------------------------------------------- -def test_generate_file_level_targets() -> None: - class MockGenerator(Target): - alias = "generator" - core_fields = (Dependencies, Tags, MultipleSourcesField) - - class MockGenerated(Target): - alias = "generated" - core_fields = (Dependencies, Tags, MultipleSourcesField) - - def generate( - generator: Target, - files: List[str], - *, - add_dependencies_on_all_siblings: bool = False, - use_generated_addr_syntax: bool = False, - overrides: Optional[Dict[str, Dict[str, Any]]] = None, - ) -> GeneratedTargets: - return generate_file_level_targets( - MockGenerated, - generator, - files, - None, - add_dependencies_on_all_siblings=add_dependencies_on_all_siblings, - use_generated_address_syntax=use_generated_addr_syntax, - use_source_field=False, - overrides=overrides, - ) - - tgt = MockGenerator( - {MultipleSourcesField.alias: ["f1.ext", "f2.ext"], Tags.alias: ["tag"]}, Address("demo") - ) - assert generate(tgt, ["demo/f1.ext", "demo/f2.ext"]) == GeneratedTargets( - tgt, - [ - MockGenerated( - {MultipleSourcesField.alias: ["f1.ext"], Tags.alias: ["tag"]}, - Address("demo", relative_file_path="f1.ext"), - residence_dir="demo", - ), - MockGenerated( - {MultipleSourcesField.alias: ["f2.ext"], Tags.alias: ["tag"]}, - Address("demo", relative_file_path="f2.ext"), - residence_dir="demo", - ), - ], - ) - assert generate( - tgt, ["demo/f1.ext", "demo/f2.ext"], add_dependencies_on_all_siblings=True - ) == GeneratedTargets( - tgt, - [ - MockGenerated( - { - MultipleSourcesField.alias: ["f1.ext"], - Dependencies.alias: ["demo/f2.ext"], - Tags.alias: ["tag"], - }, - Address("demo", relative_file_path="f1.ext"), - residence_dir="demo", - ), - MockGenerated( - { - MultipleSourcesField.alias: ["f2.ext"], - Dependencies.alias: ["demo/f1.ext"], - Tags.alias: ["tag"], - }, - Address("demo", relative_file_path="f2.ext"), - residence_dir="demo", - ), - ], - ) - - subdir_tgt = MockGenerator( - {MultipleSourcesField.alias: ["demo.f95", "subdir/demo.f95"]}, - Address("src/fortran", target_name="demo"), - ) - assert generate(subdir_tgt, ["src/fortran/subdir/demo.f95"]) == GeneratedTargets( - subdir_tgt, - [ - MockGenerated( - {MultipleSourcesField.alias: ["subdir/demo.f95"]}, - Address("src/fortran", target_name="demo", relative_file_path="subdir/demo.f95"), - residence_dir="src/fortran/subdir", - ) - ], - ) - assert generate( - subdir_tgt, ["src/fortran/subdir/demo.f95"], use_generated_addr_syntax=True - ) == GeneratedTargets( - subdir_tgt, - [ - MockGenerated( - {MultipleSourcesField.alias: ["subdir/demo.f95"]}, - Address("src/fortran", target_name="demo", generated_name="subdir/demo.f95"), - residence_dir="src/fortran/subdir", - ) - ], - ) - - # Can override fields. - assert generate( - tgt, ["demo/f1.ext"], overrides={"demo/f1.ext": {"tags": ["overridden"]}} - ) == GeneratedTargets( - tgt, - [ - MockGenerated( - { - MultipleSourcesField.alias: ["f1.ext"], - Tags.alias: ["overridden"], - }, - Address("demo", relative_file_path="f1.ext"), - ), - ], - ) - - # The file path must match the filespec of the generator target's SourcesField. - with pytest.raises(AssertionError) as exc: - generate(tgt, ["demo/fake.ext"]) - assert "does not match a file demo/fake.ext" in str(exc.value) - - class MissingFieldsTarget(Target): - alias = "missing_fields" - core_fields = (Tags,) - - missing_fields_tgt = MissingFieldsTarget( - {Tags.alias: ["tag"]}, Address("", target_name="missing_fields") - ) - with pytest.raises(AssertionError) as exc: - generate(missing_fields_tgt, ["fake.txt"]) - assert "does not have both a `dependencies` and `sources` field" in str(exc.value) - - def test_generated_targets_address_validation() -> None: """Ensure that all addresses are well formed.""" @@ -634,29 +499,6 @@ class MockTarget(Target): ) -def test_generated_targets_unused_overrides() -> None: - class MockGenerator(Target): - alias = "generator" - core_fields = (Dependencies, Tags, MultipleSourcesField) - - class MockGenerated(Target): - alias = "generated" - core_fields = (Dependencies, Tags, SingleSourceField) - - with pytest.raises(InvalidFieldException) as exc: - generate_file_level_targets( - MockGenerated, - MockGenerator( - {MultipleSourcesField.alias: ["f*.ext"]}, Address("dir", target_name="gen") - ), - ["dir/f1.ext", "dir/f2.ext"], - None, - add_dependencies_on_all_siblings=False, - overrides={"dir/fake.ext": {"tags": ["overridden"]}}, - ) - assert "Unused file paths in the `overrides` field for dir:gen: ['fake.ext']" in str(exc.value) - - # ----------------------------------------------------------------------------------------------- # Test FieldSet. Also see engine/internals/graph_test.py. # ----------------------------------------------------------------------------------------------- @@ -1449,15 +1291,25 @@ def test_overrides_field_normalization() -> None: path_field = OverridesField( {"foo.ext": tgt1_override, ("foo.ext", "bar*.ext"): tgt2_override}, Address("dir") ) - to_globs = [ - path_globs.globs for path_globs in path_field.to_path_globs(FilesNotFoundBehavior.error) + globs = OverridesField.to_path_globs( + Address("dir"), path_field.flatten(), FilesNotFoundBehavior.error + ) + assert [path_globs.globs for path_globs in globs] == [ + ("dir/foo.ext",), + ("dir/bar*.ext",), ] - assert to_globs == [("dir/foo.ext",), ("dir/bar*.ext", "dir/foo.ext")] - assert path_field.flatten_paths( - { - Paths(("dir/foo.ext",), ()): tgt1_override, - Paths(("dir/bar1.ext", "dir/bar2.ext"), ()): tgt2_override, - }, + assert OverridesField.flatten_paths( + addr, + [ + (paths, globs, overrides) + for (paths, overrides), globs in zip( + [ + (Paths(("dir/foo.ext",), ()), tgt1_override), + (Paths(("dir/bar1.ext", "dir/bar2.ext"), ()), tgt2_override), + ], + globs, + ) + ], ) == { "dir/foo.ext": tgt1_override, "dir/bar1.ext": tgt2_override, @@ -1469,9 +1321,10 @@ def test_overrides_field_normalization() -> None: } with pytest.raises(InvalidFieldException): # Same field is overridden for the same file multiple times, which is an error. - path_field.flatten_paths( - { - Paths(("dir/foo.ext",), ()): tgt1_override, - Paths(("dir/foo.ext", "dir/bar.ext"), ()): tgt1_override, - } + OverridesField.flatten_paths( + addr, + [ + (Paths(("dir/foo.ext",), ()), PathGlobs([]), tgt1_override), + (Paths(("dir/foo.ext", "dir/bar.ext"), ()), PathGlobs([]), tgt1_override), + ], ) diff --git a/src/python/pants/help/help_info_extracter.py b/src/python/pants/help/help_info_extracter.py index 821a5e77a50..13575fe2698 100644 --- a/src/python/pants/help/help_info_extracter.py +++ b/src/python/pants/help/help_info_extracter.py @@ -15,7 +15,7 @@ from pants.build_graph.build_configuration import BuildConfiguration from pants.engine.goal import GoalSubsystem from pants.engine.rules import TaskRule -from pants.engine.target import Field, RegisteredTargetTypes, StringField, Target +from pants.engine.target import Field, RegisteredTargetTypes, StringField, Target, TargetGenerator from pants.engine.unions import UnionMembership, UnionRule from pants.option.option_util import is_dict_option, is_list_option from pants.option.options import Options @@ -225,6 +225,12 @@ def create( union_membership: UnionMembership, get_field_type_provider: Callable[[type[Field]], str] | None, ) -> TargetTypeHelpInfo: + fields = list(target_type.class_field_types(union_membership=union_membership)) + if issubclass(target_type, TargetGenerator): + # NB: Even though the moved_fields will never be present on a constructed + # TargetGenerator, they are legal arguments... and that is what most help consumers + # are interested in. + fields.extend(target_type.moved_fields) return cls( alias=target_type.alias, provider=provider, @@ -237,7 +243,7 @@ def create( if get_field_type_provider is None else get_field_type_provider(field), ) - for field in target_type.class_field_types(union_membership=union_membership) + for field in fields if not field.alias.startswith("_") and field.removal_version is None ), ) diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index 3381044b4d9..6772d553e07 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -26,7 +26,7 @@ from pants.engine.rules import QueryRule, collect_rules, rule from pants.engine.streaming_workunit_handler import rules as streaming_workunit_handler_rules from pants.engine.target import RegisteredTargetTypes -from pants.engine.unions import UnionMembership +from pants.engine.unions import UnionMembership, UnionRule from pants.init import specs_calculator from pants.option.global_options import ( DEFAULT_EXECUTION_OPTIONS, @@ -219,7 +219,7 @@ def setup_graph_extended( build_root_path = build_root or get_buildroot() rules = build_configuration.rules - union_membership = UnionMembership.from_rules(build_configuration.union_rules) + union_membership: UnionMembership registered_target_types = RegisteredTargetTypes.create(build_configuration.target_types) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS @@ -280,6 +280,12 @@ def build_root_singleton() -> BuildRoot: QueryRule(Snapshot, [PathGlobs]), # Used by the SchedulerService. ) ) + union_membership = UnionMembership.from_rules( + ( + *build_configuration.union_rules, + *(r for r in rules if isinstance(r, UnionRule)), + ) + ) def ensure_absolute_path(v: str) -> str: return Path(v).resolve().as_posix()