diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d37b236c2..b01e7c206e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5869 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 diff --git a/contrib/runners/noop_runner/BUILD b/contrib/runners/noop_runner/BUILD new file mode 100644 index 0000000000..02983d4e61 --- /dev/null +++ b/contrib/runners/noop_runner/BUILD @@ -0,0 +1,7 @@ +# stevedore_extension( +# name="runner", +# namespace="st2common.runners.runner", +# entry_points={ +# "noop": "noop_runner.noop_runner", +# }, +# ) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index 8f99aa3d5e..d006ca3989 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". +These plugins might be useful outside of the StackStorm project: +- `stevedore_extensions` + These StackStorm-specific plugins might be useful in other StackStorm-related repos. - `pack_metadata` @@ -66,3 +69,12 @@ the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will be regenerated if any of the files used to generate them have changed. Also, running the `lint` goal will fail if the schemas need to be regenerated. + +### `stevedore_extensions` plugin + +This plugin teaches pants how to infer dependencies on stevedore +extensions (python plugins loaded at runtime via entry points). +This includes the `stevedore_extensions` target and the +`stevedore_namespaces` field. When necessary, it generates an +`entry_points.txt` file so that stevedore can work correctly +within pants sandboxes. diff --git a/pants-plugins/stevedore_extensions/BUILD b/pants-plugins/stevedore_extensions/BUILD new file mode 100644 index 0000000000..0eea8b1cf1 --- /dev/null +++ b/pants-plugins/stevedore_extensions/BUILD @@ -0,0 +1,5 @@ +python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/stevedore_extensions/__init__.py b/pants-plugins/stevedore_extensions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/stevedore_extensions/python_target_dependencies.py b/pants-plugins/stevedore_extensions/python_target_dependencies.py new file mode 100644 index 0000000000..6ca1d902e0 --- /dev/null +++ b/pants-plugins/stevedore_extensions/python_target_dependencies.py @@ -0,0 +1,123 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from dataclasses import dataclass +from typing import List, Mapping, Tuple + +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, + PythonTestsDependenciesField, +) +from pants.engine.addresses import Address +from pants.engine.rules import collect_rules, rule, UnionRule +from pants.engine.target import ( + AllTargets, + FieldSet, + InferDependenciesRequest, + InferredDependencies, +) +from pants.util.frozendict import FrozenDict +from pants.util.logging import LogLevel +from pants.util.ordered_set import OrderedSet + +from stevedore_extensions.target_types import ( + AllStevedoreExtensionTargets, + StevedoreEntryPointsField, + StevedoreExtension, + StevedoreNamespaceField, + StevedoreNamespacesField, +) + + +# ----------------------------------------------------------------------------------------------- +# Utility rules to analyze all `StevedoreExtension` targets +# ----------------------------------------------------------------------------------------------- + + +@rule(desc="Find all StevedoreExtension targets in project", level=LogLevel.DEBUG) +def find_all_stevedore_extension_targets( + targets: AllTargets, +) -> AllStevedoreExtensionTargets: + return AllStevedoreExtensionTargets( + tgt for tgt in targets if tgt.has_field(StevedoreEntryPointsField) + ) + + +@dataclass(frozen=True) +class StevedoreExtensions: + """A mapping of stevedore namespaces to a list of StevedoreExtension targets that provide them""" + + mapping: FrozenDict[str, Tuple[StevedoreExtension]] + + +@rule( + desc="Creating map of stevedore_extension namespaces to StevedoreExtension targets", + level=LogLevel.DEBUG, +) +async def map_stevedore_extensions( + stevedore_extensions: AllStevedoreExtensionTargets, +) -> StevedoreExtensions: + mapping: Mapping[str, List[StevedoreExtension]] = defaultdict(list) + for extension in stevedore_extensions: + mapping[extension[StevedoreNamespaceField].value].append(extension) + return StevedoreExtensions( + FrozenDict((k, tuple(v)) for k, v in sorted(mapping.items())) + ) + + +# ----------------------------------------------------------------------------------------------- +# Dependencies for `python_test` and `python_tests` targets +# ----------------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PythonTestsStevedoreNamespaceInferenceFieldSet(FieldSet): + required_fields = (PythonTestsDependenciesField, StevedoreNamespacesField) + + stevedore_namespaces: StevedoreNamespacesField + + +class InferStevedoreNamespaceDependencies(InferDependenciesRequest): + infer_from = PythonTestsStevedoreNamespaceInferenceFieldSet + + +@rule( + desc="Infer stevedore_extension target dependencies based on namespace list.", + level=LogLevel.DEBUG, +) +async def infer_stevedore_namespace_dependencies( + request: InferStevedoreNamespaceDependencies, + stevedore_extensions: StevedoreExtensions, +) -> InferredDependencies: + namespaces: StevedoreNamespacesField = request.field_set.stevedore_namespaces + if namespaces.value is None: + return InferredDependencies(()) + + addresses = [] + for namespace in namespaces.value: + extensions = stevedore_extensions.mapping.get(namespace, ()) + addresses.extend(extension.address for extension in extensions) + + result: OrderedSet[Address] = OrderedSet(addresses) + return InferredDependencies(sorted(result)) + + +def rules(): + return [ + *collect_rules(), + PythonTestsGeneratorTarget.register_plugin_field(StevedoreNamespacesField), + PythonTestTarget.register_plugin_field(StevedoreNamespacesField), + UnionRule(InferDependenciesRequest, InferStevedoreNamespaceDependencies), + ] diff --git a/pants-plugins/stevedore_extensions/python_target_dependencies_test.py b/pants-plugins/stevedore_extensions/python_target_dependencies_test.py new file mode 100644 index 0000000000..156e8d2b06 --- /dev/null +++ b/pants-plugins/stevedore_extensions/python_target_dependencies_test.py @@ -0,0 +1,182 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.addresses import Address +from pants.engine.target import InferredDependencies +from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.frozendict import FrozenDict + +from .python_target_dependencies import ( + InferStevedoreNamespaceDependencies, + PythonTestsStevedoreNamespaceInferenceFieldSet, + StevedoreExtensions, + rules as stevedore_dep_rules, +) +from .target_types import ( + AllStevedoreExtensionTargets, + StevedoreExtension, +) + + +# random set of runner names to use in tests +st2_runners = ["noop", "python", "foobar"] + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = RuleRunner( + rules=[ + *python_target_types_rules(), + *stevedore_dep_rules(), + QueryRule(AllStevedoreExtensionTargets, ()), + QueryRule(StevedoreExtensions, ()), + QueryRule(InferredDependencies, (InferStevedoreNamespaceDependencies,)), + ], + target_types=[ + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, + StevedoreExtension, + ], + ) + for runner in st2_runners: + rule_runner.write_files( + { + f"runners/{runner}_runner/BUILD": dedent( + f"""\ + stevedore_extension( + name="runner", + namespace="st2common.runners.runner", + entry_points={{ + "{runner}": "{runner}_runner.{runner}_runner", + }}, + ) + stevedore_extension( + name="thing", + namespace="some.thing.else", + entry_points={{ + "{runner}": "{runner}_runner.thing", + }}, + ) + """ + ), + f"runners/{runner}_runner/{runner}_runner/BUILD": "python_sources()", + f"runners/{runner}_runner/{runner}_runner/__init__.py": "", + f"runners/{runner}_runner/{runner}_runner/{runner}_runner.py": "", + f"runners/{runner}_runner/{runner}_runner/thing.py": "", + } + ) + args = [ + "--source-root-patterns=runners/*_runner", + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return rule_runner + + +# ----------------------------------------------------------------------------------------------- +# Tests for utility rules +# ----------------------------------------------------------------------------------------------- + + +def test_find_all_stevedore_extension_targets(rule_runner: RuleRunner) -> None: + assert rule_runner.request( + AllStevedoreExtensionTargets, [] + ) == AllStevedoreExtensionTargets( + rule_runner.get_target( + Address(f"runners/{runner}_runner", target_name=target_name) + ) + for runner in sorted(st2_runners) + for target_name in ["runner", "thing"] + ) + + +def test_map_stevedore_extensions(rule_runner: RuleRunner) -> None: + assert rule_runner.request(StevedoreExtensions, []) == StevedoreExtensions( + FrozenDict( + { + "some.thing.else": tuple( + rule_runner.get_target( + Address(f"runners/{runner}_runner", target_name="thing") + ) + for runner in sorted(st2_runners) + ), + "st2common.runners.runner": tuple( + rule_runner.get_target( + Address(f"runners/{runner}_runner", target_name="runner") + ) + for runner in sorted(st2_runners) + ), + } + ) + ) + + +# ----------------------------------------------------------------------------------------------- +# Tests for dependency inference of python targets (python_tests, etc) +# ----------------------------------------------------------------------------------------------- + + +def test_infer_stevedore_namespace_dependencies(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/foobar/BUILD": dedent( + """\ + python_tests( + name="tests", + stevedore_namespaces=["st2common.runners.runner"], + ) + """ + ), + "src/foobar/test_something.py": "", + } + ) + + def run_dep_inference(address: Address) -> InferredDependencies: + target = rule_runner.get_target(address) + return rule_runner.request( + InferredDependencies, + [ + InferStevedoreNamespaceDependencies( + PythonTestsStevedoreNamespaceInferenceFieldSet.create(target) + ) + ], + ) + + # this asserts that only the st2common.runners.runner namespace gets selected. + assert run_dep_inference( + Address( + "src/foobar", target_name="tests", relative_file_path="test_something.py" + ), + ) == InferredDependencies( + [ + Address( + f"runners/{runner}_runner", + target_name="runner", + ) + for runner in st2_runners + ], + ) diff --git a/pants-plugins/stevedore_extensions/register.py b/pants-plugins/stevedore_extensions/register.py new file mode 100644 index 0000000000..6f13676d41 --- /dev/null +++ b/pants-plugins/stevedore_extensions/register.py @@ -0,0 +1,39 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pants.backend.codegen import export_codegen_goal + +from stevedore_extensions import ( + target_types_rules, + rules as stevedore_rules, + python_target_dependencies, +) +from stevedore_extensions.target_types import StevedoreExtension + + +# TODO: add the entry_points automatically to setup_py +# TODO: add stevedore_namespaces field to python_sources? + + +def rules(): + return [ + *target_types_rules.rules(), + *stevedore_rules.rules(), + *python_target_dependencies.rules(), + *export_codegen_goal.rules(), + ] + + +def target_types(): + return [StevedoreExtension] diff --git a/pants-plugins/stevedore_extensions/target_types.py b/pants-plugins/stevedore_extensions/target_types.py new file mode 100644 index 0000000000..da4757e153 --- /dev/null +++ b/pants-plugins/stevedore_extensions/target_types.py @@ -0,0 +1,155 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# repurposed from pants.backend.python.target_types +import os + +from dataclasses import dataclass +from typing import Dict, Optional + +from pants.engine.addresses import Address +from pants.engine.collection import Collection +from pants.engine.target import ( + AsyncFieldMixin, + COMMON_TARGET_FIELDS, + Dependencies, + DictStringToStringField, + InvalidFieldException, + SecondaryOwnerMixin, + StringField, + StringSequenceField, + Target, + Targets, +) +from pants.backend.python.target_types import EntryPoint, PythonResolveField +from pants.source.filespec import Filespec +from pants.util.strutil import softwrap + + +@dataclass(frozen=True) +class StevedoreEntryPoint: + name: str + value: EntryPoint + + +class StevedoreEntryPoints(Collection[StevedoreEntryPoint]): + pass + + +class StevedoreNamespaceField(StringField): + alias = "namespace" + help = softwrap( + """ + Set the stevedore extension namespace. + + This looks like a python module 'my.stevedore.namespace', but a python module + of that name does not need to exist. This is what a stevedore ExtensionManager + uses to look up relevant entry_points from pkg_resources. + """ + ) + required = True + + +class StevedoreEntryPointsField( + AsyncFieldMixin, SecondaryOwnerMixin, DictStringToStringField +): + # based on pants.backend.python.target_types.PexEntryPointField + alias = "entry_points" + help = softwrap( + """ + Map stevedore extension names to the entry_point that implements each name. + + Specify each entry_point to a module stevedore should use for the given extension name. + You can specify a full module like 'path.to.module' and 'path.to.module:func', or use a + shorthand to specify a file name, using the same syntax as the `sources` field: + + 1) 'app.py', Pants will convert into the module `path.to.app`; + 2) 'app.py:func', Pants will convert into `path.to.app:func`. + + You must use the file name shorthand for file arguments to work with this target. + """ + ) + required = True + value: StevedoreEntryPoints + + @classmethod + def compute_value( + cls, raw_value: Optional[Dict[str, str]], address: Address + ) -> Collection[StevedoreEntryPoint]: + # TODO: maybe support raw entry point maps like ["name = path.to.module:func"] + # raw_value: Optional[Union[Dict[str, str], List[str]]] + raw_entry_points = super().compute_value(raw_value, address) + entry_points = [] + for name, value in raw_entry_points.items(): + try: + entry_point = EntryPoint.parse( + value, provenance=f"for {name} on {address}" + ) + except ValueError as e: + raise InvalidFieldException(str(e)) + entry_points.append(StevedoreEntryPoint(name=name, value=entry_point)) + return StevedoreEntryPoints(entry_points) + + @property + def filespec(self) -> Filespec: + includes = [] + for entry_point in self.value: + if not entry_point.value.module.endswith(".py"): + continue + full_glob = os.path.join(self.address.spec_path, entry_point.value.module) + includes.append(full_glob) + return {"includes": includes} + + +# See `target_types_rules.py` for the `ResolveStevedoreEntryPointsRequest -> ResolvedStevedoreEntryPoints` rule. +@dataclass(frozen=True) +class ResolvedStevedoreEntryPoints: + val: Optional[StevedoreEntryPoints] + + +@dataclass(frozen=True) +class ResolveStevedoreEntryPointsRequest: + """Determine the `entry_points` for a `stevedore_extension` after applying all syntactic sugar.""" + + entry_points_field: StevedoreEntryPointsField + + +class StevedoreExtension(Target): + alias = "stevedore_extension" + core_fields = ( + *COMMON_TARGET_FIELDS, + StevedoreNamespaceField, + StevedoreEntryPointsField, + Dependencies, + PythonResolveField, + ) + help = "Entry points used to generate setuptools metadata for stevedore." + + +# This is a lot like a SpecialCasedDependencies field, but it doesn't list targets directly. +class StevedoreNamespacesField(StringSequenceField): + alias = "stevedore_namespaces" + help = softwrap( + """ + List the stevedore namespaces required by this target. + + All stevedore_extension targets with these namespaces will be added as + dependencies so that they are available on PYTHONPATH during tests. + The stevedore namespace format (my.stevedore.extension) is similar + to a python namespace. + """ + ) + + +class AllStevedoreExtensionTargets(Targets): + pass diff --git a/pants-plugins/stevedore_extensions/target_types_rules.py b/pants-plugins/stevedore_extensions/target_types_rules.py new file mode 100644 index 0000000000..f93eb6079e --- /dev/null +++ b/pants-plugins/stevedore_extensions/target_types_rules.py @@ -0,0 +1,212 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# repurposed from pants.backend.python.target_types_rules +import dataclasses +import os +from dataclasses import dataclass + +from pants.backend.python.dependency_inference.module_mapper import ( + PythonModuleOwners, + PythonModuleOwnersRequest, +) +from pants.backend.python.dependency_inference.rules import import_rules +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField +from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths +from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule +from pants.engine.target import ( + Dependencies, + DependenciesRequest, + ExplicitlyProvidedDependencies, + FieldSet, + InferDependenciesRequest, + InferredDependencies, + InvalidFieldException, +) +from pants.source.source_root import SourceRoot, SourceRootRequest +from pants.util.logging import LogLevel + +from stevedore_extensions.target_types import ( + ResolvedStevedoreEntryPoints, + ResolveStevedoreEntryPointsRequest, + StevedoreEntryPoints, + StevedoreEntryPointsField, +) + + +# ----------------------------------------------------------------------------------------------- +# `StevedoreExtension` target rules +# ----------------------------------------------------------------------------------------------- + + +@rule( + desc="Determining the entry points for a `stevedore_extension` target", + level=LogLevel.DEBUG, +) +async def resolve_stevedore_entry_points( + request: ResolveStevedoreEntryPointsRequest, +) -> ResolvedStevedoreEntryPoints: + # based on: pants.backend.python.target_types_rules.resolve_pex_entry_point + + # supported schemes mirror those in resolve_pex_entry_point: + # 1) this does not support None, unlike pex_entry_point. + # 2) `path.to.module` => preserve exactly. + # 3) `path.to.module:func` => preserve exactly. + # 4) `app.py` => convert into `path.to.app`. + # 5) `app.py:func` => convert into `path.to.app:func`. + + address = request.entry_points_field.address + + # Use the engine to validate that any file exists + entry_point_paths_results = await MultiGet( + Get( + Paths, + PathGlobs( + [os.path.join(address.spec_path, entry_point.value.module)], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"{address}'s `{request.entry_points_field.alias}` field", + ), + ) + for entry_point in request.entry_points_field.value + if entry_point.value.module.endswith(".py") + ) + + # use iter so we can use next() below + iter_entry_point_paths_results = iter(entry_point_paths_results) + + # We will have already raised if the glob did not match, i.e. if there were no files. But + # we need to check if they used a file glob (`*` or `**`) that resolved to >1 file. + # + # It is clearer to iterate over entry_point_paths_results, but we need to include + # the original glob in the error message, so we have to check for ".py" again. + for entry_point in request.entry_points_field.value: + # We only need paths/globs for this check. Ignore any modules. + if not entry_point.value.module.endswith(".py"): + continue + + entry_point_paths = next(iter_entry_point_paths_results) + if len(entry_point_paths.files) != 1: + raise InvalidFieldException( + f"Multiple files matched for the `{request.entry_points_field.alias}` " + f"{entry_point.value.spec!r} for the target {address}, but only one file expected. Are you using " + f"a glob, rather than a file name?\n\n" + f"All matching files: {list(entry_point_paths.files)}." + ) + + source_root_results = await MultiGet( + Get( + SourceRoot, + SourceRootRequest, + SourceRootRequest.for_file(entry_point_path.files[0]), + ) + for entry_point_path in entry_point_paths_results + ) + + # use iter so we can use next() below + iter_entry_point_paths_results = iter(entry_point_paths_results) + iter_source_root_results = iter(source_root_results) + + resolved = [] + for entry_point in request.entry_points_field.value: + # If it's already a module (cases #2 and #3), we'll just use that. + # Otherwise, convert the file name into a module path (cases #4 and #5). + if not entry_point.value.module.endswith(".py"): + resolved.append(entry_point) + continue + + entry_point_path = next(iter_entry_point_paths_results).files[0] + source_root = next(iter_source_root_results) + + stripped_source_path = os.path.relpath(entry_point_path, source_root.path) + module_base, _ = os.path.splitext(stripped_source_path) + normalized_path = module_base.replace(os.path.sep, ".") + resolved_ep_val = dataclasses.replace(entry_point.value, module=normalized_path) + resolved.append(dataclasses.replace(entry_point, value=resolved_ep_val)) + return ResolvedStevedoreEntryPoints(StevedoreEntryPoints(resolved)) + + +@dataclass(frozen=True) +class StevedoreEntryPointsInferenceFieldSet(FieldSet): + required_fields = (StevedoreEntryPointsField, Dependencies, PythonResolveField) + + entry_points: StevedoreEntryPointsField + dependencies: Dependencies + resolve: PythonResolveField + + +class InferStevedoreExtensionDependencies(InferDependenciesRequest): + infer_from = StevedoreEntryPointsInferenceFieldSet + + +@rule( + desc="Inferring dependency from the stevedore_extension `entry_points` field", + level=LogLevel.DEBUG, +) +async def infer_stevedore_entry_points_dependencies( + request: InferStevedoreExtensionDependencies, + python_setup: PythonSetup, +) -> InferredDependencies: + entry_points: ResolvedStevedoreEntryPoints + explicitly_provided_deps, entry_points = await MultiGet( + Get( + ExplicitlyProvidedDependencies, + DependenciesRequest(request.field_set.dependencies), + ), + Get( + ResolvedStevedoreEntryPoints, + ResolveStevedoreEntryPointsRequest(request.field_set.entry_points), + ), + ) + if entry_points.val is None: + return InferredDependencies() + address = request.field_set.address + owners_per_entry_point = await MultiGet( + Get( + PythonModuleOwners, + PythonModuleOwnersRequest( + entry_point.value.module, + resolve=request.field_set.resolve.normalized_value(python_setup), + ), + ) + for entry_point in entry_points.val + ) + original_entry_points = request.field_set.entry_points.value + resolved_owners = [] + for entry_point, owners, original_ep in zip( + entry_points.val, owners_per_entry_point, original_entry_points + ): + explicitly_provided_deps.maybe_warn_of_ambiguous_dependency_inference( + owners.ambiguous, + address, + import_reference="module", + context=( + f"The stevedore_extension target {address} has in its entry_points field " + f'`"{entry_point.name}": "{repr(original_ep.value.spec)}"`,' + f"which maps to the Python module `{entry_point.value.module}`" + ), + ) + maybe_disambiguated = explicitly_provided_deps.disambiguated(owners.ambiguous) + unambiguous_owners = owners.unambiguous or ( + (maybe_disambiguated,) if maybe_disambiguated else () + ) + resolved_owners.extend(unambiguous_owners) + return InferredDependencies(resolved_owners) + + +def rules(): + return [ + *collect_rules(), + *import_rules(), + UnionRule(InferDependenciesRequest, InferStevedoreExtensionDependencies), + ] diff --git a/pants-plugins/stevedore_extensions/target_types_rules_test.py b/pants-plugins/stevedore_extensions/target_types_rules_test.py new file mode 100644 index 0000000000..fa27866ac2 --- /dev/null +++ b/pants-plugins/stevedore_extensions/target_types_rules_test.py @@ -0,0 +1,229 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from textwrap import dedent + +import pytest + +from pants.backend.python.target_types import ( + EntryPoint, + PythonSourceTarget, + PythonSourcesGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import InferredDependencies +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .target_types_rules import ( + InferStevedoreExtensionDependencies, + StevedoreEntryPointsInferenceFieldSet, + resolve_stevedore_entry_points, + rules as stevedore_target_types_rules, +) +from .target_types import ( + ResolveStevedoreEntryPointsRequest, + ResolvedStevedoreEntryPoints, + StevedoreEntryPoint, + StevedoreEntryPointsField, # on stevedore_extension target + StevedoreExtension, +) + + +def test_resolve_stevedore_entry_points() -> None: + # based on: pants.backend.python.target_types_test.test_resolve_pex_binary_entry_point + rule_runner = RuleRunner( + rules=[ + resolve_stevedore_entry_points, + QueryRule( + ResolvedStevedoreEntryPoints, (ResolveStevedoreEntryPointsRequest,) + ), + ] + ) + + def assert_resolved( + *, entry_point: str | None, expected: EntryPoint | None + ) -> None: + plugin_name = "plugin" + addr = Address("src/python/project") + rule_runner.write_files( + { + "src/python/project/app.py": "", + "src/python/project/f2.py": "", + } + ) + ep_field = StevedoreEntryPointsField({plugin_name: entry_point}, addr) + + result = rule_runner.request( + ResolvedStevedoreEntryPoints, [ResolveStevedoreEntryPointsRequest(ep_field)] + ) + + assert result.val is not None + assert len(result.val) == 1 + assert result.val[0].name == plugin_name + assert result.val[0].value == expected + + # Full module provided. + assert_resolved( + entry_point="custom.entry_point", expected=EntryPoint("custom.entry_point") + ) + assert_resolved( + entry_point="custom.entry_point:func", + expected=EntryPoint.parse("custom.entry_point:func"), + ) + + # File names are expanded into the full module path. + assert_resolved(entry_point="app.py", expected=EntryPoint(module="project.app")) + assert_resolved( + entry_point="app.py:func", + expected=EntryPoint(module="project.app", function="func"), + ) + + with pytest.raises(ExecutionError): + assert_resolved( + entry_point="doesnt_exist.py", expected=EntryPoint("doesnt matter") + ) + # Resolving >1 file is an error. + with pytest.raises(ExecutionError): + assert_resolved(entry_point="*.py", expected=EntryPoint("doesnt matter")) + + # Test with multiple entry points (keep indiviudual asserts above, + # despite apparent duplication below, to simplify finding bugs). + rule_runner.write_files( + { + "src/python/project/app.py": "", + "src/python/project/f2.py": "", + } + ) + entry_points_field = StevedoreEntryPointsField( + { + "a": "custom.entry_point", + "b": "custom.entry_point:func", + "c": "app.py", + "d": "app.py:func", + }, + Address("src/python/project"), + ) + + resolved = rule_runner.request( + ResolvedStevedoreEntryPoints, + [ResolveStevedoreEntryPointsRequest(entry_points_field)], + ) + + assert resolved.val is not None + assert set(resolved.val) == { + StevedoreEntryPoint("a", EntryPoint(module="custom.entry_point")), + StevedoreEntryPoint( + "b", EntryPoint(module="custom.entry_point", function="func") + ), + StevedoreEntryPoint("c", EntryPoint(module="project.app")), + StevedoreEntryPoint("d", EntryPoint(module="project.app", function="func")), + } + + +def test_infer_stevedore_entry_points_dependencies() -> None: + rule_runner = RuleRunner( + rules=[ + *python_target_types_rules(), + *stevedore_target_types_rules(), + QueryRule(InferredDependencies, (InferStevedoreExtensionDependencies,)), + ], + target_types=[ + PythonSourceTarget, + PythonSourcesGeneratorTarget, + StevedoreExtension, + ], + ) + rule_runner.write_files( + { + "runners/foobar_runner/BUILD": dedent( + """\ + stevedore_extension( + name="runner", + namespace="st2common.runners.runner", + entry_points={ + "foobar": "foobar_runner.foobar_runner", + }, + ) + + stevedore_extension( + name="foobar", + namespace="example.foobar", + entry_points={ + "thing1": "foobar_runner.thing1:ThingBackend", + "thing2": "foobar_runner.thing2:ThingBackend", + }, + ) + """ + ), + "runners/foobar_runner/foobar_runner/BUILD": "python_sources()", + "runners/foobar_runner/foobar_runner/__init__.py": "", + "runners/foobar_runner/foobar_runner/foobar_runner.py": "", + "runners/foobar_runner/foobar_runner/thing1.py": dedent( + """\ + class ThingBackend: + pass + """ + ), + "runners/foobar_runner/foobar_runner/thing2.py": dedent( + """\ + class ThingBackend: + pass + """ + ), + } + ) + + def run_dep_inference(address: Address) -> InferredDependencies: + args = [ + "--source-root-patterns=runners/*_runner", + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + target = rule_runner.get_target(address) + return rule_runner.request( + InferredDependencies, + [ + InferStevedoreExtensionDependencies( + StevedoreEntryPointsInferenceFieldSet.create(target) + ) + ], + ) + + assert run_dep_inference( + Address("runners/foobar_runner", target_name="runner") + ) == InferredDependencies( + [ + Address( + "runners/foobar_runner/foobar_runner", + relative_file_path="foobar_runner.py", + ), + ], + ) + + assert run_dep_inference( + Address("runners/foobar_runner", target_name="foobar") + ) == InferredDependencies( + [ + Address( + "runners/foobar_runner/foobar_runner", + relative_file_path="thing1.py", + ), + Address( + "runners/foobar_runner/foobar_runner", + relative_file_path="thing2.py", + ), + ], + ) diff --git a/pants.toml b/pants.toml index 375f9a1843..22a8a4b1a4 100644 --- a/pants.toml +++ b/pants.toml @@ -28,6 +28,7 @@ backend_packages = [ "pack_metadata", "sample_conf", "schemas", + #"stevedore_extensions", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index fe53c9f265..2b645538d5 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -9,6 +9,7 @@ python_tests( # several files import tests.unit.base which is ambiguous. Tell pants which one to use. "st2common/tests/unit/base.py", ], + # stevedore_namespaces=["st2common.runners.runner"], ) python_sources()