From b7d2ca30b6ebe1c6eb099e87e5894ea2b8c7de3b Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 7 Jan 2022 17:43:45 -0700 Subject: [PATCH] Move `generate-lockfiles` goal to `core/goals` # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/goals/lockfile.py | 118 +----------------- .../pants/core/goals/generate_lockfiles.py | 113 ++++++++++++++++- src/python/pants/core/register.py | 2 + 3 files changed, 118 insertions(+), 115 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 288af5e49a25..fa69cd12865f 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -7,7 +7,7 @@ from collections import defaultdict from dataclasses import dataclass from pathlib import PurePath -from typing import Iterable, cast +from typing import Iterable from pants.backend.python.pip_requirement import PipRequirement from pants.backend.python.subsystems.poetry import ( @@ -30,84 +30,27 @@ ) from pants.backend.python.util_rules.pex import PexRequest, PexRequirements, VenvPex, VenvPexProcess from pants.core.goals.generate_lockfiles import ( + GenerateLockfilesSubsystem, KnownUserResolveNames, KnownUserResolveNamesRequest, Lockfile, LockfileRequest, RequestedUserResolveNames, - ToolLockfileSentinel, UserLockfileRequests, WrappedLockfileRequest, - determine_resolves_to_generate, - filter_tool_lockfile_requests, + logger, ) -from pants.engine.fs import ( - CreateDigest, - Digest, - DigestContents, - FileContent, - MergeDigests, - Workspace, -) -from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.fs import CreateDigest, Digest, DigestContents, FileContent from pants.engine.process import ProcessCacheScope, ProcessResult -from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import AllTargets -from pants.engine.unions import UnionMembership, UnionRule +from pants.engine.unions import UnionRule from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet logger = logging.getLogger(__name__) -class GenerateLockfilesSubsystem(GoalSubsystem): - name = "generate-lockfiles" - help = "Generate lockfiles for Python third-party dependencies." - required_union_implementations = (ToolLockfileSentinel, KnownUserResolveNames) - - @classmethod - def register_options(cls, register) -> None: - super().register_options(register) - register( - "--resolve", - type=list, - member_type=str, - advanced=False, - help=( - "Only generate lockfiles for the specified resolve(s).\n\n" - "Resolves are the logical names for the different lockfiles used in your project. " - "For your own code's dependencies, these come from the option " - "`[python].experimental_resolves`. For tool lockfiles, resolve " - "names are the options scope for that tool such as `black`, `pytest`, and " - "`mypy-protobuf`.\n\n" - "For example, you can run `./pants generate-lockfiles --resolve=black " - "--resolve=pytest --resolve=data-science` to only generate lockfiles for those " - "two tools and your resolve named `data-science`.\n\n" - "If you specify an invalid resolve name, like 'fake', Pants will output all " - "possible values.\n\n" - "If not specified, Pants will generate lockfiles for all resolves." - ), - ) - register( - "--custom-command", - advanced=True, - type=str, - default=None, - help=( - "If set, lockfile headers will say to run this command to regenerate the lockfile, " - "rather than running `./pants generate-lockfiles --resolve=` like normal." - ), - ) - - @property - def resolve_names(self) -> tuple[str, ...]: - return tuple(self.options.resolve) - - @property - def custom_command(self) -> str | None: - return cast("str | None", self.options.custom_command) - - @dataclass(frozen=True) class PythonLockfileRequest(LockfileRequest): requirements: FrozenOrderedSet[str] @@ -322,55 +265,6 @@ async def setup_user_lockfile_requests( ) -class GenerateLockfilesGoal(Goal): - subsystem_cls = GenerateLockfilesSubsystem - - -@goal_rule -async def generate_lockfiles_goal( - workspace: Workspace, - union_membership: UnionMembership, - generate_lockfiles_subsystem: GenerateLockfilesSubsystem, -) -> GenerateLockfilesGoal: - known_user_resolve_names = await MultiGet( - Get(KnownUserResolveNames, KnownUserResolveNamesRequest, request()) - for request in union_membership.get(KnownUserResolveNamesRequest) - ) - requested_user_resolve_names, requested_tool_sentinels = determine_resolves_to_generate( - known_user_resolve_names, - union_membership.get(ToolLockfileSentinel), - set(generate_lockfiles_subsystem.resolve_names), - ) - - all_specified_user_requests = await MultiGet( - Get(UserLockfileRequests, RequestedUserResolveNames, resolve_names) - for resolve_names in requested_user_resolve_names - ) - specified_tool_requests = await MultiGet( - Get(WrappedLockfileRequest, ToolLockfileSentinel, sentinel()) - for sentinel in requested_tool_sentinels - ) - applicable_tool_requests = filter_tool_lockfile_requests( - specified_tool_requests, - resolve_specified=bool(generate_lockfiles_subsystem.resolve_names), - ) - - results = await MultiGet( - Get(Lockfile, LockfileRequest, req) - for req in ( - *(req for reqs in all_specified_user_requests for req in reqs), - *applicable_tool_requests, - ) - ) - - merged_digest = await Get(Digest, MergeDigests(res.digest for res in results)) - workspace.write_digest(merged_digest) - for result in results: - logger.info(f"Wrote lockfile for the resolve `{result.resolve_name}` to {result.path}") - - return GenerateLockfilesGoal(exit_code=0) - - def rules(): return ( *collect_rules(), diff --git a/src/python/pants/core/goals/generate_lockfiles.py b/src/python/pants/core/goals/generate_lockfiles.py index d35d7740d8a5..f7e7e1bd501a 100644 --- a/src/python/pants/core/goals/generate_lockfiles.py +++ b/src/python/pants/core/goals/generate_lockfiles.py @@ -4,12 +4,18 @@ from __future__ import annotations import itertools +import logging from dataclasses import dataclass -from typing import ClassVar, Iterable, Sequence +from typing import ClassVar, Iterable, Sequence, cast from pants.engine.collection import Collection -from pants.engine.fs import Digest -from pants.engine.unions import union +from pants.engine.fs import Digest, MergeDigests, Workspace +from pants.engine.goal import Goal, GoalSubsystem +from pants.engine.internals.selectors import Get, MultiGet +from pants.engine.rules import collect_rules, goal_rule +from pants.engine.unions import UnionMembership, union + +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -219,3 +225,104 @@ def filter_tool_lockfile_requests( ) return result + + +class GenerateLockfilesSubsystem(GoalSubsystem): + name = "generate-lockfiles" + help = "Generate lockfiles for Python third-party dependencies." + required_union_implementations = (ToolLockfileSentinel, KnownUserResolveNames) + + @classmethod + def register_options(cls, register) -> None: + super().register_options(register) + register( + "--resolve", + type=list, + member_type=str, + advanced=False, + help=( + "Only generate lockfiles for the specified resolve(s).\n\n" + "Resolves are the logical names for the different lockfiles used in your project. " + "For your own code's dependencies, these come from the option " + "`[python].experimental_resolves`. For tool lockfiles, resolve " + "names are the options scope for that tool such as `black`, `pytest`, and " + "`mypy-protobuf`.\n\n" + "For example, you can run `./pants generate-lockfiles --resolve=black " + "--resolve=pytest --resolve=data-science` to only generate lockfiles for those " + "two tools and your resolve named `data-science`.\n\n" + "If you specify an invalid resolve name, like 'fake', Pants will output all " + "possible values.\n\n" + "If not specified, Pants will generate lockfiles for all resolves." + ), + ) + register( + "--custom-command", + advanced=True, + type=str, + default=None, + help=( + "If set, lockfile headers will say to run this command to regenerate the lockfile, " + "rather than running `./pants generate-lockfiles --resolve=` like normal." + ), + ) + + @property + def resolve_names(self) -> tuple[str, ...]: + return tuple(self.options.resolve) + + @property + def custom_command(self) -> str | None: + return cast("str | None", self.options.custom_command) + + +class GenerateLockfilesGoal(Goal): + subsystem_cls = GenerateLockfilesSubsystem + + +@goal_rule +async def generate_lockfiles_goal( + workspace: Workspace, + union_membership: UnionMembership, + generate_lockfiles_subsystem: GenerateLockfilesSubsystem, +) -> GenerateLockfilesGoal: + known_user_resolve_names = await MultiGet( + Get(KnownUserResolveNames, KnownUserResolveNamesRequest, request()) + for request in union_membership.get(KnownUserResolveNamesRequest) + ) + requested_user_resolve_names, requested_tool_sentinels = determine_resolves_to_generate( + known_user_resolve_names, + union_membership.get(ToolLockfileSentinel), + set(generate_lockfiles_subsystem.resolve_names), + ) + + all_specified_user_requests = await MultiGet( + Get(UserLockfileRequests, RequestedUserResolveNames, resolve_names) + for resolve_names in requested_user_resolve_names + ) + specified_tool_requests = await MultiGet( + Get(WrappedLockfileRequest, ToolLockfileSentinel, sentinel()) + for sentinel in requested_tool_sentinels + ) + applicable_tool_requests = filter_tool_lockfile_requests( + specified_tool_requests, + resolve_specified=bool(generate_lockfiles_subsystem.resolve_names), + ) + + results = await MultiGet( + Get(Lockfile, LockfileRequest, req) + for req in ( + *(req for reqs in all_specified_user_requests for req in reqs), + *applicable_tool_requests, + ) + ) + + merged_digest = await Get(Digest, MergeDigests(res.digest for res in results)) + workspace.write_digest(merged_digest) + for result in results: + logger.info(f"Wrote lockfile for the resolve `{result.resolve_name}` to {result.path}") + + return GenerateLockfilesGoal(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/core/register.py b/src/python/pants/core/register.py index 09f19d57c736..41fce4f1a078 100644 --- a/src/python/pants/core/register.py +++ b/src/python/pants/core/register.py @@ -10,6 +10,7 @@ check, export, fmt, + generate_lockfiles, lint, package, publish, @@ -50,6 +51,7 @@ def rules(): *check.rules(), *export.rules(), *fmt.rules(), + *generate_lockfiles.rules(), *lint.rules(), *update_build_files.rules(), *package.rules(),