From a7b244723a8cad26bfb76b0d654be57fb00736e8 Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Fri, 19 Aug 2022 19:12:49 -0500 Subject: [PATCH] Fix performance for generating lockfiles for `pytest` and `setuptools` (#16591) Follow up to https://github.com/pantsbuild/pants/pull/15531. We didn't apply the optimization for Pytest and setuptools that it is no longer necessary to consult `TransitiveTargets` to determine interpreter constraints for `generate-lockfiles`, purely out of oversight. This also leverages `@rule_helper` to DRY some of our interpreter constraints code for lockfiles. That results in these slight behavior changes: * MyPy and Black are fixed to properly OR distinct interpreter constraints. We always meant to do this. * If MyPy does not encounter any MyPy-compatible targets in the repo, we fall back to `[python].interpreter_constraints` rather than the default `[mypy].interpreter_constraints`. This situation is unlikely to happen. In `pantsbuild/pants`, it now takes us 0.4 seconds rather than 4.7 seconds to calculate the interpreter constraints for `./pants generate-lockfiles --resolve=pytest`. [ci skip-rust] [ci skip-build-wheels] --- .../backend/python/lint/bandit/subsystem.py | 30 +--- .../python/lint/bandit/subsystem_test.py | 90 ---------- .../pants/backend/python/lint/black/rules.py | 19 +-- .../lint/black/rules_integration_test.py | 4 +- .../backend/python/lint/black/subsystem.py | 30 +++- .../backend/python/lint/flake8/subsystem.py | 64 ++------ .../python/lint/flake8/subsystem_test.py | 2 +- .../backend/python/lint/pylint/subsystem.py | 59 ++----- .../python/lint/pylint/subsystem_test.py | 2 +- .../backend/python/subsystems/ipython.py | 35 ++-- .../backend/python/subsystems/ipython_test.py | 78 --------- .../pants/backend/python/subsystems/pytest.py | 42 +---- .../backend/python/subsystems/pytest_test.py | 116 +------------ .../backend/python/subsystems/setuptools.py | 28 +--- .../python/subsystems/setuptools_test.py | 154 ------------------ .../python/typecheck/mypy/subsystem.py | 11 +- .../python/typecheck/mypy/subsystem_test.py | 6 +- .../backend/python/util_rules/partition.py | 52 +++++- .../python/util_rules/partition_test.py | 145 +++++++++++++++++ src/python/pants/core/goals/repl.py | 7 +- 20 files changed, 281 insertions(+), 693 deletions(-) delete mode 100644 src/python/pants/backend/python/lint/bandit/subsystem_test.py delete mode 100644 src/python/pants/backend/python/subsystems/ipython_test.py delete mode 100644 src/python/pants/backend/python/subsystems/setuptools_test.py create mode 100644 src/python/pants/backend/python/util_rules/partition_test.py diff --git a/src/python/pants/backend/python/lint/bandit/subsystem.py b/src/python/pants/backend/python/lint/bandit/subsystem.py index c25e6684c62..18520c93e57 100644 --- a/src/python/pants/backend/python/lint/bandit/subsystem.py +++ b/src/python/pants/backend/python/lint/bandit/subsystem.py @@ -3,7 +3,6 @@ from __future__ import annotations -import itertools from dataclasses import dataclass from pants.backend.python.goals import lockfile @@ -20,11 +19,11 @@ InterpreterConstraintsField, PythonSourceField, ) -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.util_rules.config_files import ConfigFilesRequest -from pants.engine.rules import Get, collect_rules, rule, rule_helper -from pants.engine.target import AllTargets, AllTargetsRequest, FieldSet, Target +from pants.engine.rules import collect_rules, rule +from pants.engine.target import FieldSet, Target from pants.engine.unions import UnionRule from pants.option.option_types import ArgsListOption, FileOption, SkipOption from pants.util.docutil import git_url @@ -84,25 +83,6 @@ def config_request(self) -> ConfigFilesRequest: ) -@rule_helper -async def _bandit_interpreter_constraints(python_setup: PythonSetup) -> InterpreterConstraints: - # While Bandit will run in partitions, we need a set of constraints that works with every - # partition. - # - # This ORs all unique interpreter constraints. The net effect is that every possible Python - # interpreter used will be covered. - all_tgts = await Get(AllTargets, AllTargetsRequest()) - unique_constraints = { - InterpreterConstraints.create_from_targets([tgt], python_setup) - for tgt in all_tgts - if BanditFieldSet.is_applicable(tgt) - } - constraints = InterpreterConstraints( - itertools.chain.from_iterable(ic for ic in unique_constraints if ic) - ) - return constraints or InterpreterConstraints(python_setup.interpreter_constraints) - - class BanditLockfileSentinel(GeneratePythonToolLockfileSentinel): resolve_name = Bandit.options_scope @@ -124,7 +104,7 @@ async def setup_bandit_lockfile( bandit, use_pex=python_setup.generate_lockfiles_with_pex ) - constraints = await _bandit_interpreter_constraints(python_setup) + constraints = await _find_all_unique_interpreter_constraints(python_setup, BanditFieldSet) return GeneratePythonLockfile.from_tool( bandit, constraints, @@ -150,7 +130,7 @@ async def bandit_export( ) -> ExportPythonTool: if not bandit.export: return ExportPythonTool(resolve_name=bandit.options_scope, pex_request=None) - constraints = await _bandit_interpreter_constraints(python_setup) + constraints = await _find_all_unique_interpreter_constraints(python_setup, BanditFieldSet) return ExportPythonTool( resolve_name=bandit.options_scope, pex_request=bandit.to_pex_request(interpreter_constraints=constraints), diff --git a/src/python/pants/backend/python/lint/bandit/subsystem_test.py b/src/python/pants/backend/python/lint/bandit/subsystem_test.py deleted file mode 100644 index a84854ae6f4..00000000000 --- a/src/python/pants/backend/python/lint/bandit/subsystem_test.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from textwrap import dedent - -from pants.backend.python import target_types_rules -from pants.backend.python.goals.lockfile import GeneratePythonLockfile -from pants.backend.python.lint.bandit import skip_field -from pants.backend.python.lint.bandit.subsystem import BanditLockfileSentinel -from pants.backend.python.lint.bandit.subsystem import rules as subsystem_rules -from pants.backend.python.target_types import PythonSourcesGeneratorTarget -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.core.target_types import GenericTarget -from pants.testutil.rule_runner import QueryRule, RuleRunner - - -def test_setup_lockfile_interpreter_constraints() -> None: - rule_runner = RuleRunner( - rules=[ - *subsystem_rules(), - *skip_field.rules(), - *target_types_rules.rules(), - QueryRule(GeneratePythonLockfile, [BanditLockfileSentinel]), - ], - target_types=[PythonSourcesGeneratorTarget, GenericTarget], - ) - - global_constraint = "==3.9.*" - rule_runner.set_options( - ["--bandit-lockfile=lockfile.txt"], - env={"PANTS_PYTHON_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, - ) - - def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) - lockfile_request = rule_runner.request(GeneratePythonLockfile, [BanditLockfileSentinel()]) - assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) - - assert_ics("python_sources()", [global_constraint]) - assert_ics("python_sources(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) - assert_ics( - "python_sources(interpreter_constraints=['==2.7.*', '==3.5.*'])", ["==2.7.*", "==3.5.*"] - ) - - # If no Python targets in repo, fall back to global [python] constraints. - assert_ics("target()", [global_constraint]) - - # Ignore targets that are skipped. - assert_ics( - dedent( - """\ - python_sources(name='a', interpreter_constraints=['==2.7.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*'], skip_bandit=True) - """ - ), - ["==2.7.*"], - ) - - # If there are multiple distinct ICs in the repo, we OR them. This is because Flake8 will - # group into each distinct IC. - assert_ics( - dedent( - """\ - python_sources(name='a', interpreter_constraints=['==2.7.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*']) - """ - ), - ["==2.7.*", "==3.5.*"], - ) - assert_ics( - dedent( - """\ - python_sources(name='a', interpreter_constraints=['==2.7.*', '==3.5.*']) - python_sources(name='b', interpreter_constraints=['>=3.5']) - """ - ), - ["==2.7.*", "==3.5.*", ">=3.5"], - ) - assert_ics( - dedent( - """\ - python_sources(name='a') - python_sources(name='b', interpreter_constraints=['==2.7.*']) - python_sources(name='c', interpreter_constraints=['>=3.6']) - """ - ), - ["==2.7.*", global_constraint, ">=3.6"], - ) diff --git a/src/python/pants/backend/python/lint/black/rules.py b/src/python/pants/backend/python/lint/black/rules.py index 7d5b2fe73f7..aa8e7ecd21e 100644 --- a/src/python/pants/backend/python/lint/black/rules.py +++ b/src/python/pants/backend/python/lint/black/rules.py @@ -1,12 +1,8 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from dataclasses import dataclass - -from pants.backend.python.lint.black.skip_field import SkipBlackField -from pants.backend.python.lint.black.subsystem import Black +from pants.backend.python.lint.black.subsystem import Black, BlackFieldSet from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import InterpreterConstraintsField, PythonSourceField from pants.backend.python.util_rules import pex from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.pex import PexRequest, VenvPex, VenvPexProcess @@ -16,24 +12,11 @@ from pants.engine.internals.native_engine import Snapshot from pants.engine.process import ProcessResult from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import FieldSet, Target from pants.engine.unions import UnionRule from pants.util.logging import LogLevel from pants.util.strutil import pluralize, softwrap -@dataclass(frozen=True) -class BlackFieldSet(FieldSet): - required_fields = (PythonSourceField,) - - source: PythonSourceField - interpreter_constraints: InterpreterConstraintsField - - @classmethod - def opt_out(cls, tgt: Target) -> bool: - return tgt.get(SkipBlackField).value - - class BlackRequest(FmtTargetsRequest): field_set_type = BlackFieldSet name = Black.options_scope diff --git a/src/python/pants/backend/python/lint/black/rules_integration_test.py b/src/python/pants/backend/python/lint/black/rules_integration_test.py index fe1844bab28..5456e06e377 100644 --- a/src/python/pants/backend/python/lint/black/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/black/rules_integration_test.py @@ -8,9 +8,9 @@ import pytest from pants.backend.python import target_types_rules -from pants.backend.python.lint.black.rules import BlackFieldSet, BlackRequest +from pants.backend.python.lint.black.rules import BlackRequest from pants.backend.python.lint.black.rules import rules as black_rules -from pants.backend.python.lint.black.subsystem import Black +from pants.backend.python.lint.black.subsystem import Black, BlackFieldSet from pants.backend.python.lint.black.subsystem import rules as black_subsystem_rules from pants.backend.python.target_types import PythonSourcesGeneratorTarget from pants.core.goals.fmt import FmtResult diff --git a/src/python/pants/backend/python/lint/black/subsystem.py b/src/python/pants/backend/python/lint/black/subsystem.py index 647853f0dd3..037a0473492 100644 --- a/src/python/pants/backend/python/lint/black/subsystem.py +++ b/src/python/pants/backend/python/lint/black/subsystem.py @@ -4,6 +4,7 @@ from __future__ import annotations import os.path +from dataclasses import dataclass from typing import Iterable from pants.backend.python.goals import lockfile @@ -15,12 +16,17 @@ from pants.backend.python.lint.black.skip_field import SkipBlackField from pants.backend.python.subsystems.python_tool_base import ExportToolOption, PythonToolBase from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import ConsoleScript +from pants.backend.python.target_types import ( + ConsoleScript, + InterpreterConstraintsField, + PythonSourceField, +) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.util_rules.config_files import ConfigFilesRequest -from pants.engine.rules import Get, collect_rules, rule, rule_helper -from pants.engine.target import AllTargets, AllTargetsRequest +from pants.engine.rules import collect_rules, rule, rule_helper +from pants.engine.target import FieldSet, Target from pants.engine.unions import UnionRule from pants.option.option_types import ArgsListOption, BoolOption, FileOption, SkipOption from pants.util.docutil import git_url @@ -28,6 +34,18 @@ from pants.util.strutil import softwrap +@dataclass(frozen=True) +class BlackFieldSet(FieldSet): + required_fields = (PythonSourceField,) + + source: PythonSourceField + interpreter_constraints: InterpreterConstraintsField + + @classmethod + def opt_out(cls, tgt: Target) -> bool: + return tgt.get(SkipBlackField).value + + class Black(PythonToolBase): options_scope = "black" name = "Black" @@ -92,10 +110,8 @@ async def _black_interpreter_constraints( ) -> InterpreterConstraints: constraints = black.interpreter_constraints if black.options.is_default("interpreter_constraints"): - all_tgts = await Get(AllTargets, AllTargetsRequest()) - # TODO: fix to use `FieldSet.is_applicable()`. - code_constraints = InterpreterConstraints.create_from_targets( - (tgt for tgt in all_tgts if not tgt.get(SkipBlackField).value), python_setup + code_constraints = await _find_all_unique_interpreter_constraints( + python_setup, BlackFieldSet ) if code_constraints is not None and code_constraints.requires_python38_or_newer( python_setup.interpreter_versions_universe diff --git a/src/python/pants/backend/python/lint/flake8/subsystem.py b/src/python/pants/backend/python/lint/flake8/subsystem.py index 70881f74603..8e4f8ab85df 100644 --- a/src/python/pants/backend/python/lint/flake8/subsystem.py +++ b/src/python/pants/backend/python/lint/flake8/subsystem.py @@ -3,7 +3,6 @@ from __future__ import annotations -import itertools from dataclasses import dataclass from pants.backend.python.goals import lockfile @@ -22,7 +21,7 @@ PythonSourceField, ) from pants.backend.python.util_rules import python_sources -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.backend.python.util_rules.pex_requirements import PexRequirements from pants.backend.python.util_rules.python_sources import ( PythonSourceFilesRequest, @@ -33,15 +32,8 @@ from pants.engine.addresses import Addresses, UnparsedAddressInputs from pants.engine.fs import AddPrefix, Digest from pants.engine.internals.native_engine import EMPTY_DIGEST -from pants.engine.rules import Get, collect_rules, rule, rule_helper -from pants.engine.target import ( - AllTargets, - AllTargetsRequest, - FieldSet, - Target, - TransitiveTargets, - TransitiveTargetsRequest, -) +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import FieldSet, Target, TransitiveTargets, TransitiveTargetsRequest from pants.engine.unions import UnionRule from pants.option.option_types import ( ArgsListOption, @@ -237,44 +229,6 @@ async def flake8_first_party_plugins(flake8: Flake8) -> Flake8FirstPartyPlugins: ) -# -------------------------------------------------------------------------------------- -# Interpreter constraints -# -------------------------------------------------------------------------------------- - - -@rule_helper -async def _flake8_interpreter_constraints( - first_party_plugins: Flake8FirstPartyPlugins, - python_setup: PythonSetup, -) -> InterpreterConstraints: - # While Flake8 will run in partitions, we need a set of constraints that works with every - # partition. - # - # This ORs all unique interpreter constraints. The net effect is that every possible Python - # interpreter used will be covered. - all_tgts = await Get(AllTargets, AllTargetsRequest()) - unique_constraints = { - InterpreterConstraints.create_from_compatibility_fields( - ( - tgt[InterpreterConstraintsField], - *first_party_plugins.interpreter_constraints_fields, - ), - python_setup, - ) - for tgt in all_tgts - if Flake8FieldSet.is_applicable(tgt) - } - if not unique_constraints: - unique_constraints.add( - InterpreterConstraints.create_from_compatibility_fields( - first_party_plugins.interpreter_constraints_fields, - python_setup, - ) - ) - constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) - return constraints or InterpreterConstraints(python_setup.interpreter_constraints) - - # -------------------------------------------------------------------------------------- # Lockfile # -------------------------------------------------------------------------------------- @@ -304,7 +258,11 @@ async def setup_flake8_lockfile( flake8, use_pex=python_setup.generate_lockfiles_with_pex ) - constraints = await _flake8_interpreter_constraints(first_party_plugins, python_setup) + constraints = await _find_all_unique_interpreter_constraints( + python_setup, + Flake8FieldSet, + extra_constraints_per_tgt=first_party_plugins.interpreter_constraints_fields, + ) return GeneratePythonLockfile.from_tool( flake8, constraints, @@ -339,7 +297,11 @@ async def flake8_export( ) -> ExportPythonTool: if not flake8.export: return ExportPythonTool(resolve_name=flake8.options_scope, pex_request=None) - constraints = await _flake8_interpreter_constraints(first_party_plugins, python_setup) + constraints = await _find_all_unique_interpreter_constraints( + python_setup, + Flake8FieldSet, + extra_constraints_per_tgt=first_party_plugins.interpreter_constraints_fields, + ) return ExportPythonTool( resolve_name=flake8.options_scope, pex_request=flake8.to_pex_request( diff --git a/src/python/pants/backend/python/lint/flake8/subsystem_test.py b/src/python/pants/backend/python/lint/flake8/subsystem_test.py index 01c662656de..b1871871ae3 100644 --- a/src/python/pants/backend/python/lint/flake8/subsystem_test.py +++ b/src/python/pants/backend/python/lint/flake8/subsystem_test.py @@ -105,7 +105,7 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: ) -def test_setup_lockfile_interpreter_constraints(rule_runner) -> None: +def test_setup_lockfile(rule_runner) -> None: global_constraint = "==3.9.*" def assert_lockfile_request( diff --git a/src/python/pants/backend/python/lint/pylint/subsystem.py b/src/python/pants/backend/python/lint/pylint/subsystem.py index 04383075dd8..a9105664a24 100644 --- a/src/python/pants/backend/python/lint/pylint/subsystem.py +++ b/src/python/pants/backend/python/lint/pylint/subsystem.py @@ -3,7 +3,6 @@ from __future__ import annotations -import itertools import os.path from dataclasses import dataclass from typing import Iterable @@ -23,7 +22,7 @@ PythonRequirementsField, PythonSourceField, ) -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.backend.python.util_rules.pex_requirements import PexRequirements from pants.backend.python.util_rules.python_sources import ( PythonSourceFilesRequest, @@ -33,10 +32,8 @@ from pants.core.util_rules.config_files import ConfigFilesRequest from pants.engine.addresses import Addresses, UnparsedAddressInputs from pants.engine.fs import EMPTY_DIGEST, AddPrefix, Digest -from pants.engine.rules import Get, collect_rules, rule, rule_helper +from pants.engine.rules import Get, collect_rules, rule from pants.engine.target import ( - AllTargets, - AllTargetsRequest, Dependencies, FieldSet, Target, @@ -228,46 +225,6 @@ async def pylint_first_party_plugins(pylint: Pylint) -> PylintFirstPartyPlugins: ) -# -------------------------------------------------------------------------------------- -# Interpreter constraints -# -------------------------------------------------------------------------------------- - - -@rule_helper -async def _pylint_interpreter_constraints( - first_party_plugins: PylintFirstPartyPlugins, - python_setup: PythonSetup, -) -> InterpreterConstraints: - # While Pylint will run in partitions, we need a set of constraints that works with every - # partition. We must also consider any 3rd-party requirements used by 1st-party plugins. - # - # This first computes the constraints for each individual target. Then, it ORs all unique - # resulting interpreter constraints. The net effect is that every possible Python interpreter - # used will be covered. - all_tgts = await Get(AllTargets, AllTargetsRequest()) - - unique_constraints = { - InterpreterConstraints.create_from_compatibility_fields( - ( - tgt[InterpreterConstraintsField], - *first_party_plugins.interpreter_constraints_fields, - ), - python_setup, - ) - for tgt in all_tgts - if PylintFieldSet.is_applicable(tgt) - } - if not unique_constraints: - unique_constraints.add( - InterpreterConstraints.create_from_compatibility_fields( - first_party_plugins.interpreter_constraints_fields, - python_setup, - ) - ) - constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) - return constraints or InterpreterConstraints(python_setup.interpreter_constraints) - - # -------------------------------------------------------------------------------------- # Lockfile # -------------------------------------------------------------------------------------- @@ -297,7 +254,11 @@ async def setup_pylint_lockfile( pylint, use_pex=python_setup.generate_lockfiles_with_pex ) - constraints = await _pylint_interpreter_constraints(first_party_plugins, python_setup) + constraints = await _find_all_unique_interpreter_constraints( + python_setup, + PylintFieldSet, + extra_constraints_per_tgt=first_party_plugins.interpreter_constraints_fields, + ) return GeneratePythonLockfile.from_tool( pylint, constraints, @@ -332,7 +293,11 @@ async def pylint_export( ) -> ExportPythonTool: if not pylint.export: return ExportPythonTool(resolve_name=pylint.options_scope, pex_request=None) - constraints = await _pylint_interpreter_constraints(first_party_plugins, python_setup) + constraints = await _find_all_unique_interpreter_constraints( + python_setup, + PylintFieldSet, + extra_constraints_per_tgt=first_party_plugins.interpreter_constraints_fields, + ) return ExportPythonTool( resolve_name=pylint.options_scope, pex_request=pylint.to_pex_request( diff --git a/src/python/pants/backend/python/lint/pylint/subsystem_test.py b/src/python/pants/backend/python/lint/pylint/subsystem_test.py index 950c3bb9ecf..b9133fb1236 100644 --- a/src/python/pants/backend/python/lint/pylint/subsystem_test.py +++ b/src/python/pants/backend/python/lint/pylint/subsystem_test.py @@ -105,7 +105,7 @@ def test_first_party_plugins(rule_runner: RuleRunner) -> None: ) -def test_setup_lockfile_interpreter_constraints(rule_runner: RuleRunner) -> None: +def test_setup_lockfile(rule_runner: RuleRunner) -> None: global_constraint = "==3.9.*" def assert_lockfile_request( diff --git a/src/python/pants/backend/python/subsystems/ipython.py b/src/python/pants/backend/python/subsystems/ipython.py index a39f7243ef7..7b7fbfa2856 100644 --- a/src/python/pants/backend/python/subsystems/ipython.py +++ b/src/python/pants/backend/python/subsystems/ipython.py @@ -3,8 +3,6 @@ from __future__ import annotations -import itertools - from pants.backend.python.goals import lockfile from pants.backend.python.goals.lockfile import ( GeneratePythonLockfile, @@ -13,10 +11,10 @@ from pants.backend.python.subsystems.python_tool_base import PythonToolBase from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import ConsoleScript, InterpreterConstraintsField -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel -from pants.engine.rules import Get, collect_rules, rule -from pants.engine.target import AllTargets, AllTargetsRequest +from pants.engine.rules import collect_rules, rule +from pants.engine.target import FieldSet from pants.engine.unions import UnionRule from pants.option.option_types import BoolOption from pants.util.docutil import git_url @@ -58,6 +56,10 @@ class IPythonLockfileSentinel(GeneratePythonToolLockfileSentinel): resolve_name = IPython.options_scope +class _IpythonFieldSetForLockfiles(FieldSet): + required_fields = (InterpreterConstraintsField,) + + @rule( desc=softwrap( """ @@ -75,26 +77,11 @@ async def setup_ipython_lockfile( ipython, use_pex=python_setup.generate_lockfiles_with_pex ) - # IPython is often run against the whole repo (`./pants repl ::`), but it is possible to run - # on subsets of the codebase with disjoint interpreter constraints, such as - # `./pants repl py2::` and then `./pants repl py3::`. Still, even with those subsets possible, - # we need a single lockfile that works with all possible Python interpreters in use. - # - # This ORs all unique interpreter constraints. The net effect is that every possible Python - # interpreter used will be covered. - all_tgts = await Get(AllTargets, AllTargetsRequest()) - unique_constraints = { - InterpreterConstraints.create_from_compatibility_fields( - [tgt[InterpreterConstraintsField]], python_setup - ) - for tgt in all_tgts - if tgt.has_field(InterpreterConstraintsField) - } - constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) + interpreter_constraints = await _find_all_unique_interpreter_constraints( + python_setup, _IpythonFieldSetForLockfiles + ) return GeneratePythonLockfile.from_tool( - ipython, - constraints or InterpreterConstraints(python_setup.interpreter_constraints), - use_pex=python_setup.generate_lockfiles_with_pex, + ipython, interpreter_constraints, use_pex=python_setup.generate_lockfiles_with_pex ) diff --git a/src/python/pants/backend/python/subsystems/ipython_test.py b/src/python/pants/backend/python/subsystems/ipython_test.py deleted file mode 100644 index 4e6f6c116ce..00000000000 --- a/src/python/pants/backend/python/subsystems/ipython_test.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from textwrap import dedent - -from pants.backend.python import target_types_rules -from pants.backend.python.goals.lockfile import GeneratePythonLockfile -from pants.backend.python.subsystems.ipython import IPythonLockfileSentinel -from pants.backend.python.subsystems.ipython import rules as subsystem_rules -from pants.backend.python.target_types import PythonSourcesGeneratorTarget -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.core.target_types import GenericTarget -from pants.testutil.rule_runner import QueryRule, RuleRunner - - -def test_setup_lockfile_interpreter_constraints() -> None: - rule_runner = RuleRunner( - rules=[ - *subsystem_rules(), - *target_types_rules.rules(), - QueryRule(GeneratePythonLockfile, [IPythonLockfileSentinel]), - ], - target_types=[PythonSourcesGeneratorTarget, GenericTarget], - ) - - global_constraint = "==3.9.*" - rule_runner.set_options( - ["--ipython-lockfile=lockfile.txt", "--no-python-infer-imports"], - env={"PANTS_PYTHON_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, - ) - - def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) - lockfile_request = rule_runner.request(GeneratePythonLockfile, [IPythonLockfileSentinel()]) - assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) - - assert_ics("python_sources()", [global_constraint]) - assert_ics("python_sources(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) - assert_ics( - "python_sources(interpreter_constraints=['==2.7.*', '==3.5.*'])", ["==2.7.*", "==3.5.*"] - ) - - # If no Python targets in repo, fall back to global [python] constraints. - assert_ics("target()", [global_constraint]) - - # If there are multiple distinct ICs in the repo, we OR them. Even though the user might AND - # them by running `./pants repl ::`, they could also run on more precise subsets like - # `./pants repl py2::` and then `./pants repl py3::` - assert_ics( - dedent( - """\ - python_sources(name='a', interpreter_constraints=['==2.7.*']) - python_sources(name='b', interpreter_constraints=['==3.5.*']) - """ - ), - ["==2.7.*", "==3.5.*"], - ) - assert_ics( - dedent( - """\ - python_sources(name='a', interpreter_constraints=['==2.7.*', '==3.5.*']) - python_sources(name='b', interpreter_constraints=['>=3.5']) - """ - ), - ["==2.7.*", "==3.5.*", ">=3.5"], - ) - assert_ics( - dedent( - """\ - python_sources(name='a') - python_sources(name='b', interpreter_constraints=['==2.7.*']) - python_sources(name='c', interpreter_constraints=['>=3.6']) - """ - ), - ["==2.7.*", global_constraint, ">=3.6"], - ) diff --git a/src/python/pants/backend/python/subsystems/pytest.py b/src/python/pants/backend/python/subsystems/pytest.py index 8846e0dbf51..f7f81223e07 100644 --- a/src/python/pants/backend/python/subsystems/pytest.py +++ b/src/python/pants/backend/python/subsystems/pytest.py @@ -3,7 +3,6 @@ from __future__ import annotations -import itertools import os.path from dataclasses import dataclass from typing import Iterable @@ -28,18 +27,12 @@ PythonTestsXdistConcurrencyField, SkipPythonTestsField, ) -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.goals.test import RuntimePackageDependenciesField, TestFieldSet from pants.core.util_rules.config_files import ConfigFilesRequest -from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper -from pants.engine.target import ( - AllTargets, - AllTargetsRequest, - Target, - TransitiveTargets, - TransitiveTargetsRequest, -) +from pants.engine.rules import collect_rules, rule +from pants.engine.target import Target from pants.engine.unions import UnionRule from pants.option.option_types import ArgsListOption, BoolOption, FileOption, IntOption, StrOption from pants.util.docutil import bin_name, doc_url, git_url @@ -232,31 +225,6 @@ def validate_pytest_cov_included(self) -> None: ) -@rule_helper -async def _pytest_interpreter_constraints(python_setup: PythonSetup) -> InterpreterConstraints: - # Even though we run each python_tests target in isolation, we need a single set of constraints - # that works with them all (and their transitive deps). - # - # This first computes the constraints for each individual `python_test` target - # (which will AND across each target in the closure). Then, it ORs all unique resulting - # interpreter constraints. The net effect is that every possible Python interpreter used will - # be covered. - all_tgts = await Get(AllTargets, AllTargetsRequest()) - transitive_targets_per_test = await MultiGet( - Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) - for tgt in all_tgts - if PythonTestFieldSet.is_applicable(tgt) - ) - unique_constraints = { - InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup) - for transitive_targets in transitive_targets_per_test - } - constraints = InterpreterConstraints( - itertools.chain.from_iterable(ic for ic in unique_constraints if ic) - ) - return constraints or InterpreterConstraints(python_setup.interpreter_constraints) - - class PytestLockfileSentinel(GeneratePythonToolLockfileSentinel): resolve_name = PyTest.options_scope @@ -278,7 +246,7 @@ async def setup_pytest_lockfile( pytest, use_pex=python_setup.generate_lockfiles_with_pex ) - constraints = await _pytest_interpreter_constraints(python_setup) + constraints = await _find_all_unique_interpreter_constraints(python_setup, PythonTestFieldSet) return GeneratePythonLockfile.from_tool( pytest, constraints, @@ -304,7 +272,7 @@ async def pytest_export( ) -> ExportPythonTool: if not pytest.export: return ExportPythonTool(resolve_name=pytest.options_scope, pex_request=None) - constraints = await _pytest_interpreter_constraints(python_setup) + constraints = await _find_all_unique_interpreter_constraints(python_setup, PythonTestFieldSet) return ExportPythonTool( resolve_name=pytest.options_scope, pex_request=pytest.to_pex_request(interpreter_constraints=constraints), diff --git a/src/python/pants/backend/python/subsystems/pytest_test.py b/src/python/pants/backend/python/subsystems/pytest_test.py index 188625a2426..535b1978fd5 100644 --- a/src/python/pants/backend/python/subsystems/pytest_test.py +++ b/src/python/pants/backend/python/subsystems/pytest_test.py @@ -3,125 +3,11 @@ from __future__ import annotations -from textwrap import dedent - import pytest -from pants.backend.python import target_types_rules -from pants.backend.python.goals.lockfile import GeneratePythonLockfile -from pants.backend.python.subsystems.pytest import PyTest, PytestLockfileSentinel -from pants.backend.python.subsystems.pytest import rules as subsystem_rules -from pants.backend.python.target_types import ( - PythonSourcesGeneratorTarget, - PythonTestsGeneratorTarget, -) -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.core.target_types import GenericTarget +from pants.backend.python.subsystems.pytest import PyTest from pants.option.ranked_value import Rank, RankedValue from pants.testutil.option_util import create_subsystem -from pants.testutil.rule_runner import QueryRule, RuleRunner - - -def test_setup_lockfile_interpreter_constraints() -> None: - rule_runner = RuleRunner( - rules=[ - *subsystem_rules(), - *target_types_rules.rules(), - QueryRule(GeneratePythonLockfile, [PytestLockfileSentinel]), - ], - target_types=[PythonSourcesGeneratorTarget, PythonTestsGeneratorTarget, GenericTarget], - ) - - global_constraint = "==3.9.*" - rule_runner.set_options( - ["--pytest-lockfile=lockfile.txt", "--no-python-infer-imports"], - env={"PANTS_PYTHON_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, - ) - - def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files( - {"project/BUILD": build_file, "project/f.py": "", "project/f_test.py": ""} - ) - lockfile_request = rule_runner.request(GeneratePythonLockfile, [PytestLockfileSentinel()]) - assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) - - assert_ics("python_tests()", [global_constraint]) - assert_ics("python_tests(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) - assert_ics( - "python_tests(interpreter_constraints=['==2.7.*', '==3.8.*'])", ["==2.7.*", "==3.8.*"] - ) - - # If no Python targets in repo, fall back to global [python] constraints. - assert_ics("target()", [global_constraint]) - - # Only care about `python_tests` and their transitive deps, not unused `python_sources`s. - assert_ics("python_sources(interpreter_constraints=['==2.7.*'])", [global_constraint]) - - # Ignore targets that are skipped. - assert_ics( - dedent( - """\ - python_tests(name='a', interpreter_constraints=['==2.7.*']) - python_tests(name='b', interpreter_constraints=['==3.8.*'], skip_tests=True) - """ - ), - ["==2.7.*"], - ) - - # If there are multiple distinct ICs in the repo, we OR them because the lockfile needs to be - # compatible with every target. - assert_ics( - dedent( - """\ - python_tests(name='a', interpreter_constraints=['==2.7.*']) - python_tests(name='b', interpreter_constraints=['==3.8.*']) - """ - ), - ["==2.7.*", "==3.8.*"], - ) - assert_ics( - dedent( - """\ - python_tests(name='a', interpreter_constraints=['==2.7.*', '==3.8.*']) - python_tests(name='b', interpreter_constraints=['>=3.8']) - """ - ), - ["==2.7.*", "==3.8.*", ">=3.8"], - ) - assert_ics( - dedent( - """\ - python_tests(name='a') - python_tests(name='b', interpreter_constraints=['==2.7.*']) - python_tests(name='c', interpreter_constraints=['>=3.6']) - """ - ), - ["==2.7.*", global_constraint, ">=3.6"], - ) - - # Also consider transitive deps. They should be ANDed within each python_tests's transitive - # closure like normal, but then ORed across each python_tests closure. - assert_ics( - dedent( - """\ - python_sources(name='lib', interpreter_constraints=['==2.7.*', '==3.6.*']) - python_tests(name='tests', dependencies=[":lib"], interpreter_constraints=['==2.7.*']) - """ - ), - ["==2.7.*", "==2.7.*,==3.6.*"], - ) - assert_ics( - dedent( - """\ - python_sources(name='lib1', interpreter_constraints=['==2.7.*', '==3.6.*']) - python_tests(name='tests1', dependencies=[":lib1"], interpreter_constraints=['==2.7.*']) - - python_sources(name='lib2', interpreter_constraints=['>=3.7']) - python_tests(name='tests2', dependencies=[":lib2"], interpreter_constraints=['==3.8.*']) - """ - ), - ["==2.7.*", "==2.7.*,==3.6.*", ">=3.7,==3.8.*"], - ) def test_validate_pytest_cov_included() -> None: diff --git a/src/python/pants/backend/python/subsystems/setuptools.py b/src/python/pants/backend/python/subsystems/setuptools.py index c0db5e9fbd3..7ef1de1498c 100644 --- a/src/python/pants/backend/python/subsystems/setuptools.py +++ b/src/python/pants/backend/python/subsystems/setuptools.py @@ -1,7 +1,6 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import itertools from dataclasses import dataclass from pants.backend.python.goals import lockfile @@ -12,16 +11,10 @@ from pants.backend.python.subsystems.python_tool_base import PythonToolRequirementsBase from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PythonProvidesField -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel from pants.core.goals.package import PackageFieldSet -from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import ( - AllTargets, - AllTargetsRequest, - TransitiveTargets, - TransitiveTargetsRequest, -) +from pants.engine.rules import collect_rules, rule from pants.engine.unions import UnionRule from pants.util.docutil import git_url from pants.util.logging import LogLevel @@ -69,22 +62,11 @@ async def setup_setuptools_lockfile( setuptools, use_pex=python_setup.generate_lockfiles_with_pex ) - all_tgts = await Get(AllTargets, AllTargetsRequest()) - transitive_targets_per_python_dist = await MultiGet( - Get(TransitiveTargets, TransitiveTargetsRequest([tgt.address])) - for tgt in all_tgts - if PythonDistributionFieldSet.is_applicable(tgt) + interpreter_constraints = await _find_all_unique_interpreter_constraints( + python_setup, PythonDistributionFieldSet ) - unique_constraints = { - InterpreterConstraints.create_from_targets(transitive_targets.closure, python_setup) - or InterpreterConstraints(python_setup.interpreter_constraints) - for transitive_targets in transitive_targets_per_python_dist - } - constraints = InterpreterConstraints(itertools.chain.from_iterable(unique_constraints)) return GeneratePythonLockfile.from_tool( - setuptools, - constraints or InterpreterConstraints(python_setup.interpreter_constraints), - use_pex=python_setup.generate_lockfiles_with_pex, + setuptools, interpreter_constraints, use_pex=python_setup.generate_lockfiles_with_pex ) diff --git a/src/python/pants/backend/python/subsystems/setuptools_test.py b/src/python/pants/backend/python/subsystems/setuptools_test.py deleted file mode 100644 index 92c173e4399..00000000000 --- a/src/python/pants/backend/python/subsystems/setuptools_test.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -from textwrap import dedent - -from pants.backend.python import target_types_rules -from pants.backend.python.goals.lockfile import GeneratePythonLockfile -from pants.backend.python.macros.python_artifact import PythonArtifact -from pants.backend.python.subsystems import setuptools -from pants.backend.python.subsystems.setuptools import SetuptoolsLockfileSentinel -from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.testutil.rule_runner import QueryRule, RuleRunner - - -def test_setup_lockfile_interpreter_constraints() -> None: - rule_runner = RuleRunner( - rules=[ - *setuptools.rules(), - *target_types_rules.rules(), - QueryRule(GeneratePythonLockfile, [SetuptoolsLockfileSentinel]), - ], - target_types=[PythonSourcesGeneratorTarget, PythonDistribution], - objects={"python_artifact": PythonArtifact}, - ) - - global_constraint = "==3.9.*" - rule_runner.set_options( - ["--setuptools-lockfile=lockfile.txt", "--no-python-infer-imports"], - env={"PANTS_PYTHON_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, - ) - - def assert_ics(build_file: str, expected: list[str]) -> None: - rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) - lockfile_request = rule_runner.request( - GeneratePythonLockfile, [SetuptoolsLockfileSentinel()] - ) - assert lockfile_request.interpreter_constraints == InterpreterConstraints(expected) - - # If no dependencies for python_distribution, fall back to global [python] constraints. - assert_ics("python_distribution(provides=python_artifact(name='dist'))", [global_constraint]) - - assert_ics( - dedent( - """\ - python_sources(name="lib") - python_distribution( - name="dist", - dependencies=[":lib"], - provides=python_artifact(name="dist"), - ) - """ - ), - [global_constraint], - ) - assert_ics( - dedent( - """\ - python_sources(name="lib", interpreter_constraints=["==2.7.*"]) - python_distribution( - name="dist", - dependencies=[":lib"], - interpreter_constraints=["==2.7.*"], - provides=python_artifact(name="dist"), - ) - """ - ), - ["==2.7.*"], - ) - assert_ics( - dedent( - """\ - python_distribution( - name="dist", - interpreter_constraints=["==2.7.*", "==3.5.*"], - provides=python_artifact(name="dist"), - ) - """ - ), - ["==2.7.*", "==3.5.*"], - ) - - # If no python_distribution targets in repo, fall back to global [python] constraints. - assert_ics("python_sources()", [global_constraint]) - - # If there are multiple distinct ICs in the repo, we OR them. This is because setup_py.py will - # build each Python distribution independently. - assert_ics( - dedent( - """\ - python_distribution( - name="dist1", - interpreter_constraints=["==2.7.*"], - provides=python_artifact(name="dist"), - ) - - python_distribution( - name="dist2", - interpreter_constraints=["==3.5.*"], - provides=python_artifact(name="dist"), - ) - """ - ), - ["==2.7.*", "==3.5.*"], - ) - assert_ics( - dedent( - """\ - python_distribution( - name="dist1", - interpreter_constraints=["==2.7.*", "==3.5.*"], - provides=python_artifact(name="dist"), - ) - - python_distribution( - name="dist2", - interpreter_constraints=[">=3.5"], - provides=python_artifact(name="dist"), - ) - """ - ), - ["==2.7.*", "==3.5.*", ">=3.5"], - ) - assert_ics( - dedent( - """\ - python_sources(name="lib1") - python_distribution( - name="dist1", - dependencies=[":lib1"], - provides=python_artifact(name="dist"), - ) - - python_sources(name="lib2", interpreter_constraints=["==2.7.*"]) - python_distribution( - name="dist2", - dependencies=[":lib2"], - interpreter_constraints=["==2.7.*"], - provides=python_artifact(name="dist"), - ) - - python_sources(name="lib3", interpreter_constraints=[">=3.6"]) - python_distribution( - name="dist3", - dependencies=[":lib3"], - interpreter_constraints=[">=3.6"], - provides=python_artifact(name="dist"), - ) - """ - ), - ["==2.7.*", global_constraint, ">=3.6"], - ) diff --git a/src/python/pants/backend/python/typecheck/mypy/subsystem.py b/src/python/pants/backend/python/typecheck/mypy/subsystem.py index 36296fbd249..510ef194edb 100644 --- a/src/python/pants/backend/python/typecheck/mypy/subsystem.py +++ b/src/python/pants/backend/python/typecheck/mypy/subsystem.py @@ -25,6 +25,7 @@ from pants.backend.python.typecheck.mypy.skip_field import SkipMyPyField from pants.backend.python.util_rules import partition from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints from pants.backend.python.util_rules.pex import PexRequest from pants.backend.python.util_rules.pex_requirements import ( EntireLockfile, @@ -348,14 +349,8 @@ async def _mypy_interpreter_constraints( ) -> InterpreterConstraints: constraints = mypy.interpreter_constraints if mypy.options.is_default("interpreter_constraints"): - all_tgts = await Get(AllTargets, AllTargetsRequest()) - unique_constraints = { - InterpreterConstraints.create_from_targets([tgt], python_setup) - for tgt in all_tgts - if MyPyFieldSet.is_applicable(tgt) - } - code_constraints = InterpreterConstraints( - itertools.chain.from_iterable(ic for ic in unique_constraints if ic) + code_constraints = await _find_all_unique_interpreter_constraints( + python_setup, MyPyFieldSet ) if code_constraints.requires_python38_or_newer(python_setup.interpreter_versions_universe): constraints = code_constraints diff --git a/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py b/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py index 1a08c17fe3e..186018687a1 100644 --- a/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py +++ b/src/python/pants/backend/python/typecheck/mypy/subsystem_test.py @@ -196,8 +196,8 @@ def assert_lockfile_request( MyPy.default_interpreter_constraints, ) - # If no Python targets in repo, fall back to MyPy constraints. - assert_lockfile_request("target()", MyPy.default_interpreter_constraints) + # If no Python targets in repo, fall back to global Python constraint. + assert_lockfile_request("target()", [global_constraint]) # Ignore targets that are skipped. assert_lockfile_request( @@ -245,7 +245,7 @@ def assert_lockfile_request( python_requirement(name="thirdparty", requirements=["ansicolors"]) """ ), - MyPy.default_interpreter_constraints, + [global_constraint], extra_args=["--mypy-source-plugins=project"], extra_expected_requirements=["ansicolors"], ) diff --git a/src/python/pants/backend/python/util_rules/partition.py b/src/python/pants/backend/python/util_rules/partition.py index b4657bbd96d..84fcb6d9700 100644 --- a/src/python/pants/backend/python/util_rules/partition.py +++ b/src/python/pants/backend/python/util_rules/partition.py @@ -3,14 +3,22 @@ from __future__ import annotations +import itertools from collections import defaultdict -from typing import Mapping, Sequence, TypeVar +from typing import Iterable, Mapping, Sequence, TypeVar from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import PythonResolveField +from pants.backend.python.target_types import InterpreterConstraintsField, PythonResolveField from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.engine.rules import Get, rule_helper -from pants.engine.target import CoarsenedTarget, CoarsenedTargets, CoarsenedTargetsRequest, FieldSet +from pants.engine.target import ( + AllTargets, + AllTargetsRequest, + CoarsenedTarget, + CoarsenedTargets, + CoarsenedTargetsRequest, + FieldSet, +) from pants.util.ordered_set import OrderedSet ResolveName = str @@ -58,3 +66,41 @@ async def _by_interpreter_constraints_and_resolve( root_cts.add(ct) return resolve_and_interpreter_constraints_to_coarsened_targets + + +@rule_helper +async def _find_all_unique_interpreter_constraints( + python_setup: PythonSetup, + field_set_type: type[FieldSet], + *, + extra_constraints_per_tgt: Iterable[InterpreterConstraintsField] = (), +) -> InterpreterConstraints: + """Find all unique interpreter constraints used by given field set. + + This will find the constraints for each individual matching field set, and then OR across all + unique constraints. Usually, Pants partitions when necessary so that conflicting interpreter + constraints can be handled gracefully. But in some cases, like the `generate-lockfiles` goal, + we need to combine those targets into a single value. This ORs, so that if you have a + ==2.7 partition and ==3.6 partition, for example, we return ==2.7 OR ==3.6. + + Returns the global interpreter constraints if no relevant targets were matched. + """ + all_tgts = await Get(AllTargets, AllTargetsRequest()) + unique_constraints = { + InterpreterConstraints.create_from_compatibility_fields( + [tgt[InterpreterConstraintsField], *extra_constraints_per_tgt], python_setup + ) + for tgt in all_tgts + if tgt.has_field(InterpreterConstraintsField) and field_set_type.is_applicable(tgt) + } + if not unique_constraints and extra_constraints_per_tgt: + unique_constraints.add( + InterpreterConstraints.create_from_compatibility_fields( + extra_constraints_per_tgt, + python_setup, + ) + ) + constraints = InterpreterConstraints( + itertools.chain.from_iterable(ic for ic in unique_constraints if ic) + ) + return constraints or InterpreterConstraints(python_setup.interpreter_constraints) diff --git a/src/python/pants/backend/python/util_rules/partition_test.py b/src/python/pants/backend/python/util_rules/partition_test.py new file mode 100644 index 00000000000..deb9b5342b9 --- /dev/null +++ b/src/python/pants/backend/python/util_rules/partition_test.py @@ -0,0 +1,145 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python import target_types_rules +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import ( + InterpreterConstraintsField, + PythonSourceField, + PythonSourcesGeneratorTarget, +) +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.partition import _find_all_unique_interpreter_constraints +from pants.build_graph.address import Address +from pants.core.target_types import GenericTarget +from pants.engine.rules import QueryRule, SubsystemRule, rule +from pants.engine.target import FieldSet, Target +from pants.testutil.rule_runner import RuleRunner + + +@dataclass(frozen=True) +class UniqueICsRequest: + include_extra_fields: bool + + +def test_find_unique_interpreter_constraints() -> None: + class AnotherMockFieldSet(FieldSet): + required_fields = (PythonSourceField,) + + @classmethod + def opt_out(cls, tgt: Target) -> bool: + return tgt.address.target_name == "skip_me" + + extra_fields_ic = "==1.2" + + @rule + async def run_rule( + request: UniqueICsRequest, python_setup: PythonSetup + ) -> InterpreterConstraints: + return await _find_all_unique_interpreter_constraints( + python_setup, + AnotherMockFieldSet, + extra_constraints_per_tgt=[ + InterpreterConstraintsField([extra_fields_ic], Address("foo")) + ] + if request.include_extra_fields + else [], + ) + + rule_runner = RuleRunner( + rules=[ + run_rule, + *target_types_rules.rules(), + SubsystemRule(PythonSetup), + QueryRule(InterpreterConstraints, [UniqueICsRequest]), + ], + target_types=[PythonSourcesGeneratorTarget, GenericTarget], + ) + + global_constraint = "==3.9.*" + rule_runner.set_options( + [], + env={"PANTS_PYTHON_INTERPRETER_CONSTRAINTS": f"['{global_constraint}']"}, + ) + + def assert_ics( + build_file: str, expected: list[str], *, include_extra_fields: bool = False + ) -> None: + rule_runner.write_files({"project/BUILD": build_file, "project/f.py": ""}) + result = rule_runner.request( + InterpreterConstraints, [UniqueICsRequest(include_extra_fields)] + ) + assert result == InterpreterConstraints(expected) + + assert_ics("python_sources()", [global_constraint]) + assert_ics("python_sources(interpreter_constraints=['==2.7.*'])", ["==2.7.*"]) + assert_ics( + "python_sources(interpreter_constraints=['==2.7.*', '==3.5.*'])", ["==2.7.*", "==3.5.*"] + ) + + # If no Python targets in repo, fall back to global [python] constraints. + assert_ics("target()", [global_constraint]) + + # Ignore targets that are skipped. + assert_ics( + dedent( + """\ + python_sources(name='a', interpreter_constraints=['==2.7.*']) + python_sources(name='skip_me', interpreter_constraints=['==3.5.*']) + """ + ), + ["==2.7.*"], + ) + + # If there are multiple distinct ICs in the repo, we OR them. This is because Flake8 will + # group into each distinct IC. + assert_ics( + dedent( + """\ + python_sources(name='a', interpreter_constraints=['==2.7.*']) + python_sources(name='b', interpreter_constraints=['==3.5.*']) + """ + ), + ["==2.7.*", "==3.5.*"], + ) + assert_ics( + dedent( + """\ + python_sources(name='a', interpreter_constraints=['==2.7.*', '==3.5.*']) + python_sources(name='b', interpreter_constraints=['>=3.5']) + """ + ), + ["==2.7.*", "==3.5.*", ">=3.5"], + ) + assert_ics( + dedent( + """\ + python_sources(name='a') + python_sources(name='b', interpreter_constraints=['==2.7.*']) + python_sources(name='c', interpreter_constraints=['>=3.6']) + """ + ), + ["==2.7.*", global_constraint, ">=3.6"], + ) + + # Extra requirements get ANDed with each value. + assert_ics( + "python_sources(interpreter_constraints=['==2.7.*'])", + include_extra_fields=True, + expected=[f"{extra_fields_ic},==2.7.*"], + ) + assert_ics( + dedent( + """\ + python_sources(name='a', interpreter_constraints=['==2.7.*']) + python_sources(name='b', interpreter_constraints=['==3.5.*']) + """ + ), + include_extra_fields=True, + expected=[f"{extra_fields_ic},==2.7.*", f"{extra_fields_ic},==3.5.*"], + ) diff --git a/src/python/pants/core/goals/repl.py b/src/python/pants/core/goals/repl.py index 6b04b62ffda..310c1111212 100644 --- a/src/python/pants/core/goals/repl.py +++ b/src/python/pants/core/goals/repl.py @@ -7,17 +7,15 @@ from dataclasses import dataclass from typing import ClassVar, Iterable, Mapping, Optional, Sequence, Tuple -from pants.base.build_root import BuildRoot from pants.engine.addresses import Addresses from pants.engine.console import Console from pants.engine.environment import CompleteEnvironment -from pants.engine.fs import Digest, Workspace +from pants.engine.fs import Digest from pants.engine.goal import Goal, GoalSubsystem from pants.engine.process import InteractiveProcess, InteractiveProcessResult from pants.engine.rules import Effect, Get, collect_rules, goal_rule from pants.engine.target import FilteredTargets, Target from pants.engine.unions import UnionMembership, union -from pants.option.global_options import GlobalOptions from pants.option.option_types import BoolOption, StrOption from pants.util.frozendict import FrozenDict from pants.util.memo import memoized_property @@ -97,12 +95,9 @@ def __init__( @goal_rule async def run_repl( console: Console, - workspace: Workspace, repl_subsystem: ReplSubsystem, specified_targets: FilteredTargets, - build_root: BuildRoot, union_membership: UnionMembership, - global_options: GlobalOptions, complete_env: CompleteEnvironment, ) -> Repl: # TODO: When we support multiple languages, detect the default repl to use based