diff --git a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py b/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py index 6b1d445aa49..f972c952eab 100644 --- a/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py +++ b/src/python/pants/backend/awslambda/python/awslambda_python_rules_test.py @@ -24,17 +24,24 @@ from pants.engine.selectors import Params from pants.rules.core import strip_source_root from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class TestPythonAWSLambdaCreation(TestBase): + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + @classmethod def rules(cls): return ( *super().rules(), *awslambda_python_rules(), - *download_pex_bin.rules(), + # If we pull in the subsystem_rule() as well from this file, we get an error saying the scope + # 'download-pex-bin' was not found when trying to fetch the appropriate scope. + download_pex_bin.download_pex_bin, *inject_init.rules(), *pex.rules(), *pex_from_target_closure.rules(), @@ -43,6 +50,7 @@ def rules(cls): *strip_source_root.rules(), *subprocess_environment.rules(), RootRule(PythonAWSLambdaAdaptor), + RootRule(download_pex_bin.DownloadedPexBin.Factory), ) def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]: @@ -52,6 +60,7 @@ def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]: Params( target.adaptor, create_options_bootstrapper(args=["--backend-packages2=pants.backend.awslambda.python"]), + download_pex_bin.DownloadedPexBin.Factory.global_instance(), ), ) files_content = list(self.request_single_product(FilesContent, diff --git a/src/python/pants/backend/graph_info/subsystems/BUILD b/src/python/pants/backend/graph_info/subsystems/BUILD index 5e4cda20a1c..770ba15dcd8 100644 --- a/src/python/pants/backend/graph_info/subsystems/BUILD +++ b/src/python/pants/backend/graph_info/subsystems/BUILD @@ -4,6 +4,7 @@ python_library( dependencies = [ 'src/python/pants/binaries', + 'src/python/pants/engine:fs', ], tags = {"partially_type_checked"}, ) diff --git a/src/python/pants/backend/graph_info/subsystems/cloc_binary.py b/src/python/pants/backend/graph_info/subsystems/cloc_binary.py index 213ecbc8270..4c8fb234956 100644 --- a/src/python/pants/backend/graph_info/subsystems/cloc_binary.py +++ b/src/python/pants/backend/graph_info/subsystems/cloc_binary.py @@ -1,7 +1,9 @@ # Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.binaries.binary_tool import Script +from pants.binaries.binary_tool import Script, ToolForPlatform, ToolVersion +from pants.engine.fs import Digest +from pants.engine.platform import PlatformConstraint class ClocBinary(Script): @@ -12,3 +14,11 @@ class ClocBinary(Script): replaces_scope = 'cloc' replaces_name = 'version' + + default_versions_and_digests = { + PlatformConstraint.none: ToolForPlatform( + digest=Digest('2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d', + 546279), + version=ToolVersion('1.80'), + ), + } diff --git a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py index aad8dc3eeba..1bc89dfa4bc 100644 --- a/src/python/pants/backend/python/lint/bandit/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/bandit/rules_integration_test.py @@ -7,6 +7,8 @@ from pants.backend.python.lint.bandit.rules import BanditTarget from pants.backend.python.lint.bandit.rules import rules as bandit_rules +from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.subsystems import python_native_code, subprocess_environment from pants.backend.python.targets.python_library import PythonLibrary from pants.build_graph.address import Address from pants.build_graph.build_file_aliases import BuildFileAliases @@ -17,10 +19,16 @@ from pants.rules.core.lint import LintResult from pants.source.wrapped_globs import EagerFilesetWithSpec from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class BanditIntegrationTest(TestBase): + + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + good_source = FileContent(path="test/good.py", content=b"hashlib.sha256()\n") bad_source = FileContent(path="test/bad.py", content=b"hashlib.md5()\n") # MD5 is a insecure hashing function @@ -31,7 +39,16 @@ def alias_groups(cls) -> BuildFileAliases: @classmethod def rules(cls): - return (*super().rules(), *bandit_rules(), RootRule(BanditTarget)) + return ( + *super().rules(), + *bandit_rules(), + download_pex_bin.download_pex_bin, + *pex.rules(), + *python_native_code.rules(), + *subprocess_environment.rules(), + RootRule(BanditTarget), + RootRule(download_pex_bin.DownloadedPexBin.Factory), + ) def run_bandit( self, @@ -60,7 +77,11 @@ def run_bandit( ) ) return self.request_single_product( - LintResult, Params(target, create_options_bootstrapper(args=args)), + LintResult, Params( + target, + create_options_bootstrapper(args=args), + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + ), ) def test_single_passing_source(self) -> None: diff --git a/src/python/pants/backend/python/lint/black/rules_integration_test.py b/src/python/pants/backend/python/lint/black/rules_integration_test.py index af3c20ade70..acd2cb831a8 100644 --- a/src/python/pants/backend/python/lint/black/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/black/rules_integration_test.py @@ -7,6 +7,8 @@ from pants.backend.python.lint.black.rules import BlackTarget from pants.backend.python.lint.black.rules import rules as black_rules +from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.subsystems import python_native_code, subprocess_environment from pants.build_graph.address import Address from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot from pants.engine.legacy.structs import TargetAdaptor @@ -16,11 +18,16 @@ from pants.rules.core.lint import LintResult from pants.source.wrapped_globs import EagerFilesetWithSpec from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class BlackIntegrationTest(TestBase): + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + good_source = FileContent(path="test/good.py", content=b'animal = "Koala"\n') bad_source = FileContent(path="test/bad.py", content=b'name= "Anakin"\n') fixed_bad_source = FileContent(path="test/bad.py", content=b'name = "Anakin"\n') @@ -30,7 +37,16 @@ class BlackIntegrationTest(TestBase): @classmethod def rules(cls): - return (*super().rules(), *black_rules(), RootRule(BlackTarget)) + return ( + *super().rules(), + *black_rules(), + download_pex_bin.download_pex_bin, + *pex.rules(), + *python_native_code.rules(), + *subprocess_environment.rules(), + RootRule(BlackTarget), + RootRule(download_pex_bin.DownloadedPexBin.Factory), + ) def run_black( self, @@ -56,8 +72,16 @@ def run_black( lint_target = BlackTarget(target_adaptor) fmt_target = BlackTarget(target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest) options_bootstrapper = create_options_bootstrapper(args=args) - lint_result = self.request_single_product(LintResult, Params(lint_target, options_bootstrapper)) - fmt_result = self.request_single_product(FmtResult, Params(fmt_target, options_bootstrapper)) + lint_result = self.request_single_product(LintResult, Params( + lint_target, + options_bootstrapper, + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + )) + fmt_result = self.request_single_product(FmtResult, Params( + fmt_target, + options_bootstrapper, + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + )) return lint_result, fmt_result def get_digest(self, source_files: List[FileContent]) -> Digest: diff --git a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py index 801d2d88dbc..4a78e3fd621 100644 --- a/src/python/pants/backend/python/lint/flake8/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/flake8/rules_integration_test.py @@ -7,6 +7,8 @@ from pants.backend.python.lint.flake8.rules import Flake8Target from pants.backend.python.lint.flake8.rules import rules as flake8_rules +from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.subsystems import python_native_code, subprocess_environment from pants.backend.python.targets.python_library import PythonLibrary from pants.build_graph.address import Address from pants.build_graph.build_file_aliases import BuildFileAliases @@ -18,10 +20,16 @@ from pants.source.wrapped_globs import EagerFilesetWithSpec from pants.testutil.interpreter_selection_utils import skip_unless_python27_and_python3_present from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class Flake8IntegrationTest(TestBase): + + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + good_source = FileContent(path="test/good.py", content=b"print('Nothing suspicious here..')\n") bad_source = FileContent(path="test/bad.py", content=b"import typing\n") # unused import py3_only_source = FileContent(path="test/py3.py", content=b"version: str = 'Py3 > Py2'\n") @@ -32,7 +40,16 @@ def alias_groups(cls) -> BuildFileAliases: @classmethod def rules(cls): - return (*super().rules(), *flake8_rules(), RootRule(Flake8Target)) + return ( + *super().rules(), + *flake8_rules(), + download_pex_bin.download_pex_bin, + *pex.rules(), + *python_native_code.rules(), + *subprocess_environment.rules(), + RootRule(Flake8Target), + RootRule(download_pex_bin.DownloadedPexBin.Factory), + ) def run_flake8( self, @@ -61,7 +78,11 @@ def run_flake8( ) ) return self.request_single_product( - LintResult, Params(target, create_options_bootstrapper(args=args)), + LintResult, Params( + target, + create_options_bootstrapper(args=args), + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + ), ) def test_single_passing_source(self) -> None: diff --git a/src/python/pants/backend/python/lint/isort/rules_integration_test.py b/src/python/pants/backend/python/lint/isort/rules_integration_test.py index aa300e50030..3e0f4a9d5e9 100644 --- a/src/python/pants/backend/python/lint/isort/rules_integration_test.py +++ b/src/python/pants/backend/python/lint/isort/rules_integration_test.py @@ -5,6 +5,8 @@ from pants.backend.python.lint.isort.rules import IsortTarget from pants.backend.python.lint.isort.rules import rules as isort_rules +from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.subsystems import python_native_code, subprocess_environment from pants.build_graph.address import Address from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot from pants.engine.legacy.structs import TargetAdaptor @@ -14,11 +16,16 @@ from pants.rules.core.lint import LintResult from pants.source.wrapped_globs import EagerFilesetWithSpec from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class IsortIntegrationTest(TestBase): + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + good_source = FileContent(path="test/good.py", content=b'from animals import cat, dog\n') bad_source = FileContent(path="test/bad.py", content=b'from colors import green, blue\n') fixed_bad_source = FileContent(path="test/bad.py", content=b'from colors import blue, green\n') @@ -36,7 +43,16 @@ class IsortIntegrationTest(TestBase): @classmethod def rules(cls): - return (*super().rules(), *isort_rules(), RootRule(IsortTarget)) + return ( + *super().rules(), + *isort_rules(), + download_pex_bin.download_pex_bin, + *pex.rules(), + *python_native_code.rules(), + *subprocess_environment.rules(), + RootRule(IsortTarget), + RootRule(download_pex_bin.DownloadedPexBin.Factory), + ) def run_isort( self, @@ -64,8 +80,16 @@ def run_isort( target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest, ) options_bootstrapper = create_options_bootstrapper(args=args) - lint_result = self.request_single_product(LintResult, Params(lint_target, options_bootstrapper)) - fmt_result = self.request_single_product(FmtResult, Params(fmt_target, options_bootstrapper)) + lint_result = self.request_single_product(LintResult, Params( + lint_target, + options_bootstrapper, + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + )) + fmt_result = self.request_single_product(FmtResult, Params( + fmt_target, + options_bootstrapper, + download_pex_bin.DownloadedPexBin.Factory.global_instance(), + )) return lint_result, fmt_result def get_digest(self, source_files: List[FileContent]) -> Digest: diff --git a/src/python/pants/backend/python/lint/python_format_target_integration_test.py b/src/python/pants/backend/python/lint/python_format_target_integration_test.py index 07adab2f1f6..eb250f4b0c8 100644 --- a/src/python/pants/backend/python/lint/python_format_target_integration_test.py +++ b/src/python/pants/backend/python/lint/python_format_target_integration_test.py @@ -11,6 +11,8 @@ _ConcretePythonFormatTarget, format_python_target, ) +from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.subsystems import python_native_code, subprocess_environment from pants.build_graph.address import Address from pants.engine.fs import Digest, FileContent, InputFilesContent, Snapshot from pants.engine.legacy.structs import TargetAdaptor @@ -19,11 +21,16 @@ from pants.rules.core.fmt import AggregatedFmtResults from pants.source.wrapped_globs import EagerFilesetWithSpec from pants.testutil.option.util import create_options_bootstrapper +from pants.testutil.subsystem.util import init_subsystems from pants.testutil.test_base import TestBase class PythonFormatTargetIntegrationTest(TestBase): + def setUp(self): + super().setUp() + init_subsystems([download_pex_bin.DownloadedPexBin.Factory]) + @classmethod def rules(cls): return ( @@ -31,9 +38,14 @@ def rules(cls): format_python_target, *black_rules(), *isort_rules(), + download_pex_bin.download_pex_bin, + *pex.rules(), + *python_native_code.rules(), + *subprocess_environment.rules(), RootRule(_ConcretePythonFormatTarget), RootRule(BlackTarget), RootRule(IsortTarget), + RootRule(download_pex_bin.DownloadedPexBin.Factory), ) def run_black_and_isort( @@ -55,7 +67,8 @@ def run_black_and_isort( args=[ "--backend-packages2=['pants.backend.python.lint.black', 'pants.backend.python.lint.isort']" ], - ) + ), + download_pex_bin.DownloadedPexBin.Factory.global_instance(), ) ) return results diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py index bda9ac952bc..08726fbd0e3 100644 --- a/src/python/pants/backend/python/rules/download_pex_bin.py +++ b/src/python/pants/backend/python/rules/download_pex_bin.py @@ -7,13 +7,21 @@ from pants.backend.python.rules.hermetic_pex import HermeticPex from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment -from pants.engine.fs import Digest, SingleFileExecutable, Snapshot, UrlToFetch +from pants.binaries.binary_tool import BinaryToolFetchRequest, Script, ToolForPlatform, ToolVersion +from pants.binaries.binary_util import BinaryToolUrlGenerator +from pants.engine.fs import Digest, SingleFileExecutable, Snapshot from pants.engine.isolated_process import ExecuteProcessRequest -from pants.engine.rules import rule +from pants.engine.platform import PlatformConstraint +from pants.engine.rules import rule, subsystem_rule from pants.engine.selectors import Get from pants.python.python_setup import PythonSetup +class PexBinUrlGenerator(BinaryToolUrlGenerator): + def generate_urls(self, version, host_platform): + return [f'https://github.com/pantsbuild/pex/releases/download/{version}/pex'] + + @dataclass(frozen=True) class DownloadedPexBin(HermeticPex): exe: SingleFileExecutable @@ -26,7 +34,23 @@ def executable(self) -> str: def directory_digest(self) -> Digest: return self.exe.directory_digest - def create_execute_request( # type: ignore[override] + class Factory(Script): + options_scope = 'download-pex-bin' + name = 'pex' + default_version = 'v1.6.12' + + default_versions_and_digests = { + PlatformConstraint.none: ToolForPlatform( + digest=Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', + 1842359), + version=ToolVersion('v1.6.12'), + ), + } + + def get_external_url_generator(self): + return PexBinUrlGenerator() + + def create_execute_request( # type: ignore[override] self, python_setup: PythonSetup, subprocess_encoding_environment: SubprocessEncodingEnvironment, @@ -65,15 +89,13 @@ def create_execute_request( # type: ignore[override] @rule -async def download_pex_bin() -> DownloadedPexBin: - # TODO: Inject versions and digests here through some option, rather than hard-coding it. - url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex' - digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359) - snapshot = await Get[Snapshot](UrlToFetch(url, digest)) +async def download_pex_bin(pex_binary_tool: DownloadedPexBin.Factory) -> DownloadedPexBin: + snapshot = await Get[Snapshot](BinaryToolFetchRequest(pex_binary_tool)) return DownloadedPexBin(SingleFileExecutable(snapshot)) def rules(): return [ download_pex_bin, + subsystem_rule(DownloadedPexBin.Factory), ] diff --git a/src/python/pants/binaries/BUILD b/src/python/pants/binaries/BUILD index c73bbbf5f35..ce5857972eb 100644 --- a/src/python/pants/binaries/BUILD +++ b/src/python/pants/binaries/BUILD @@ -8,6 +8,8 @@ python_library( 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', 'src/python/pants/engine:fs', + 'src/python/pants/engine:rules', + 'src/python/pants/engine:selectors', 'src/python/pants/fs', 'src/python/pants/net', 'src/python/pants/option', diff --git a/src/python/pants/binaries/binary_tool.py b/src/python/pants/binaries/binary_tool.py index 750c0b7d0b2..35dcce9f3db 100644 --- a/src/python/pants/binaries/binary_tool.py +++ b/src/python/pants/binaries/binary_tool.py @@ -4,18 +4,67 @@ import logging import os import threading -from typing import Optional - -from pants.binaries.binary_util import BinaryRequest, BinaryUtil -from pants.engine.fs import PathGlobs, PathGlobsAndRoot +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union, cast + +from pants.binaries.binary_util import ( + BinaryRequest, + BinaryToolUrlGenerator, + BinaryUtil, + HostPlatform, +) +from pants.engine.fs import Digest, PathGlobs, PathGlobsAndRoot, Snapshot, UrlToFetch +from pants.engine.platform import Platform, PlatformConstraint +from pants.engine.rules import RootRule, rule +from pants.engine.selectors import Get from pants.fs.archive import XZCompressedTarArchiver, create_archiver from pants.subsystem.subsystem import Subsystem +from pants.util.enums import match from pants.util.memo import memoized_method, memoized_property +from pants.util.meta import frozen_after_init +from pants.util.osutil import get_closest_mac_host_platform_pair logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class ToolVersion: + version: str + + +@dataclass(frozen=True) +class ToolForPlatform: + version: ToolVersion + digest: Digest + + def into_tuple(self) -> Tuple[str, str, int]: + return (self.version.version, self.digest.fingerprint, self.digest.serialized_bytes_length) + + +@rule +def translate_host_platform( + platform_constraint: PlatformConstraint, + binary_util: BinaryUtil, +) -> HostPlatform: + # This method attempts to provide a uname function to BinaryUtil.host_platform() so that the + # download urls can be calculated. For platforms that are different than the current host, we try + # to "spoof" the most appropriate value. + if Platform.current == Platform.darwin: + darwin_uname: Any = os.uname + linux_uname: Any = lambda: ('linux', None, None, None, 'x86_64') + else: + assert Platform.current == Platform.linux + darwin_uname = lambda: ('darwin', None, get_closest_mac_host_platform_pair(), None, 'x86_64') + linux_uname = os.uname + + return cast(HostPlatform, match(platform_constraint, { + PlatformConstraint.none: lambda: HostPlatform.empty, + PlatformConstraint.darwin: lambda: binary_util.host_platform(uname=darwin_uname()), + PlatformConstraint.linux: lambda: binary_util.host_platform(uname=linux_uname()), + })()) + + # TODO: Add integration tests for this file. class BinaryToolBase(Subsystem): """Base class for subsytems that configure binary tools. @@ -28,7 +77,9 @@ class BinaryToolBase(Subsystem): # They must also set options_scope appropriately. platform_dependent: Optional[bool] = None archive_type: Optional[str] = None # See pants.fs.archive.archive for valid string values. + default_version: Optional[str] = None + default_versions_and_digests: Dict[PlatformConstraint, ToolForPlatform] = {} # Subclasses may set this to the tool name as understood by BinaryUtil. # If unset, it defaults to the value of options_scope. @@ -115,6 +166,22 @@ def register_options(cls, register): version_registration_kwargs['fingerprint'] = True register('--version', **version_registration_kwargs) + register('--version-digest-mapping', type=dict, + default={ + # "Serialize" the default value dict into "basic" types that can be easily specified + # in pants.ini. + platform_constraint.value: tool.into_tuple() + for platform_constraint, tool in cls.default_versions_and_digests.items() + }, + fingerprint=True, + help='A dict mapping -> (, , ).' + f'A "platform constraint" is any of {[c.value for c in PlatformConstraint]}, and ' + 'is the platform to fetch the tool for. A platform-independent tool should ' + f'use {PlatformConstraint.none.value}, while a platform-dependent tool should specify ' + 'all environments it needs to be used for. The "fingerprint" and "size_bytes" ' + 'arguments are the result printed when running `sha256sum` and `wc -c` on ' + 'the downloaded file, respectively.') + @memoized_method def select(self, context=None): """Returns the path to the specified binary tool. @@ -166,7 +233,7 @@ def get_support_dir(cls): def _name_to_fetch(cls): return '{}{}'.format(cls._get_name(), cls.suffix) - def _make_binary_request(self, version): + def make_binary_request(self, version): return BinaryRequest( supportdir=self.get_support_dir(), version=version, @@ -176,7 +243,7 @@ def _make_binary_request(self, version): archiver=self._get_archiver()) def _select_for_version(self, version): - binary_request = self._make_binary_request(version) + binary_request = self.make_binary_request(version) return self._binary_util.select(binary_request) @memoized_method @@ -230,3 +297,112 @@ def tar_xz_extractor(self): def _executable_location(self): return os.path.join(self.select(), 'bin', 'xz') + + +@frozen_after_init +@dataclass(unsafe_hash=True) +class VersionDigestMapping: + """Parse the --version-digest-mapping option back into a dictionary.""" + version_digest_mapping: Tuple[Tuple[str, Tuple[str, str, int]], ...] + + def __init__(self, version_digest_mapping: Dict[str, List[Union[str, int]]]) -> None: + self.version_digest_mapping = tuple( + (platform_constraint, tuple(data)) # type: ignore[misc] + for platform_constraint, data in version_digest_mapping.items() + ) + + @memoized_property + def _deserialized_mapping( + self, + ) -> Dict[PlatformConstraint, ToolForPlatform]: + deserialized: Dict[PlatformConstraint, ToolForPlatform] = {} + for platform_constraint, (version, fingerprint, size_bytes) in self.version_digest_mapping: + deserialized[PlatformConstraint(platform_constraint)] = ToolForPlatform( + version=ToolVersion(version), + digest=Digest(fingerprint, size_bytes), + ) + return deserialized + + def get(self, platform_constraint: PlatformConstraint) -> ToolForPlatform: + return self._deserialized_mapping[platform_constraint] + + +@dataclass(frozen=True) +class BinaryToolUrlSet: + tool_for_platform: ToolForPlatform + host_platform: HostPlatform + url_generator: BinaryToolUrlGenerator + + def get_urls(self) -> List[str]: + return self.url_generator.generate_urls( + version=self.tool_for_platform.version.version, + host_platform=self.host_platform if self.host_platform != HostPlatform.empty else None) + + +@frozen_after_init +@dataclass(unsafe_hash=True) +class BinaryToolFetchRequest: + tool: BinaryToolBase + platform_constraint: PlatformConstraint + + def __init__( + self, + tool: BinaryToolBase, + platform_constraint: Optional[PlatformConstraint] = None, + ) -> None: + self.tool = tool + if platform_constraint is None: + if tool.platform_dependent: + platform_constraint = PlatformConstraint.local_platform + else: + platform_constraint = PlatformConstraint.none + + self.platform_constraint = platform_constraint + + +@rule +async def get_binary_tool_urls( + req: BinaryToolFetchRequest, + binary_util: BinaryUtil, +) -> BinaryToolUrlSet: + tool = req.tool + platform_constraint = req.platform_constraint + + mapping = VersionDigestMapping(tool.get_options().version_digest_mapping) + tool_for_platform = mapping.get(platform_constraint) + + version = tool_for_platform.version.version + url_generator = binary_util.get_url_generator(tool.make_binary_request(version)) + + host_platform = await Get[HostPlatform](PlatformConstraint, platform_constraint) + + return BinaryToolUrlSet( + tool_for_platform=tool_for_platform, + host_platform=host_platform, + url_generator=url_generator, + ) + + +@rule +async def fetch_binary_tool(req: BinaryToolFetchRequest, url_set: BinaryToolUrlSet) -> Snapshot: + digest = url_set.tool_for_platform.digest + urls = url_set.get_urls() + + if not urls: + raise ValueError(f'binary tool url generator {url_set.url_generator} produced an empty list of ' + f'urls for the request {req}') + # TODO: allow fetching a UrlToFetch with failure! Consider FallibleUrlToFetch analog to + # FallibleExecuteProcessResult! + url_to_fetch = urls[0] + + return await Get[Snapshot](UrlToFetch(url_to_fetch, digest)) + + +def rules(): + return [ + RootRule(PlatformConstraint), + translate_host_platform, + get_binary_tool_urls, + fetch_binary_tool, + RootRule(BinaryToolFetchRequest), + ] diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py index 86f92ed0b46..c64357695b5 100644 --- a/src/python/pants/binaries/binary_util.py +++ b/src/python/pants/binaries/binary_util.py @@ -7,16 +7,17 @@ import posixpath import shutil import sys -from abc import abstractmethod +from abc import ABC, abstractmethod from contextlib import contextmanager from dataclasses import dataclass from functools import reduce -from typing import Any, Optional, Tuple +from typing import Any, List, Optional, Tuple, cast from twitter.common.collections import OrderedSet from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError +from pants.engine.rules import rule from pants.fs.archive import archiver_for_path from pants.net.http.fetcher import Fetcher from pants.option.global_options import GlobalOptionsRegistrar @@ -24,7 +25,7 @@ from pants.subsystem.subsystem import Subsystem from pants.util.contextutil import temporary_file from pants.util.dirutil import chmod_plus_x, safe_concurrent_creation, safe_open -from pants.util.memo import memoized_method, memoized_property +from pants.util.memo import memoized_classproperty, memoized_method, memoized_property from pants.util.osutil import ( SUPPORTED_PLATFORM_NORMALIZED_NAMES, get_closest_mac_host_platform_pair, @@ -40,8 +41,12 @@ class HostPlatform: :class:`BinaryToolUrlGenerator` instances receive this to generate download urls. """ - os_name: Any - arch_or_version: Any + os_name: Optional[str] + arch_or_version: Optional[str] + + @memoized_classproperty + def empty(cls): + return cls(None, None) def binary_path_components(self): """These strings are used as consecutive components of the path where a binary is fetched. @@ -50,7 +55,7 @@ def binary_path_components(self): return [self.os_name, self.arch_or_version] -class BinaryToolUrlGenerator: +class BinaryToolUrlGenerator(ABC): """Encapsulates the selection of urls to download for some binary tool. :API: public @@ -61,7 +66,7 @@ class BinaryToolUrlGenerator: """ @abstractmethod - def generate_urls(self, version, host_platform): + def generate_urls(self, version, host_platform) -> List[str]: """Return a list of urls to download some binary tool from given a version and platform. Each url is tried in order to resolve the binary -- if the list of urls is empty, or downloading @@ -101,7 +106,7 @@ def __init__(self, binary_request, baseurls): .format(binary_request.name)) self._baseurls = baseurls - def generate_urls(self, _version, host_platform): + def generate_urls(self, version, host_platform): """Append the file's download path to each of --binaries-baseurls. This assumes that the urls in --binaries-baseurls point somewhere that mirrors Pants's @@ -253,9 +258,9 @@ class Factory(Subsystem): options_scope = 'binaries' @classmethod - def create(cls): + def create(cls) -> 'BinaryUtil': # NB: create is a class method to ~force binary fetch location to be global. - return cls._create_for_cls(BinaryUtil) + return cast(BinaryUtil, cls._create_for_cls(BinaryUtil)) @classmethod def _create_for_cls(cls, binary_util_cls): @@ -324,8 +329,8 @@ def __init__(self, baseurls, binary_tool_fetcher, path_by_id=None, # to fail until a binary is requested. The HostPlatform should be a parameter that gets lazily # resolved by the v2 engine. @memoized_method - def _host_platform(self): - uname_result = self._uname_func() + def host_platform(self, uname=None): + uname_result = uname if uname else self._uname_func() sysname, _, release, _, machine = uname_result os_id_key = sysname.lower() try: @@ -364,9 +369,9 @@ def _host_platform(self): .format(os_id_tuple, self._path_by_id)) def _get_download_path(self, binary_request): - return binary_request.get_download_path(self._host_platform()) + return binary_request.get_download_path(self.host_platform()) - def _get_url_generator(self, binary_request): + def get_url_generator(self, binary_request): external_url_generator = binary_request.external_url_generator @@ -384,7 +389,7 @@ def _get_url_generator(self, binary_request): return url_generator def _get_urls(self, url_generator, binary_request): - return url_generator.generate_urls(binary_request.version, self._host_platform()) + return url_generator.generate_urls(binary_request.version, self.host_platform()) def select(self, binary_request): """Fetches a file, unpacking it if necessary.""" @@ -397,7 +402,7 @@ def select(self, binary_request): raise self.BinaryResolutionError(binary_request, e) try: - url_generator = self._get_url_generator(binary_request) + url_generator = self.get_url_generator(binary_request) except self.NoBaseUrlsError as e: raise self.BinaryResolutionError(binary_request, e) @@ -518,3 +523,14 @@ def select(argv): if __name__ == '__main__': print(select(sys.argv)) + + +@rule +def provide_binary_util() -> BinaryUtil: + return BinaryUtil.Factory.create() + + +def rules(): + return [ + provide_binary_util, + ] diff --git a/src/python/pants/init/engine_initializer.py b/src/python/pants/init/engine_initializer.py index 57d69ad2613..4d7530c61e3 100644 --- a/src/python/pants/init/engine_initializer.py +++ b/src/python/pants/init/engine_initializer.py @@ -18,6 +18,8 @@ from pants.base.exiter import PANTS_SUCCEEDED_EXIT_CODE from pants.base.file_system_project_tree import FileSystemProjectTree from pants.base.specs import Specs +from pants.binaries.binary_tool import rules as binary_tool_rules +from pants.binaries.binary_util import rules as binary_util_rules from pants.build_graph.build_configuration import BuildConfiguration from pants.build_graph.build_file_aliases import BuildFileAliases from pants.build_graph.remote_sources import RemoteSources @@ -450,6 +452,8 @@ def build_root_singleton() -> BuildRoot: *create_options_parsing_rules(), *structs_rules(), *changed_rules(), + *binary_tool_rules(), + *binary_util_rules(), *rules, ) diff --git a/src/python/pants/rules/core/BUILD b/src/python/pants/rules/core/BUILD index 97f7bbac41f..0ee75f29725 100644 --- a/src/python/pants/rules/core/BUILD +++ b/src/python/pants/rules/core/BUILD @@ -5,8 +5,10 @@ python_library( dependencies = [ '3rdparty/python:dataclasses', 'src/python/pants:version', + 'src/python/pants/backend/graph_info/subsystems', 'src/python/pants/base:build_root', 'src/python/pants/base:exiter', + 'src/python/pants/binaries', 'src/python/pants/build_graph', 'src/python/pants/engine:goal', 'src/python/pants/engine:rules', diff --git a/src/python/pants/rules/core/cloc.py b/src/python/pants/rules/core/cloc.py index ac6bfaf2523..ad2299fb386 100644 --- a/src/python/pants/rules/core/cloc.py +++ b/src/python/pants/rules/core/cloc.py @@ -4,6 +4,8 @@ import itertools from dataclasses import dataclass +from pants.backend.graph_info.subsystems.cloc_binary import ClocBinary +from pants.binaries.binary_tool import BinaryToolFetchRequest from pants.engine.console import Console from pants.engine.fs import ( Digest, @@ -13,12 +15,11 @@ InputFilesContent, SingleFileExecutable, Snapshot, - UrlToFetch, ) from pants.engine.goal import Goal, GoalSubsystem from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult from pants.engine.legacy.graph import SourcesSnapshots -from pants.engine.rules import goal_rule, rule +from pants.engine.rules import goal_rule, rule, subsystem_rule from pants.engine.selectors import Get @@ -36,14 +37,9 @@ def digest(self) -> Digest: return self.exe.directory_digest -#TODO(#7790) - We can't call this feature-complete with the v1 version of cloc -# until we have a way to download the cloc binary without hardcoding it @rule -async def download_cloc_script() -> DownloadedClocScript: - url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc" - sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d" - digest = Digest(sha_256, 546279) - snapshot = await Get[Snapshot](UrlToFetch(url, digest)) +async def download_cloc_script(cloc_binary_tool: ClocBinary) -> DownloadedClocScript: + snapshot = await Get[Snapshot](BinaryToolFetchRequest(cloc_binary_tool)) return DownloadedClocScript(SingleFileExecutable(snapshot)) @@ -129,6 +125,7 @@ async def run_cloc( def rules(): return [ - run_cloc, - download_cloc_script, - ] + run_cloc, + download_cloc_script, + subsystem_rule(ClocBinary), + ] diff --git a/src/python/pants/util/osutil.py b/src/python/pants/util/osutil.py index bc901b00949..c6d70259da5 100644 --- a/src/python/pants/util/osutil.py +++ b/src/python/pants/util/osutil.py @@ -99,12 +99,17 @@ def safe_kill(pid: Pid, signum: int) -> None: def get_closest_mac_host_platform_pair( - darwin_version_upper_bound: str, + darwin_version_upper_bound: Optional[str] = None, platform_name_map: Dict[Tuple[str, str], Tuple[str, str]] = SUPPORTED_PLATFORM_NORMALIZED_NAMES ) -> Tuple[Optional[str], Optional[str]]: """Return the (host, platform) pair for the highest known darwin version less than the bound.""" darwin_versions = [int(x[1]) for x in platform_name_map if x[0] == 'darwin'] - bounded_darwin_versions = [v for v in darwin_versions if v <= int(darwin_version_upper_bound)] + + if darwin_version_upper_bound is not None: + bounded_darwin_versions = [v for v in darwin_versions if v <= int(darwin_version_upper_bound)] + else: + bounded_darwin_versions = darwin_versions + if not bounded_darwin_versions: return None, None max_darwin_version = str(max(bounded_darwin_versions)) diff --git a/tests/python/pants_test/binaries/test_binary_tool.py b/tests/python/pants_test/binaries/test_binary_tool.py index f65f938dc61..a385b25fc03 100644 --- a/tests/python/pants_test/binaries/test_binary_tool.py +++ b/tests/python/pants_test/binaries/test_binary_tool.py @@ -40,7 +40,7 @@ class ReplacingLegacyOptionsTool(BinaryToolBase): class BinaryUtilFakeUname(BinaryUtil): - def _host_platform(self): + def host_platform(self): return HostPlatform('xxx', 'yyy') @@ -71,7 +71,7 @@ def get_external_url_generator(self): return CustomUrlGenerator() def _select_for_version(self, version): - binary_request = self._make_binary_request(version) + binary_request = self.make_binary_request(version) return BinaryUtilFakeUname.Factory._create_for_cls(BinaryUtilFakeUname).select(binary_request) diff --git a/tests/python/pants_test/binaries/test_binary_util.py b/tests/python/pants_test/binaries/test_binary_util.py index b4a093f022a..14e4dc4f4ee 100644 --- a/tests/python/pants_test/binaries/test_binary_util.py +++ b/tests/python/pants_test/binaries/test_binary_util.py @@ -70,7 +70,7 @@ def _fake_url(cls, binaries, base, binary_key): supportdir, version, name = binaries[binary_key] binary_request = binary_util._make_deprecated_binary_request(supportdir, version, name) - binary_path = binary_request.get_download_path(binary_util._host_platform()) + binary_path = binary_request.get_download_path(binary_util.host_platform()) return f'{base}/{binary_path}' @classmethod @@ -148,7 +148,7 @@ def test_support_url_multi(self): version='2.4.1', name='protoc') - binary_path = binary_request.get_download_path(binary_util._host_platform()) + binary_path = binary_request.get_download_path(binary_util.host_platform()) contents = b'proof' with safe_open(os.path.join(valid_local_files, binary_path), 'wb') as fp: fp.write(contents) diff --git a/tests/python/pants_test/util/test_osutil.py b/tests/python/pants_test/util/test_osutil.py index a137ccabbb3..24cf17fbe4f 100644 --- a/tests/python/pants_test/util/test_osutil.py +++ b/tests/python/pants_test/util/test_osutil.py @@ -50,7 +50,7 @@ def test_get_closest_mac_host_platform_pair(self) -> None: ('darwin', '17'): ('mac', '10.13'), } - def get_macos_version(darwin_version: str) -> Optional[str]: + def get_macos_version(darwin_version: Optional[str]) -> Optional[str]: host, version = get_closest_mac_host_platform_pair( darwin_version, platform_name_map=platform_name_map) if host is not None: @@ -68,3 +68,7 @@ def get_macos_version(darwin_version: str) -> Optional[str]: self.assertEqual('10.6', get_macos_version('11')) self.assertEqual('10.6', get_macos_version('10')) self.assertEqual(None, get_macos_version('9')) + + # When a version bound of `None` is provided, it should select the most recent OSX platform + # available. + self.assertEqual('10.13', get_macos_version(None))