From 0afdbb861798a6cbfaee1b577ed5aac48c45eb32 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 28 Jul 2020 05:58:14 -0600 Subject: [PATCH 01/18] Control PATH and PEX_PYTHON_PATH seperately. Previously we used PATH to steer interpreter selection for hermetic PEX bootstrap and runtime. This led to fragile or impossible to solve setups. Switch to using PEX_PYTHON_PATH to both discover a bootstrap interpreter and steer runtime interpreter selection. Introduce a find_binary rule to facilitate this that uses ~portable bash for now. Also introduce a PexRuntimeEnvironment to steer bootstrap interpreter probing and allow specification of the PATH that should be exposed to the PEX runtime environment. Fixes #9760 # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/awslambda_python_rules.py | 6 +- .../pants/backend/python/lint/bandit/rules.py | 5 +- .../pants/backend/python/lint/black/rules.py | 6 +- .../backend/python/lint/docformatter/rules.py | 6 +- .../pants/backend/python/lint/flake8/rules.py | 5 +- .../pants/backend/python/lint/isort/rules.py | 6 +- .../pants/backend/python/lint/pylint/rules.py | 5 +- .../pants/backend/python/rules/coverage.py | 10 +- .../backend/python/rules/download_pex_bin.py | 10 +- .../backend/python/rules/hermetic_pex.py | 154 ++++++++++++++++-- src/python/pants/backend/python/rules/pex.py | 7 +- .../backend/python/rules/pytest_runner.py | 5 +- .../backend/python/rules/run_setup_py.py | 5 +- .../backend/python/typecheck/mypy/rules.py | 6 +- src/python/pants/engine/platform.py | 8 +- src/python/pants/engine/process.py | 89 +++++++++- src/python/pants/util/strutil.py | 4 +- 17 files changed, 277 insertions(+), 60 deletions(-) diff --git a/src/python/pants/backend/awslambda/python/awslambda_python_rules.py b/src/python/pants/backend/awslambda/python/awslambda_python_rules.py index daf391f71c0..77a743e217a 100644 --- a/src/python/pants/backend/awslambda/python/awslambda_python_rules.py +++ b/src/python/pants/backend/awslambda/python/awslambda_python_rules.py @@ -13,6 +13,7 @@ PythonAwsLambdaRuntime, ) from pants.backend.python.rules import download_pex_bin, pex, pex_from_targets, python_sources +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -33,7 +34,6 @@ from pants.engine.rules import collect_rules, rule from pants.engine.selectors import Get from pants.engine.unions import UnionRule -from pants.python.python_setup import PythonSetup @dataclass(frozen=True) @@ -53,7 +53,7 @@ class LambdexSetup: async def create_python_awslambda( field_set: PythonAwsLambdaFieldSet, lambdex_setup: LambdexSetup, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> CreatedAWSLambda: # Lambdas typically use the .zip suffix, so we use that instead of .pex. @@ -92,7 +92,7 @@ async def create_python_awslambda( # NB: Lambdex modifies its input pex in-place, so the input file is also the output file. lambdex_args = ("build", "-e", field_set.handler.value, pex_filename) process = lambdex_setup.requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./lambdex.pex", pex_args=lambdex_args, diff --git a/src/python/pants/backend/python/lint/bandit/rules.py b/src/python/pants/backend/python/lint/bandit/rules.py index 48db3928dda..42672f61a7a 100644 --- a/src/python/pants/backend/python/lint/bandit/rules.py +++ b/src/python/pants/backend/python/lint/bandit/rules.py @@ -6,6 +6,7 @@ from pants.backend.python.lint.bandit.subsystem import Bandit from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -63,7 +64,7 @@ def generate_args(*, specified_source_files: SourceFiles, bandit: Bandit) -> Tup async def bandit_lint_partition( partition: BanditPartition, bandit: Bandit, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: requirements_pex_request = Get( @@ -116,7 +117,7 @@ async def bandit_lint_partition( ) process = requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./bandit.pex", pex_args=generate_args(specified_source_files=specified_source_files, bandit=bandit), diff --git a/src/python/pants/backend/python/lint/black/rules.py b/src/python/pants/backend/python/lint/black/rules.py index 1ffb6e4578d..81f0cf285c8 100644 --- a/src/python/pants/backend/python/lint/black/rules.py +++ b/src/python/pants/backend/python/lint/black/rules.py @@ -9,6 +9,7 @@ from pants.backend.python.lint.black.subsystem import Black from pants.backend.python.lint.python_fmt import PythonFmtRequest from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -32,7 +33,6 @@ from pants.engine.selectors import Get, MultiGet from pants.engine.target import FieldSetWithOrigin from pants.engine.unions import UnionRule -from pants.python.python_setup import PythonSetup from pants.util.strutil import pluralize @@ -82,7 +82,7 @@ def generate_args( async def setup( setup_request: SetupRequest, black: Black, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> Setup: requirements_pex_request = Get( @@ -140,7 +140,7 @@ async def setup( ) process = requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./black.pex", pex_args=generate_args( diff --git a/src/python/pants/backend/python/lint/docformatter/rules.py b/src/python/pants/backend/python/lint/docformatter/rules.py index 6cbbb7787c4..19d2fd80faf 100644 --- a/src/python/pants/backend/python/lint/docformatter/rules.py +++ b/src/python/pants/backend/python/lint/docformatter/rules.py @@ -7,6 +7,7 @@ from pants.backend.python.lint.docformatter.subsystem import Docformatter from pants.backend.python.lint.python_fmt import PythonFmtRequest from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -30,7 +31,6 @@ from pants.engine.selectors import Get, MultiGet from pants.engine.target import FieldSetWithOrigin from pants.engine.unions import UnionRule -from pants.python.python_setup import PythonSetup from pants.util.strutil import pluralize @@ -71,7 +71,7 @@ def generate_args( async def setup( setup_request: SetupRequest, docformatter: Docformatter, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> Setup: requirements_pex_request = Get( @@ -118,7 +118,7 @@ async def setup( ) process = requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./docformatter.pex", pex_args=generate_args( diff --git a/src/python/pants/backend/python/lint/flake8/rules.py b/src/python/pants/backend/python/lint/flake8/rules.py index 9b817161d79..bf0175e819e 100644 --- a/src/python/pants/backend/python/lint/flake8/rules.py +++ b/src/python/pants/backend/python/lint/flake8/rules.py @@ -6,6 +6,7 @@ from pants.backend.python.lint.flake8.subsystem import Flake8 from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -81,7 +82,7 @@ async def flake8_lint_partition( partition: Flake8Partition, flake8: Flake8, lint_subsystem: LintSubsystem, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: requirements_pex_request = Get( @@ -141,7 +142,7 @@ async def flake8_lint_partition( output_file=report_path.name if report_path else None, ) process = requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./flake8.pex", pex_args=flake8_args, diff --git a/src/python/pants/backend/python/lint/isort/rules.py b/src/python/pants/backend/python/lint/isort/rules.py index 69b56ea273a..3dacc89e78e 100644 --- a/src/python/pants/backend/python/lint/isort/rules.py +++ b/src/python/pants/backend/python/lint/isort/rules.py @@ -7,6 +7,7 @@ from pants.backend.python.lint.isort.subsystem import Isort from pants.backend.python.lint.python_fmt import PythonFmtRequest from pants.backend.python.rules import download_pex_bin, pex +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -37,7 +38,6 @@ from pants.engine.selectors import Get, MultiGet from pants.engine.target import FieldSetWithOrigin from pants.engine.unions import UnionRule -from pants.python.python_setup import PythonSetup from pants.util.strutil import pluralize @@ -81,7 +81,7 @@ def generate_args( async def setup( setup_request: SetupRequest, isort: Isort, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> Setup: requirements_pex_request = Get( @@ -144,7 +144,7 @@ async def setup( ) process = requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./isort.pex", pex_args=generate_args( diff --git a/src/python/pants/backend/python/lint/pylint/rules.py b/src/python/pants/backend/python/lint/pylint/rules.py index 86bec251069..ac056d30301 100644 --- a/src/python/pants/backend/python/lint/pylint/rules.py +++ b/src/python/pants/backend/python/lint/pylint/rules.py @@ -8,6 +8,7 @@ from pants.backend.python.lint.pylint.subsystem import Pylint from pants.backend.python.rules import download_pex_bin, pex, python_sources +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -111,7 +112,7 @@ def generate_args(*, specified_source_files: SourceFiles, pylint: Pylint) -> Tup async def pylint_lint_partition( partition: PylintPartition, pylint: Pylint, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> LintResult: # We build one PEX with Pylint requirements and another with all direct 3rd-party dependencies. @@ -233,7 +234,7 @@ async def pylint_lint_partition( ) process = pylint_runner_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./pylint_runner.pex", env={"PEX_EXTRA_SYS_PATH": ":".join(pythonpath)}, diff --git a/src/python/pants/backend/python/rules/coverage.py b/src/python/pants/backend/python/rules/coverage.py index 0ab068a3631..456734f03bb 100644 --- a/src/python/pants/backend/python/rules/coverage.py +++ b/src/python/pants/backend/python/rules/coverage.py @@ -7,6 +7,7 @@ from pathlib import PurePath from typing import List, Optional, Sequence, Tuple +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -45,7 +46,6 @@ from pants.engine.target import TransitiveTargets from pants.engine.unions import UnionRule from pants.option.custom_types import file_option -from pants.python.python_setup import PythonSetup """ @@ -215,7 +215,7 @@ class MergedCoverageData: async def merge_coverage_data( data_collection: PytestCoverageDataCollection, coverage_setup: CoverageSetup, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> MergedCoverageData: if len(data_collection) == 1: @@ -233,7 +233,7 @@ async def merge_coverage_data( input_digest=input_digest, output_files=(".coverage",), description=f"Merge {len(prefixes)} Pytest coverage reports.", - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, ) result = await Get(ProcessResult, Process, process) @@ -247,7 +247,7 @@ async def generate_coverage_reports( coverage_config: CoverageConfig, coverage_subsystem: CoverageSubsystem, transitive_targets: TransitiveTargets, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> CoverageReports: """Takes all Python test results and generates a single coverage report.""" @@ -292,7 +292,7 @@ async def generate_coverage_reports( output_directories=("htmlcov",) if report_type == CoverageReportType.HTML else None, output_files=("coverage.xml",) if report_type == CoverageReportType.XML else None, description=f"Generate Pytest {report_type.report_name} coverage report.", - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, ) ) diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py index 7f9191f884a..c32689cff3a 100644 --- a/src/python/pants/backend/python/rules/download_pex_bin.py +++ b/src/python/pants/backend/python/rules/download_pex_bin.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any, Iterable, Mapping, Optional -from pants.backend.python.rules.hermetic_pex import HermeticPex +from pants.backend.python.rules.hermetic_pex import HermeticPex, PexEnvironment from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment from pants.core.util_rules.external_tool import ( @@ -17,7 +17,6 @@ from pants.engine.process import Process from pants.engine.rules import collect_rules, rule from pants.engine.selectors import Get -from pants.python.python_setup import PythonSetup class PexBin(ExternalTool): @@ -50,7 +49,7 @@ def digest(self) -> Digest: def create_process( # type: ignore[override] self, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, pex_build_environment: PexBuildEnvironment, *, @@ -62,8 +61,7 @@ def create_process( # type: ignore[override] ) -> Process: """Creates an Process that will run the pex CLI tool hermetically. - :param python_setup: The parameters for selecting python interpreters to use when invoking - the pex tool. + :param pex_environment: The environment needed to bootstrap the PEX runtime. :param subprocess_encoding_environment: The locale settings to use for the pex tool invocation. :param pex_build_environment: The build environment for the pex tool. @@ -85,7 +83,7 @@ def create_process( # type: ignore[override] pex_args = ("--pex-root", pex_root_path) + tuple(pex_args) return super().create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=self.executable, pex_args=pex_args, diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 9bc0dd1ca02..efb44a3a65a 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -1,21 +1,149 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - -from typing import Any, Iterable, Mapping, Optional +import logging +import os +from dataclasses import dataclass +from typing import Any, Iterable, Mapping, Optional, Tuple from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment +from pants.engine import process +from pants.engine.engine_aware import EngineAware from pants.engine.fs import Digest -from pants.engine.process import Process +from pants.engine.process import BinaryPathRequest, BinaryPaths, Process +from pants.engine.rules import collect_rules, rule +from pants.engine.selectors import Get, MultiGet from pants.python.python_setup import PythonSetup +from pants.subsystem.subsystem import Subsystem +from pants.util.logging import LogLevel +from pants.util.memo import memoized_property +from pants.util.ordered_set import OrderedSet from pants.util.strutil import create_path_env_var +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PexEnvironment(EngineAware): + path: Iterable[str] + interpreter_search_paths: Iterable[str] + bootstrap_python: Optional[str] = None + + def create_argv(self, pex_path: str, *args: str) -> Iterable[str]: + argv = [self.bootstrap_python] if self.bootstrap_python else [] + argv.extend((pex_path, *args)) + return argv + + @property + def environment_dict(self) -> Mapping[str, str]: + return dict( + PATH=create_path_env_var(self.path), + PEX_PYTHON_PATH=create_path_env_var(self.interpreter_search_paths), + ) + + def level(self) -> Optional[LogLevel]: + return LogLevel.INFO if self.bootstrap_python else LogLevel.WARN + + def message(self) -> Optional[str]: + if not self.bootstrap_python: + return ( + "No bootstrap python executable could be found. " + "Will attempt to run PEXes directly." + ) + else: + return f"Selected {self.bootstrap_python} to bootstrap PEXes with." + + +class PexRuntimeEnvironment(Subsystem): + options_scope = "pex" + + @classmethod + def register_options(cls, register): + super().register_options(register) + register( + "--binary-search-path", + advanced=True, + type=list, + default=[""], + metavar="", + help=( + "A list of paths to search for binaries needed by PEX subprocess. The special " + 'string "" will expand to the contents of the PATH env var.' + ), + ) + register( + "--bootstrap-interpreter-names", + advanced=True, + type=list, + default=["python", "python3", "python2"], + metavar="", + help=( + "The names of python binaries to search for to bootstrap PEX files with. Earlier " + "names in the list will be preferred over later names and the first matching " + "interpreter found will be used to execute the PEX bootstrap stage. If no matching " + "interpreters are found, PEXes will be executed directly relying on their embedded " + "shebang and the $PATH (see --binary-search-path) to locate a bootstrap " + "interpreter." + ), + ) + + @memoized_property + def path(self) -> Tuple[str, ...]: + def iter_path_entries(): + for entry in self.options.binary_search_path: + if entry == "": + path = os.environ.get("PATH") + if path: + for path_entry in os.pathsep.split(path): + yield path_entry + else: + yield entry + + return tuple(OrderedSet(iter_path_entries())) + + @property + def bootstrap_interpreter_names(self) -> Tuple[str, ...]: + return tuple(self.options.bootstrap_interpreter_names) + + +@rule(desc="Find PEX Python") +async def find_pex_python( + python_setup: PythonSetup, pex_runtime_environment: PexRuntimeEnvironment +) -> PexEnvironment: + # PEX files are compatible with bootstrapping via python2.7 or python 3.5+. The bootstrap + # code will then re-exec itself if the underlying PEX user code needs a more specific python + # interpreter. As such, we look for many pythons usable by the PEX bootstrap code here for + # maximum flexibility. + all_python_binary_paths = await MultiGet( + [ + Get( + BinaryPaths, + BinaryPathRequest( + search_path=python_setup.interpreter_search_paths, binary_name=binary_name + ), + ) + for binary_name in pex_runtime_environment.bootstrap_interpreter_names + ] + ) + + def first_python_binary(): + for binary_paths in all_python_binary_paths: + for binary_path in binary_paths.paths: + return binary_path + return None + + return PexEnvironment( + path=pex_runtime_environment.path, + interpreter_search_paths=python_setup.interpreter_search_paths, + bootstrap_python=first_python_binary(), + ) + class HermeticPex: """A mixin for types that provide an executable Pex that should be executed hermetically.""" def create_process( self, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, *, pex_path: str, @@ -23,12 +151,11 @@ def create_process( description: str, input_digest: Digest, env: Optional[Mapping[str, str]] = None, - **kwargs: Any + **kwargs: Any, ) -> Process: """Creates an Process that will run a PEX hermetically. - :param python_setup: The parameters for selecting python interpreters to use when invoking - the PEX. + :param pex_environment: The environment needed to bootstrap the PEX runtime. :param subprocess_encoding_environment: The locale settings to use for the PEX invocation. :param pex_path: The path within `input_files` of the PEX file (or directory if a loose pex). @@ -40,19 +167,14 @@ def create_process( :param **kwargs: Any additional :class:`Process` kwargs to pass through. """ - # NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like - # `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote - # execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable - # somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not - # necessarily the interpreter that PEX will use to execute the generated .pex file. # TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target # platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735. - argv = ("python", pex_path, *pex_args) + argv = pex_environment.create_argv(pex_path, *pex_args) hermetic_env = dict( - PATH=create_path_env_var(python_setup.interpreter_search_paths), PEX_INHERIT_PATH="false", PEX_IGNORE_RCFILES="true", + **pex_environment.environment_dict, **subprocess_encoding_environment.invocation_environment_dict, ) if env: @@ -65,3 +187,7 @@ def create_process( env=hermetic_env, **kwargs, ) + + +def rules(): + return [*collect_rules(), *process.rules()] diff --git a/src/python/pants/backend/python/rules/pex.py b/src/python/pants/backend/python/rules/pex.py index 1b4be294b7d..c92c2095b62 100644 --- a/src/python/pants/backend/python/rules/pex.py +++ b/src/python/pants/backend/python/rules/pex.py @@ -20,8 +20,9 @@ from typing_extensions import Protocol +from pants.backend.python.rules import hermetic_pex from pants.backend.python.rules.download_pex_bin import DownloadedPexBin -from pants.backend.python.rules.hermetic_pex import HermeticPex +from pants.backend.python.rules.hermetic_pex import HermeticPex, PexEnvironment from pants.backend.python.rules.util import parse_interpreter_constraint from pants.backend.python.subsystems.python_native_code import PexBuildEnvironment from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment @@ -306,6 +307,7 @@ def log(self, *args, **kwargs) -> None: async def create_pex( request: PexRequest, pex_bin: DownloadedPexBin, + pex_environment: PexEnvironment, python_setup: PythonSetup, python_repos: PythonRepos, subprocess_encoding_environment: SubprocessEncodingEnvironment, @@ -415,7 +417,7 @@ async def create_pex( PlatformConstraint(platform.value), PlatformConstraint(platform.value), ): pex_bin.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_build_environment=pex_build_environment, pex_args=argv, @@ -483,6 +485,7 @@ async def two_step_create_pex(two_step_pex_request: TwoStepPexRequest) -> TwoSte def rules(): return [ *collect_rules(), + *hermetic_pex.rules(), RootRule(PexRequest), RootRule(TwoStepPexRequest), ] diff --git a/src/python/pants/backend/python/rules/pytest_runner.py b/src/python/pants/backend/python/rules/pytest_runner.py index a13ec1ddde1..46911676e6a 100644 --- a/src/python/pants/backend/python/rules/pytest_runner.py +++ b/src/python/pants/backend/python/rules/pytest_runner.py @@ -13,6 +13,7 @@ CoverageSubsystem, PytestCoverageData, ) +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -211,7 +212,7 @@ async def setup_pytest_for_target( async def run_python_test( field_set: PythonTestFieldSet, test_setup: TestTargetSetup, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, global_options: GlobalOptions, test_subsystem: TestSubsystem, @@ -251,7 +252,7 @@ async def run_python_test( env["__PANTS_FORCE_TEST_RUN__"] = str(uuid) process = test_setup.test_runner_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=f"./{test_setup.test_runner_pex.output_filename}", pex_args=test_setup.args, diff --git a/src/python/pants/backend/python/rules/run_setup_py.py b/src/python/pants/backend/python/rules/run_setup_py.py index e89b18f0f53..8fc1a3fc2be 100644 --- a/src/python/pants/backend/python/rules/run_setup_py.py +++ b/src/python/pants/backend/python/rules/run_setup_py.py @@ -9,6 +9,7 @@ from typing import List, Set, Tuple, cast from pants.backend.python.python_artifact import PythonArtifact +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -373,7 +374,7 @@ async def run_setup_pys( async def run_setup_py( req: RunSetupPyRequest, setuptools_setup: SetuptoolsSetup, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> RunSetupPyResult: """Run a setup.py command on a single exported target.""" @@ -384,7 +385,7 @@ async def run_setup_py( # pants's own dist dir, at the buildroot). dist_dir = "dist/" process = setuptools_setup.requirements_pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path="./setuptools.pex", pex_args=("setup.py", *req.args), diff --git a/src/python/pants/backend/python/typecheck/mypy/rules.py b/src/python/pants/backend/python/typecheck/mypy/rules.py index eaaf36381bf..65404d55dcc 100644 --- a/src/python/pants/backend/python/typecheck/mypy/rules.py +++ b/src/python/pants/backend/python/typecheck/mypy/rules.py @@ -5,6 +5,7 @@ from typing import Tuple from pants.backend.python.rules import download_pex_bin, pex, python_sources +from pants.backend.python.rules.hermetic_pex import PexEnvironment from pants.backend.python.rules.pex import ( Pex, PexInterpreterConstraints, @@ -35,7 +36,6 @@ from pants.engine.selectors import Get, MultiGet from pants.engine.target import FieldSetWithOrigin, TransitiveTargets from pants.engine.unions import UnionRule -from pants.python.python_setup import PythonSetup from pants.util.strutil import pluralize @@ -65,7 +65,7 @@ def generate_args(mypy: MyPy, *, file_list_path: str) -> Tuple[str, ...]: async def mypy_lint( request: MyPyRequest, mypy: MyPy, - python_setup: PythonSetup, + pex_environment: PexEnvironment, subprocess_encoding_environment: SubprocessEncodingEnvironment, ) -> TypecheckResults: if mypy.skip: @@ -118,7 +118,7 @@ async def mypy_lint( ) process = pex.create_process( - python_setup=python_setup, + pex_environment=pex_environment, subprocess_encoding_environment=subprocess_encoding_environment, pex_path=pex.output_filename, pex_args=generate_args(mypy, file_list_path=file_list_path), diff --git a/src/python/pants/engine/platform.py b/src/python/pants/engine/platform.py index ff88ba10e28..8f491243522 100644 --- a/src/python/pants/engine/platform.py +++ b/src/python/pants/engine/platform.py @@ -2,9 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from enum import Enum -from typing import Callable, List +from typing import Iterable -from pants.engine.rules import rule +from pants.engine.rules import Rule, collect_rules, rule from pants.util.memo import memoized_classproperty from pants.util.osutil import get_normalized_os_name @@ -36,5 +36,5 @@ def materialize_platform() -> Platform: return Platform.current -def create_platform_rules() -> List[Callable]: - return [materialize_platform] +def create_platform_rules() -> Iterable[Rule]: + return collect_rules() diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index ab9faa5a711..b67824c68c8 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -1,15 +1,20 @@ # Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - import itertools import logging from dataclasses import dataclass +from textwrap import dedent from typing import Dict, Iterable, Mapping, Optional, Tuple, Union -from pants.engine.fs import EMPTY_DIGEST, Digest +from pants.engine.engine_aware import EngineAware +from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent from pants.engine.platform import Platform, PlatformConstraint from pants.engine.rules import RootRule, collect_rules, rule +from pants.engine.selectors import Get +from pants.util.logging import LogLevel from pants.util.meta import frozen_after_init +from pants.util.ordered_set import OrderedSet +from pants.util.strutil import create_path_env_var, pluralize logger = logging.getLogger(__name__) @@ -230,6 +235,86 @@ def remove_platform_information(res: FallibleProcessResultWithPlatform,) -> Fall ) +@frozen_after_init +@dataclass(unsafe_hash=True) +class BinaryPathRequest: + search_path: Tuple[str, ...] + binary_name: str + + def __init__(self, search_path: Iterable[str], binary_name: str): + self.search_path = tuple(OrderedSet(search_path)) + self.binary_name = binary_name + + +@frozen_after_init +@dataclass(unsafe_hash=True) +class BinaryPaths(EngineAware): + binary_name: str + paths: Tuple[str, ...] + + def __init__(self, binary_name: str, paths: Iterable[str]): + self.binary_name = binary_name + self.paths = tuple(OrderedSet(paths)) + + def level(self) -> Optional[LogLevel]: + return LogLevel.DEBUG if self.paths else LogLevel.WARN + + def message(self) -> Optional[str]: + if not self.paths: + return f"failed to find {self.binary_name}" + found_msg = f"found {self.binary_name} at {self.paths[0]}" + if len(self.paths) > 1: + found_msg = f"{found_msg} and {pluralize(len(self.paths) - 1, 'other location')}" + return found_msg + + +@rule(desc="Find binary path") +async def find_binary(request: BinaryPathRequest) -> BinaryPaths: + # TODO(John Sirois): Replace this script with a statically linked native binary so we don't + # depend on either /usr/bin/env or bash being available on the Process host. + script_path = "./script.sh" + script_digest = await Get( + Digest, + CreateDigest( + [ + FileContent( + script_path, + content=dedent( + """ + #!/usr/bin/env bash + + set -euo pipefail + + if command -v which > /dev/null; then + command which -a $1 + else + command -v $1 + fi + """ + ).encode(), + is_executable=True, + ) + ] + ), + ) + + paths = [] + search_path = create_path_env_var(request.search_path) + result = await Get( + FallibleProcessResult, + Process( + description=f"Searching for `{request.binary_name}` on PATH={search_path}", + input_digest=script_digest, + argv=[script_path, request.binary_name], + env={"PATH": search_path}, + ), + ) + if result.exit_code == 0: + paths.extend(result.stdout.decode().splitlines()) + + return BinaryPaths(binary_name=request.binary_name, paths=paths) + + def rules(): """Creates rules that consume the intrinsic filesystem types.""" return [ diff --git a/src/python/pants/util/strutil.py b/src/python/pants/util/strutil.py index ddfacdf3e79..8bd845edde1 100644 --- a/src/python/pants/util/strutil.py +++ b/src/python/pants/util/strutil.py @@ -3,7 +3,7 @@ import re import shlex -from typing import Dict, Iterable, List, Optional, Sequence, Union +from typing import Dict, Iterable, List, Optional, Union def ensure_binary(text_or_binary: Union[bytes, str]) -> bytes: @@ -59,7 +59,7 @@ def safe_shlex_join(arg_list: Iterable[str]) -> str: def create_path_env_var( - new_entries: Sequence[str], + new_entries: Iterable[str], env: Optional[Dict[str, str]] = None, env_var: str = "PATH", delimiter: str = ":", From b5d4009212eca42902139c0349ce02e035583f88 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 28 Jul 2020 06:34:15 -0600 Subject: [PATCH 02/18] Fix up remoting config. # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- pants.remote.toml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pants.remote.toml b/pants.remote.toml index 5b96a40ce75..18f1bef54a4 100644 --- a/pants.remote.toml +++ b/pants.remote.toml @@ -32,13 +32,22 @@ process_execution_speculation_strategy = "none" # p95 of RBE appears to be ~ 2 seconds, but we need to factor in local queue time which can be much longer, but no metrics yet. process_execution_speculation_delay = 15 +[pex] +# TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote +# platforms. This should be replaced by a proper mechanism to differentiate between the two. +binary_search_path = [ + # The remote container has binaries like `ld` and `gcc` in /usr/bin. + "/usr/bin", + # We include the host PATH so that speculation still works. + '', +] + [python-setup] # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote # platforms. This should be replaced by a proper mechanism to differentiate between the two. interpreter_search_paths = [ - # These are the interpreter paths we set up on the remote container, plus `/usr/bin`, so that - # pip can find `ld` if necessary. - "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin:/usr/bin", + # These are the interpreter paths we set up on the remote container. + "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin", # We include the host PATH and PEXRC values so that speculation still works. # NOTE: These come after the remote paths. Putting them before the remote paths means generic # bin dirs like /usr/bin will be on the PATH ahead of the pyenv dirs we actually want to use From 0f23fe903ebf3ee36940e11ca1314c2fc7b33dd9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 28 Jul 2020 22:44:52 -0700 Subject: [PATCH 03/18] Small tweaks + add a requested TODO [ci skip-rust] [ci skip-build-wheels] --- .../backend/python/rules/download_pex_bin.py | 2 +- .../pants/backend/python/rules/hermetic_pex.py | 16 +++++++++------- .../python/subsystems/python_native_code.py | 2 +- .../python/subsystems/subprocess_environment.py | 2 +- src/python/pants/engine/process.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py index 7e65bb3519b..77af873e2ee 100644 --- a/src/python/pants/backend/python/rules/download_pex_bin.py +++ b/src/python/pants/backend/python/rules/download_pex_bin.py @@ -75,7 +75,7 @@ def create_process( # type: ignore[override] """ pex_root_path = ".cache/pex_root" - env = {**(env or {}), **python_native_code.invocation_environment} + env = {**(env or {}), **python_native_code.environment_dict} if "--pex-root" in pex_args: raise ValueError("--pex-root flag not allowed. We set its value for you.") pex_args = ("--pex-root", pex_root_path) + tuple(pex_args) diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index b6fbdedeff1..28e07c7cf3a 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -49,8 +49,7 @@ def message(self) -> Optional[str]: "No bootstrap python executable could be found. " "Will attempt to run PEXes directly." ) - else: - return f"Selected {self.bootstrap_python} to bootstrap PEXes with." + return f"Selected {self.bootstrap_python} to bootstrap PEXes with." class PexRuntimeEnvironment(Subsystem): @@ -91,6 +90,8 @@ def path(self) -> Tuple[str, ...]: def iter_path_entries(): for entry in self.options.binary_search_path: if entry == "": + # TODO(#9760): This is not very robust. We want to be able to read an env var + # safely via the engine. path = os.environ.get("PATH") if path: for path_entry in os.pathsep.split(path): @@ -111,7 +112,7 @@ async def find_pex_python( ) -> PexEnvironment: # PEX files are compatible with bootstrapping via python2.7 or python 3.5+. The bootstrap # code will then re-exec itself if the underlying PEX user code needs a more specific python - # interpreter. As such, we look for many pythons usable by the PEX bootstrap code here for + # interpreter. As such, we look for many Pythons usable by the PEX bootstrap code here for # maximum flexibility. all_python_binary_paths = await MultiGet( [ @@ -125,10 +126,11 @@ async def find_pex_python( ] ) - def first_python_binary(): + def first_python_binary() -> Optional[str]: for binary_paths in all_python_binary_paths: - for binary_path in binary_paths.paths: - return binary_path + first_path = next(iter(binary_paths.paths), None) + if first_path: + return first_path return None return PexEnvironment( @@ -175,7 +177,7 @@ def create_process( PEX_INHERIT_PATH="false", PEX_IGNORE_RCFILES="true", **pex_environment.environment_dict, - **subprocess_environment.invocation_environment, + **subprocess_environment.environment_dict, ) if env: hermetic_env.update(env) diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index f3d02dce41f..f921b95790a 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -42,7 +42,7 @@ def ld_flags(self) -> Tuple[str, ...]: return tuple(self.options.ld_flags) @property - def invocation_environment(self) -> Dict[str, str]: + def environment_dict(self) -> Dict[str, str]: return { "CPPFLAGS": safe_shlex_join(self.cpp_flags), "LDFLAGS": safe_shlex_join(self.ld_flags), diff --git a/src/python/pants/backend/python/subsystems/subprocess_environment.py b/src/python/pants/backend/python/subsystems/subprocess_environment.py index 90f82245e59..a158de40323 100644 --- a/src/python/pants/backend/python/subsystems/subprocess_environment.py +++ b/src/python/pants/backend/python/subsystems/subprocess_environment.py @@ -41,7 +41,7 @@ def lc_all(self) -> Optional[str]: return cast(Optional[str], self.options.lc_all) @property - def invocation_environment(self) -> Dict[str, str]: + def environment_dict(self) -> Dict[str, str]: return {"LANG": self.lang or "", "LC_ALL": self.lc_all or ""} diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index f792fec4535..ed155d2858a 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -301,7 +301,7 @@ class BinaryPathRequest: search_path: Tuple[str, ...] binary_name: str - def __init__(self, search_path: Iterable[str], binary_name: str): + def __init__(self, *, search_path: Iterable[str], binary_name: str) -> None: self.search_path = tuple(OrderedSet(search_path)) self.binary_name = binary_name From 6ed4e421cef4137d6589f60db78087ccc69787dc Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 28 Jul 2020 23:04:43 -0700 Subject: [PATCH 04/18] Other small fixes, including adding a missing docstring [ci skip-rust] [ci skip-build-wheels] --- .travis.yml | 3 +-- build-support/bin/generate_travis_yml.py | 6 +----- pants.travis-ci.toml | 3 +-- src/python/pants/backend/python/rules/hermetic_pex.py | 2 ++ src/python/pants/python/python_setup.py | 4 ++-- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c275b00ff9..8945dc4a1ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ deploy: env: global: - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.toml" + - PANTS_DYNAMIC_UI=false - LC_ALL="en_US.UTF-8" - AWS_BUCKET=ci-public.pantsbuild.org - BOOTSTRAPPED_PEX_KEY_PREFIX=daily/${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex @@ -327,7 +328,6 @@ jobs: env: - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem - - PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]" - CACHE_NAME=lint.py37 language: python name: Self-checks and lint (Python 3.7) @@ -487,7 +487,6 @@ jobs: env: - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux - PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem - - PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]" - CACHE_NAME=python_tests.py37 language: python name: Python tests (Python 3.7) diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index 3742c688753..fca9999d908 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -67,6 +67,7 @@ def all_entries(cls) -> List[Dict[str, str]]: GLOBAL_ENV_VARS = [ 'PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.toml"', + 'PANTS_DYNAMIC_UI=false', 'LC_ALL="en_US.UTF-8"', "AWS_BUCKET=ci-public.pantsbuild.org", # The ci-public.pantsbuild.org bucket has expiration policies set up for key prefixes @@ -355,11 +356,6 @@ def linux_shard( "PANTS_REMOTE_CA_CERTS_PATH=/usr/lib/google-cloud-sdk/lib/third_party/grpc/_cython/_credentials/roots.pem", ] setup = {**setup, **CACHE_PANTS_RUN} - if python_version.is_py37: - # 3.7.2 for Linux uses the new C++ ABI, which may be an error. - setup["env"].append( - 'PANTS_NATIVE_BUILD_STEP_CPP_COMPILE_SETTINGS_DEFAULT_COMPILER_OPTION_SETS="[]"' - ) if use_docker: setup["services"] = ["docker"] return setup diff --git a/pants.travis-ci.toml b/pants.travis-ci.toml index 57103c23bd0..347ff2f2c47 100644 --- a/pants.travis-ci.toml +++ b/pants.travis-ci.toml @@ -1,4 +1,3 @@ -# Overrides for TravisCI runs. [DEFAULT] # If we use typical default process parallelism tied to core count, we see too many cores under # travis and either get oomkilled from launching too many processes with too much total memory @@ -25,4 +24,4 @@ resolver_jobs = 2 use_coverage = true [coverage-py] -report=["raw"] \ No newline at end of file +report = ["raw"] diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 28e07c7cf3a..3d0a5d5e972 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -53,6 +53,8 @@ def message(self) -> Optional[str]: class PexRuntimeEnvironment(Subsystem): + """How Pants uses Pex to run Python subprocesses.""" + options_scope = "pex" @classmethod diff --git a/src/python/pants/python/python_setup.py b/src/python/pants/python/python_setup.py index c19648a3d42..eaba34dab3e 100644 --- a/src/python/pants/python/python_setup.py +++ b/src/python/pants/python/python_setup.py @@ -84,7 +84,7 @@ def register_options(cls, register): "If unspecified, a standard path under the workdir is used.", removal_version="2.1.0.dev0", removal_hint=( - "The option `--python-setup-interpreter-cache-dir` does not anything anymore." + "The option `--python-setup-interpreter-cache-dir` does not do anything anymore." ), ) register( @@ -96,7 +96,7 @@ def register_options(cls, register): "If unspecified, a standard path under the workdir is used.", removal_version="2.1.0.dev0", removal_hint=( - "The option `--python-setup-resolver-cache-dir` does not anything anymore." + "The option `--python-setup-resolver-cache-dir` does not do anything anymore." ), ) register( From 19c21545e118cdb28b3f3116d8a1e1474ce15481 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 28 Jul 2020 23:05:30 -0700 Subject: [PATCH 05/18] Debug the $PATH for docker linux build [ci skip-rust] [ci skip-build-wheels] --- .travis.yml | 12 ++++++------ build-support/bin/generate_travis_yml.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8945dc4a1ad..1adb686efe6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -132,7 +132,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -176,8 +176,8 @@ jobs: os: osx osx_image: xcode8 script: - - ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 - --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} + - echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version + 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants - after_failure: @@ -217,8 +217,8 @@ jobs: os: osx osx_image: xcode8 script: - - ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 - --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} + - echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version + 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants (Cron) - addons: diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index fca9999d908..34d398f5732 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -445,7 +445,7 @@ def osx_shard( def _bootstrap_command(*, python_version: PythonVersion) -> str: return ( - f"./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " + "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " f"{python_version.decimal} --aws-bucket ${{AWS_BUCKET}} --native-engine-so-key-prefix " "${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key " "${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" From 2b8757bc13818ded6acceddc64783473972b36dc Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 09:28:51 -0700 Subject: [PATCH 06/18] More debugging [ci skip-rust] [ci skip-build-wheels] --- .travis.yml | 19 +++++++++++-------- build-support/bin/generate_travis_yml.py | 2 +- .../backend/python/rules/hermetic_pex.py | 3 +++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1adb686efe6..01042ef9728 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -132,7 +132,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -176,9 +176,9 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version - 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} - --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + - echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix + ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants - after_failure: - ./build-support/bin/ci-failure.sh @@ -217,9 +217,9 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version - 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} - --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + - echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix + ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants (Cron) - addons: apt: @@ -974,3 +974,6 @@ stages: - if: tag IS NOT present AND type NOT IN (pull_request, cron) name: Deploy Pants Pex Unstable +HERE +['/Users/eric/DocsLocal/code/projects/pants/build-support/virtualenvs/Darwin/pants_dev_deps.py36.venv/bin/python', 'pex', '--pex-root', '.cache/pex_root', '--output-file', 'generate_travis_yml.pex', '--no-pypi', '--index=https://pypi.org/simple/', '--interpreter-constraint', 'CPython>=3.6', '--no-emit-warnings', '--manylinux', 'manylinux2014', '--constraints', '3rdparty/python/requirements.txt', '--sources-directory=source_files', 'PyYAML<5.4,>=5.3.1'] +{'PEX_INHERIT_PATH': 'false', 'PEX_IGNORE_RCFILES': 'true', 'PATH': ':', 'PEX_PYTHON_PATH': '/Users/eric/DocsLocal/code/projects/pants/build-support/virtualenvs/Darwin/pants_dev_deps.py36.venv/bin:/Users/eric/.pyenv/shims:/usr/local/opt/openssl/bin:/Users/eric/.pyenv/shims:/Users/eric/.poetry/bin:/Users/eric/.cargo/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/TeX/texbin:/Users/eric/.local/bin', 'LANG': 'en_US.UTF-8', 'LC_ALL': '', 'CPPFLAGS': '-I/usr/local/opt/openssl/include', 'LDFLAGS': '-L/usr/local/opt/openssl/lib'} diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index 34d398f5732..e7923e81200 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -445,7 +445,7 @@ def osx_shard( def _bootstrap_command(*, python_version: PythonVersion) -> str: return ( - "echo ${PATH} && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " + "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " f"{python_version.decimal} --aws-bucket ${{AWS_BUCKET}} --native-engine-so-key-prefix " "${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key " "${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 9d1816b73f9..b78a619654e 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -184,6 +184,9 @@ def create_process( if env: hermetic_env.update(env) + print("HERE") + print(argv) + print(hermetic_env) return Process( argv=argv, input_digest=input_digest, From 5d4cc116c52b40b9ffb995ed92844edbf090213e Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 10:17:02 -0700 Subject: [PATCH 07/18] Remove bad .travis.yml value --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01042ef9728..3ff7b458136 100644 --- a/.travis.yml +++ b/.travis.yml @@ -973,7 +973,3 @@ stages: name: Deploy Pants Pex - if: tag IS NOT present AND type NOT IN (pull_request, cron) name: Deploy Pants Pex Unstable - -HERE -['/Users/eric/DocsLocal/code/projects/pants/build-support/virtualenvs/Darwin/pants_dev_deps.py36.venv/bin/python', 'pex', '--pex-root', '.cache/pex_root', '--output-file', 'generate_travis_yml.pex', '--no-pypi', '--index=https://pypi.org/simple/', '--interpreter-constraint', 'CPython>=3.6', '--no-emit-warnings', '--manylinux', 'manylinux2014', '--constraints', '3rdparty/python/requirements.txt', '--sources-directory=source_files', 'PyYAML<5.4,>=5.3.1'] -{'PEX_INHERIT_PATH': 'false', 'PEX_IGNORE_RCFILES': 'true', 'PATH': ':', 'PEX_PYTHON_PATH': '/Users/eric/DocsLocal/code/projects/pants/build-support/virtualenvs/Darwin/pants_dev_deps.py36.venv/bin:/Users/eric/.pyenv/shims:/usr/local/opt/openssl/bin:/Users/eric/.pyenv/shims:/Users/eric/.poetry/bin:/Users/eric/.cargo/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/TeX/texbin:/Users/eric/.local/bin', 'LANG': 'en_US.UTF-8', 'LC_ALL': '', 'CPPFLAGS': '-I/usr/local/opt/openssl/include', 'LDFLAGS': '-L/usr/local/opt/openssl/lib'} From 2d0525d1f2a55d8a21d6108eeb0170c6a232d825 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 10:29:41 -0700 Subject: [PATCH 08/18] Apparently our Docker doesn't have `which` --- .travis.yml | 8 ++++---- build-support/bin/generate_travis_yml.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ff7b458136..b3dccc32c35 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -132,7 +132,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -176,7 +176,7 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + - echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants @@ -217,7 +217,7 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + - echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants (Cron) diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index e7923e81200..71b7c81fabc 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -445,7 +445,7 @@ def osx_shard( def _bootstrap_command(*, python_version: PythonVersion) -> str: return ( - "echo ${PATH} && which -a gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " + "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " f"{python_version.decimal} --aws-bucket ${{AWS_BUCKET}} --native-engine-so-key-prefix " "${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key " "${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" From 53d5780c1b64e3756a0289880692478b37b597b9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 11:06:29 -0700 Subject: [PATCH 09/18] Fix bug in setting PATH --- src/python/pants/backend/python/rules/hermetic_pex.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index b78a619654e..b86bdad33a9 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -96,11 +96,10 @@ def iter_path_entries(): # safely via the engine. path = os.environ.get("PATH") if path: - for path_entry in os.pathsep.split(path): + for path_entry in path.split(os.pathsep): yield path_entry else: yield entry - return tuple(OrderedSet(iter_path_entries())) @property From 264aa516e59ff78a358930112c13a5dcfdf476e9 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 11:07:35 -0700 Subject: [PATCH 10/18] Remove debugging --- .travis.yml | 17 +++++++++-------- build-support/bin/generate_travis_yml.py | 2 +- .../pants/backend/python/rules/hermetic_pex.py | 3 --- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3dccc32c35..8945dc4a1ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -132,7 +132,7 @@ jobs: "TRAVIS_UID=$(id -u)" --build-arg "TRAVIS_GROUP=$(id -gn)" --build-arg "TRAVIS_GID=$(id -g)" build-support/docker/travis_ci/ - docker run --rm -t -v "${HOME}:/travis/home" -v "${TRAVIS_BUILD_DIR}:/travis/workdir" - travis_ci:latest sh -c "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py + travis_ci:latest sh -c "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" services: @@ -176,9 +176,9 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py - --python-version 3.6 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix - ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + - ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.6 + --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} + --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants - after_failure: - ./build-support/bin/ci-failure.sh @@ -217,9 +217,9 @@ jobs: os: osx osx_image: xcode8 script: - - echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py - --python-version 3.7 --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix - ${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} + - ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version 3.7 + --aws-bucket ${AWS_BUCKET} --native-engine-so-key-prefix ${NATIVE_ENGINE_SO_KEY_PREFIX} + --pex-key ${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX} stage: Bootstrap Pants (Cron) - addons: apt: @@ -973,3 +973,4 @@ stages: name: Deploy Pants Pex - if: tag IS NOT present AND type NOT IN (pull_request, cron) name: Deploy Pants Pex Unstable + diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index 71b7c81fabc..80b50c0b158 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -445,7 +445,7 @@ def osx_shard( def _bootstrap_command(*, python_version: PythonVersion) -> str: return ( - "echo ${PATH} && command -v gcc && ./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " + "./build-support/bin/bootstrap_and_deploy_ci_pants_pex.py --python-version " f"{python_version.decimal} --aws-bucket ${{AWS_BUCKET}} --native-engine-so-key-prefix " "${NATIVE_ENGINE_SO_KEY_PREFIX} --pex-key " "${BOOTSTRAPPED_PEX_KEY_PREFIX}.${BOOTSTRAPPED_PEX_KEY_SUFFIX}" diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index b86bdad33a9..4a76ee9702f 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -183,9 +183,6 @@ def create_process( if env: hermetic_env.update(env) - print("HERE") - print(argv) - print(hermetic_env) return Process( argv=argv, input_digest=input_digest, From 2411f46536d259ae39dc84db1b24eb9d196e2d60 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 11:10:04 -0700 Subject: [PATCH 11/18] Remove unused logger --- src/python/pants/backend/python/rules/hermetic_pex.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 4a76ee9702f..2b2f735fd30 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -1,7 +1,6 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import logging import os from dataclasses import dataclass from typing import Any, Iterable, Mapping, Optional, Tuple @@ -19,8 +18,6 @@ from pants.util.ordered_set import OrderedSet from pants.util.strutil import create_path_env_var -logger = logging.getLogger(__name__) - @dataclass(frozen=True) class PexEnvironment(EngineAware): From c81afad2b426852b9a91c2efbc50faa2157dfb71 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 12:21:54 -0700 Subject: [PATCH 12/18] Try to fix Code 26 error seen with RBE --- src/python/pants/engine/process.py | 40 +++++++++++++----------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index ed155d2858a..f2bca78e2bc 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -331,31 +331,25 @@ def message(self) -> Optional[str]: @rule(desc="Find binary path") async def find_binary(request: BinaryPathRequest) -> BinaryPaths: # TODO(John Sirois): Replace this script with a statically linked native binary so we don't - # depend on either /usr/bin/env or bash being available on the Process host. + # depend on either /bin/bash being available on the Process host. + # TODO: Once we stop using Google RBE, add a shebang to the script `/usr/bin/env bash`, rather + # than running the binary `/bin/bash`. We hit a 26 "Text File Busy" error, which we suspect is + # from RBE not having fsynced the ./script.sh file. script_path = "./script.sh" + script_content = dedent( + """ + set -euo pipefail + + if command -v which > /dev/null; then + command which -a $1 + else + command -v $1 + fi + """ + ) script_digest = await Get( Digest, - CreateDigest( - [ - FileContent( - script_path, - content=dedent( - """ - #!/usr/bin/env bash - - set -euo pipefail - - if command -v which > /dev/null; then - command which -a $1 - else - command -v $1 - fi - """ - ).encode(), - is_executable=True, - ) - ] - ), + CreateDigest([FileContent(script_path, script_content.encode(), is_executable=True)]), ) paths = [] @@ -365,7 +359,7 @@ async def find_binary(request: BinaryPathRequest) -> BinaryPaths: Process( description=f"Searching for `{request.binary_name}` on PATH={search_path}", input_digest=script_digest, - argv=[script_path, request.binary_name], + argv=["/bin/bash", script_path, request.binary_name], env={"PATH": search_path}, ), ) From ade7efb7c310556c227da129d231a02d10a805d1 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 12:46:29 -0700 Subject: [PATCH 13/18] Debug if relative path fixes the File Not Found error --- src/python/pants/backend/python/rules/download_pex_bin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/python/rules/download_pex_bin.py b/src/python/pants/backend/python/rules/download_pex_bin.py index 77af873e2ee..9bbdd69441e 100644 --- a/src/python/pants/backend/python/rules/download_pex_bin.py +++ b/src/python/pants/backend/python/rules/download_pex_bin.py @@ -39,7 +39,7 @@ class DownloadedPexBin(HermeticPex): @property def executable(self) -> str: - return self.downloaded_tool.exe + return f"./{self.downloaded_tool.exe}" @property def digest(self) -> Digest: From 4ed9ea0362167897b7594290384e2b245612b31f Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 14:09:40 -0700 Subject: [PATCH 14/18] Fix fmt --- build-support/bin/generate_travis_yml.py | 2 +- src/python/pants/backend/python/rules/hermetic_pex.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build-support/bin/generate_travis_yml.py b/build-support/bin/generate_travis_yml.py index 80b50c0b158..47cfef95666 100644 --- a/build-support/bin/generate_travis_yml.py +++ b/build-support/bin/generate_travis_yml.py @@ -67,7 +67,7 @@ def all_entries(cls) -> List[Dict[str, str]]: GLOBAL_ENV_VARS = [ 'PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.toml"', - 'PANTS_DYNAMIC_UI=false', + "PANTS_DYNAMIC_UI=false", 'LC_ALL="en_US.UTF-8"', "AWS_BUCKET=ci-public.pantsbuild.org", # The ci-public.pantsbuild.org bucket has expiration policies set up for key prefixes diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 2b2f735fd30..bb42df73940 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -11,8 +11,8 @@ from pants.engine.fs import Digest from pants.engine.process import BinaryPathRequest, BinaryPaths, Process from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.python.python_setup import PythonSetup from pants.option.subsystem import Subsystem +from pants.python.python_setup import PythonSetup from pants.util.logging import LogLevel from pants.util.memo import memoized_property from pants.util.ordered_set import OrderedSet @@ -97,6 +97,7 @@ def iter_path_entries(): yield path_entry else: yield entry + return tuple(OrderedSet(iter_path_entries())) @property From e3d0332a757849038a1b2abe812bae048b74d46a Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 14:14:39 -0700 Subject: [PATCH 15/18] Fix test rule registration --- src/python/pants/engine/process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index f2bca78e2bc..938dbe9e32e 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -372,6 +372,7 @@ async def find_binary(request: BinaryPathRequest) -> BinaryPaths: def rules(): return [ *collect_rules(), + RootRule(BinaryPathRequest), RootRule(Process), RootRule(InteractiveRunner), RootRule(MultiPlatformProcess), From 4ac4fb79775b0b980c84546e50a933e550a3e2b7 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 14:25:58 -0700 Subject: [PATCH 16/18] Generalize finding the first path out of all matching paths This will be useful in other contexts too. --- src/python/pants/backend/python/rules/hermetic_pex.py | 5 ++--- src/python/pants/engine/process.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index bb42df73940..26257e0f7e0 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -127,9 +127,8 @@ async def find_pex_python( def first_python_binary() -> Optional[str]: for binary_paths in all_python_binary_paths: - first_path = next(iter(binary_paths.paths), None) - if first_path: - return first_path + if binary_paths.first_path: + return binary_paths.first_path return None return PexEnvironment( diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index 938dbe9e32e..e80ec8f3f4d 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -327,6 +327,11 @@ def message(self) -> Optional[str]: found_msg = f"{found_msg} and {pluralize(len(self.paths) - 1, 'other location')}" return found_msg + @property + def first_path(self) -> Optional[str]: + """Return the first path to the binary that was discovered, if any.""" + return next(iter(self.paths), None) + @rule(desc="Find binary path") async def find_binary(request: BinaryPathRequest) -> BinaryPaths: From 1643cb70a2f8648b5a6296ba184b54f65386cf75 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 15:26:35 -0700 Subject: [PATCH 17/18] Fix RBE not being able to discover Python interpreters during unit tests --- pants.remote.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pants.remote.toml b/pants.remote.toml index 18f1bef54a4..948fc5b5c3f 100644 --- a/pants.remote.toml +++ b/pants.remote.toml @@ -36,6 +36,9 @@ process_execution_speculation_delay = 15 # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote # platforms. This should be replaced by a proper mechanism to differentiate between the two. binary_search_path = [ + # These are the interpreter paths we set up on the remote container. We need to specify these + # because many of our tests for the Python backend need to discover Python interpreters. + "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin", # The remote container has binaries like `ld` and `gcc` in /usr/bin. "/usr/bin", # We include the host PATH so that speculation still works. From 80519557a81a3dea74a0084f3e905d1868e2c2e2 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 29 Jul 2020 21:26:10 -0700 Subject: [PATCH 18/18] Review feedback [ci skip-rust] [ci skip-build-wheels] --- pants.remote.toml | 2 +- .../backend/python/rules/hermetic_pex.py | 27 +++++++++---------- src/python/pants/engine/process.py | 5 ++-- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/pants.remote.toml b/pants.remote.toml index 948fc5b5c3f..0002b43e375 100644 --- a/pants.remote.toml +++ b/pants.remote.toml @@ -35,7 +35,7 @@ process_execution_speculation_delay = 15 [pex] # TODO(#7735): This config is not ideal, that we must specify the PATH for both local and remote # platforms. This should be replaced by a proper mechanism to differentiate between the two. -binary_search_path = [ +executable_search_paths = [ # These are the interpreter paths we set up on the remote container. We need to specify these # because many of our tests for the Python backend need to discover Python interpreters. "/pyenv-docker-build/versions/3.7.3/bin:/pyenv-docker-build/versions/3.6.8/bin:/pyenv-docker-build/versions/2.7.15/bin", diff --git a/src/python/pants/backend/python/rules/hermetic_pex.py b/src/python/pants/backend/python/rules/hermetic_pex.py index 26257e0f7e0..ec31bd79a29 100644 --- a/src/python/pants/backend/python/rules/hermetic_pex.py +++ b/src/python/pants/backend/python/rules/hermetic_pex.py @@ -38,12 +38,12 @@ def environment_dict(self) -> Mapping[str, str]: ) def level(self) -> Optional[LogLevel]: - return LogLevel.INFO if self.bootstrap_python else LogLevel.WARN + return LogLevel.DEBUG if self.bootstrap_python else LogLevel.WARN def message(self) -> Optional[str]: if not self.bootstrap_python: return ( - "No bootstrap python executable could be found. " + "No bootstrap Python executable could be found. " "Will attempt to run PEXes directly." ) return f"Selected {self.bootstrap_python} to bootstrap PEXes with." @@ -57,15 +57,18 @@ class PexRuntimeEnvironment(Subsystem): @classmethod def register_options(cls, register): super().register_options(register) + # TODO(#9760): We'll want to deprecate this in favor of a global option which allows for a + # per-process override. register( - "--binary-search-path", + "--executable-search-paths", advanced=True, type=list, default=[""], metavar="", help=( - "A list of paths to search for binaries needed by PEX subprocess. The special " - 'string "" will expand to the contents of the PATH env var.' + "The PATH value that will be used by the PEX subprocess and any subprocesses it " + 'spawns. The special string "" will expand to the contents of the PATH env ' + "var." ), ) register( @@ -75,22 +78,18 @@ def register_options(cls, register): default=["python", "python3", "python2"], metavar="", help=( - "The names of python binaries to search for to bootstrap PEX files with. Earlier " - "names in the list will be preferred over later names and the first matching " - "interpreter found will be used to execute the PEX bootstrap stage. If no matching " - "interpreters are found, PEXes will be executed directly relying on their embedded " - "shebang and the $PATH (see --binary-search-path) to locate a bootstrap " - "interpreter." + "The names of Python binaries to search for to bootstrap PEX files with. This does " + "not impact which Python interpreter is used to run your code, only what is used " + "to run the PEX tool. See the `interpreter_search_paths` option in " + "`[python-setup]` to influence where interpreters are searched for." ), ) @memoized_property def path(self) -> Tuple[str, ...]: def iter_path_entries(): - for entry in self.options.binary_search_path: + for entry in self.options.executable_search_paths: if entry == "": - # TODO(#9760): This is not very robust. We want to be able to read an env var - # safely via the engine. path = os.environ.get("PATH") if path: for path_entry in path.split(os.pathsep): diff --git a/src/python/pants/engine/process.py b/src/python/pants/engine/process.py index e80ec8f3f4d..2dbb5871403 100644 --- a/src/python/pants/engine/process.py +++ b/src/python/pants/engine/process.py @@ -337,9 +337,8 @@ def first_path(self) -> Optional[str]: async def find_binary(request: BinaryPathRequest) -> BinaryPaths: # TODO(John Sirois): Replace this script with a statically linked native binary so we don't # depend on either /bin/bash being available on the Process host. - # TODO: Once we stop using Google RBE, add a shebang to the script `/usr/bin/env bash`, rather - # than running the binary `/bin/bash`. We hit a 26 "Text File Busy" error, which we suspect is - # from RBE not having fsynced the ./script.sh file. + # TODO(#10507): Running the script directly from a shebang sometimes results in a "Text file + # busy" error. script_path = "./script.sh" script_content = dedent( """