From da76a138c9ebc91d7522a9692f1299febc6e46e9 Mon Sep 17 00:00:00 2001 From: Joshua Cannon Date: Wed, 29 Jun 2022 20:23:30 -0500 Subject: [PATCH] Add ability to `run` any `PythonSourceField` (#15849) Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> [ci skip-build-wheels] --- .../Python/python-goals/python-run-goal.md | 29 ++- src/python/pants/backend/python/goals/BUILD | 3 +- .../python/goals/package_pex_binary.py | 2 - .../pants/backend/python/goals/run_helper.py | 172 +++++++++++++++ .../backend/python/goals/run_pex_binary.py | 190 ++++------------- .../goals/run_pex_binary_integration_test.py | 141 +++++++------ .../backend/python/goals/run_python_source.py | 66 ++++++ .../run_python_source_integration_test.py | 198 ++++++++++++++++++ src/python/pants/backend/python/register.py | 2 + .../pants/backend/python/target_types.py | 61 ++++-- .../pants/engine/internals/options_parsing.py | 16 +- .../pants/engine/internals/specs_rules.py | 77 ++++++- src/python/pants/option/global_options.py | 73 +++++++ testprojects/src/python/print_env/BUILD | 2 +- .../pantsd/pantsd_integration_test.py | 4 +- 15 files changed, 793 insertions(+), 243 deletions(-) create mode 100644 src/python/pants/backend/python/goals/run_helper.py create mode 100644 src/python/pants/backend/python/goals/run_python_source.py create mode 100644 src/python/pants/backend/python/goals/run_python_source_integration_test.py diff --git a/docs/markdown/Python/python-goals/python-run-goal.md b/docs/markdown/Python/python-goals/python-run-goal.md index 4f107954b5f..bd989e2f988 100644 --- a/docs/markdown/Python/python-goals/python-run-goal.md +++ b/docs/markdown/Python/python-goals/python-run-goal.md @@ -6,15 +6,22 @@ hidden: false createdAt: "2020-03-16T16:19:56.403Z" updatedAt: "2022-01-29T16:45:29.511Z" --- -To run an executable/script, use `./pants run` on a [`pex_binary`](doc:reference-pex_binary) target. (See [package](doc:python-package-goal) for more on the `pex_binary` target.) +To run an executable/script, use `./pants run` on one of the following target types: + +* [`pex_binary`](doc:reference-pex_binary) +* [`python_source`](doc:reference-python_source) + +(See [package](doc:python-package-goal) for more on the `pex_binary` target.) ```bash +# A python_source target (usually referred to by the filename) $ ./pants run project/app.py ``` or ```bash +# A pex_binary target (must be referred to by target name) $ ./pants run project:app ``` @@ -36,6 +43,26 @@ The program will have access to the same environment used by the parent `./pants > > Run `./pants dependencies --transitive path/to/binary.py` to ensure that all the files you need are showing up, including for any [assets](doc:assets) you intend to use. +Execution Semantics +------------------- + +Running a `pex_binary` is equivalent to `package`-ing the target followed by executing the built PEX +from the repo root. + +Running a `python_source` with the `run_goal_use_sandbox` field set to `True` (the default) runs your +code in an ephemeral sandbox (temporary directory) with your firstparty code and and +Pants-generated files (such as a `relocated_files` or `archive`) copied inside. If you are using +generated files like this, you may need to set the `run_goal_use_sandbox` to `True` for file loading +to work properly. + +Running a `python_source` with the `run_goal_use_sandbox` field set to `False` is equivalent to +running the source directly (a la `python ...`) with the set of third-party dependencies exposed to +the interpreter. This is comparable to using a virtual environment or Poetry to run your script +(E.g. `venv/bin/python ...` or `poetry run python ...`). When scripts write in-repo files—such as +Django's `manage.py makemigrations` - it is often necessary to set `run_goal_use_sandbox` to `False` +so that the file is written into the expected location. + + Watching the filesystem ----------------------- diff --git a/src/python/pants/backend/python/goals/BUILD b/src/python/pants/backend/python/goals/BUILD index eda30b9fd4b..34b87d349ae 100644 --- a/src/python/pants/backend/python/goals/BUILD +++ b/src/python/pants/backend/python/goals/BUILD @@ -25,7 +25,8 @@ python_tests( "tags": ["platform_specific_behavior"], "timeout": 300, }, - "run_pex_binary_integration_test.py": {"timeout": 180}, + "run_pex_binary_integration_test.py": {"timeout": 400}, + "run_python_source_integration_test.py": {"timeout": 180}, "setup_py_integration_test.py": { "dependencies": ["testprojects/src/python:native_directory"], "tags": ["platform_specific_behavior"], diff --git a/src/python/pants/backend/python/goals/package_pex_binary.py b/src/python/pants/backend/python/goals/package_pex_binary.py index 2b69ed2d910..ec2770a9e55 100644 --- a/src/python/pants/backend/python/goals/package_pex_binary.py +++ b/src/python/pants/backend/python/goals/package_pex_binary.py @@ -25,7 +25,6 @@ PexStripEnvField, ResolvedPexEntryPoint, ResolvePexEntryPointRequest, - RunInSandboxField, ) from pants.backend.python.util_rules.pex import CompletePlatforms, Pex, PexPlatforms from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest @@ -71,7 +70,6 @@ class PexBinaryFieldSet(PackageFieldSet, RunFieldSet): execution_mode: PexExecutionModeField include_requirements: PexIncludeRequirementsField include_tools: PexIncludeToolsField - run_in_sandbox: RunInSandboxField @property def _execution_mode(self) -> PexExecutionMode: diff --git a/src/python/pants/backend/python/goals/run_helper.py b/src/python/pants/backend/python/goals/run_helper.py new file mode 100644 index 00000000000..4336aa1e9a8 --- /dev/null +++ b/src/python/pants/backend/python/goals/run_helper.py @@ -0,0 +1,172 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + +import os +from typing import Iterable, Optional + +from pants.backend.python.subsystems.debugpy import DebugPy +from pants.backend.python.target_types import ( + ConsoleScript, + PexEntryPointField, + ResolvedPexEntryPoint, + ResolvePexEntryPointRequest, +) +from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints +from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest +from pants.backend.python.util_rules.pex import Pex, PexRequest +from pants.backend.python.util_rules.pex_environment import PexEnvironment +from pants.backend.python.util_rules.pex_from_targets import ( + InterpreterConstraintsRequest, + PexFromTargetsRequest, +) +from pants.backend.python.util_rules.python_sources import ( + PythonSourceFiles, + PythonSourceFilesRequest, +) +from pants.core.goals.run import RunDebugAdapterRequest, RunRequest +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem +from pants.engine.addresses import Address +from pants.engine.fs import Digest, MergeDigests +from pants.engine.rules import Get, MultiGet, rule_helper +from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest + + +def _in_chroot(relpath: str) -> str: + return os.path.join("{chroot}", relpath) + + +@rule_helper +async def _create_python_source_run_request( + address: Address, + *, + entry_point_field: PexEntryPointField, + pex_env: PexEnvironment, + run_in_sandbox: bool, + console_script: Optional[ConsoleScript] = None, + additional_pex_args: Iterable[str] = (), +) -> RunRequest: + addresses = [address] + entry_point, transitive_targets = await MultiGet( + Get( + ResolvedPexEntryPoint, + ResolvePexEntryPointRequest(entry_point_field), + ), + Get(TransitiveTargets, TransitiveTargetsRequest(addresses)), + ) + + interpreter_constraints = await Get( + InterpreterConstraints, InterpreterConstraintsRequest(addresses) + ) + + pex_filename = ( + address.generated_name.replace(".", "_") if address.generated_name else address.target_name + ) + pex_get = Get( + Pex, + PexFromTargetsRequest( + addresses, + output_filename=f"{pex_filename}.pex", + internal_only=True, + include_source_files=False, + # `PEX_EXTRA_SYS_PATH` should contain this entry_point's module. + main=console_script or entry_point.val, + additional_args=( + *additional_pex_args, + # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH + # below, it's important for any app that re-executes itself that these environment + # variables are not stripped. + "--no-strip-pex-env", + ), + ), + ) + sources_get = Get( + PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True) + ) + pex, sources = await MultiGet(pex_get, sources_get) + + local_dists = await Get( + LocalDistsPex, + LocalDistsPexRequest( + addresses, + internal_only=True, + interpreter_constraints=interpreter_constraints, + sources=sources, + ), + ) + + input_digests = [ + pex.digest, + local_dists.pex.digest, + # Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied + # into the chroot when using inline sources. But they do, because some of them might be + # codegenned, and those won't exist in the inline source tree. Rather than incurring the + # complexity of figuring out here which sources were codegenned, we copy everything. + # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline + # sources will take precedence and their copies in the chroot will be ignored. + local_dists.remaining_sources.source_files.snapshot.digest, + ] + merged_digest = await Get(Digest, MergeDigests(input_digests)) + + complete_pex_env = pex_env.in_workspace() + args = complete_pex_env.create_argv(_in_chroot(pex.name), python=pex.python) + + chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots] + # The order here is important: we want the in-repo sources to take precedence over their + # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). + source_roots = [ + *([] if run_in_sandbox else sources.source_roots), + *chrooted_source_roots, + ] + extra_env = { + **pex_env.in_workspace().environment_dict(python_configured=pex.python is not None), + "PEX_PATH": _in_chroot(local_dists.pex.name), + "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), + } + + return RunRequest( + digest=merged_digest, + args=args, + extra_env=extra_env, + ) + + +@rule_helper +async def _create_python_source_run_dap_request( + regular_run_request: RunRequest, + *, + entry_point_field: PexEntryPointField, + debugpy: DebugPy, + debug_adapter: DebugAdapterSubsystem, + console_script: Optional[ConsoleScript] = None, +) -> RunDebugAdapterRequest: + entry_point, debugpy_pex = await MultiGet( + Get( + ResolvedPexEntryPoint, + ResolvePexEntryPointRequest(entry_point_field), + ), + Get(Pex, PexRequest, debugpy.to_pex_request()), + ) + + merged_digest = await Get( + Digest, MergeDigests([regular_run_request.digest, debugpy_pex.digest]) + ) + extra_env = dict(regular_run_request.extra_env) + extra_env["PEX_PATH"] = os.pathsep.join( + [ + extra_env["PEX_PATH"], + # For debugpy to work properly, we need to have just one "environment" for our + # command to run in. Therefore, we cobble one together by exeucting debugpy's PEX, and + # shoehorning in the original PEX through PEX_PATH. + _in_chroot(os.path.basename(regular_run_request.args[1])), + ] + ) + main = console_script or entry_point.val + assert main is not None + args = [ + regular_run_request.args[0], # python executable + _in_chroot(debugpy_pex.name), + *debugpy.get_args(debug_adapter, main), + ] + + return RunDebugAdapterRequest(digest=merged_digest, args=args, extra_env=extra_env) diff --git a/src/python/pants/backend/python/goals/run_pex_binary.py b/src/python/pants/backend/python/goals/run_pex_binary.py index 454c72edc4d..c1c0a866fd9 100644 --- a/src/python/pants/backend/python/goals/run_pex_binary.py +++ b/src/python/pants/backend/python/goals/run_pex_binary.py @@ -1,183 +1,73 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -import logging import os from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet -from pants.backend.python.subsystems.debugpy import DebugPy -from pants.backend.python.target_types import ( - PexBinaryDefaults, - ResolvedPexEntryPoint, - ResolvePexEntryPointRequest, +from pants.backend.python.goals.run_helper import ( + _create_python_source_run_dap_request, + _create_python_source_run_request, ) -from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints -from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest -from pants.backend.python.util_rules.pex import Pex, PexRequest +from pants.backend.python.subsystems.debugpy import DebugPy +from pants.backend.python.target_types import PexBinaryDefaults from pants.backend.python.util_rules.pex_environment import PexEnvironment -from pants.backend.python.util_rules.pex_from_targets import ( - InterpreterConstraintsRequest, - PexFromTargetsRequest, -) -from pants.backend.python.util_rules.python_sources import ( - PythonSourceFiles, - PythonSourceFilesRequest, -) +from pants.core.goals.package import BuiltPackage from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem -from pants.engine.fs import Digest, MergeDigests -from pants.engine.rules import Get, MultiGet, collect_rules, rule -from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest +from pants.engine.rules import Get, collect_rules, rule from pants.engine.unions import UnionRule +from pants.option.global_options import UseDeprecatedPexBinaryRunSemanticsOption from pants.util.logging import LogLevel -from pants.util.strutil import softwrap - -logger = logging.getLogger(__name__) - - -def _in_chroot(relpath: str) -> str: - return os.path.join("{chroot}", relpath) @rule(level=LogLevel.DEBUG) async def create_pex_binary_run_request( - field_set: PexBinaryFieldSet, pex_binary_defaults: PexBinaryDefaults, pex_env: PexEnvironment + field_set: PexBinaryFieldSet, + use_deprecated_pex_binary_run_semantics: UseDeprecatedPexBinaryRunSemanticsOption, + pex_binary_defaults: PexBinaryDefaults, + pex_env: PexEnvironment, ) -> RunRequest: - run_in_sandbox = field_set.run_in_sandbox.value - entry_point, transitive_targets = await MultiGet( - Get( - ResolvedPexEntryPoint, - ResolvePexEntryPointRequest(field_set.entry_point), - ), - Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address])), - ) - - addresses = [field_set.address] - interpreter_constraints = await Get( - InterpreterConstraints, InterpreterConstraintsRequest(addresses) - ) - - pex_filename = ( - field_set.address.generated_name.replace(".", "_") - if field_set.address.generated_name - else field_set.address.target_name - ) - pex_get = Get( - Pex, - PexFromTargetsRequest( - [field_set.address], - output_filename=f"{pex_filename}.pex", - internal_only=True, - include_source_files=False, - # Note that the file for first-party entry points is not in the PEX itself. In that - # case, it's loaded by setting `PEX_EXTRA_SYS_PATH`. - main=entry_point.val or field_set.script.value, - additional_args=( - *field_set.generate_additional_args(pex_binary_defaults), - # N.B.: Since we cobble together the runtime environment via PEX_EXTRA_SYS_PATH - # below, it's important for any app that re-executes itself that these environment - # variables are not stripped. - "--no-strip-pex-env", - ), - ), - ) - sources_get = Get( - PythonSourceFiles, PythonSourceFilesRequest(transitive_targets.closure, include_files=True) - ) - pex, sources = await MultiGet(pex_get, sources_get) + if not use_deprecated_pex_binary_run_semantics.val: + built_pex = await Get(BuiltPackage, PexBinaryFieldSet, field_set) + relpath = built_pex.artifacts[0].relpath + assert relpath is not None + return RunRequest( + digest=built_pex.digest, + args=[os.path.join("{chroot}", relpath)], + ) - local_dists = await Get( - LocalDistsPex, - LocalDistsPexRequest( - [field_set.address], - internal_only=True, - interpreter_constraints=interpreter_constraints, - sources=sources, - ), + return await _create_python_source_run_request( + field_set.address, + entry_point_field=field_set.entry_point, + pex_env=pex_env, + run_in_sandbox=True, + console_script=field_set.script.value, + additional_pex_args=field_set.generate_additional_args(pex_binary_defaults), ) - input_digests = [ - pex.digest, - local_dists.pex.digest, - # Note regarding inline mode: You might think that the sources don't need to be copied - # into the chroot when using inline sources. But they do, because some of them might be - # codegenned, and those won't exist in the inline source tree. Rather than incurring the - # complexity of figuring out here which sources were codegenned, we copy everything. - # The inline source roots precede the chrooted ones in PEX_EXTRA_SYS_PATH, so the inline - # sources will take precedence and their copies in the chroot will be ignored. - local_dists.remaining_sources.source_files.snapshot.digest, - ] - merged_digest = await Get(Digest, MergeDigests(input_digests)) - - complete_pex_env = pex_env.in_workspace() - # NB. If this changes, please consider how it affects the `DebugRequest` below - # (which is not easy to write automated tests for) - args = complete_pex_env.create_argv(_in_chroot(pex.name), python=pex.python) - - chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots] - # The order here is important: we want the in-repo sources to take precedence over their - # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). - source_roots = [ - *([] if run_in_sandbox else sources.source_roots), - *chrooted_source_roots, - ] - extra_env = { - **complete_pex_env.environment_dict(python_configured=pex.python is not None), - "PEX_PATH": _in_chroot(local_dists.pex.name), - "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), - } - - return RunRequest(digest=merged_digest, args=args, extra_env=extra_env) - @rule async def run_pex_debug_adapter_binary( field_set: PexBinaryFieldSet, + use_deprecated_pex_binary_run_semantics: UseDeprecatedPexBinaryRunSemanticsOption, debugpy: DebugPy, debug_adapter: DebugAdapterSubsystem, ) -> RunDebugAdapterRequest: - if field_set.run_in_sandbox: - logger.warning( - softwrap( - f""" - Using `run --debug-adapter` on the target {field_set.address}, which sets the field - `run_in_sandbox` to `True`. This will likely cause your breakpoints to not be hit, - as your code will be run under the sandbox's path. - """ - ) + if not use_deprecated_pex_binary_run_semantics.val: + # NB: Technically we could run this using `debugpy`, however it is unclear how the user + # would be able to debug the code, as the client and server will disagree on the code's path. + raise NotImplementedError( + "Debugging a `pex_binary` using a debug adapter has not yet been implemented." ) - entry_point, regular_run_request, debugpy_pex = await MultiGet( - Get( - ResolvedPexEntryPoint, - ResolvePexEntryPointRequest(field_set.entry_point), - ), - Get(RunRequest, PexBinaryFieldSet, field_set), - Get(Pex, PexRequest, debugpy.to_pex_request()), - ) - - entry_point_or_script = entry_point.val or field_set.script.value - assert entry_point_or_script is not None - merged_digest = await Get( - Digest, MergeDigests([regular_run_request.digest, debugpy_pex.digest]) - ) - extra_env = dict(regular_run_request.extra_env) - extra_env["PEX_PATH"] = os.pathsep.join( - [ - extra_env["PEX_PATH"], - # For debugpy to work properly, we need to have just one "environment" for our - # command to run in. Therefore, we cobble one together by exeucting debugpy's PEX, and - # shoehorning in the original PEX through PEX_PATH. - _in_chroot(os.path.basename(regular_run_request.args[1])), - ] + run_request = await Get(RunRequest, PexBinaryFieldSet, field_set) + return await _create_python_source_run_dap_request( + run_request, + entry_point_field=field_set.entry_point, + debugpy=debugpy, + debug_adapter=debug_adapter, + console_script=field_set.script.value, ) - args = [ - regular_run_request.args[0], # python executable - _in_chroot(debugpy_pex.name), - *debugpy.get_args(debug_adapter, entry_point_or_script), - ] - - return RunDebugAdapterRequest(digest=merged_digest, args=args, extra_env=extra_env) def rules(): diff --git a/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py b/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py index b7b9486d924..08ed2b93992 100644 --- a/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py +++ b/src/python/pants/backend/python/goals/run_pex_binary_integration_test.py @@ -1,39 +1,47 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations import json -import os from textwrap import dedent -from typing import Optional, Tuple +from typing import Optional import pytest from pants.backend.python.target_types import PexExecutionMode from pants.testutil.pants_integration_test import PantsResult, run_pants, setup_tmpdir +use_new_semantics_args = pytest.mark.parametrize( + "use_new_semantics_args", + [ + (), + ("--no-use-deprecated-pex-binary-run-semantics",), + ], +) + @pytest.mark.parametrize( - ("entry_point", "execution_mode", "include_tools", "run_in_sandbox"), + ("entry_point", "execution_mode", "include_tools"), [ - ("app.py", None, True, True), - ("app.py", None, True, False), - ("app.py", PexExecutionMode.VENV, False, True), - ("app.py:main", PexExecutionMode.ZIPAPP, True, True), - ("app.py:main", None, False, True), + ("app.py", None, True), + ("app.py", None, True), + ("app.py", PexExecutionMode.VENV, False), + ("app.py:main", PexExecutionMode.ZIPAPP, True), + ("app.py:main", None, False), ], ) +@use_new_semantics_args def test_run_sample_script( entry_point: str, execution_mode: Optional[PexExecutionMode], include_tools: bool, - run_in_sandbox: bool, + use_new_semantics_args: tuple[str, ...], ) -> None: """Test that we properly run a `pex_binary` target. This checks a few things: - We can handle source roots. - We properly load third party requirements. - - We run in-repo when requested, and handle codegen correctly. - We propagate the error code. """ sources = { @@ -56,10 +64,10 @@ def main(): f"""\ python_sources(name='lib') pex_binary( + name="binary", entry_point={entry_point!r}, execution_mode={execution_mode.value if execution_mode is not None else None!r}, include_tools={include_tools!r}, - run_in_sandbox={run_in_sandbox!r}, ) """ ), @@ -79,42 +87,48 @@ def my_file(): ), } - def run(*extra_args: str, **extra_env: str) -> Tuple[PantsResult, str]: + def run(*extra_args: str, **extra_env: str) -> PantsResult: with setup_tmpdir(sources) as tmpdir: args = [ "--backend-packages=pants.backend.python", "--backend-packages=pants.backend.codegen.protobuf.python", + *use_new_semantics_args, f"--source-root-patterns=['/{tmpdir}/src_root1', '/{tmpdir}/src_root2']", "--pants-ignore=__pycache__", "--pants-ignore=/src/python", "run", - f"{tmpdir}/src_root1/project/app.py", + f"{tmpdir}/src_root1/project:binary", *extra_args, ] - return run_pants(args, extra_env=extra_env), tmpdir + return run_pants(args, extra_env=extra_env) - result, test_repo_root = run() + result = run() assert "Hola, mundo.\n" in result.stderr file = result.stdout.strip() - if run_in_sandbox: + if use_new_semantics_args: + assert file.endswith("utils/strutil.py") + assert ".pants.d/tmp" not in file + else: assert file.endswith("src_root2/utils/strutil.py") assert ".pants.d/tmp" in file - else: - assert file == os.path.join(test_repo_root, "src_root2/utils/strutil.py") assert result.exit_code == 23 if include_tools: - result, _ = run("--", "info", PEX_TOOLS="1") + result = run("--", "info", PEX_TOOLS="1") assert result.exit_code == 0 pex_info = json.loads(result.stdout) assert (execution_mode is PexExecutionMode.VENV) == pex_info["venv"] assert ("prepend" if execution_mode is PexExecutionMode.VENV else "false") == pex_info[ "venv_bin_path" ] - assert pex_info["strip_pex_env"] is False + if use_new_semantics_args: + assert pex_info["strip_pex_env"] + else: + assert not pex_info["strip_pex_env"] -def test_no_strip_pex_env_issues_12057() -> None: +@use_new_semantics_args +def test_no_strip_pex_env_issues_12057(use_new_semantics_args: tuple[str, ...]) -> None: sources = { "src/app.py": dedent( """\ @@ -133,55 +147,27 @@ def test_no_strip_pex_env_issues_12057() -> None: "src/BUILD": dedent( """\ python_sources(name="lib") - pex_binary(entry_point="app.py") + pex_binary( + name="binary", + entry_point="app.py" + ) """ ), } with setup_tmpdir(sources) as tmpdir: args = [ "--backend-packages=pants.backend.python", + *use_new_semantics_args, f"--source-root-patterns=['/{tmpdir}/src']", "run", - f"{tmpdir}/src/app.py", + f"{tmpdir}/src:binary", ] result = run_pants(args) assert result.exit_code == 42, result.stderr -def test_no_leak_pex_root_issues_12055() -> None: - read_config_result = run_pants(["help-all"]) - read_config_result.assert_success() - config_data = json.loads(read_config_result.stdout) - global_advanced_options = { - option["config_key"]: [ - ranked_value["value"] for ranked_value in option["value_history"]["ranked_values"] - ][-1] - for option in config_data["scope_to_help_info"][""]["advanced"] - } - named_caches_dir = global_advanced_options["named_caches_dir"] - - sources = { - "src/app.py": "import os; print(os.environ['PEX_ROOT'])", - "src/BUILD": dedent( - """\ - python_sources(name="lib") - pex_binary(entry_point="app.py") - """ - ), - } - with setup_tmpdir(sources) as tmpdir: - args = [ - "--backend-packages=pants.backend.python", - f"--source-root-patterns=['/{tmpdir}/src']", - "run", - f"{tmpdir}/src/app.py", - ] - result = run_pants(args) - result.assert_success() - assert os.path.join(named_caches_dir, "pex_root") == result.stdout.strip() - - -def test_local_dist() -> None: +@use_new_semantics_args +def test_local_dist(use_new_semantics_args: tuple[str, ...]) -> None: sources = { "foo/bar.py": "BAR = 'LOCAL DIST'", "foo/setup.py": dedent( @@ -218,15 +204,17 @@ def test_local_dist() -> None: with setup_tmpdir(sources) as tmpdir: args = [ "--backend-packages=pants.backend.python", + *use_new_semantics_args, f"--source-root-patterns=['/{tmpdir}']", "run", - f"{tmpdir}/foo/main.py", + f"{tmpdir}/foo:bin", ] result = run_pants(args) assert result.stdout == "LOCAL DIST\n" -def test_run_script_from_3rdparty_dist_issue_13747() -> None: +@use_new_semantics_args +def test_run_script_from_3rdparty_dist_issue_13747(use_new_semantics_args) -> None: sources = { "src/BUILD": dedent( """\ @@ -239,6 +227,7 @@ def test_run_script_from_3rdparty_dist_issue_13747() -> None: SAY = "moooo" args = [ "--backend-packages=pants.backend.python", + *use_new_semantics_args, f"--source-root-patterns=['/{tmpdir}/src']", "run", f"{tmpdir}/src:test", @@ -248,3 +237,37 @@ def test_run_script_from_3rdparty_dist_issue_13747() -> None: result = run_pants(args) result.assert_success() assert SAY in result.stdout.strip() + + +# NB: Can be removed in 2.15 +@use_new_semantics_args +def test_filename_spec_ambiutity(use_new_semantics_args) -> None: + sources = { + "src/app.py": dedent( + """\ + if __name__ == "__main__": + print(__file__) + """ + ), + "src/BUILD": dedent( + """\ + python_sources(name="lib") + pex_binary( + name="binary", + entry_point="app.py" + ) + """ + ), + } + with setup_tmpdir(sources) as tmpdir: + args = [ + "--backend-packages=pants.backend.python", + *use_new_semantics_args, + f"--source-root-patterns=['/{tmpdir}/src']", + "run", + f"{tmpdir}/src/app.py", + ] + result = run_pants(args) + file = result.stdout.strip() + assert file.endswith("src/app.py") + assert ".pants.d/tmp" in file diff --git a/src/python/pants/backend/python/goals/run_python_source.py b/src/python/pants/backend/python/goals/run_python_source.py new file mode 100644 index 00000000000..324702074de --- /dev/null +++ b/src/python/pants/backend/python/goals/run_python_source.py @@ -0,0 +1,66 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass + +from pants.backend.python.goals.run_helper import ( + _create_python_source_run_dap_request, + _create_python_source_run_request, +) +from pants.backend.python.subsystems.debugpy import DebugPy +from pants.backend.python.target_types import ( + PexEntryPointField, + PythonRunGoalUseSandboxField, + PythonSourceField, +) +from pants.backend.python.util_rules.pex_environment import PexEnvironment +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem +from pants.engine.internals.selectors import Get +from pants.engine.rules import collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + + +@dataclass(frozen=True) +class PythonSourceFieldSet(RunFieldSet): + required_fields = (PythonSourceField, PythonRunGoalUseSandboxField) + + source: PythonSourceField + run_goal_use_sandbox: PythonRunGoalUseSandboxField + + +@rule(level=LogLevel.DEBUG) +async def create_python_source_run_request( + field_set: PythonSourceFieldSet, pex_env: PexEnvironment +) -> RunRequest: + return await _create_python_source_run_request( + field_set.address, + entry_point_field=PexEntryPointField(field_set.source.value, field_set.address), + pex_env=pex_env, + run_in_sandbox=field_set.run_goal_use_sandbox.value, + # Setting --venv is kosher because the PEX we create is just for the thirdparty deps. + additional_pex_args=["--venv"], + ) + + +@rule +async def create_python_source_debug_adapter_request( + field_set: PythonSourceFieldSet, + debugpy: DebugPy, + debug_adapter: DebugAdapterSubsystem, +) -> RunDebugAdapterRequest: + run_request = await Get(RunRequest, PythonSourceFieldSet, field_set) + return await _create_python_source_run_dap_request( + run_request, + entry_point_field=PexEntryPointField(field_set.source.value, field_set.address), + debugpy=debugpy, + debug_adapter=debug_adapter, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(RunFieldSet, PythonSourceFieldSet), + ] diff --git a/src/python/pants/backend/python/goals/run_python_source_integration_test.py b/src/python/pants/backend/python/goals/run_python_source_integration_test.py new file mode 100644 index 00000000000..2555f8558d9 --- /dev/null +++ b/src/python/pants/backend/python/goals/run_python_source_integration_test.py @@ -0,0 +1,198 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import json +import os +from textwrap import dedent +from typing import Tuple + +import pytest + +from pants.testutil.pants_integration_test import PantsResult, run_pants, setup_tmpdir + + +@pytest.mark.parametrize( + "run_in_sandbox", + [True, False], +) +def test_run_sample_script( + run_in_sandbox: bool, +) -> None: + """Test that we properly run a `python_source` target. + + This checks a few things: + - We can handle source roots. + - We run in-repo when requested, and handle codegen correctly. + - We propagate the error code. + """ + sources = { + "src_root1/project/app.py": dedent( + """\ + import sys + from utils.strutil import my_file + from codegen.hello_pb2 import Hi + + def main(): + print("Hola, mundo.", file=sys.stderr) + print(my_file()) + sys.exit(23) + + if __name__ == "__main__": + main() + """ + ), + "src_root1/project/BUILD": dedent( + f"""\ + python_sources( + name='lib', + run_goal_use_sandbox={run_in_sandbox}, + ) + """ + ), + "src_root2/utils/strutil.py": dedent( + """\ + def my_file(): + return __file__ + """ + ), + "src_root2/utils/BUILD": "python_sources()", + "src_root2/codegen/hello.proto": 'syntax = "proto3";\nmessage Hi {{}}', + "src_root2/codegen/BUILD": dedent( + """\ + protobuf_sources() + python_requirement(name='protobuf', requirements=['protobuf']) + """ + ), + } + + def run(*extra_args: str, **extra_env: str) -> Tuple[PantsResult, str]: + with setup_tmpdir(sources) as tmpdir: + args = [ + "--backend-packages=pants.backend.python", + "--backend-packages=pants.backend.codegen.protobuf.python", + f"--source-root-patterns=['/{tmpdir}/src_root1', '/{tmpdir}/src_root2']", + "--pants-ignore=__pycache__", + "--pants-ignore=/src/python", + "run", + f"{tmpdir}/src_root1/project/app.py", + *extra_args, + ] + return run_pants(args, extra_env=extra_env), tmpdir + + result, test_repo_root = run() + assert "Hola, mundo.\n" in result.stderr + file = result.stdout.strip() + if run_in_sandbox: + assert file.endswith("src_root2/utils/strutil.py") + assert ".pants.d/tmp" in file + else: + assert file.endswith(os.path.join(test_repo_root, "src_root2/utils/strutil.py")) + assert result.exit_code == 23 + + +def test_no_strip_pex_env_issues_12057() -> None: + sources = { + "src/app.py": dedent( + """\ + import os + import sys + + + if __name__ == "__main__": + exit_code = os.environ.get("PANTS_ISSUES_12057") + if exit_code is None: + os.environ["PANTS_ISSUES_12057"] = "42" + os.execv(sys.executable, [sys.executable, *sys.argv]) + sys.exit(int(exit_code)) + """ + ), + "src/BUILD": dedent( + """\ + python_sources(name="lib") + """ + ), + } + with setup_tmpdir(sources) as tmpdir: + args = [ + "--backend-packages=pants.backend.python", + f"--source-root-patterns=['/{tmpdir}/src']", + "run", + f"{tmpdir}/src/app.py", + ] + result = run_pants(args) + assert result.exit_code == 42, result.stderr + + +def test_no_leak_pex_root_issues_12055() -> None: + read_config_result = run_pants(["help-all"]) + read_config_result.assert_success() + config_data = json.loads(read_config_result.stdout) + global_advanced_options = { + option["config_key"]: [ + ranked_value["value"] for ranked_value in option["value_history"]["ranked_values"] + ][-1] + for option in config_data["scope_to_help_info"][""]["advanced"] + } + named_caches_dir = global_advanced_options["named_caches_dir"] + + sources = { + "src/app.py": "import os; print(os.environ['PEX_ROOT'])", + "src/BUILD": dedent( + """\ + python_sources(name="lib") + """ + ), + } + with setup_tmpdir(sources) as tmpdir: + args = [ + "--backend-packages=pants.backend.python", + f"--source-root-patterns=['/{tmpdir}/src']", + "run", + f"{tmpdir}/src/app.py", + ] + result = run_pants(args) + result.assert_success() + assert os.path.join(named_caches_dir, "pex_root") == result.stdout.strip() + + +def test_local_dist() -> None: + sources = { + "foo/bar.py": "BAR = 'LOCAL DIST'", + "foo/setup.py": dedent( + """\ + from setuptools import setup + + # Double-brace the package_dir to avoid setup_tmpdir treating it as a format. + setup(name="foo", version="9.8.7", packages=["foo"], package_dir={{"foo": "."}},) + """ + ), + "foo/main.py": "from foo.bar import BAR; print(BAR)", + "foo/BUILD": dedent( + """\ + python_sources(name="lib", sources=["bar.py", "setup.py"]) + + python_distribution( + name="dist", + dependencies=[":lib"], + provides=python_artifact(name="foo", version="9.8.7"), + sdist=False, + generate_setup=False, + ) + + python_sources(name="main_lib", + sources=["main.py"], + # Force-exclude any dep on bar.py, so the only way to consume it is via the dist. + dependencies=[":dist", "!:lib"], + ) + """ + ), + } + with setup_tmpdir(sources) as tmpdir: + args = [ + "--backend-packages=pants.backend.python", + f"--source-root-patterns=['/{tmpdir}']", + "run", + f"{tmpdir}/foo/main.py", + ] + result = run_pants(args) + assert result.stdout == "LOCAL DIST\n" diff --git a/src/python/pants/backend/python/register.py b/src/python/pants/backend/python/register.py index 813d88bc5de..9e11e0ccd7e 100644 --- a/src/python/pants/backend/python/register.py +++ b/src/python/pants/backend/python/register.py @@ -16,6 +16,7 @@ pytest_runner, repl, run_pex_binary, + run_python_source, setup_py, tailor, ) @@ -78,6 +79,7 @@ def rules(): *python_sources.rules(), *repl.rules(), *run_pex_binary.rules(), + *run_python_source.rules(), *setup_py.rules(), *setuptools.rules(), *tailor.rules(), diff --git a/src/python/pants/backend/python/target_types.py b/src/python/pants/backend/python/target_types.py index 9cfbc36d9b9..9574ac3e599 100644 --- a/src/python/pants/backend/python/target_types.py +++ b/src/python/pants/backend/python/target_types.py @@ -144,6 +144,30 @@ def normalized_value(self, python_setup: PythonSetup) -> str: return resolve +class PythonRunGoalUseSandboxField(BoolField): + alias = "run_goal_use_sandbox" + default = True + help = softwrap( + """ + If true, runs of this target with the `run` goal will copy the needed first-party sources + into a temporary sandbox and run from there. + + If false, runs of this target with the `run` goal will use the in-repo sources + directly. + + The former mode is more hermetic, and is closer to building and running the source as it + were packaged in a `pex_binary`. Additionally, it may be necessary if your sources depend + transitively on "generated" files which will be materialized in the sandbox in a source + root, but are not in-repo. + + The latter mode is similar to creating, activating, and using a virtual environment when + running your files. It may also be necessary if the source being run writes files into the + repo and computes their location relative to the executed files. Django's makemigrations + command is an example of such a process. + """ + ) + + # ----------------------------------------------------------------------------------------------- # Target generation support # ----------------------------------------------------------------------------------------------- @@ -258,6 +282,7 @@ def spec(self) -> str: return self.name +# @TODO: Remove the SecondaryOwnerMixin in Pants 2.15.0 class PexEntryPointField(AsyncFieldMixin, SecondaryOwnerMixin, Field): alias = "entry_point" default = None @@ -568,25 +593,6 @@ class PexIncludeToolsField(BoolField): ) -class RunInSandboxField(BoolField): - alias = "run_in_sandbox" - default = True - help = softwrap( - """ - If true, runs of this target with the `run` goal will copy the needed first-party sources - into a temporary chroot and run from there. - If false, runs of this target with the `run` goal will use the in-repo sources directly. - - The former mode is more hermetic, and is closer to building and running a standalone binary. - - The latter mode may be necessary if the binary being run writes files into the repo and - computes their location relative to the executed files. Django's makemigrations command - is an example of such a process. It may also have lower latency, since no files need - to be copied into a chroot. - """ - ) - - _PEX_BINARY_COMMON_FIELDS = ( InterpreterConstraintsField, PythonResolveField, @@ -603,7 +609,6 @@ class RunInSandboxField(BoolField): PexExecutionModeField, PexIncludeRequirementsField, PexIncludeToolsField, - RunInSandboxField, RestartableField, ) @@ -829,6 +834,7 @@ class SkipPythonTestsField(BoolField): _PYTHON_TEST_MOVED_FIELDS = ( PythonTestsDependenciesField, PythonResolveField, + PythonRunGoalUseSandboxField, PythonTestsTimeoutField, RuntimePackageDependenciesField, PythonTestsExtraEnvVarsField, @@ -926,6 +932,7 @@ class PythonSourceTarget(Target): InterpreterConstraintsField, Dependencies, PythonResolveField, + PythonRunGoalUseSandboxField, PythonSourceField, ) help = "A single Python source file." @@ -972,7 +979,12 @@ class PythonTestUtilsGeneratorTarget(TargetFilesGenerator): ) generated_target_cls = PythonSourceTarget copied_fields = COMMON_TARGET_FIELDS - moved_fields = (PythonResolveField, Dependencies, InterpreterConstraintsField) + moved_fields = ( + PythonResolveField, + PythonRunGoalUseSandboxField, + Dependencies, + InterpreterConstraintsField, + ) settings_request_cls = PythonFilesGeneratorSettingsRequest help = softwrap( """ @@ -998,7 +1010,12 @@ class PythonSourcesGeneratorTarget(TargetFilesGenerator): ) generated_target_cls = PythonSourceTarget copied_fields = COMMON_TARGET_FIELDS - moved_fields = (PythonResolveField, Dependencies, InterpreterConstraintsField) + moved_fields = ( + PythonResolveField, + PythonRunGoalUseSandboxField, + Dependencies, + InterpreterConstraintsField, + ) settings_request_cls = PythonFilesGeneratorSettingsRequest help = softwrap( """ diff --git a/src/python/pants/engine/internals/options_parsing.py b/src/python/pants/engine/internals/options_parsing.py index 40c9f2343e9..b580cc7b23c 100644 --- a/src/python/pants/engine/internals/options_parsing.py +++ b/src/python/pants/engine/internals/options_parsing.py @@ -6,7 +6,12 @@ from pants.build_graph.build_configuration import BuildConfiguration from pants.engine.internals.session import SessionValues from pants.engine.rules import collect_rules, rule -from pants.option.global_options import GlobalOptions, NamedCachesDirOption, ProcessCleanupOption +from pants.option.global_options import ( + GlobalOptions, + NamedCachesDirOption, + ProcessCleanupOption, + UseDeprecatedPexBinaryRunSemanticsOption, +) from pants.option.options import Options from pants.option.options_bootstrapper import OptionsBootstrapper from pants.option.scope import Scope, ScopedOptions @@ -63,5 +68,14 @@ def extract_named_caches_dir_option(global_options: GlobalOptions) -> NamedCache return NamedCachesDirOption(global_options.named_caches_dir) +@rule +def extract_use_deprecated_pex_binary_run_semantics( + global_options: GlobalOptions, +) -> UseDeprecatedPexBinaryRunSemanticsOption: + return UseDeprecatedPexBinaryRunSemanticsOption( + global_options.use_deprecated_pex_binary_run_semantics + ) + + def rules(): return collect_rules() diff --git a/src/python/pants/engine/internals/specs_rules.py b/src/python/pants/engine/internals/specs_rules.py index c6c3f4bb2e2..fcd9d97aa25 100644 --- a/src/python/pants/engine/internals/specs_rules.py +++ b/src/python/pants/engine/internals/specs_rules.py @@ -9,9 +9,12 @@ import os from collections import defaultdict from pathlib import PurePath -from typing import Iterable +from typing import Iterable, cast from pants.backend.project_info.filter_targets import FilterSubsystem +from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet +from pants.backend.python.goals.run_python_source import PythonSourceFieldSet +from pants.base.deprecated import warn_or_error from pants.base.specs import ( AddressLiteralSpec, AncestorGlobSpec, @@ -54,12 +57,16 @@ WrappedTargetRequest, ) from pants.engine.unions import UnionMembership -from pants.option.global_options import GlobalOptions, OwnersNotFoundBehavior +from pants.option.global_options import ( + GlobalOptions, + OwnersNotFoundBehavior, + UseDeprecatedPexBinaryRunSemanticsOption, +) from pants.util.dirutil import recursive_dirname from pants.util.docutil import bin_name from pants.util.logging import LogLevel from pants.util.ordered_set import FrozenOrderedSet, OrderedSet -from pants.util.strutil import bullet_list +from pants.util.strutil import bullet_list, softwrap logger = logging.getLogger(__name__) @@ -480,9 +487,62 @@ def __init__( ) +def _handle_ambiguous_result( + request: TargetRootsToFieldSetsRequest, + result: TargetRootsToFieldSets, + field_set_to_default_to: type[FieldSet], +) -> TargetRootsToFieldSets: + assert len(result.targets) > 1 + field_set_types = [type(field_set) for field_set in result.field_sets] + + # NB: See https://github.com/pantsbuild/pants/pull/15849. We don't want to break clients as + # we shift behavior, so we add this temporary hackery here. + if ( + # (We check for 2 targets because 3 would've been ambiguous pre-our-change) + len(result.targets) == 2 + and PexBinaryFieldSet in field_set_types + and PythonSourceFieldSet in field_set_types + ): + if field_set_to_default_to is PexBinaryFieldSet: + warn_or_error( + "2.14.0dev1", + "referring to a `pex_binary` by using the filename specified in `entry_point`", + softwrap( + """ + In Pants 2.14, a `pex_binary` can no longer be referred to by the filename that + the `entry_point` field uses. + + This is due to a change in Pants 2.13, which allows you to use the `run` goal + directly on a `python_source` target without requiring a `pex_binary`. As a + consequence the ability to refer to the `pex_binary` via its `entry_point` is + being removed, as otherwise it would be ambiguous which target to use. + + Note that because of this change you are able to remove any `pex_binary` targets + you have declared just to support the `run` goal + (usually these are developer scripts), as using `run` on the `python_source` will + have the equivalent behavior. + + To fix this deprecation, you can use the `pex_binary`'s address to refer to + the `pex_binary`, or set the `[GLOBAL].use_deprecated_pex_binary_run_semantics` + option to `false` (which, among other things, will have `run` on a Python + filename run the `python_source`). + """ + ), + ) + return TargetRootsToFieldSets( + { + target: field_sets + for target, field_sets in result.mapping.items() + if field_set_to_default_to in {type(field_set) for field_set in field_sets} + } + ) + raise TooManyTargetsException(result.targets, goal_description=request.goal_description) + + @rule async def find_valid_field_sets_for_target_roots( request: TargetRootsToFieldSetsRequest, + use_deprecated_pex_binary_run_semantics: UseDeprecatedPexBinaryRunSemanticsOption, specs: Specs, union_membership: UnionMembership, registered_target_types: RegisteredTargetTypes, @@ -541,7 +601,16 @@ async def find_valid_field_sets_for_target_roots( if not request.expect_single_field_set: return result if len(result.targets) > 1: - raise TooManyTargetsException(result.targets, goal_description=request.goal_description) + return _handle_ambiguous_result( + request, + result, + cast( + "type[FieldSet]", + PexBinaryFieldSet + if use_deprecated_pex_binary_run_semantics.val + else PythonSourceFieldSet, + ), + ) if len(result.field_sets) > 1: raise AmbiguousImplementationsException( result.targets[0], result.field_sets, goal_description=request.goal_description diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index b81d301de2d..bd98f547047 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -23,6 +23,7 @@ is_in_container, pants_version, ) +from pants.base.deprecated import warn_or_error from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.engine.environment import CompleteEnvironment from pants.engine.internals.native_engine import PyExecutor @@ -1711,6 +1712,68 @@ class GlobalOptions(BootstrapOptions, Subsystem): ), ) + _use_deprecated_pex_binary_run_semantics = BoolOption( + "--use-deprecated-pex-binary-run-semantics", + default=True, + help=softwrap( + """ + If `true`, `run`ning a `pex_binary` will run your firstparty code by copying sources to + a sandbox (while still using a PEX for thirdparty dependencies). Additionally, you can + refer to the `pex_binary` using the value of its `entry_point` field (if it is a filename). + + If `false`, `run`ning a `pex_binary` will build the PEX via `package` and run it directly. + This makes `run` equivalent to using `package` and running the artifact. Additionally, + the binary must be `run` using the `pex_binary`'s address, as passing a filename to `run` + will run the `python_source`. + + Note that support has been added to Pants to allow you to `run` any `python_source`, + so setting this to `true` should be reserved for maintaining backwards-compatibility + with previous versions of Pants. Additionally, you can remove any `pex_binary` targets + that exist solely for running Python code (and aren't meant to be packaged). + """ + ), + removal_version="2.15.0.dev0", + removal_hint=softwrap( + """ + If `use_deprecated_pex_binary_run_semantics` is already set explicitly to `false`, + simply delete the option from `pants.toml` because `false` is now the default. + + If set to `true`, removing the option will cause `run` on a `pex_binary` to package and + run the built PEX file. Additionally, the `pex_binary` must be referred to by its address. + To keep the old `run` semantics, use `run` on the relevant `python_source` target. + """ + ), + ) + + @property + def use_deprecated_pex_binary_run_semantics(self) -> bool: + if self.options.is_default("use_deprecated_pex_binary_run_semantics"): + warn_or_error( + "2.14.0.dev1", + "the option --use-deprecated-pex-binary-run-semantics defaulting to true", + softwrap( + f""" + Currently, running a `pex_binary` by default will not include the source files + in the PEX, and will instead put them in a temporary sandbox. + + In Pants 2.14, the default will change to instead build the PEX like you had run + the `package` goal, and then execute that PEX. This is more consistent and + intuitive behavior. + + To fix this deprecation, explictly set `use_deprecated_pex_binary_run_semantics` + in the `[GLOBAL]` section of `pants.toml`. + Set it to `true` to use the "old" behavior. + Set it to `false` to use the "new" behavior. + + When set to `false`, you can still run the binary as before because you can now + run on a `python_source` target. The simplest way to do this is to use + `{bin_name()} run path/to/file.py`, which will find the owning `python_source`. + Pants will run the file the same way it used to with `pex_binary` targets. + """ + ), + ) + return self._use_deprecated_pex_binary_run_semantics + @classmethod def validate_instance(cls, opts): """Validates an instance of global options for cases that are not prohibited via @@ -1945,3 +2008,13 @@ class NamedCachesDirOption: """ val: PurePath + + +@dataclass(frozen=True) +class UseDeprecatedPexBinaryRunSemanticsOption: + """A wrapper around the global option `use_deprecated_pex_binary_run_semantics`. + + Prefer to use this rather than requesting `GlobalOptions` for more precise invalidation. + """ + + val: bool diff --git a/testprojects/src/python/print_env/BUILD b/testprojects/src/python/print_env/BUILD index eb7e9b05852..f6018298ffa 100644 --- a/testprojects/src/python/print_env/BUILD +++ b/testprojects/src/python/print_env/BUILD @@ -1,6 +1,6 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -pex_binary(entry_point="print_env.main", dependencies=[":lib"]) +pex_binary(name="binary", entry_point="print_env.main", dependencies=[":lib"]) python_sources(name="lib") diff --git a/tests/python/pants_test/pantsd/pantsd_integration_test.py b/tests/python/pants_test/pantsd/pantsd_integration_test.py index 6dfdbfa3eee..228761a2831 100644 --- a/tests/python/pants_test/pantsd/pantsd_integration_test.py +++ b/tests/python/pants_test/pantsd/pantsd_integration_test.py @@ -307,7 +307,7 @@ def test_pantsd_client_env_var_is_inherited_by_pantsd_runner_children(self): } with environment_as(**env): result = ctx.runner( - ["run", "testprojects/src/python/print_env", "--", expected_key] + ["run", "testprojects/src/python/print_env:binary", "--", expected_key] ) ctx.checker.assert_running() @@ -322,7 +322,7 @@ def test_pantsd_launch_env_var_is_not_inherited_by_pantsd_runner_children(self): checker.assert_started() self.run_pants_with_workdir( - ["run", "testprojects/src/python/print_env", "--", "NO_LEAKS"], + ["run", "testprojects/src/python/print_env:binary", "--", "NO_LEAKS"], workdir=workdir, config=pantsd_config, ).assert_failure()