Always provide Python-for-Pants-scripts

src/python/pants/backend/python/goals/export.py

@@ -233,10 +233,9 @@ async def do_export(
                             "--collisions-ok",
                             output_path,
                         ],
-                        python=requirements_pex.python,
                     ),
                     {
-                        **complete_pex_env.environment_dict(python_configured=True),
+                        **complete_pex_env.environment_dict(python=requirements_pex.python),
                         "PEX_MODULE": "pex.tools",
                     },
                 ),

src/python/pants/backend/python/goals/repl.py

@@ -109,13 +109,11 @@ async def create_python_repl_request(
     )
 
     complete_pex_env = pex_env.in_workspace()
-    args = complete_pex_env.create_argv(
-        request.in_chroot(requirements_pex.name), python=requirements_pex.python
-    )
+    args = complete_pex_env.create_argv(request.in_chroot(requirements_pex.name))
 
     chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
-        **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
+        **complete_pex_env.environment_dict(python=requirements_pex.python),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
         "PEX_PATH": request.in_chroot(local_dists.pex.name),
         "PEX_INTERPRETER_HISTORY": "1" if python_setup.repl_history else "0",
@@ -175,15 +173,13 @@ async def create_ipython_repl_request(
     )
 
     complete_pex_env = pex_env.in_workspace()
-    args = list(
-        complete_pex_env.create_argv(request.in_chroot(ipython_pex.name), python=ipython_pex.python)
-    )
+    args = list(complete_pex_env.create_argv(request.in_chroot(ipython_pex.name)))
     if ipython.ignore_cwd:
         args.append("--ignore-cwd")
 
     chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
-        **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
+        **complete_pex_env.environment_dict(python=ipython_pex.python),
         "PEX_PATH": os.pathsep.join(
             [
                 request.in_chroot(requirements_pex.name),

src/python/pants/backend/python/goals/run_helper.py

@@ -110,7 +110,7 @@ async def _create_python_source_run_request(
         *chrooted_source_roots,
     ]
     extra_env = {
-        **complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
+        **complete_pex_environment.environment_dict(python=None),
         "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
     }
     append_only_caches = (
@@ -125,6 +125,7 @@ async def _create_python_source_run_request(
             **complete_pex_environment.append_only_caches,
             **append_only_caches,
         },
+        immutable_input_digests=complete_pex_environment.immutable_input_digests,
     )
 
 
@@ -202,4 +203,5 @@ def patched_resolve_remote_root(self, local_root, remote_root):
         args=args,
         extra_env=extra_env,
         append_only_caches=regular_run_request.append_only_caches,
+        immutable_input_digests=regular_run_request.immutable_input_digests,
     )

src/python/pants/backend/python/goals/run_python_requirement.py

@@ -162,7 +162,7 @@ async def create_python_requirement_run_request(
     input_digest = venv_pex.digest
 
     extra_env = {
-        **complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
+        **complete_pex_environment.environment_dict(python=None),
     }
 
     return RunRequest(

src/python/pants/backend/python/util_rules/pex.py

@@ -622,7 +622,6 @@ async def build_pex(
     result = await Get(
         ProcessResult,
         PexCliProcess(
-            python=pex_python_setup.python,
             subcommand=(),
             extra_args=argv,
             additional_input_digest=merged_digest,
@@ -751,15 +750,15 @@ def _create_venv_script(
         env_vars = (
             f"{name}={shlex.quote(value)}"
             for name, value in self.complete_pex_env.environment_dict(
-                python_configured=True
+                python=self.pex.python
             ).items()
         )
 
         target_venv_executable = shlex.quote(str(venv_executable))
         venv_dir = shlex.quote(str(self.venv_dir))
         execute_pex_args = " ".join(
             f"$(adjust_relative_paths {shlex.quote(arg)})"
-            for arg in self.complete_pex_env.create_argv(self.pex.name, python=self.pex.python)
+            for arg in self.complete_pex_env.create_argv(self.pex.name)
         )
 
         script = dedent(
@@ -1034,9 +1033,9 @@ def __init__(
 async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process:
     pex = request.pex
     complete_pex_env = pex_environment.in_sandbox(working_directory=request.working_directory)
-    argv = complete_pex_env.create_argv(pex.name, *request.argv, python=pex.python)
+    argv = complete_pex_env.create_argv(pex.name, *request.argv)
     env = {
-        **complete_pex_env.environment_dict(python_configured=pex.python is not None),
+        **complete_pex_env.environment_dict(python=pex.python),
         **request.extra_env,
     }
     input_digest = (
@@ -1060,6 +1059,7 @@ async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment
             **complete_pex_env.append_only_caches,
             **append_only_caches,
         },
+        immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
         timeout_seconds=request.timeout_seconds,
         execution_slot_variable=request.execution_slot_variable,
         concurrency_available=request.concurrency_available,
@@ -1153,6 +1153,7 @@ async def setup_venv_pex_process(
         output_files=request.output_files,
         output_directories=request.output_directories,
         append_only_caches=append_only_caches,
+        immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
         timeout_seconds=request.timeout_seconds,
         execution_slot_variable=request.execution_slot_variable,
         concurrency_available=request.concurrency_available,

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -15,7 +15,8 @@
     PexSubsystem,
     PythonExecutable,
 )
-from pants.core.util_rules import external_tool
+from pants.core.util_rules import adhoc_binaries_rules, external_tool
+from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary
 from pants.core.util_rules.external_tool import (
     DownloadedExternalTool,
     ExternalToolRequest,
@@ -61,7 +62,6 @@ def default_known_versions(cls):
 class PexCliProcess:
     subcommand: tuple[str, ...]
     extra_args: tuple[str, ...]
-    set_resolve_args: bool
     description: str = dataclasses.field(compare=False)
     additional_input_digest: Optional[Digest]
     extra_env: Optional[FrozenDict[str, str]]
@@ -78,7 +78,6 @@ def __init__(
         subcommand: Iterable[str],
         extra_args: Iterable[str],
         description: str,
-        set_resolve_args: bool = True,
         additional_input_digest: Optional[Digest] = None,
         extra_env: Optional[Mapping[str, str]] = None,
         output_files: Optional[Iterable[str]] = None,
@@ -90,7 +89,6 @@ def __init__(
     ) -> None:
         object.__setattr__(self, "subcommand", tuple(subcommand))
         object.__setattr__(self, "extra_args", tuple(extra_args))
-        object.__setattr__(self, "set_resolve_args", set_resolve_args)
         object.__setattr__(self, "description", description)
         object.__setattr__(self, "additional_input_digest", additional_input_digest)
         object.__setattr__(self, "extra_env", FrozenDict(extra_env) if extra_env else None)
@@ -125,6 +123,7 @@ async def setup_pex_cli_process(
     request: PexCliProcess,
     pex_pex: PexPEX,
     pex_env: PexEnvironment,
+    bootstrap_python: PythonBuildStandaloneBinary,
     python_native_code: PythonNativeCodeSubsystem.EnvironmentAware,
     global_options: GlobalOptions,
     pex_subsystem: PexSubsystem,
@@ -164,11 +163,13 @@ async def setup_pex_cli_process(
 
     verbosity_args = [f"-{'v' * pex_subsystem.verbosity}"] if pex_subsystem.verbosity > 0 else []
 
-    resolve_args = (
-        [*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
-        if request.set_resolve_args
-        else []
-    )
+    # NB: We should always pass `--python-path`, as that tells Pex where to look for interpreters
+    # when `--python` isn't an absolute path.
+    resolve_args = [
+        *cert_args,
+        "--python-path",
+        create_path_env_var(pex_env.interpreter_search_paths),
+    ]
     # All old-style pex runs take the --pip-version flag, but only certain subcommands of the
     # `pex3` console script do. So if invoked with a subcommand, the caller must selectively
     # set --pip-version only on subcommands that take it.
@@ -187,15 +188,14 @@ async def setup_pex_cli_process(
     ]
 
     complete_pex_env = pex_env.in_sandbox(working_directory=None)
-    normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
+    normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args)
     env = {
-        **complete_pex_env.environment_dict(python_configured=request.python is not None),
+        **complete_pex_env.environment_dict(python=request.python),
         **python_native_code.subprocess_env_vars,
         **(request.extra_env or {}),
         # If a subcommand is used, we need to use the `pex3` console script.
         **({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
     }
-    append_only_caches = request.python.append_only_caches if request.python else FrozenDict({})
 
     return Process(
         normalized_argv,
@@ -204,10 +204,8 @@ async def setup_pex_cli_process(
         env=env,
         output_files=request.output_files,
         output_directories=request.output_directories,
-        append_only_caches={
-            **complete_pex_env.append_only_caches,
-            **append_only_caches,
-        },
+        append_only_caches=complete_pex_env.append_only_caches,
+        immutable_input_digests=bootstrap_python.immutable_input_digests,
         level=request.level,
         concurrency_available=request.concurrency_available,
         cache_scope=request.cache_scope,
@@ -219,4 +217,5 @@ def rules():
         *collect_rules(),
         *external_tool.rules(),
         *pex_environment.rules(),
+        *adhoc_binaries_rules.rules(),
     ]

src/python/pants/backend/python/util_rules/pex_environment.py

@@ -10,9 +10,11 @@
 
 from pants.core.subsystems.python_bootstrap import PythonBootstrap
 from pants.core.util_rules import subprocess_environment, system_binaries
+from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary
 from pants.core.util_rules.subprocess_environment import SubprocessEnvironmentVars
-from pants.core.util_rules.system_binaries import BinaryPath, PythonBinary
+from pants.core.util_rules.system_binaries import BinaryPath
 from pants.engine.engine_aware import EngineAwareReturnType
+from pants.engine.internals.native_engine import Digest
 from pants.engine.rules import collect_rules, rule
 from pants.option.global_options import NamedCachesDirOption
 from pants.option.option_types import BoolOption, IntOption, StrListOption
@@ -91,17 +93,20 @@ def verbosity(self) -> int:
 
 @dataclass(frozen=True)
 class PythonExecutable(BinaryPath, EngineAwareReturnType):
-    """The BinaryPath of a Python executable, along with some extras."""
+    """The BinaryPath of a Python executable for user code, along with some extras."""
 
     append_only_caches: FrozenDict[str, str] = FrozenDict({})
+    immutable_input_digests: FrozenDict[str, str] = FrozenDict({})
 
     def __init__(
         self,
         path: str,
         fingerprint: str | None = None,
         append_only_caches: Mapping[str, str] = FrozenDict({}),
+        immutable_input_digests: Mapping[str, str] = FrozenDict({}),
     ) -> None:
         object.__setattr__(self, "append_only_caches", FrozenDict(append_only_caches))
+        object.__setattr__(self, "immutable_input_digests", FrozenDict(immutable_input_digests))
         super().__init__(path, fingerprint)
         self.__post_init__()
 
@@ -124,49 +129,26 @@ def __post_init__(self) -> None:
     def message(self) -> str:
         return f"Selected {self.path} to run PEXes with."
 
-    @classmethod
-    def from_python_binary(cls, python_binary: PythonBinary) -> PythonExecutable:
-        """Converts from PythonBinary to PythonExecutable.
-
-        The PythonBinary type is a singleton representing the Python that is used for script
-        execution by `@rule`s. On the other hand, there may be multiple PythonExecutables, since
-        they are subject to a user's interpreter constraints.
-        """
-        return cls(path=python_binary.path, fingerprint=python_binary.fingerprint)
-
 
 @dataclass(frozen=True)
 class PexEnvironment(EngineAwareReturnType):
     path: tuple[str, ...]
     interpreter_search_paths: tuple[str, ...]
     subprocess_environment_dict: FrozenDict[str, str]
     named_caches_dir: PurePath
-    bootstrap_python: PythonExecutable | None = None
+    bootstrap_python: PythonBuildStandaloneBinary
     venv_use_symlinks: bool = False
 
     _PEX_ROOT_DIRNAME = "pex_root"
 
-    def level(self) -> LogLevel:
-        return LogLevel.DEBUG if self.bootstrap_python else LogLevel.WARN
-
-    def message(self) -> str:
-        if not self.bootstrap_python:
-            return softwrap(
-                """
-                No bootstrap Python executable could be found from the option
-                `interpreter_search_paths` in the `[python]` scope. Will attempt to run
-                PEXes directly.
-                """
-            )
-        return f"Selected {self.bootstrap_python.path} to bootstrap PEXes with."
-
     def in_sandbox(self, *, working_directory: str | None) -> CompletePexEnvironment:
         pex_root = PurePath(".cache") / self._PEX_ROOT_DIRNAME
         return CompletePexEnvironment(
             _pex_environment=self,
             pex_root=pex_root,
             _working_directory=PurePath(working_directory) if working_directory else None,
             append_only_caches=FrozenDict({self._PEX_ROOT_DIRNAME: str(pex_root)}),
+            immutable_input_digests=self.bootstrap_python.immutable_input_digests,
         )
 
     def in_workspace(self) -> CompletePexEnvironment:
@@ -181,6 +163,7 @@ def in_workspace(self) -> CompletePexEnvironment:
             pex_root=pex_root,
             _working_directory=None,
             append_only_caches=FrozenDict(),
+            immutable_input_digests=self.bootstrap_python.immutable_input_digests,
         )
 
     def venv_site_packages_copies_option(self, use_copies: bool) -> str:
@@ -192,7 +175,7 @@ def venv_site_packages_copies_option(self, use_copies: bool) -> str:
 @rule(desc="Prepare environment for running PEXes", level=LogLevel.DEBUG)
 async def find_pex_python(
     python_bootstrap: PythonBootstrap,
-    python_binary: PythonBinary,
+    python_binary: PythonBuildStandaloneBinary,
     pex_subsystem: PexSubsystem,
     pex_environment_aware: PexSubsystem.EnvironmentAware,
     subprocess_env_vars: SubprocessEnvironmentVars,
@@ -203,7 +186,7 @@ async def find_pex_python(
         interpreter_search_paths=python_bootstrap.interpreter_search_paths,
         subprocess_environment_dict=subprocess_env_vars.vars,
         named_caches_dir=named_caches_dir.val,
-        bootstrap_python=PythonExecutable.from_python_binary(python_binary),
+        bootstrap_python=python_binary,
         venv_use_symlinks=pex_subsystem.venv_use_symlinks,
     )
 
@@ -214,8 +197,7 @@ class CompletePexEnvironment:
     pex_root: PurePath
     _working_directory: PurePath | None
     append_only_caches: FrozenDict[str, str]
-
-    _PEX_ROOT_DIRNAME = "pex_root"
+    immutable_input_digests: FrozenDict[str, Digest]
 
     @property
     def interpreter_search_paths(self) -> tuple[str, ...]:
@@ -229,14 +211,9 @@ def create_argv(
             if self._working_directory
             else pex_filepath
         )
-        python = python or self._pex_environment.bootstrap_python
-        if python:
-            return (python.path, pex_relpath, *args)
-        if os.path.basename(pex_relpath) == pex_relpath:
-            return (f"./{pex_relpath}", *args)
-        return (pex_relpath, *args)
+        return (self._pex_environment.bootstrap_python.path, pex_relpath, *args)
 
-    def environment_dict(self, *, python_configured: bool) -> Mapping[str, str]:
+    def environment_dict(self, *, python: PythonExecutable | None = None) -> Mapping[str, str]:
         """The environment to use for running anything with PEX.
 
         If the Process is run with a pre-selected Python interpreter, set `python_configured=True`
@@ -252,10 +229,10 @@ def environment_dict(self, *, python_configured: bool) -> Mapping[str, str]:
             ),
             **self._pex_environment.subprocess_environment_dict,
         )
-        # NB: We only set `PEX_PYTHON_PATH` if the Python interpreter has not already been
-        # pre-selected by Pants. Otherwise, Pex would inadvertently try to find another interpreter
-        # when running PEXes. (Creating a PEX will ignore this env var in favor of `--python-path`.)
-        if not python_configured:
+        if python:
+            assert isinstance(python.path, str)
+            d["PEX_PYTHON"] = python.path
+        else:
             d["PEX_PYTHON_PATH"] = create_path_env_var(self.interpreter_search_paths)
         return d