Always provide Python-for-Pants-scripts (Cherry-pick of #18433) (#18495)

This PR decouples the Python Pants uses for its own nefarious purposes
(like running PEX or `gunzip`) from the user search paths by either
using `sys.executable` locally or downloading and using Python Build
Standalone in a Docker environment.

Additionally when making this change, the Python/pex code was refactored
so that we always use this Python to run pex, with Python either being
chosen by pex, or by using `PEX_PYTHON` env var at runtime. I think this
is a nice cleanup of the handshake between
`CompletePexEnvironment.create_argv` and
`CompletePexEnvironment.environment_dict` used to have.

On a side note, this (with `scie-pants` and
#18352) this marks a complete
divorce from any Python on the user system (or lack thereof) and running
Python code.

pants.toml

@@ -137,6 +137,8 @@ venv_use_symlinks = true
 # `python_distrobution` targets, currently:
 # + src/python/pants:pants-packaged
 # + src/python/pants/testutil:testutil_wheel
+# And update the PythonBuildStandalone version/URL:
+# + src/python/pants/core/subsystems/python_bootstrap.py
 interpreter_constraints = [">=3.7,<3.10"]
 macos_big_sur_compatibility = true
 enable_resolves = true

src/python/pants/backend/python/goals/export.py

@@ -233,10 +233,9 @@ async def do_export(
                             "--collisions-ok",
                             output_path,
                         ],
-                        python=requirements_pex.python,
                     ),
                     {
-                        **complete_pex_env.environment_dict(python_configured=True),
+                        **complete_pex_env.environment_dict(python=requirements_pex.python),
                         "PEX_MODULE": "pex.tools",
                     },
                 ),

src/python/pants/backend/python/goals/repl.py

@@ -109,13 +109,11 @@ async def create_python_repl_request(
     )
 
     complete_pex_env = pex_env.in_workspace()
-    args = complete_pex_env.create_argv(
-        request.in_chroot(requirements_pex.name), python=requirements_pex.python
-    )
+    args = complete_pex_env.create_argv(request.in_chroot(requirements_pex.name))
 
     chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
-        **complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
+        **complete_pex_env.environment_dict(python=requirements_pex.python),
         "PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
         "PEX_PATH": request.in_chroot(local_dists.pex.name),
         "PEX_INTERPRETER_HISTORY": "1" if python_setup.repl_history else "0",
@@ -175,15 +173,13 @@ async def create_ipython_repl_request(
     )
 
     complete_pex_env = pex_env.in_workspace()
-    args = list(
-        complete_pex_env.create_argv(request.in_chroot(ipython_pex.name), python=ipython_pex.python)
-    )
+    args = list(complete_pex_env.create_argv(request.in_chroot(ipython_pex.name)))
     if ipython.ignore_cwd:
         args.append("--ignore-cwd")
 
     chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
     extra_env = {
-        **complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
+        **complete_pex_env.environment_dict(python=ipython_pex.python),
         "PEX_PATH": os.pathsep.join(
             [
                 request.in_chroot(requirements_pex.name),

src/python/pants/backend/python/goals/run_helper.py

@@ -14,8 +14,9 @@
     ResolvedPexEntryPoint,
     ResolvePexEntryPointRequest,
 )
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexRequest
-from pants.backend.python.util_rules.pex_environment import PexEnvironment
+from pants.backend.python.util_rules.pex_environment import PexEnvironment, PythonExecutable
 from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
 from pants.backend.python.util_rules.python_sources import (
     PythonSourceFiles,
@@ -89,7 +90,10 @@ async def _create_python_source_run_request(
         complete_pex_environment = pex_env.in_sandbox(working_directory=None)
     else:
         complete_pex_environment = pex_env.in_workspace()
-    venv_pex = await Get(VenvPex, VenvPexRequest(pex_request, complete_pex_environment))
+    venv_pex, python = await MultiGet(
+        Get(VenvPex, VenvPexRequest(pex_request, complete_pex_environment)),
+        Get(PythonExecutable, InterpreterConstraints, pex_request.interpreter_constraints),
+    )
     input_digests = [
         venv_pex.digest,
         # Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
@@ -110,7 +114,7 @@ async def _create_python_source_run_request(
         *chrooted_source_roots,
     ]
     extra_env = {
-        **complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
+        **complete_pex_environment.environment_dict(python=python),
         "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
     }
     append_only_caches = (
@@ -125,6 +129,7 @@ async def _create_python_source_run_request(
             **complete_pex_environment.append_only_caches,
             **append_only_caches,
         },
+        immutable_input_digests=complete_pex_environment.immutable_input_digests,
     )
 
 
@@ -202,4 +207,5 @@ def patched_resolve_remote_root(self, local_root, remote_root):
         args=args,
         extra_env=extra_env,
         append_only_caches=regular_run_request.append_only_caches,
+        immutable_input_digests=regular_run_request.immutable_input_digests,
     )

src/python/pants/backend/python/goals/run_python_requirement.py

@@ -160,7 +160,7 @@ async def create_python_requirement_run_request(
     input_digest = venv_pex.digest
 
     extra_env = {
-        **complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
+        **complete_pex_environment.environment_dict(python=None),
     }
 
     return RunRequest(

src/python/pants/backend/python/providers/pyenv/rules_integration_test.py

@@ -37,6 +37,7 @@ def rule_runner() -> RuleRunner:
             PythonSourcesGeneratorTarget,
             PyenvInstall,
         ],
+        preserve_tmpdirs=True,
     )
 
 
@@ -113,7 +114,7 @@ def test_venv_pex_reconstruction(rule_runner):
                 print(venv_location)
                 """
             ),
-            "src/BUILD": "python_sources()",
+            "src/BUILD": "python_sources(interpreter_constraints=['==3.9.*'])",
         }
     )
 

src/python/pants/backend/python/util_rules/local_dists.py

@@ -117,7 +117,7 @@ def __init__(
         addresses: Iterable[Address],
         *,
         internal_only: bool,
-        interpreter_constraints: InterpreterConstraints = InterpreterConstraints(),
+        interpreter_constraints: InterpreterConstraints,
         sources: PythonSourceFiles = PythonSourceFiles.empty(),
     ) -> None:
         object.__setattr__(self, "addresses", Addresses(addresses))

src/python/pants/backend/python/util_rules/local_dists_test.py

@@ -15,8 +15,10 @@
 from pants.backend.python.macros.python_artifact import PythonArtifact
 from pants.backend.python.subsystems.setuptools import rules as setuptools_rules
 from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget
-from pants.backend.python.util_rules import local_dists
+from pants.backend.python.util_rules import local_dists, pex_from_targets
+from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest
+from pants.backend.python.util_rules.pex_from_targets import InterpreterConstraintsRequest
 from pants.backend.python.util_rules.python_sources import PythonSourceFiles
 from pants.build_graph.address import Address
 from pants.core.util_rules.source_files import SourceFiles
@@ -32,6 +34,8 @@ def rule_runner() -> RuleRunner:
             *setup_py_rules(),
             *setuptools_rules(),
             *target_types_rules.rules(),
+            *pex_from_targets.rules(),
+            QueryRule(InterpreterConstraints, (InterpreterConstraintsRequest,)),
             QueryRule(LocalDistsPex, (LocalDistsPexRequest,)),
         ],
         target_types=[PythonSourcesGeneratorTarget, PythonDistribution],
@@ -79,8 +83,15 @@ def test_build_local_dists(rule_runner: RuleRunner) -> None:
     )
     sources_snapshot = rule_runner.request(Snapshot, [sources_digest])
     sources = PythonSourceFiles(SourceFiles(sources_snapshot, tuple()), ("srcroot",))
+    addresses = [Address("foo", target_name="dist")]
+    interpreter_constraints = rule_runner.request(
+        InterpreterConstraints, [InterpreterConstraintsRequest(addresses)]
+    )
     request = LocalDistsPexRequest(
-        [Address("foo", target_name="dist")], internal_only=True, sources=sources
+        addresses,
+        internal_only=True,
+        sources=sources,
+        interpreter_constraints=interpreter_constraints,
     )
     result = rule_runner.request(LocalDistsPex, [request])
 

src/python/pants/backend/python/util_rules/pex.py

@@ -622,7 +622,6 @@ async def build_pex(
     result = await Get(
         ProcessResult,
         PexCliProcess(
-            python=pex_python_setup.python,
             subcommand=(),
             extra_args=argv,
             additional_input_digest=merged_digest,
@@ -751,15 +750,15 @@ def _create_venv_script(
         env_vars = (
             f"{name}={shlex.quote(value)}"
             for name, value in self.complete_pex_env.environment_dict(
-                python_configured=True
+                python=self.pex.python
             ).items()
         )
 
         target_venv_executable = shlex.quote(str(venv_executable))
         venv_dir = shlex.quote(str(self.venv_dir))
         execute_pex_args = " ".join(
             f"$(adjust_relative_paths {shlex.quote(arg)})"
-            for arg in self.complete_pex_env.create_argv(self.pex.name, python=self.pex.python)
+            for arg in self.complete_pex_env.create_argv(self.pex.name)
         )
 
         script = dedent(
@@ -1034,9 +1033,9 @@ def __init__(
 async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process:
     pex = request.pex
     complete_pex_env = pex_environment.in_sandbox(working_directory=request.working_directory)
-    argv = complete_pex_env.create_argv(pex.name, *request.argv, python=pex.python)
+    argv = complete_pex_env.create_argv(pex.name, *request.argv)
     env = {
-        **complete_pex_env.environment_dict(python_configured=pex.python is not None),
+        **complete_pex_env.environment_dict(python=pex.python),
         **request.extra_env,
     }
     input_digest = (
@@ -1060,6 +1059,7 @@ async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment
             **complete_pex_env.append_only_caches,
             **append_only_caches,
         },
+        immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
         timeout_seconds=request.timeout_seconds,
         execution_slot_variable=request.execution_slot_variable,
         concurrency_available=request.concurrency_available,
@@ -1153,6 +1153,7 @@ async def setup_venv_pex_process(
         output_files=request.output_files,
         output_directories=request.output_directories,
         append_only_caches=append_only_caches,
+        immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
         timeout_seconds=request.timeout_seconds,
         execution_slot_variable=request.execution_slot_variable,
         concurrency_available=request.concurrency_available,

src/python/pants/backend/python/util_rules/pex_cli.py

@@ -15,7 +15,8 @@
     PexSubsystem,
     PythonExecutable,
 )
-from pants.core.util_rules import external_tool
+from pants.core.util_rules import adhoc_binaries, external_tool
+from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary
 from pants.core.util_rules.external_tool import (
     DownloadedExternalTool,
     ExternalToolRequest,
@@ -61,7 +62,6 @@ def default_known_versions(cls):
 class PexCliProcess:
     subcommand: tuple[str, ...]
     extra_args: tuple[str, ...]
-    set_resolve_args: bool
     description: str = dataclasses.field(compare=False)
     additional_input_digest: Optional[Digest]
     extra_env: Optional[FrozenDict[str, str]]
@@ -78,7 +78,6 @@ def __init__(
         subcommand: Iterable[str],
         extra_args: Iterable[str],
         description: str,
-        set_resolve_args: bool = True,
         additional_input_digest: Optional[Digest] = None,
         extra_env: Optional[Mapping[str, str]] = None,
         output_files: Optional[Iterable[str]] = None,
@@ -90,7 +89,6 @@ def __init__(
     ) -> None:
         object.__setattr__(self, "subcommand", tuple(subcommand))
         object.__setattr__(self, "extra_args", tuple(extra_args))
-        object.__setattr__(self, "set_resolve_args", set_resolve_args)
         object.__setattr__(self, "description", description)
         object.__setattr__(self, "additional_input_digest", additional_input_digest)
         object.__setattr__(self, "extra_env", FrozenDict(extra_env) if extra_env else None)
@@ -125,6 +123,7 @@ async def setup_pex_cli_process(
     request: PexCliProcess,
     pex_pex: PexPEX,
     pex_env: PexEnvironment,
+    bootstrap_python: PythonBuildStandaloneBinary,
     python_native_code: PythonNativeCodeSubsystem.EnvironmentAware,
     global_options: GlobalOptions,
     pex_subsystem: PexSubsystem,
@@ -164,11 +163,13 @@ async def setup_pex_cli_process(
 
     verbosity_args = [f"-{'v' * pex_subsystem.verbosity}"] if pex_subsystem.verbosity > 0 else []
 
-    resolve_args = (
-        [*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
-        if request.set_resolve_args
-        else []
-    )
+    # NB: We should always pass `--python-path`, as that tells Pex where to look for interpreters
+    # when `--python` isn't an absolute path.
+    resolve_args = [
+        *cert_args,
+        "--python-path",
+        create_path_env_var(pex_env.interpreter_search_paths),
+    ]
     # All old-style pex runs take the --pip-version flag, but only certain subcommands of the
     # `pex3` console script do. So if invoked with a subcommand, the caller must selectively
     # set --pip-version only on subcommands that take it.
@@ -187,15 +188,14 @@ async def setup_pex_cli_process(
     ]
 
     complete_pex_env = pex_env.in_sandbox(working_directory=None)
-    normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
+    normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args)
     env = {
-        **complete_pex_env.environment_dict(python_configured=request.python is not None),
+        **complete_pex_env.environment_dict(python=request.python),
         **python_native_code.subprocess_env_vars,
         **(request.extra_env or {}),
         # If a subcommand is used, we need to use the `pex3` console script.
         **({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
     }
-    append_only_caches = request.python.append_only_caches if request.python else FrozenDict({})
 
     return Process(
         normalized_argv,
@@ -204,10 +204,8 @@ async def setup_pex_cli_process(
         env=env,
         output_files=request.output_files,
         output_directories=request.output_directories,
-        append_only_caches={
-            **complete_pex_env.append_only_caches,
-            **append_only_caches,
-        },
+        append_only_caches=complete_pex_env.append_only_caches,
+        immutable_input_digests=bootstrap_python.immutable_input_digests,
         level=request.level,
         concurrency_available=request.concurrency_available,
         cache_scope=request.cache_scope,
@@ -219,4 +217,5 @@ def rules():
         *collect_rules(),
         *external_tool.rules(),
         *pex_environment.rules(),
+        *adhoc_binaries.rules(),
     ]