Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Always provide Python-for-Pants-scripts #18433

Merged
merged 27 commits into from
Mar 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pants.toml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ venv_use_symlinks = true
# `python_distrobution` targets, currently:
# + src/python/pants:pants-packaged
# + src/python/pants/testutil:testutil_wheel
# And update the PythonBuildStandalone version/URL:
# + src/python/pants/core/subsystems/python_bootstrap.py
interpreter_constraints = [">=3.7,<3.10"]
macos_big_sur_compatibility = true
enable_resolves = true
Expand Down
3 changes: 1 addition & 2 deletions src/python/pants/backend/python/goals/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,10 +233,9 @@ async def do_export(
"--collisions-ok",
output_path,
],
python=requirements_pex.python,
),
{
**complete_pex_env.environment_dict(python_configured=True),
**complete_pex_env.environment_dict(python=requirements_pex.python),
"PEX_MODULE": "pex.tools",
},
),
Expand Down
12 changes: 4 additions & 8 deletions src/python/pants/backend/python/goals/repl.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,11 @@ async def create_python_repl_request(
)

complete_pex_env = pex_env.in_workspace()
args = complete_pex_env.create_argv(
request.in_chroot(requirements_pex.name), python=requirements_pex.python
)
args = complete_pex_env.create_argv(request.in_chroot(requirements_pex.name))

chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=requirements_pex.python is not None),
**complete_pex_env.environment_dict(python=requirements_pex.python),
"PEX_EXTRA_SYS_PATH": ":".join(chrooted_source_roots),
"PEX_PATH": request.in_chroot(local_dists.pex.name),
"PEX_INTERPRETER_HISTORY": "1" if python_setup.repl_history else "0",
Expand Down Expand Up @@ -175,15 +173,13 @@ async def create_ipython_repl_request(
)

complete_pex_env = pex_env.in_workspace()
args = list(
complete_pex_env.create_argv(request.in_chroot(ipython_pex.name), python=ipython_pex.python)
)
args = list(complete_pex_env.create_argv(request.in_chroot(ipython_pex.name)))
if ipython.ignore_cwd:
args.append("--ignore-cwd")

chrooted_source_roots = [request.in_chroot(sr) for sr in sources.source_roots]
extra_env = {
**complete_pex_env.environment_dict(python_configured=ipython_pex.python is not None),
**complete_pex_env.environment_dict(python=ipython_pex.python),
"PEX_PATH": os.pathsep.join(
[
request.in_chroot(requirements_pex.name),
Expand Down
12 changes: 9 additions & 3 deletions src/python/pants/backend/python/goals/run_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
ResolvedPexEntryPoint,
ResolvePexEntryPointRequest,
)
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.pex import Pex, PexRequest, VenvPex, VenvPexRequest
from pants.backend.python.util_rules.pex_environment import PexEnvironment
from pants.backend.python.util_rules.pex_environment import PexEnvironment, PythonExecutable
from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest
from pants.backend.python.util_rules.python_sources import (
PythonSourceFiles,
Expand Down Expand Up @@ -89,7 +90,10 @@ async def _create_python_source_run_request(
complete_pex_environment = pex_env.in_sandbox(working_directory=None)
else:
complete_pex_environment = pex_env.in_workspace()
venv_pex = await Get(VenvPex, VenvPexRequest(pex_request, complete_pex_environment))
venv_pex, python = await MultiGet(
Get(VenvPex, VenvPexRequest(pex_request, complete_pex_environment)),
Get(PythonExecutable, InterpreterConstraints, pex_request.interpreter_constraints),
)
input_digests = [
venv_pex.digest,
# Note regarding not-in-sandbox mode: You might think that the sources don't need to be copied
Expand All @@ -110,7 +114,7 @@ async def _create_python_source_run_request(
*chrooted_source_roots,
]
extra_env = {
**complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
**complete_pex_environment.environment_dict(python=python),
"PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
}
append_only_caches = (
Expand All @@ -125,6 +129,7 @@ async def _create_python_source_run_request(
**complete_pex_environment.append_only_caches,
**append_only_caches,
},
immutable_input_digests=complete_pex_environment.immutable_input_digests,
)


Expand Down Expand Up @@ -202,4 +207,5 @@ def patched_resolve_remote_root(self, local_root, remote_root):
args=args,
extra_env=extra_env,
append_only_caches=regular_run_request.append_only_caches,
immutable_input_digests=regular_run_request.immutable_input_digests,
)
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ async def create_python_requirement_run_request(
input_digest = venv_pex.digest

extra_env = {
**complete_pex_environment.environment_dict(python_configured=venv_pex.python is not None),
**complete_pex_environment.environment_dict(python=None),
}

return RunRequest(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def rule_runner() -> RuleRunner:
PythonSourcesGeneratorTarget,
PyenvInstall,
],
preserve_tmpdirs=True,
)


Expand Down Expand Up @@ -113,7 +114,7 @@ def test_venv_pex_reconstruction(rule_runner):
print(venv_location)
"""
),
"src/BUILD": "python_sources()",
"src/BUILD": "python_sources(interpreter_constraints=['==3.9.*'])",
}
)

Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/backend/python/util_rules/local_dists.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def __init__(
addresses: Iterable[Address],
*,
internal_only: bool,
interpreter_constraints: InterpreterConstraints = InterpreterConstraints(),
interpreter_constraints: InterpreterConstraints,
sources: PythonSourceFiles = PythonSourceFiles.empty(),
) -> None:
object.__setattr__(self, "addresses", Addresses(addresses))
Expand Down
15 changes: 13 additions & 2 deletions src/python/pants/backend/python/util_rules/local_dists_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
from pants.backend.python.macros.python_artifact import PythonArtifact
from pants.backend.python.subsystems.setuptools import rules as setuptools_rules
from pants.backend.python.target_types import PythonDistribution, PythonSourcesGeneratorTarget
from pants.backend.python.util_rules import local_dists
from pants.backend.python.util_rules import local_dists, pex_from_targets
from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest
from pants.backend.python.util_rules.pex_from_targets import InterpreterConstraintsRequest
from pants.backend.python.util_rules.python_sources import PythonSourceFiles
from pants.build_graph.address import Address
from pants.core.util_rules.source_files import SourceFiles
Expand All @@ -32,6 +34,8 @@ def rule_runner() -> RuleRunner:
*setup_py_rules(),
*setuptools_rules(),
*target_types_rules.rules(),
*pex_from_targets.rules(),
QueryRule(InterpreterConstraints, (InterpreterConstraintsRequest,)),
QueryRule(LocalDistsPex, (LocalDistsPexRequest,)),
],
target_types=[PythonSourcesGeneratorTarget, PythonDistribution],
Expand Down Expand Up @@ -79,8 +83,15 @@ def test_build_local_dists(rule_runner: RuleRunner) -> None:
)
sources_snapshot = rule_runner.request(Snapshot, [sources_digest])
sources = PythonSourceFiles(SourceFiles(sources_snapshot, tuple()), ("srcroot",))
addresses = [Address("foo", target_name="dist")]
interpreter_constraints = rule_runner.request(
InterpreterConstraints, [InterpreterConstraintsRequest(addresses)]
)
request = LocalDistsPexRequest(
[Address("foo", target_name="dist")], internal_only=True, sources=sources
addresses,
internal_only=True,
sources=sources,
interpreter_constraints=interpreter_constraints,
)
result = rule_runner.request(LocalDistsPex, [request])

Expand Down
11 changes: 6 additions & 5 deletions src/python/pants/backend/python/util_rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,6 @@ async def build_pex(
result = await Get(
ProcessResult,
PexCliProcess(
python=pex_python_setup.python,
subcommand=(),
extra_args=argv,
additional_input_digest=merged_digest,
Expand Down Expand Up @@ -751,15 +750,15 @@ def _create_venv_script(
env_vars = (
f"{name}={shlex.quote(value)}"
for name, value in self.complete_pex_env.environment_dict(
python_configured=True
python=self.pex.python
).items()
)

target_venv_executable = shlex.quote(str(venv_executable))
venv_dir = shlex.quote(str(self.venv_dir))
execute_pex_args = " ".join(
f"$(adjust_relative_paths {shlex.quote(arg)})"
for arg in self.complete_pex_env.create_argv(self.pex.name, python=self.pex.python)
for arg in self.complete_pex_env.create_argv(self.pex.name)
)

script = dedent(
Expand Down Expand Up @@ -1034,9 +1033,9 @@ def __init__(
async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment) -> Process:
pex = request.pex
complete_pex_env = pex_environment.in_sandbox(working_directory=request.working_directory)
argv = complete_pex_env.create_argv(pex.name, *request.argv, python=pex.python)
argv = complete_pex_env.create_argv(pex.name, *request.argv)
env = {
**complete_pex_env.environment_dict(python_configured=pex.python is not None),
**complete_pex_env.environment_dict(python=pex.python),
**request.extra_env,
}
input_digest = (
Expand All @@ -1060,6 +1059,7 @@ async def setup_pex_process(request: PexProcess, pex_environment: PexEnvironment
**complete_pex_env.append_only_caches,
**append_only_caches,
},
immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
concurrency_available=request.concurrency_available,
Expand Down Expand Up @@ -1153,6 +1153,7 @@ async def setup_venv_pex_process(
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches=append_only_caches,
immutable_input_digests=pex_environment.bootstrap_python.immutable_input_digests,
timeout_seconds=request.timeout_seconds,
execution_slot_variable=request.execution_slot_variable,
concurrency_available=request.concurrency_available,
Expand Down
31 changes: 15 additions & 16 deletions src/python/pants/backend/python/util_rules/pex_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
PexSubsystem,
PythonExecutable,
)
from pants.core.util_rules import external_tool
from pants.core.util_rules import adhoc_binaries, external_tool
from pants.core.util_rules.adhoc_binaries import PythonBuildStandaloneBinary
from pants.core.util_rules.external_tool import (
DownloadedExternalTool,
ExternalToolRequest,
Expand Down Expand Up @@ -61,7 +62,6 @@ def default_known_versions(cls):
class PexCliProcess:
subcommand: tuple[str, ...]
extra_args: tuple[str, ...]
set_resolve_args: bool
description: str = dataclasses.field(compare=False)
additional_input_digest: Optional[Digest]
extra_env: Optional[FrozenDict[str, str]]
Expand All @@ -78,7 +78,6 @@ def __init__(
subcommand: Iterable[str],
extra_args: Iterable[str],
description: str,
set_resolve_args: bool = True,
additional_input_digest: Optional[Digest] = None,
extra_env: Optional[Mapping[str, str]] = None,
output_files: Optional[Iterable[str]] = None,
Expand All @@ -90,7 +89,6 @@ def __init__(
) -> None:
object.__setattr__(self, "subcommand", tuple(subcommand))
object.__setattr__(self, "extra_args", tuple(extra_args))
object.__setattr__(self, "set_resolve_args", set_resolve_args)
object.__setattr__(self, "description", description)
object.__setattr__(self, "additional_input_digest", additional_input_digest)
object.__setattr__(self, "extra_env", FrozenDict(extra_env) if extra_env else None)
Expand Down Expand Up @@ -125,6 +123,7 @@ async def setup_pex_cli_process(
request: PexCliProcess,
pex_pex: PexPEX,
pex_env: PexEnvironment,
bootstrap_python: PythonBuildStandaloneBinary,
python_native_code: PythonNativeCodeSubsystem.EnvironmentAware,
global_options: GlobalOptions,
pex_subsystem: PexSubsystem,
Expand Down Expand Up @@ -164,11 +163,13 @@ async def setup_pex_cli_process(

verbosity_args = [f"-{'v' * pex_subsystem.verbosity}"] if pex_subsystem.verbosity > 0 else []

resolve_args = (
[*cert_args, "--python-path", create_path_env_var(pex_env.interpreter_search_paths)]
if request.set_resolve_args
else []
)
# NB: We should always pass `--python-path`, as that tells Pex where to look for interpreters
# when `--python` isn't an absolute path.
resolve_args = [
*cert_args,
"--python-path",
create_path_env_var(pex_env.interpreter_search_paths),
]
# All old-style pex runs take the --pip-version flag, but only certain subcommands of the
# `pex3` console script do. So if invoked with a subcommand, the caller must selectively
# set --pip-version only on subcommands that take it.
Expand All @@ -187,15 +188,14 @@ async def setup_pex_cli_process(
]

complete_pex_env = pex_env.in_sandbox(working_directory=None)
normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args, python=request.python)
normalized_argv = complete_pex_env.create_argv(pex_pex.exe, *args)
env = {
**complete_pex_env.environment_dict(python_configured=request.python is not None),
**complete_pex_env.environment_dict(python=request.python),
**python_native_code.subprocess_env_vars,
**(request.extra_env or {}),
# If a subcommand is used, we need to use the `pex3` console script.
**({"PEX_SCRIPT": "pex3"} if request.subcommand else {}),
}
append_only_caches = request.python.append_only_caches if request.python else FrozenDict({})

return Process(
normalized_argv,
Expand All @@ -204,10 +204,8 @@ async def setup_pex_cli_process(
env=env,
output_files=request.output_files,
output_directories=request.output_directories,
append_only_caches={
**complete_pex_env.append_only_caches,
**append_only_caches,
},
append_only_caches=complete_pex_env.append_only_caches,
immutable_input_digests=bootstrap_python.immutable_input_digests,
level=request.level,
concurrency_available=request.concurrency_available,
cache_scope=request.cache_scope,
Expand All @@ -219,4 +217,5 @@ def rules():
*collect_rules(),
*external_tool.rules(),
*pex_environment.rules(),
*adhoc_binaries.rules(),
]
Loading