Skip to content

Commit

Permalink
Refactors PythonBootstrap to pre-calculate expanded interpreter pat…
Browse files Browse the repository at this point in the history 
…hs (#17030)

Previously, `PythonBootstrap` had a method that would calculate the expanded interpreter paths when they are first consumed. This change moves the calculation of those expanded interpreter paths to the initial creation of `PythonBootstrap`.

Relatedly, a bunch of static methods defined on `PythonBootstrap` are now private functions at module scope. This is prework that will significantly improve the clarity of my next piece of work on `PythonBootstrap`, namely converting all of the various filesystem-dependent functions to `@rule`s.

**None of the helper methods have been changed**

Addresses #16800.
  • Loading branch information
Christopher Neugebauer authored Sep 28, 2022
1 parent c944831 commit 7020ae2
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 142 deletions.
18 changes: 10 additions & 8 deletions src/python/pants/backend/python/util_rules/pex_environment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,13 @@
from pants.core.util_rules.subprocess_environment import SubprocessEnvironmentVars
from pants.core.util_rules.system_binaries import BinaryPath, PythonBinary
from pants.engine.engine_aware import EngineAwareReturnType
from pants.engine.env_vars import EnvironmentVars
from pants.engine.rules import collect_rules, rule
from pants.option.global_options import NamedCachesDirOption
from pants.option.option_types import BoolOption, IntOption, StrListOption
from pants.option.subsystem import Subsystem
from pants.util.frozendict import FrozenDict
from pants.util.logging import LogLevel
from pants.util.memo import memoized_method
from pants.util.memo import memoized_property
from pants.util.ordered_set import OrderedSet
from pants.util.strutil import create_path_env_var, softwrap

Expand All @@ -29,9 +28,12 @@ class PexSubsystem(Subsystem):
options_scope = "pex"
help = "How Pants uses Pex to run Python subprocesses."

class EnvironmentAware:
class EnvironmentAware(Subsystem.EnvironmentAware):
# TODO(#9760): We'll want to deprecate this in favor of a global option which allows for a
# per-process override.

depends_on_env_vars = ("PATH",)

_executable_search_paths = StrListOption(
default=["<PATH>"],
help=softwrap(
Expand All @@ -46,12 +48,12 @@ class EnvironmentAware:
metavar="<binary-paths>",
)

@memoized_method
def path(self, env: EnvironmentVars) -> tuple[str, ...]:
@memoized_property
def path(self) -> tuple[str, ...]:
def iter_path_entries():
for entry in self._executable_search_paths:
if entry == "<PATH>":
path = env.get("PATH")
path = self.env_vars.get("PATH")
if path:
yield from path.split(os.pathsep)
else:
Expand Down Expand Up @@ -168,8 +170,8 @@ async def find_pex_python(
named_caches_dir: NamedCachesDirOption,
) -> PexEnvironment:
return PexEnvironment(
path=pex_environment_aware.path(python_bootstrap.environment),
interpreter_search_paths=tuple(python_bootstrap.interpreter_search_paths()),
path=pex_environment_aware.path,
interpreter_search_paths=python_bootstrap.interpreter_search_paths,
subprocess_environment_dict=subprocess_env_vars.vars,
named_caches_dir=named_caches_dir.val,
bootstrap_python=PythonExecutable.from_python_binary(python_binary),
Expand Down
236 changes: 113 additions & 123 deletions src/python/pants/core/subsystems/python_bootstrap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from pants.engine.rules import Get, collect_rules, rule
from pants.option.option_types import StrListOption
from pants.option.subsystem import Subsystem
from pants.util.memo import memoized_method
from pants.util.strutil import softwrap

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -86,135 +85,122 @@ class PythonBootstrap:
EXTRA_ENV_VAR_NAMES = ("PATH", "PYENV_ROOT")

interpreter_names: tuple[str, ...]
raw_interpreter_search_paths: tuple[str, ...]
environment: EnvironmentVars
asdf_standard_tool_paths: tuple[str, ...]
asdf_local_tool_paths: tuple[str, ...]

@memoized_method
def interpreter_search_paths(self):
return self._expand_interpreter_search_paths(
self.raw_interpreter_search_paths,
self.environment,
self.asdf_standard_tool_paths,
self.asdf_local_tool_paths,
interpreter_search_paths: tuple[str, ...]


def _expand_interpreter_search_paths(
interpreter_search_paths: Sequence[str],
env: EnvironmentVars,
asdf_standard_tool_paths: tuple[str, ...],
asdf_local_tool_paths: tuple[str, ...],
) -> tuple[str, ...]:

special_strings = {
"<PEXRC>": _get_pex_python_paths,
"<PATH>": lambda: _get_environment_paths(env),
"<ASDF>": lambda: asdf_standard_tool_paths,
"<ASDF_LOCAL>": lambda: asdf_local_tool_paths,
"<PYENV>": lambda: _get_pyenv_paths(env),
"<PYENV_LOCAL>": lambda: _get_pyenv_paths(env, pyenv_local=True),
}
expanded: list[str] = []
from_pexrc = None
for s in interpreter_search_paths:
if s in special_strings:
special_paths = special_strings[s]()
if s == "<PEXRC>":
from_pexrc = special_paths
expanded.extend(special_paths)
else:
expanded.append(s)
# Some special-case logging to avoid misunderstandings.
if from_pexrc and len(expanded) > len(from_pexrc):
logger.info(
softwrap(
f"""
pexrc interpreters requested and found, but other paths were also specified,
so interpreters may not be restricted to the pexrc ones. Full search path is:
{":".join(expanded)}
"""
)
)
return tuple(expanded)

@classmethod
def _expand_interpreter_search_paths(
cls,
interpreter_search_paths: Sequence[str],
env: EnvironmentVars,
asdf_standard_tool_paths: tuple[str, ...],
asdf_local_tool_paths: tuple[str, ...],
):
special_strings = {
"<PEXRC>": cls.get_pex_python_paths,
"<PATH>": lambda: cls.get_environment_paths(env),
"<ASDF>": lambda: asdf_standard_tool_paths,
"<ASDF_LOCAL>": lambda: asdf_local_tool_paths,
"<PYENV>": lambda: cls.get_pyenv_paths(env),
"<PYENV_LOCAL>": lambda: cls.get_pyenv_paths(env, pyenv_local=True),
}
expanded = []
from_pexrc = None
for s in interpreter_search_paths:
if s in special_strings:
special_paths = special_strings[s]()
if s == "<PEXRC>":
from_pexrc = special_paths
expanded.extend(special_paths)
else:
expanded.append(s)
# Some special-case logging to avoid misunderstandings.
if from_pexrc and len(expanded) > len(from_pexrc):
logger.info(
softwrap(
f"""
pexrc interpreters requested and found, but other paths were also specified,
so interpreters may not be restricted to the pexrc ones. Full search path is:

{":".join(expanded)}
"""
)
)
return expanded

@staticmethod
def get_environment_paths(env: EnvironmentVars):
"""Returns a list of paths specified by the PATH env var."""
pathstr = env.get("PATH")
if pathstr:
return pathstr.split(os.pathsep)
def _get_environment_paths(env: EnvironmentVars):
"""Returns a list of paths specified by the PATH env var."""
pathstr = env.get("PATH")
if pathstr:
return pathstr.split(os.pathsep)
return []


def _get_pex_python_paths():
"""Returns a list of paths to Python interpreters as defined in a pexrc file.
These are provided by a PEX_PYTHON_PATH in either of '/etc/pexrc', '~/.pexrc'. PEX_PYTHON_PATH
defines a colon-separated list of paths to interpreters that a pex can be built and run against.
"""
ppp = Variables.from_rc().get("PEX_PYTHON_PATH")
if ppp:
return ppp.split(os.pathsep)
else:
return []

@staticmethod
def get_pex_python_paths():
"""Returns a list of paths to Python interpreters as defined in a pexrc file.

These are provided by a PEX_PYTHON_PATH in either of '/etc/pexrc', '~/.pexrc'.
PEX_PYTHON_PATH defines a colon-separated list of paths to interpreters that a pex can be
built and run against.
"""
ppp = Variables.from_rc().get("PEX_PYTHON_PATH")
if ppp:
return ppp.split(os.pathsep)
else:
return []
def _contains_asdf_path_tokens(interpreter_search_paths: Iterable[str]) -> tuple[bool, bool]:
"""Returns tuple of whether the path list contains standard or local ASDF path tokens."""
standard_path_token = False
local_path_token = False
for interpreter_search_path in interpreter_search_paths:
if interpreter_search_path == "<ASDF>":
standard_path_token = True
elif interpreter_search_path == "<ASDF_LOCAL>":
local_path_token = True
return standard_path_token, local_path_token

@staticmethod
def contains_asdf_path_tokens(interpreter_search_paths: Iterable[str]) -> tuple[bool, bool]:
"""Returns tuple of whether the path list contains standard or local ASDF path tokens."""
standard_path_token = False
local_path_token = False
for interpreter_search_path in interpreter_search_paths:
if interpreter_search_path == "<ASDF>":
standard_path_token = True
elif interpreter_search_path == "<ASDF_LOCAL>":
local_path_token = True
return standard_path_token, local_path_token

@staticmethod
def get_pyenv_paths(env: EnvironmentVars, *, pyenv_local: bool = False) -> list[str]:
"""Returns a list of paths to Python interpreters managed by pyenv.
:param env: The environment to use to look up pyenv.
:param bool pyenv_local: If True, only use the interpreter specified by
'.python-version' file under `build_root`.
"""
pyenv_root = get_pyenv_root(env)
if not pyenv_root:
return []

versions_dir = Path(pyenv_root, "versions")
if not versions_dir.is_dir():
return []
def _get_pyenv_paths(env: EnvironmentVars, *, pyenv_local: bool = False) -> list[str]:
"""Returns a list of paths to Python interpreters managed by pyenv.
if pyenv_local:
local_version_file = Path(get_buildroot(), ".python-version")
if not local_version_file.exists():
logger.warning(
softwrap(
"""
No `.python-version` file found in the build root,
but <PYENV_LOCAL> was set in `[python-bootstrap].search_path`.
"""
)
)
return []
:param env: The environment to use to look up pyenv.
:param bool pyenv_local: If True, only use the interpreter specified by
'.python-version' file under `build_root`.
"""
pyenv_root = get_pyenv_root(env)
if not pyenv_root:
return []

local_version = local_version_file.read_text().strip()
path = Path(versions_dir, local_version, "bin")
if path.is_dir():
return [str(path)]
versions_dir = Path(pyenv_root, "versions")
if not versions_dir.is_dir():
return []

if pyenv_local:
local_version_file = Path(get_buildroot(), ".python-version")
if not local_version_file.exists():
logger.warning(
softwrap(
"""
No `.python-version` file found in the build root,
but <PYENV_LOCAL> was set in `[python-bootstrap].search_path`.
"""
)
)
return []

paths = []
for version in sorted(versions_dir.iterdir()):
path = Path(versions_dir, version, "bin")
if path.is_dir():
paths.append(str(path))
return paths
local_version = local_version_file.read_text().strip()
path = Path(versions_dir, local_version, "bin")
if path.is_dir():
return [str(path)]
return []

paths = []
for version in sorted(versions_dir.iterdir()):
path = Path(versions_dir, version, "bin")
if path.is_dir():
paths.append(str(path))
return paths


def get_pyenv_root(env: EnvironmentVars) -> str | None:
Expand All @@ -236,7 +222,7 @@ async def python_bootstrap(
interpreter_search_paths = python_bootstrap_subsystem.search_path
interpreter_names = python_bootstrap_subsystem.names

has_standard_path_token, has_local_path_token = PythonBootstrap.contains_asdf_path_tokens(
has_standard_path_token, has_local_path_token = _contains_asdf_path_tokens(
interpreter_search_paths
)
result = await Get(
Expand All @@ -251,12 +237,16 @@ async def python_bootstrap(
),
)

expanded_paths = _expand_interpreter_search_paths(
interpreter_search_paths,
result.env,
result.standard_tool_paths,
result.local_tool_paths,
)

return PythonBootstrap(
interpreter_names=interpreter_names,
raw_interpreter_search_paths=interpreter_search_paths,
environment=result.env,
asdf_standard_tool_paths=result.standard_tool_paths,
asdf_local_tool_paths=result.local_tool_paths,
interpreter_search_paths=expanded_paths,
)


Expand Down
Loading

0 comments on commit 7020ae2

Please sign in to comment.