diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index 3eb6d724e..f6590d2b7 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -1,19 +1,51 @@ # Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import absolute_import + +import itertools +import logging import os.path from argparse import ArgumentParser, _ActionsContainer +from pex import pex_warnings from pex.cli.command import BuildTimeCommand from pex.commands.command import JsonMixin, OutputMixin -from pex.common import is_script +from pex.common import DETERMINISTIC_DATETIME, is_script, open_zip, pluralize +from pex.dist_metadata import Distribution +from pex.enum import Enum +from pex.executor import Executor +from pex.pex import PEX from pex.pex_info import PexInfo -from pex.result import Error, Ok, Result +from pex.resolve import configured_resolve, requirement_options, resolver_options, target_options +from pex.resolve.resolver_configuration import ( + LockRepositoryConfiguration, + PexRepositoryConfiguration, +) +from pex.result import Error, Ok, Result, try_ +from pex.targets import LocalInterpreter, Target, Targets +from pex.tracer import TRACER from pex.typing import TYPE_CHECKING +from pex.venv import installer, installer_options +from pex.venv.install_scope import InstallScope +from pex.venv.installer import Provenance +from pex.venv.installer_configuration import InstallerConfiguration from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Any, Dict + from typing import Any, Dict, Iterable, Optional + + +logger = logging.getLogger(__name__) + + +class InstallLayout(Enum["InstallLayout.Value"]): + class Value(Enum.Value): + pass + + VENV = Value("venv") + FLAT = Value("flat") + FLAT_ZIPPED = Value("flat-zipped") class Venv(OutputMixin, JsonMixin, BuildTimeCommand): @@ -27,6 +59,60 @@ def _add_inspect_arguments(cls, parser): cls.add_output_option(parser, entity="venv information") cls.add_json_options(parser, entity="venv information") + @classmethod + def _add_create_arguments(cls, parser): + # type: (_ActionsContainer) -> None + parser.add_argument( + "-d", + "--dir", + "--dest-dir", + dest="dest_dir", + metavar="VENV_DIR", + required=True, + help=( + "The directory to install the venv or flat layout in. If the layout is " + "{flat_zipped}, then the directory will be installed to and then the zip created " + "at the same path with a '.zip' extension.".format( + flat_zipped=InstallLayout.FLAT_ZIPPED + ) + ), + ) + parser.add_argument( + "--prefix", + dest="prefix", + help=( + "A prefix directory to nest the installation in under the dest dir. This is mainly " + "useful in the {flat_zipped} layout to inject a fixed prefix to all zip " + "entries".format(flat_zipped=InstallLayout.FLAT_ZIPPED) + ), + ) + parser.add_argument( + "--layout", + default=InstallLayout.VENV, + choices=InstallLayout.values(), + type=InstallLayout.for_value, + help=( + "The layout to create. By default, this is a standard {venv} layout including " + "activation scripts and a hermetic `sys.path`. The {flat} and {flat_zipped} " + "layouts can be selected when just the `sys.path` entries are desired. This" + "effectively exports what would otherwise be the venv `site-packages` directory as " + "a flat directory that can be joined to the `sys.path` of a compatible" + "interpreter. These layouts are useful for runtimes that supply an isolated Python " + "runtime already like AWS Lambda. As a technical detail, these flat layouts " + "emulate the result of `pip install --target` and include non `site-packages` " + "installation artifacts at the top level. The common example being a top-level " + "`bin/` dir containing console scripts.".format( + venv=InstallLayout.VENV, + flat=InstallLayout.FLAT, + flat_zipped=InstallLayout.FLAT_ZIPPED, + ) + ), + ) + installer_options.register(parser) + target_options.register(parser, include_platforms=True) + resolver_options.register(parser, include_pex_repository=True, include_lock=True) + requirement_options.register(parser) + @classmethod def add_extra_arguments( cls, @@ -44,6 +130,13 @@ def add_extra_arguments( include_verbosity=False, ) as inspect_parser: cls._add_inspect_arguments(inspect_parser) + with subcommands.parser( + name="create", + help="Create a venv.", + func=cls._create, + include_verbosity=True, + ) as create_parser: + cls._add_create_arguments(create_parser) def _inspect(self): # type: () -> Result @@ -101,3 +194,218 @@ def _inspect(self): out.write("\n") return Ok() + + def _create(self): + # type: () -> Result + + targets = target_options.configure(self.options).resolve_targets() + installer_configuration = installer_options.configure(self.options) + + dest_dir = ( + os.path.join(self.options.dest_dir, self.options.prefix) + if self.options.prefix + else self.options.dest_dir + ) + update = os.path.exists(dest_dir) and not installer_configuration.force + layout = self.options.layout + + subject = "venv" if layout is InstallLayout.VENV else "flat sys.path directory entry" + + venv = None # type: Optional[Virtualenv] + if update and layout is InstallLayout.VENV: + venv = Virtualenv(venv_dir=dest_dir) + target = LocalInterpreter.create(venv.interpreter) # type: Target + specified_target = try_( + targets.require_at_most_one_target( + purpose="updating venv at {dest_dir}".format(dest_dir=dest_dir) + ) + ) + if specified_target: + if specified_target.is_foreign: + return Error( + "Cannot update a local venv using a foreign platform. Given: " + "{platform}.".format(platform=specified_target.platform) + ) + original_interpreter = venv.interpreter.resolve_base_interpreter() + specified_interpreter = ( + specified_target.get_interpreter().resolve_base_interpreter() + ) + if specified_interpreter != original_interpreter: + return Error( + "Cannot update venv at {dest_dir} created with {original_python} using " + "{specified_python}".format( + dest_dir=dest_dir, + original_python=original_interpreter.binary, + specified_python=specified_interpreter.binary, + ) + ) + targets = Targets.from_target(target) + else: + target = try_( + targets.require_unique_target( + purpose="creating a {subject}".format(subject=subject) + ) + ) + if layout is InstallLayout.VENV: + if target.is_foreign: + return Error( + "Cannot create a local venv for foreign platform {platform}.".format( + platform=target.platform + ) + ) + + venv = Virtualenv.create( + venv_dir=dest_dir, + interpreter=target.get_interpreter(), + force=installer_configuration.force, + copies=installer_configuration.copies, + prompt=installer_configuration.prompt, + ) + + requirement_configuration = requirement_options.configure(self.options) + resolver_configuration = resolver_options.configure(self.options) + with TRACER.timed("Resolving distributions"): + installed = configured_resolve.resolve( + targets=targets, + requirement_configuration=requirement_configuration, + resolver_configuration=resolver_configuration, + ) + + pex = None # type: Optional[PEX] + lock = None # type: Optional[str] + if isinstance(resolver_configuration, PexRepositoryConfiguration): + pex = PEX(resolver_configuration.pex_repository, interpreter=target.get_interpreter()) + elif isinstance(resolver_configuration, LockRepositoryConfiguration): + lock = resolver_configuration.lock_file_path + + with TRACER.timed( + "Installing {count} {wheels} in {subject} at {dest_dir}".format( + count=len(installed.installed_distributions), + wheels=pluralize(installed.installed_distributions, "wheel"), + subject=subject, + dest_dir=dest_dir, + ) + ): + hermetic_scripts = not update and installer_configuration.hermetic_scripts + distributions = tuple( + installed_distribution.distribution + for installed_distribution in installed.installed_distributions + ) + provenance = ( + Provenance.create(venv=venv) + if venv + else Provenance(target_dir=dest_dir, target_python=target.get_interpreter().binary) + ) + if pex: + _install_from_pex( + pex=pex, + installer_configuration=installer_configuration, + provenance=provenance, + distributions=distributions, + dest_dir=dest_dir, + hermetic_scripts=hermetic_scripts, + venv=venv, + ) + elif venv: + installer.populate_venv_distributions( + venv=venv, + distributions=distributions, + provenance=provenance, + symlink=False, + hermetic_scripts=hermetic_scripts, + ) + else: + installer.populate_flat_distributions( + dest_dir=dest_dir, + distributions=distributions, + provenance=provenance, + symlink=False, + ) + source = ( + "PEX at {pex}".format(pex=pex.path()) + if pex + else "lock at {lock}".format(lock=lock) + if lock + else "resolved requirements" + ) + provenance.check_collisions( + collisions_ok=installer_configuration.collisions_ok, source=source + ) + + if venv and installer_configuration.pip: + with TRACER.timed("Installing Pip"): + try_( + installer.ensure_pip_installed( + venv, + distributions=distributions, + scope=installer_configuration.scope, + collisions_ok=installer_configuration.collisions_ok, + source=source, + ) + ) + + if installer_configuration.compile: + with TRACER.timed("Compiling venv sources"): + try: + target.get_interpreter().execute(["-m", "compileall", dest_dir]) + except Executor.NonZeroExit as non_zero_exit: + pex_warnings.warn("ignoring compile error {}".format(repr(non_zero_exit))) + + if layout is InstallLayout.FLAT_ZIPPED: + paths = sorted( + os.path.join(root, path) + for root, dirs, files in os.walk(dest_dir) + for path in itertools.chain(dirs, files) + ) + unprefixed_dest_dir = self.options.dest_dir + with open_zip("{dest_dir}.zip".format(dest_dir=unprefixed_dest_dir), "w") as zf: + for path in paths: + zip_entry = zf.zip_entry_from_file( + filename=path, + arcname=os.path.relpath(path, unprefixed_dest_dir), + date_time=DETERMINISTIC_DATETIME.timetuple(), + ) + zf.writestr(zip_entry.info, zip_entry.data) + + return Ok() + + +def _install_from_pex( + pex, # type: PEX + installer_configuration, # type: InstallerConfiguration + provenance, # type: Provenance + distributions, # type: Iterable[Distribution] + dest_dir, # type: str + hermetic_scripts, # type: bool + venv=None, # type: Optional[Virtualenv] +): + # type: (...) -> None + + if installer_configuration.scope in (InstallScope.ALL, InstallScope.DEPS_ONLY): + if venv: + installer.populate_venv_distributions( + venv=venv, + distributions=distributions, + provenance=provenance, + symlink=False, + hermetic_scripts=hermetic_scripts, + ) + else: + installer.populate_flat_distributions( + dest_dir=dest_dir, + distributions=distributions, + provenance=provenance, + symlink=False, + ) + + if installer_configuration.scope in (InstallScope.ALL, InstallScope.SOURCE_ONLY): + if venv: + installer.populate_venv_sources( + venv=venv, + pex=pex, + provenance=provenance, + bin_path=installer_configuration.bin_path, + hermetic_scripts=hermetic_scripts, + ) + else: + installer.populate_flat_sources(dst=dest_dir, pex=pex, provenance=provenance) diff --git a/pex/pep_376.py b/pex/pep_376.py index 98557a39e..d472eeeae 100644 --- a/pex/pep_376.py +++ b/pex/pep_376.py @@ -229,7 +229,65 @@ def stashed_path(self, *components): # type: (*str) -> str return os.path.join(self.prefix_dir, self.stash_dir, *components) - def reinstall( + @staticmethod + def _create_installed_file( + path, # type: str + dest_dir, # type: str + ): + # type: (...) -> InstalledFile + hasher = hashlib.sha256() + hashing.file_hash(path, digest=hasher) + return InstalledFile( + path=os.path.relpath(path, dest_dir), + hash=Hash.create(hasher), + size=os.stat(path).st_size, + ) + + def create_record( + self, + dst, # type: str + installed_files, # type: Iterable[InstalledFile] + ): + # type: (...) -> None + + # The RECORD is a csv file with the path to each installed file in the 1st column. + # See: https://www.python.org/dev/peps/pep-0376/#record + with safe_open(os.path.join(dst, self.record_relpath), "w") as fp: + csv_writer = cast( + "CSVWriter", + csv.writer(fp, delimiter=",", quotechar='"', lineterminator="\n"), + ) + for installed_file in sorted(installed_files, key=lambda installed: installed.path): + csv_writer.writerow(attr.astuple(installed_file, recurse=False)) + + def reinstall_flat( + self, + target_dir, # type: str + symlink=False, # type: bool + ): + # type: (...) -> Iterator[Tuple[str, str]] + """Re-installs the installed wheel in a flat target directory. + + N.B.: A record of reinstalled files is returned in the form of an iterator that must be + consumed to drive the installation to completion. + + If there is an error re-installing a file due to it already existing in the target + directory, the error is suppressed, and it's expected that the caller detects this by + comparing the record of installed files against those installed previously. + + :return: An iterator over src -> dst pairs. + """ + installed_files = [InstalledFile(self.record_relpath)] + for src, dst in itertools.chain( + self._reinstall_stash(dest_dir=target_dir), + self._reinstall_site_packages(target_dir, symlink=symlink), + ): + installed_files.append(self._create_installed_file(path=dst, dest_dir=target_dir)) + yield src, dst + + self.create_record(target_dir, installed_files) + + def reinstall_venv( self, venv, # type: Virtualenv symlink=False, # type: bool @@ -255,50 +313,36 @@ def reinstall( installed_files = [InstalledFile(self.record_relpath)] for src, dst in itertools.chain( - self._reinstall_stash(venv), + self._reinstall_stash(dest_dir=venv.venv_dir, interpreter=venv.interpreter), self._reinstall_site_packages(site_packages_dir, symlink=symlink), ): - hasher = hashlib.sha256() - hashing.file_hash(dst, digest=hasher) installed_files.append( - InstalledFile( - path=os.path.relpath(dst, site_packages_dir), - hash=Hash.create(hasher), - size=os.stat(dst).st_size, - ) + self._create_installed_file(path=dst, dest_dir=site_packages_dir) ) - yield src, dst - # The RECORD is a csv file with the path to each installed file in the 1st column. - # See: https://www.python.org/dev/peps/pep-0376/#record - with safe_open(os.path.join(site_packages_dir, self.record_relpath), "w") as fp: - csv_writer = cast( - "CSVWriter", - csv.writer(fp, delimiter=",", quotechar='"', lineterminator="\n"), - ) - for installed_file in sorted(installed_files, key=lambda installed: installed.path): - csv_writer.writerow(attr.astuple(installed_file, recurse=False)) + self.create_record(site_packages_dir, installed_files) - def _reinstall_stash(self, venv): - # type: (Virtualenv) -> Iterator[Tuple[str, str]] + def _reinstall_stash( + self, + dest_dir, # type: str + interpreter=None, # type: Optional[PythonInterpreter] + ): + # type: (...) -> Iterator[Tuple[str, str]] link = True stash_abs_path = os.path.join(self.prefix_dir, self.stash_dir) for root, dirs, files in os.walk(stash_abs_path, topdown=True, followlinks=True): - for d in dirs: - src_relpath = os.path.relpath(os.path.join(root, d), stash_abs_path) - dst = InstalledFile.denormalized_path( - path=os.path.join(venv.venv_dir, src_relpath), interpreter=venv.interpreter - ) - safe_mkdir(dst) - + dir_created = False for f in files: src = os.path.join(root, f) src_relpath = os.path.relpath(src, stash_abs_path) dst = InstalledFile.denormalized_path( - path=os.path.join(venv.venv_dir, src_relpath), interpreter=venv.interpreter + path=os.path.join(dest_dir, src_relpath), interpreter=interpreter ) + if not dir_created: + safe_mkdir(os.path.dirname(dst)) + dir_created = True try: # We only try to link regular files since linking a symlink on Linux can produce # another symlink, which leaves open the possibility the src target could later diff --git a/pex/pex_bootstrapper.py b/pex/pex_bootstrapper.py index dd755dd04..996f91244 100644 --- a/pex/pex_bootstrapper.py +++ b/pex/pex_bootstrapper.py @@ -24,6 +24,7 @@ from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast from pex.variables import ENV +from pex.venv import installer if TYPE_CHECKING: from typing import Iterable, Iterator, List, NoReturn, Optional, Set, Tuple, Union @@ -492,7 +493,6 @@ def ensure_venv( ) with atomic_directory(venv_dir) as venv: if not venv.is_finalized(): - from pex.venv.pex import populate_venv from pex.venv.virtualenv import Virtualenv virtualenv = Virtualenv.create_atomic( @@ -542,7 +542,7 @@ def ensure_venv( # modification of the source loose PEX. symlink = pex.layout != Layout.LOOSE and not pex_info.venv_site_packages_copies - shebang = populate_venv( + shebang = installer.populate_venv_from_pex( virtualenv, pex, bin_path=pex_info.venv_bin_path, diff --git a/pex/resolve/pex_repository_resolver.py b/pex/resolve/pex_repository_resolver.py index 64feb991e..f27f67d3d 100644 --- a/pex/resolve/pex_repository_resolver.py +++ b/pex/resolve/pex_repository_resolver.py @@ -12,7 +12,13 @@ from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_503 import ProjectName -from pex.requirements import Constraint, LocalProjectRequirement +from pex.pex_info import PexInfo +from pex.requirements import ( + Constraint, + LocalProjectRequirement, + parse_requirement_string, + parse_requirement_strings, +) from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolvers import Installed, InstalledDistribution, Unsatisfiable, Untranslatable from pex.targets import Targets @@ -39,12 +45,14 @@ def resolve_from_pex( requirement_files=requirement_files, constraint_files=constraint_files, ) + root_reqs = requirement_configuration.parse_requirements( + network_configuration=network_configuration + ) or parse_requirement_strings(PexInfo.from_pex(pex).requirements) + direct_requirements_by_project_name = ( OrderedDict() ) # type: OrderedDict[ProjectName, List[Requirement]] - for direct_requirement in requirement_configuration.parse_requirements( - network_configuration=network_configuration - ): + for direct_requirement in root_reqs: if isinstance(direct_requirement, LocalProjectRequirement): raise Untranslatable( "Cannot resolve local projects from PEX repositories. Asked to resolve {path} " diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index fb19eda80..40733993c 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -95,4 +95,5 @@ class PexRepositoryConfiguration(object): @attr.s(frozen=True) class LockRepositoryConfiguration(object): parse_lock = attr.ib() # type: Callable[[], Union[Lockfile, Error]] + lock_file_path = attr.ib() # type: str pip_configuration = attr.ib() # type: PipConfiguration diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 23a4c0e2c..d1b83f29e 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -416,7 +416,8 @@ def configure(options): pip_configuration = create_pip_configuration(options) if lock: return LockRepositoryConfiguration( - parse_lock=lambda: parse_lockfile(options), + parse_lock=lambda: parse_lockfile(options, lock_file_path=lock), + lock_file_path=lock, pip_configuration=pip_configuration, ) return pip_configuration diff --git a/pex/targets.py b/pex/targets.py index c2afb858e..8a103e072 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -356,3 +356,24 @@ def require_unique_target(self, purpose): ) ) return cast(Target, next(iter(resolved_targets))) + + def require_at_most_one_target(self, purpose): + # type: (str) -> Union[Optional[Target], Error] + resolved_targets = self.unique_targets(only_explicit=False) + if len(resolved_targets) > 1: + return Error( + "At most a single target is required for {purpose}.\n" + "There were {count} targets selected:\n" + "{targets}".format( + purpose=purpose, + count=len(resolved_targets), + targets="\n".join( + "{index}. {target}".format(index=index, target=target) + for index, target in enumerate(resolved_targets, start=1) + ), + ) + ) + try: + return cast(Target, next(iter(resolved_targets))) + except StopIteration: + return None diff --git a/pex/tools/commands/venv.py b/pex/tools/commands/venv.py index a289bfd51..f61bcc45a 100644 --- a/pex/tools/commands/venv.py +++ b/pex/tools/commands/venv.py @@ -6,30 +6,22 @@ import errno import logging import os -import subprocess from argparse import ArgumentParser -from collections import OrderedDict -from subprocess import CalledProcessError from pex import pex_warnings from pex.common import safe_delete, safe_rmtree -from pex.dist_metadata import Distribution from pex.enum import Enum from pex.executor import Executor -from pex.pep_440 import Version -from pex.pep_503 import ProjectName from pex.pex import PEX -from pex.result import Error, Ok, Result, try_ +from pex.result import Ok, Result, try_ from pex.tools.command import PEXCommand -from pex.tracer import TRACER from pex.typing import TYPE_CHECKING -from pex.venv.bin_path import BinPath +from pex.venv import installer, installer_options from pex.venv.install_scope import InstallScope -from pex.venv.pex import populate_venv -from pex.venv.virtualenv import PipUnavailableError, Virtualenv +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Iterable, Optional, Union + from typing import Optional import attr # vendor:skip else: @@ -80,104 +72,6 @@ def save(self, install_scope): fp.write(str(install_scope)) -def find_dist( - project_name, # type: ProjectName - dists, # type: Iterable[Distribution] -): - # type: (...) -> Optional[Version] - for dist in dists: - if project_name == dist.metadata.project_name: - return dist.metadata.version - return None - - -_PIP = ProjectName("pip") -_SETUPTOOLS = ProjectName("setuptools") - - -def ensure_pip_installed( - venv, # type: Virtualenv - pex, # type: PEX - scope, # type: InstallScope.Value - collisions_ok, # type: bool -): - # type: (...) -> Union[Version, Error] - - venv_pip_version = find_dist(_PIP, venv.iter_distributions()) - if venv_pip_version: - TRACER.log( - "The venv at {venv_dir} already has Pip {version} installed.".format( - venv_dir=venv.venv_dir, version=venv_pip_version - ) - ) - else: - try: - venv.install_pip() - except PipUnavailableError as e: - return Error( - "The virtual environment was successfully created, but Pip was not " - "installed:\n{}".format(e) - ) - venv_pip_version = find_dist(_PIP, venv.iter_distributions()) - if not venv_pip_version: - return Error( - "Failed to install pip into venv at {venv_dir}".format(venv_dir=venv.venv_dir) - ) - - if InstallScope.SOURCE_ONLY == scope: - return venv_pip_version - - uninstall = OrderedDict() - pex_pip_version = find_dist(_PIP, pex.resolve()) - if pex_pip_version and pex_pip_version != venv_pip_version: - uninstall[_PIP] = pex_pip_version - - venv_setuptools_version = find_dist(_SETUPTOOLS, venv.iter_distributions()) - if venv_setuptools_version: - pex_setuptools_version = find_dist(_SETUPTOOLS, pex.resolve()) - if pex_setuptools_version and venv_setuptools_version != pex_setuptools_version: - uninstall[_SETUPTOOLS] = pex_setuptools_version - - if not uninstall: - return venv_pip_version - - message = ( - "You asked for --pip to be installed in the venv at {venv_dir},\n" - "but the PEX at {pex} already contains:\n{distributions}" - ).format( - venv_dir=venv.venv_dir, - pex=pex.path(), - distributions="\n".join( - "{project_name} {version}".format(project_name=project_name, version=version) - for project_name, version in uninstall.items() - ), - ) - if not collisions_ok: - return Error( - "{message}\nConsider re-running either without --pip or with --collisions-ok.".format( - message=message - ) - ) - - pex_warnings.warn( - "{message}\nUninstalling venv versions and using versions from the PEX.".format( - message=message - ) - ) - projects_to_uninstall = sorted(str(project_name) for project_name in uninstall) - try: - subprocess.check_call( - args=[venv.interpreter.binary, "-m", "pip", "uninstall", "-y"] + projects_to_uninstall - ) - except CalledProcessError as e: - return Error( - "Failed to uninstall venv versions of {projects}: {err}".format( - projects=" and ".join(projects_to_uninstall), err=e - ) - ) - return pex_pip_version or venv_pip_version - - class Venv(PEXCommand): """Creates a venv from the PEX file.""" @@ -190,77 +84,7 @@ def add_arguments(cls, parser): metavar="PATH", help="The directory to create the virtual environment in.", ) - parser.add_argument( - "--scope", - default=InstallScope.ALL.value, - choices=InstallScope.values(), - type=InstallScope.for_value, - help=( - "The scope of code contained in the Pex that is installed in the venv. By default" - "{all} code is installed and this is generally what you want. However, in some " - "situations it's beneficial to split the venv installation into {deps} and " - "{sources} steps. This is particularly useful when installing a PEX in a container " - "image. See " - "https://pex.readthedocs.io/en/latest/recipes.html#pex-app-in-a-container for more " - "information.".format( - all=InstallScope.ALL, - deps=InstallScope.DEPS_ONLY, - sources=InstallScope.SOURCE_ONLY, - ) - ), - ) - parser.add_argument( - "-b", - "--bin-path", - default=BinPath.FALSE.value, - choices=BinPath.values(), - type=BinPath.for_value, - help="Add the venv bin dir to the PATH in the __main__.py script.", - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help="If the venv directory already exists, overwrite it.", - ) - parser.add_argument( - "--collisions-ok", - action="store_true", - default=False, - help=( - "Don't error if population of the venv encounters distributions in the PEX file " - "with colliding files, just emit a warning." - ), - ) - parser.add_argument( - "-p", - "--pip", - action="store_true", - default=False, - help=( - "Add pip (and setuptools) to the venv. If the PEX already contains its own " - "conflicting versions pip (or setuptools), the command will error and you must " - "pass --collisions-ok to have the PEX versions over-ride the natural venv versions " - "installed by --pip." - ), - ) - parser.add_argument( - "--copies", - action="store_true", - default=False, - help="Create the venv using copies of system files instead of symlinks", - ) - parser.add_argument( - "--compile", - action="store_true", - default=False, - help="Compile all `.py` files in the venv.", - ) - parser.add_argument( - "--prompt", - help="A custom prompt for the venv activation scripts to use.", - ) + installer_options.register(parser, include_force_switch=True) parser.add_argument( "--rm", "--remove", @@ -276,60 +100,55 @@ def add_arguments(cls, parser): ) ), ) - parser.add_argument( - "--non-hermetic-scripts", - dest="hermetic_scripts", - action="store_false", - default=True, - help=( - "Don't rewrite Python script shebangs in the venv to pass `-sE` to the " - "interpreter; for example, to enable running the venv PEX itself or its Python " - "scripts with a custom `PYTHONPATH`." - ), - ) cls.register_global_arguments(parser, include_verbosity=False) def run(self, pex): # type: (PEX) -> Result + installer_configuration = installer_options.configure(self.options) + venv_dir = self.options.venv[0] install_scope_state = InstallScopeState.load(venv_dir) - if install_scope_state.is_partial_install and not self.options.force: + if install_scope_state.is_partial_install and not installer_configuration.force: venv = Virtualenv(venv_dir) else: venv = Virtualenv.create( venv_dir, interpreter=pex.interpreter, - force=self.options.force, - copies=self.options.copies, - prompt=self.options.prompt, + force=installer_configuration.force, + copies=installer_configuration.copies, + prompt=installer_configuration.prompt, ) - if self.options.pip: + if installer_configuration.pip: try_( - ensure_pip_installed( - venv, pex, scope=self.options.scope, collisions_ok=self.options.collisions_ok + installer.ensure_pip_installed( + venv, + distributions=tuple(pex.resolve()), + scope=installer_configuration.scope, + collisions_ok=installer_configuration.collisions_ok, + source="PEX at {pex}".format(pex=pex.path()), ) ) - if self.options.prompt != venv.custom_prompt: + if installer_configuration.prompt != venv.custom_prompt: logger.warning( "Unable to apply custom --prompt {prompt!r} in {python} venv; continuing with the " "default prompt.".format( - prompt=self.options.prompt, python=venv.interpreter.identity + prompt=installer_configuration.prompt, python=venv.interpreter.identity ) ) - populate_venv( + installer.populate_venv_from_pex( venv, pex, - bin_path=self.options.bin_path, - collisions_ok=self.options.collisions_ok, + bin_path=installer_configuration.bin_path, + collisions_ok=installer_configuration.collisions_ok, symlink=False, - scope=self.options.scope, - hermetic_scripts=self.options.hermetic_scripts, + scope=installer_configuration.scope, + hermetic_scripts=installer_configuration.hermetic_scripts, ) - if self.options.compile: + if installer_configuration.compile: try: pex.interpreter.execute(["-m", "compileall", venv_dir]) except Executor.NonZeroExit as non_zero_exit: @@ -343,5 +162,5 @@ def run(self, pex): if self.options.remove is RemoveScope.PEX_AND_PEX_ROOT: safe_rmtree(pex.pex_info().pex_root) - install_scope_state.save(self.options.scope) + install_scope_state.save(installer_configuration.scope) return Ok() diff --git a/pex/venv/pex.py b/pex/venv/installer.py similarity index 71% rename from pex/venv/pex.py rename to pex/venv/installer.py index 46f736dbb..58bef2119 100644 --- a/pex/venv/pex.py +++ b/pex/venv/installer.py @@ -1,4 +1,4 @@ -# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import absolute_import, print_function @@ -7,7 +7,8 @@ import itertools import os import shutil -from collections import Counter, defaultdict +import subprocess +from collections import Counter, OrderedDict, defaultdict from textwrap import dedent from pex import layout, pex_warnings @@ -17,17 +18,119 @@ from pex.environment import PEXEnvironment from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel, LoadError +from pex.pep_440 import Version +from pex.pep_503 import ProjectName from pex.pex import PEX +from pex.result import Error from pex.tracer import TRACER from pex.typing import TYPE_CHECKING from pex.util import CacheHelper from pex.venv.bin_path import BinPath from pex.venv.install_scope import InstallScope -from pex.venv.virtualenv import Virtualenv +from pex.venv.virtualenv import PipUnavailableError, Virtualenv if TYPE_CHECKING: import typing - from typing import Iterable, Iterator, Optional, Tuple + from typing import DefaultDict, Iterable, Iterator, List, Optional, Tuple, Union + + +def find_dist( + project_name, # type: ProjectName + dists, # type: Iterable[Distribution] +): + # type: (...) -> Optional[Version] + for dist in dists: + if project_name == dist.metadata.project_name: + return dist.metadata.version + return None + + +_PIP = ProjectName("pip") +_SETUPTOOLS = ProjectName("setuptools") + + +def ensure_pip_installed( + venv, # type: Virtualenv + distributions, # type: Iterable[Distribution] + scope, # type: InstallScope.Value + collisions_ok, # type: bool + source, # type: str +): + # type: (...) -> Union[Version, Error] + + venv_pip_version = find_dist(_PIP, venv.iter_distributions()) + if venv_pip_version: + TRACER.log( + "The venv at {venv_dir} already has Pip {version} installed.".format( + venv_dir=venv.venv_dir, version=venv_pip_version + ) + ) + else: + try: + venv.install_pip() + except PipUnavailableError as e: + return Error( + "The virtual environment was successfully created, but Pip was not " + "installed:\n{}".format(e) + ) + venv_pip_version = find_dist(_PIP, venv.iter_distributions()) + if not venv_pip_version: + return Error( + "Failed to install pip into venv at {venv_dir}".format(venv_dir=venv.venv_dir) + ) + + if InstallScope.SOURCE_ONLY == scope: + return venv_pip_version + + uninstall = OrderedDict() + pex_pip_version = find_dist(_PIP, distributions) + if pex_pip_version and pex_pip_version != venv_pip_version: + uninstall[_PIP] = pex_pip_version + + venv_setuptools_version = find_dist(_SETUPTOOLS, venv.iter_distributions()) + if venv_setuptools_version: + pex_setuptools_version = find_dist(_SETUPTOOLS, distributions) + if pex_setuptools_version and venv_setuptools_version != pex_setuptools_version: + uninstall[_SETUPTOOLS] = pex_setuptools_version + + if not uninstall: + return venv_pip_version + + message = ( + "You asked for --pip to be installed in the venv at {venv_dir},\n" + "but the {source} already contains:\n{distributions}" + ).format( + venv_dir=venv.venv_dir, + source=source, + distributions="\n".join( + "{project_name} {version}".format(project_name=project_name, version=version) + for project_name, version in uninstall.items() + ), + ) + if not collisions_ok: + return Error( + "{message}\nConsider re-running either without --pip or with --collisions-ok.".format( + message=message + ) + ) + + pex_warnings.warn( + "{message}\nUninstalling venv versions and using versions from the PEX.".format( + message=message + ) + ) + projects_to_uninstall = sorted(str(project_name) for project_name in uninstall) + try: + subprocess.check_call( + args=[venv.interpreter.binary, "-m", "pip", "uninstall", "-y"] + projects_to_uninstall + ) + except subprocess.CalledProcessError as e: + return Error( + "Failed to uninstall venv versions of {projects}: {err}".format( + projects=" and ".join(projects_to_uninstall), err=e + ) + ) + return pex_pip_version or venv_pip_version def _relative_symlink( @@ -42,7 +145,7 @@ def _relative_symlink( # N.B.: We can't use shutil.copytree since we copy from multiple source locations to the same site # packages directory destination. Since we're forced to stray from the stdlib here, support for -# hardlinks is added to provide a measurable speed up and disk space savings when possible. +# hardlinks is added to provide a measurable speed-up and disk space savings when possible. def _copytree( src, # type: str dst, # type: str @@ -95,12 +198,187 @@ class CollisionError(Exception): """Indicates multiple distributions provided the same file when merging a PEX into a venv.""" +class Provenance(object): + @classmethod + def create( + cls, + venv, # type: Virtualenv + python=None, # type: Optional[str] + ): + # type: (...) -> Provenance + venv_bin_dir = os.path.dirname(python) if python else venv.bin_dir + venv_dir = os.path.dirname(venv_bin_dir) if python else venv.venv_dir + + venv_python = python or venv.interpreter.binary + return cls(target_dir=venv_dir, target_python=venv_python) + + def __init__( + self, + target_dir, # type: str + target_python, # type: str + ): + # type: (...) -> None + self._target_dir = target_dir + self._target_python = target_python + self._provenance = defaultdict(list) # type: DefaultDict[str, List[str]] + + @property + def target_python(self): + # type: () -> str + return self._target_python + + def calculate_shebang(self, hermetic_scripts=True): + # type: (bool) -> str + + shebang_argv = [self.target_python] + python_args = _script_python_args(hermetic=hermetic_scripts) + if python_args: + shebang_argv.append(python_args) + return "#!{shebang}".format(shebang=" ".join(shebang_argv)) + + def record(self, src_to_dst): + # type: (Iterable[Tuple[str, str]]) -> None + for src, dst in src_to_dst: + self._provenance[dst].append(src) + + def check_collisions( + self, + collisions_ok=False, # type: bool + source=None, # type: Optional[str] + ): + # type: (...) -> None + + potential_collisions = { + dst: srcs for dst, srcs in self._provenance.items() if len(srcs) > 1 + } + if not potential_collisions: + return + + collisions = {} + for dst, srcs in potential_collisions.items(): + contents = defaultdict(list) + for src in srcs: + contents[CacheHelper.hash(src)].append(src) + if len(contents) > 1: + collisions[dst] = contents + + if not collisions: + return + + message_lines = [ + "Encountered {collision} populating {target_dir}{source}:".format( + collision=pluralize(collisions, "collision"), + target_dir=self._target_dir, + source=" from {source}".format(source=source) if source else "", + ) + ] + for index, (dst, contents) in enumerate(collisions.items(), start=1): + message_lines.append( + "{index}. {dst} was provided by:\n\t{srcs}".format( + index=index, + dst=dst, + srcs="\n\t".join( + "sha1:{fingerprint} -> {srcs}".format( + fingerprint=fingerprint, srcs=", ".join(srcs) + ) + for fingerprint, srcs in contents.items() + ), + ) + ) + message = "\n".join(message_lines) + if not collisions_ok: + raise CollisionError(message) + pex_warnings.warn(message) + + def _script_python_args(hermetic): # type: (bool) -> Optional[str] return "-sE" if hermetic else None -def populate_venv( +def _populate_flat_deps( + dest_dir, # type: str + distributions, # type: Iterable[Distribution] + symlink=False, # type: bool +): + # type: (...) -> Iterator[Tuple[str, str]] + for dist in distributions: + try: + installed_wheel = InstalledWheel.load(dist.location) + for src, dst in installed_wheel.reinstall_flat(target_dir=dest_dir, symlink=symlink): + yield src, dst + except LoadError: + for src, dst in _populate_legacy_dist( + dest_dir=dest_dir, bin_dir=dest_dir, dist=dist, symlink=symlink + ): + yield src, dst + + +def populate_flat_distributions( + dest_dir, # type: str + distributions, # type: Iterable[Distribution] + provenance, # type: Provenance + symlink=False, # type: bool +): + # type: (...) -> None + + provenance.record( + _populate_flat_deps(dest_dir=dest_dir, distributions=distributions, symlink=symlink) + ) + + +def populate_venv_distributions( + venv, # type: Virtualenv + distributions, # type: Iterable[Distribution] + provenance, # type: Provenance + symlink=False, # type: bool + hermetic_scripts=True, # type: bool +): + # type: (...) -> None + + provenance.record( + _populate_venv_deps( + venv=venv, + distributions=distributions, + venv_python=provenance.target_python, + symlink=symlink, + hermetic_scripts=hermetic_scripts, + ) + ) + + +def populate_flat_sources( + dst, # type: str + pex, # type: PEX + provenance, # type: Provenance +): + provenance.record(_populate_sources(pex=pex, dst=dst)) + + +def populate_venv_sources( + venv, # type: Virtualenv + pex, # type: PEX + provenance, # type: Provenance + bin_path=BinPath.FALSE, # type: BinPath.Value + hermetic_scripts=True, # type: bool + shebang=None, # type: Optional[str] +): + # type: (...) -> str + + shebang = shebang or provenance.calculate_shebang(hermetic_scripts=hermetic_scripts) + provenance.record( + _populate_first_party( + venv=venv, + pex=pex, + shebang=shebang, + venv_python=provenance.target_python, + bin_path=bin_path, + ) + ) + return shebang + + +def populate_venv_from_pex( venv, # type: Virtualenv pex, # type: PEX bin_path=BinPath.FALSE, # type: BinPath.Value @@ -112,78 +390,39 @@ def populate_venv( ): # type: (...) -> str - venv_python = python or venv.interpreter.binary - - shebang_argv = [venv_python] - python_args = _script_python_args(hermetic=hermetic_scripts) - if python_args: - shebang_argv.append(python_args) - shebang = "#!{shebang}".format(shebang=" ".join(shebang_argv)) - - provenance = defaultdict(list) - - def record_provenance(src_to_dst): - # type: (Iterable[Tuple[str, str]]) -> None - for src, dst in src_to_dst: - provenance[dst].append(src) + provenance = Provenance.create(venv, python=python) + shebang = provenance.calculate_shebang(hermetic_scripts=hermetic_scripts) if scope in (InstallScope.ALL, InstallScope.DEPS_ONLY): - record_provenance(_populate_deps(venv, pex, venv_python, symlink, hermetic_scripts)) + populate_venv_distributions( + venv=venv, + distributions=pex.resolve(), + symlink=symlink, + hermetic_scripts=hermetic_scripts, + provenance=provenance, + ) if scope in (InstallScope.ALL, InstallScope.SOURCE_ONLY): - record_provenance(_populate_sources(venv, pex, shebang, venv_python, bin_path)) - - potential_collisions = {dst: srcs for dst, srcs in provenance.items() if len(srcs) > 1} - if potential_collisions: - collisions = {} - for dst, srcs in potential_collisions.items(): - contents = defaultdict(list) - for src in srcs: - contents[CacheHelper.hash(src)].append(src) - if len(contents) > 1: - collisions[dst] = contents + populate_venv_sources( + venv=venv, + pex=pex, + bin_path=bin_path, + hermetic_scripts=hermetic_scripts, + provenance=provenance, + shebang=shebang, + ) - if collisions: - venv_bin_dir = os.path.dirname(python) if python else venv.bin_dir - venv_dir = os.path.dirname(venv_bin_dir) if python else venv.venv_dir - message_lines = [ - "Encountered {collision} building venv at {venv_dir} from {pex}:".format( - collision=pluralize(collisions, "collision"), venv_dir=venv_dir, pex=pex.path() - ) - ] - for index, (dst, contents) in enumerate(collisions.items(), start=1): - message_lines.append( - "{index}. {dst} was provided by:\n\t{srcs}".format( - index=index, - dst=dst, - srcs="\n\t".join( - "sha1:{fingerprint} -> {srcs}".format( - fingerprint=fingerprint, srcs=", ".join(srcs) - ) - for fingerprint, srcs in contents.items() - ), - ) - ) - message = "\n".join(message_lines) - if not collisions_ok: - raise CollisionError(message) - pex_warnings.warn(message) + provenance.check_collisions(collisions_ok, source="PEX at {pex}".format(pex=pex.path())) return shebang def _populate_legacy_dist( - venv, # type: Virtualenv + dest_dir, # type: str + bin_dir, # type: str dist, # type: Distribution symlink=False, # type: bool - rel_extra_path=None, # type: Optional[str] ): - dst = ( - os.path.join(venv.site_packages_dir, rel_extra_path) - if rel_extra_path - else venv.site_packages_dir - ) - # N.B.: We do not include the top_level __pycache__ for a dist since there may be # multiple dists with top-level modules. In that case, one dists top-level __pycache__ # would be symlinked and all dists with top-level modules would have the .pyc files for @@ -191,19 +430,19 @@ def _populate_legacy_dist( # just 1 top-level module, we keep .pyc anchored to their associated dists when shared # and accept the cost of re-compiling top-level modules in each venv that uses them. for src, dst in _copytree( - src=dist.location, dst=dst, exclude=("bin", "__pycache__"), symlink=symlink + src=dist.location, dst=dest_dir, exclude=("bin", "__pycache__"), symlink=symlink ): yield src, dst dist_bin_dir = os.path.join(dist.location, "bin") if os.path.isdir(dist_bin_dir): - for src, dst in _copytree(src=dist_bin_dir, dst=venv.bin_dir, symlink=symlink): + for src, dst in _copytree(src=dist_bin_dir, dst=bin_dir, symlink=symlink): yield src, dst -def _populate_deps( +def _populate_venv_deps( venv, # type: Virtualenv - pex, # type: PEX + distributions, # type: Iterable[Distribution] venv_python, # type: str symlink=False, # type: bool hermetic_scripts=True, # type: bool @@ -215,7 +454,7 @@ def _populate_deps( # created in ~/.pex/venvs. top_level_packages = Counter() # type: typing.Counter[str] rel_extra_paths = OrderedSet() # type: OrderedSet[str] - for dist in pex.resolve(): + for dist in distributions: rel_extra_path = None if symlink: # In the symlink case, in order to share all generated *.pyc files for a given @@ -257,13 +496,18 @@ def _populate_deps( try: installed_wheel = InstalledWheel.load(dist.location) - for src, dst in installed_wheel.reinstall( + for src, dst in installed_wheel.reinstall_venv( venv, symlink=symlink, rel_extra_path=rel_extra_path ): yield src, dst except LoadError: + dst = ( + os.path.join(venv.site_packages_dir, rel_extra_path) + if rel_extra_path + else venv.site_packages_dir + ) for src, dst in _populate_legacy_dist( - venv, dist, symlink=symlink, rel_extra_path=rel_extra_path + dest_dir=dst, bin_dir=venv.bin_dir, dist=dist, symlink=symlink ): yield src, dst @@ -293,27 +537,19 @@ def _populate_deps( def _populate_sources( - venv, # type: Virtualenv pex, # type: PEX - shebang, # type: str - venv_python, # type: str - bin_path, # type: BinPath.Value + dst, # type: str ): # type: (...) -> Iterator[Tuple[str, str]] - # We want the venv at rest to reflect the PEX it was created from at rest; as such we use the - # PEX's at-rest PEX-INFO to perform the layout. The venv can then be executed with various PEX - # environment variables in-play that it respects (e.g.: PEX_EXTRA_SYS_PATH, PEX_INTERPRETER, - # PEX_MODULE, etc.). - pex_info = pex.pex_info(include_env_overrides=False) - # Since the pex.path() is ~always outside our control (outside ~/.pex), we copy all PEX user # sources into the venv. for src, dst in _copytree( src=PEXEnvironment.mount(pex.path()).path, - dst=venv.site_packages_dir, + dst=dst, exclude=( "__main__.py", + "__pex__", "__pycache__", layout.BOOTSTRAP_DIR, layout.DEPS_DIR, @@ -324,6 +560,25 @@ def _populate_sources( ): yield src, dst + +def _populate_first_party( + venv, # type: Virtualenv + pex, # type: PEX + shebang, # type: str + venv_python, # type: str + bin_path, # type: BinPath.Value +): + # type: (...) -> Iterator[Tuple[str, str]] + + # We want the venv at rest to reflect the PEX it was created from at rest; as such we use the + # PEX's at-rest PEX-INFO to perform the layout. The venv can then be executed with various PEX + # environment variables in-play that it respects (e.g.: PEX_EXTRA_SYS_PATH, PEX_INTERPRETER, + # PEX_MODULE, etc.). + pex_info = pex.pex_info(include_env_overrides=False) + + for src, dst in _populate_sources(pex=pex, dst=venv.site_packages_dir): + yield src, dst + with open(os.path.join(venv.site_packages_dir, "PEX_EXTRA_SYS_PATH.pth"), "w") as fp: # N.B.: .pth import lines must be single lines: https://docs.python.org/3/library/site.html for env_var in "PEX_EXTRA_SYS_PATH", "__PEX_EXTRA_SYS_PATH__": diff --git a/pex/venv/installer_configuration.py b/pex/venv/installer_configuration.py new file mode 100644 index 000000000..74fb965d8 --- /dev/null +++ b/pex/venv/installer_configuration.py @@ -0,0 +1,28 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.typing import TYPE_CHECKING +from pex.venv.bin_path import BinPath +from pex.venv.install_scope import InstallScope + +if TYPE_CHECKING: + from typing import Optional + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class InstallerConfiguration(object): + scope = attr.ib(default=InstallScope.ALL) # type: InstallScope.Value + bin_path = attr.ib(default=BinPath.FALSE) # type: BinPath.Value + force = attr.ib(default=False) # type: bool + collisions_ok = attr.ib(default=False) # type: bool + pip = attr.ib(default=False) # type: bool + copies = attr.ib(default=False) # type: bool + compile = attr.ib(default=False) # type: bool + prompt = attr.ib(default=None) # type: Optional[str] + hermetic_scripts = attr.ib(default=False) # type: bool diff --git a/pex/venv/installer_options.py b/pex/venv/installer_options.py new file mode 100644 index 000000000..a2a82a607 --- /dev/null +++ b/pex/venv/installer_options.py @@ -0,0 +1,115 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from argparse import Namespace, _ActionsContainer + +from pex.venv.bin_path import BinPath +from pex.venv.install_scope import InstallScope +from pex.venv.installer_configuration import InstallerConfiguration + + +def register( + parser, # type: _ActionsContainer + include_force_switch=False, # type: bool +): + # type: (...) -> None + default_configuration = InstallerConfiguration() + parser.add_argument( + "--scope", + default=default_configuration.scope, + choices=InstallScope.values(), + type=InstallScope.for_value, + help=( + "The scope of code contained in the Pex that is installed in the venv. By default" + "{all} code is installed and this is generally what you want. However, in some " + "situations it's beneficial to split the venv installation into {deps} and " + "{sources} steps. This is particularly useful when installing a PEX in a container " + "image. See " + "https://pex.readthedocs.io/en/latest/recipes.html#pex-app-in-a-container for more " + "information.".format( + all=InstallScope.ALL, + deps=InstallScope.DEPS_ONLY, + sources=InstallScope.SOURCE_ONLY, + ) + ), + ) + parser.add_argument( + "-b", + "--bin-path", + default=default_configuration.bin_path, + choices=BinPath.values(), + type=BinPath.for_value, + help="Add the venv bin dir to the PATH in the __main__.py script.", + ) + force_flags = ["-f", "--force"] if include_force_switch else ["--force"] + parser.add_argument( + *force_flags, + action="store_true", + default=False, + help="If the venv directory already exists, overwrite it." + ) + parser.add_argument( + "--collisions-ok", + action="store_true", + default=False, + help=( + "Don't error if population of the ven-v encounters distributions in the PEX file " + "with colliding files, just emit a warning." + ), + ) + parser.add_argument( + "-p", + "--pip", + action="store_true", + default=False, + help=( + "Add pip (and setuptools) to the venv. If the PEX already contains its own " + "conflicting versions pip (or setuptools), the command will error and you must " + "pass --collisions-ok to have the PEX versions over-ride the natural venv versions " + "installed by --pip." + ), + ) + parser.add_argument( + "--copies", + action="store_true", + default=False, + help="Create the venv using copies of system files instead of symlinks", + ) + parser.add_argument( + "--compile", + action="store_true", + default=False, + help="Compile all `.py` files in the venv.", + ) + parser.add_argument( + "--prompt", + help="A custom prompt for the venv activation scripts to use.", + ) + parser.add_argument( + "--non-hermetic-scripts", + dest="hermetic_scripts", + action="store_false", + default=True, + help=( + "Don't rewrite Python script shebangs in the venv to pass `-sE` to the " + "interpreter; for example, to enable running the venv PEX itself or its Python " + "scripts with a custom `PYTHONPATH`." + ), + ) + + +def configure(options): + # type: (Namespace) -> InstallerConfiguration + return InstallerConfiguration( + scope=options.scope, + bin_path=options.bin_path, + force=options.force, + collisions_ok=options.collisions_ok, + pip=options.pip, + copies=options.copies, + compile=options.compile, + prompt=options.prompt, + hermetic_scripts=options.hermetic_scripts, + ) diff --git a/tests/integration/cli/commands/test_venv_create.py b/tests/integration/cli/commands/test_venv_create.py new file mode 100644 index 000000000..880875f5e --- /dev/null +++ b/tests/integration/cli/commands/test_venv_create.py @@ -0,0 +1,521 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import shutil +import subprocess +import sys +from textwrap import dedent + +import colors +import pytest + +from pex import dist_metadata +from pex.cli.commands.venv import InstallLayout +from pex.cli.testing import run_pex3 +from pex.common import open_zip, safe_open +from pex.compatibility import commonpath +from pex.interpreter import PythonInterpreter +from pex.pep_440 import Version +from pex.pep_503 import ProjectName +from pex.pex import PEX +from pex.platforms import Platform +from pex.testing import IS_MAC, PY39, PY310, ensure_python_interpreter, make_env, run_pex_command +from pex.typing import TYPE_CHECKING +from pex.venv.virtualenv import Virtualenv + +if TYPE_CHECKING: + from typing import Any + + +@pytest.fixture(scope="module") +def td(tmpdir_factory): + # type: (Any) -> Any + + return tmpdir_factory.mktemp("td") + + +@pytest.fixture(scope="module") +def lock(td): + # type: (Any) -> str + + lock = str(td.join("lock.json")) + run_pex3( + "lock", "create", "cowsay==5.0", "ansicolors==1.1.8", "-o", lock, "--indent", "2" + ).assert_success() + return lock + + +@pytest.fixture(scope="module") +def cowsay_pex( + td, # type: Any + lock, # type: str +): + # type: (...) -> str + + pex = str(td.join("pex")) + run_pex_command(args=["--lock", lock, "-o", pex]).assert_success() + assert sorted( + [(ProjectName("cowsay"), Version("5.0")), (ProjectName("ansicolors"), Version("1.1.8"))] + ) == [(dist.metadata.project_name, dist.metadata.version) for dist in PEX(pex).resolve()] + return pex + + +def test_venv_empty(tmpdir): + # type: (Any) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "-d", dest).assert_success() + venv = Virtualenv(dest) + assert ( + PythonInterpreter.get().resolve_base_interpreter() + == venv.interpreter.resolve_base_interpreter() + ) + assert [] == list(venv.iter_distributions()) + + +def assert_venv( + tmpdir, # type: Any + *extra_args # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "cowsay==5.0", "-d", dest, *extra_args).assert_success() + + venv = Virtualenv(dest) + _, stdout, _ = venv.interpreter.execute( + args=["-c", "import cowsay, os; print(os.path.realpath(cowsay.__file__))"] + ) + assert venv.site_packages_dir == commonpath([venv.site_packages_dir, stdout.strip()]) + + _, stdout, _ = venv.interpreter.execute(args=["-m", "cowsay", "--version"]) + assert "5.0" == stdout.strip() + + assert ( + "5.0" + == subprocess.check_output(args=[venv.bin_path("cowsay"), "--version"]) + .decode("utf-8") + .strip() + ) + + assert [(ProjectName("cowsay"), Version("5.0"))] == [ + (dist.metadata.project_name, dist.metadata.version) for dist in venv.iter_distributions() + ] + + +def test_venv( + tmpdir, # type: Any + lock, # type: str + cowsay_pex, # type: str +): + # type: (...) -> None + + assert_venv(tmpdir) + assert_venv(tmpdir, "--lock", lock) + assert_venv(tmpdir, "--pex-repository", cowsay_pex) + + +def test_flat_empty(tmpdir): + # type: (Any) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "--layout", "flat", "-d", dest).assert_success() + assert [] == list(dist_metadata.find_distributions(search_path=[dest])) + + +def test_flat_zipped_empty(tmpdir): + # type: (Any) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "--layout", "flat-zipped", "-d", dest).assert_success() + assert [] == list(dist_metadata.find_distributions(search_path=[dest])) + with open_zip("{dest}.zip".format(dest=dest)) as zf: + assert [] == zf.namelist() + + +def assert_flat( + tmpdir, # type: Any + layout, # type: InstallLayout.Value + *extra_args # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3( + "venv", "create", "--layout", str(layout), "cowsay==5.0", "-d", dest, *extra_args + ).assert_success() + + sys_path_entry = dest if layout is InstallLayout.FLAT else "{dest}.zip".format(dest=dest) + env = make_env(PYTHONPATH=sys_path_entry) + + assert sys_path_entry == commonpath( + [ + sys_path_entry, + subprocess.check_output( + args=[sys.executable, "-c", "import cowsay; print(cowsay.__file__)"], env=env + ) + .decode("utf-8") + .strip(), + ] + ) + + assert ( + "5.0" + == subprocess.check_output(args=[sys.executable, "-m", "cowsay", "--version"], env=env) + .decode("utf-8") + .strip() + ) + + if layout is InstallLayout.FLAT_ZIPPED: + search_path_entry = os.path.join(str(tmpdir), "zip_contents") + with open_zip(sys_path_entry) as zf: + zf.extractall(search_path_entry) + else: + search_path_entry = dest + + assert [(ProjectName("cowsay"), Version("5.0"))] == [ + (dist.metadata.project_name, dist.metadata.version) + for dist in dist_metadata.find_distributions(search_path=[search_path_entry]) + ] + + +@pytest.mark.parametrize( + "layout", + [ + pytest.param(layout, id=str(layout)) + for layout in (InstallLayout.FLAT, InstallLayout.FLAT_ZIPPED) + ], +) +def test_flat( + tmpdir, # type: Any + layout, # type: InstallLayout.Value + lock, # type: str + cowsay_pex, # type: str +): + # type: (...) -> None + + assert_flat(tmpdir, layout) + assert_flat(tmpdir, layout, "--lock", lock) + assert_flat(tmpdir, layout, "--pex-repository", cowsay_pex) + + +def test_flat_zipped_prefix( + tmpdir, # type: Any + lock, # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3( + "venv", + "create", + "ansicolors", + "--lock", + lock, + "--layout", + "flat-zipped", + "--prefix", + "python", + "-d", + dest, + ).assert_success() + + sys_path_entry = os.path.join("{dest}.zip".format(dest=dest), "python") + assert ( + colors.cyan("ide") + == subprocess.check_output( + args=[sys.executable, "-c", "import colors; print(colors.cyan('ide'))"], + env=make_env(PYTHONPATH=sys_path_entry), + ) + .decode("utf-8") + .strip() + ) + + +def test_venv_pip(tmpdir): + # type: (Any) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "-d", dest).assert_success() + + venv = Virtualenv(dest) + assert "pip" not in [os.path.basename(exe) for exe in venv.iter_executables()] + assert [] == list(venv.iter_distributions()) + + run_pex3("venv", "create", "-d", dest, "--pip").assert_success() + assert "pip" in [os.path.basename(exe) for exe in venv.iter_executables()] + distributions = { + dist.metadata.project_name: dist.metadata.version for dist in venv.iter_distributions() + } + pip_version = distributions[ProjectName("pip")] + expected_prefix = "pip {version} from {prefix}".format(version=pip_version.raw, prefix=dest) + assert ( + subprocess.check_output(args=[venv.bin_path("pip"), "--version"]) + .decode("utf-8") + .startswith(expected_prefix) + ) + + +@pytest.fixture(scope="module") +def colors_pex( + td, # type: Any + lock, # type: str +): + # type: (...) -> str + + src = str(td.join("src")) + with safe_open(os.path.join(src, "exe.py"), "w") as fp: + fp.write( + dedent( + """\ + import colors + + + print(colors.magenta("Red Dwarf")) + """ + ) + ) + pex = str(td.join("pex")) + run_pex_command( + args=["--lock", lock, "ansicolors", "-D", src, "-m", "exe", "-o", pex] + ).assert_success() + return pex + + +def assert_deps_only( + interpreter, # type: PythonInterpreter + expected_prefix, # type: str + **extra_env # type: str +): + # type: (...) -> None + + assert expected_prefix == commonpath( + [ + expected_prefix, + subprocess.check_output( + args=[ + interpreter.binary, + "-c", + dedent( + """\ + import os + import sys + + import colors + + try: + import exe + sys.exit("The deps scope should not have included the exe.py src.") + except ImportError: + pass + + print(os.path.realpath(colors.__file__)) + """ + ), + ], + env=make_env(**extra_env), + ) + .decode("utf-8") + .strip(), + ] + ) + + +def assert_srcs_only( + sys_path_entry, # type: str + *extra_srcs # type: str +): + # type: (...) -> None + + assert sorted(list(extra_srcs) + ["exe.py"]) == sorted( + os.path.relpath(os.path.join(root, f), sys_path_entry) + for root, _, files in os.walk(sys_path_entry) + for f in files + ) + + +def test_pex_scope_venv( + tmpdir, # type: Any + colors_pex, # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3( + "venv", "create", "-d", dest, "--pex-repository", colors_pex, "--scope", "deps" + ).assert_success() + + venv_pex_script = os.path.join(dest, "pex") + assert not os.path.exists(venv_pex_script) + + venv = Virtualenv(dest) + assert_deps_only(interpreter=venv.interpreter, expected_prefix=venv.site_packages_dir) + + def install_srcs(): + # type: () -> None + run_pex3( + "venv", "create", "-d", dest, "--pex-repository", colors_pex, "--scope", "srcs" + ).assert_success() + + install_srcs() + assert ( + colors.magenta("Red Dwarf") + == subprocess.check_output(args=[venv_pex_script]).decode("utf-8").strip() + ) + + shutil.rmtree(dest) + install_srcs() + assert_srcs_only( + venv.site_packages_dir, + # Any venv created from a PEX supports the PEX_EXTRA_SYS_PATH runtime env var via this + # `.pth` file. + "PEX_EXTRA_SYS_PATH.pth", + ) + + +def test_pex_scope_flat( + tmpdir, # type: Any + colors_pex, # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3( + "venv", + "create", + "-d", + dest, + "--pex-repository", + colors_pex, + "--scope", + "deps", + "--layout", + "flat", + ).assert_success() + + venv_pex_script = os.path.join(dest, "pex") + assert not os.path.exists(venv_pex_script) + assert_deps_only(interpreter=PythonInterpreter.get(), expected_prefix=dest, PYTHONPATH=dest) + + def install_srcs(): + # type: () -> None + run_pex3( + "venv", + "create", + "-d", + dest, + "--pex-repository", + colors_pex, + "--scope", + "srcs", + "--layout", + "flat", + ).assert_success() + + install_srcs() + assert not os.path.exists(venv_pex_script) + assert ( + colors.magenta("Red Dwarf") + == subprocess.check_output( + args=[sys.executable, "-m", "exe"], env=make_env(PYTHONPATH=dest) + ) + .decode("utf-8") + .strip() + ) + + shutil.rmtree(dest) + install_srcs() + assert_srcs_only(sys_path_entry=dest) + + +@pytest.fixture +def foreign_platform(): + # type: () -> str + return "linux_x86_64-cp-310-cp310" if IS_MAC else "macosx_10.9_x86_64-cp-310-cp310" + + +def test_foreign_target( + tmpdir, # type: Any + foreign_platform, # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + result = run_pex3( + "venv", + "create", + "psutil==5.9.5", + "-d", + dest, + "--platform", + foreign_platform, + ) + result.assert_failure() + assert ( + "Cannot create a local venv for foreign platform {platform}.".format( + platform=Platform.create(foreign_platform) + ) + == result.error.strip() + ) + + run_pex3( + "venv", + "create", + "--layout", + "flat", + "psutil==5.9.5", + "-d", + dest, + "--platform", + foreign_platform, + ).assert_success() + + distributions = list(dist_metadata.find_distributions(search_path=[dest])) + assert 1 == len(distributions) + + dist = distributions[0] + assert ProjectName("psutil") == dist.metadata.project_name + assert Version("5.9.5") == dist.metadata.version + + +def test_venv_update_target_mismatch( + tmpdir, # type: Any + foreign_platform, # type: str +): + # type: (...) -> None + + dest = os.path.join(str(tmpdir), "dest") + run_pex3("venv", "create", "-d", dest).assert_success() + + result = run_pex3( + "venv", "create", "ansicolors==1.1.8", "-d", dest, "--platform", foreign_platform + ) + result.assert_failure() + assert ( + "Cannot update a local venv using a foreign platform. Given: {foreign_platform}.".format( + foreign_platform=Platform.create(foreign_platform) + ) + == result.error.strip() + ), result.error + + python = ensure_python_interpreter(PY310 if sys.version_info[:2] != (3, 10) else PY39) + result = run_pex3("venv", "create", "ansicolors==1.1.8", "-d", dest, "--python", python) + result.assert_failure() + assert ( + "Cannot update venv at {dest} created with {created_with} using {using}".format( + dest=dest, + created_with=PythonInterpreter.get().resolve_base_interpreter().binary, + using=PythonInterpreter.from_binary(python).resolve_base_interpreter().binary, + ) + in result.error.strip() + ), result.error + + assert [] == list(Virtualenv(dest).iter_distributions()) + run_pex3("venv", "create", "ansicolors==1.1.8", "-d", dest).assert_success() + assert [(ProjectName("ansicolors"), Version("1.1.8"))] == [ + (dist.metadata.project_name, dist.metadata.version) + for dist in Virtualenv(dest).iter_distributions() + ] diff --git a/tests/integration/test_pex_bootstrapper.py b/tests/integration/test_pex_bootstrapper.py index a884ab20b..133c66180 100644 --- a/tests/integration/test_pex_bootstrapper.py +++ b/tests/integration/test_pex_bootstrapper.py @@ -18,7 +18,7 @@ from pex.pex_info import PexInfo from pex.testing import PY38, PY39, PY_VER, ensure_python_interpreter, make_env, run_pex_command from pex.typing import TYPE_CHECKING -from pex.venv.pex import CollisionError +from pex.venv.installer import CollisionError from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: diff --git a/tests/integration/tools/commands/test_venv.py b/tests/integration/tools/commands/test_venv.py index b1c4c0dd6..a206727f8 100644 --- a/tests/integration/tools/commands/test_venv.py +++ b/tests/integration/tools/commands/test_venv.py @@ -91,16 +91,16 @@ def verb(): result = run_pex_tools(collisions_pex, "venv", venv_dir) result.assert_failure() assert ( - "Encountered collision building venv at {venv_dir} from {pex}:\n" + "Encountered collision populating {venv_dir} from PEX at {pex}:\n" "1. {venv_dir}/bin/pex was provided by:".format(venv_dir=venv_dir, pex=collisions_pex) - ) in result.error + ) in result.error, result.error result = run_pex_tools(collisions_pex, "venv", "--collisions-ok", "--force", venv_dir) result.assert_success() assert ( - "PEXWarning: Encountered collision building venv at {venv_dir} from {pex}:\n" + "PEXWarning: Encountered collision populating {venv_dir} from PEX at {pex}:\n" "1. {venv_dir}/bin/pex was provided by:".format(venv_dir=venv_dir, pex=collisions_pex) - ) in result.error + ) in result.error, result.error assert 42 == subprocess.call(args=[Virtualenv(venv_dir=venv_dir).bin_path("pex")]) diff --git a/tests/integration/venv_ITs/test_issue_1630.py b/tests/integration/venv_ITs/test_issue_1630.py index 1d49db66d..a615895aa 100644 --- a/tests/integration/venv_ITs/test_issue_1630.py +++ b/tests/integration/venv_ITs/test_issue_1630.py @@ -48,7 +48,7 @@ def test_data_files(tmpdir): pex_venv = Virtualenv.create( os.path.join(str(tmpdir), "pex.venv"), interpreter=PythonInterpreter.from_binary(py38) ) - installed = list(InstalledWheel.load(nbconvert_dist.location).reinstall(pex_venv)) + installed = list(InstalledWheel.load(nbconvert_dist.location).reinstall_venv(pex_venv)) assert installed # Single out one known data file to check diff --git a/tests/integration/venv_ITs/test_issue_1637.py b/tests/integration/venv_ITs/test_issue_1637.py index b65627bff..db6e7c1d5 100644 --- a/tests/integration/venv_ITs/test_issue_1637.py +++ b/tests/integration/venv_ITs/test_issue_1637.py @@ -167,7 +167,7 @@ def assert_venv_collision(*extra_args): # The --venv mode should warn about collisions but succeed. assert 0 == process.returncode, decoded_stderr - assert "PEXWarning: Encountered collision building venv at " in decoded_stderr + assert "PEXWarning: Encountered collision populating " in decoded_stderr assert "site-packages/colors.py was provided by:" in decoded_stderr assert "sha1:17772af8295ffb7f4d6c3353665b5c542be332a2 -> " in decoded_stderr assert "sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709 -> " in decoded_stderr