From 0768b82c29e08de940bb8eac2c89a3281615270c Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 28 Apr 2023 07:45:30 -0700 Subject: [PATCH 1/2] Factor out configured resolves from `pex`. This code as well as selection of a unique target to resolve for will be re-used by the forthcoming `pex3 venv create` command. Along the way, clean up confusion noted in #2135 in the introduction of the `pex3 venv` subcommand surrounding `pyvenv.cfg` files generated by `Virtualenv`. Work towards #1752 and #2110. --- pex/bin/pex.py | 93 +++----------------------- pex/cli/commands/lock.py | 18 +----- pex/resolve/configured_resolve.py | 104 ++++++++++++++++++++++++++++++ pex/targets.py | 22 ++++++- pex/venv/virtualenv.py | 9 ++- 5 files changed, 144 insertions(+), 102 deletions(-) create mode 100644 pex/resolve/configured_resolve.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 406b74e14..281467c6c 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -35,17 +35,10 @@ from pex.pex_info import PexInfo from pex.resolve import requirement_options, resolver_options, target_configuration, target_options from pex.resolve.config import finalize as finalize_resolve_config -from pex.resolve.configured_resolver import ConfiguredResolver -from pex.resolve.lock_resolver import resolve_from_lock -from pex.resolve.pex_repository_resolver import resolve_from_pex +from pex.resolve.configured_resolve import resolve from pex.resolve.requirement_configuration import RequirementConfiguration -from pex.resolve.resolver_configuration import ( - LockRepositoryConfiguration, - PexRepositoryConfiguration, -) from pex.resolve.resolvers import Unsatisfiable -from pex.resolver import resolve -from pex.result import catch, try_ +from pex.result import catch from pex.targets import Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast @@ -677,81 +670,13 @@ def build_pex( ) ): try: - if isinstance(resolver_configuration, LockRepositoryConfiguration): - lock = try_(resolver_configuration.parse_lock()) - with TRACER.timed( - "Resolving requirements from lock file {lock_file}".format( - lock_file=lock.source - ) - ): - pip_configuration = resolver_configuration.pip_configuration - result = try_( - resolve_from_lock( - targets=targets, - lock=lock, - resolver=ConfiguredResolver(pip_configuration=pip_configuration), - requirements=requirement_configuration.requirements, - requirement_files=requirement_configuration.requirement_files, - constraint_files=requirement_configuration.constraint_files, - transitive=pip_configuration.transitive, - indexes=pip_configuration.repos_configuration.indexes, - find_links=pip_configuration.repos_configuration.find_links, - resolver_version=pip_configuration.resolver_version, - network_configuration=pip_configuration.network_configuration, - password_entries=pip_configuration.repos_configuration.password_entries, - build=pip_configuration.allow_builds, - use_wheel=pip_configuration.allow_wheels, - prefer_older_binary=pip_configuration.prefer_older_binary, - use_pep517=pip_configuration.use_pep517, - build_isolation=pip_configuration.build_isolation, - compile=options.compile, - max_parallel_jobs=pip_configuration.max_jobs, - pip_version=lock.pip_version, - ) - ) - elif isinstance(resolver_configuration, PexRepositoryConfiguration): - with TRACER.timed( - "Resolving requirements from PEX {pex_repository}.".format( - pex_repository=resolver_configuration.pex_repository - ) - ): - result = resolve_from_pex( - targets=targets, - pex=resolver_configuration.pex_repository, - requirements=requirement_configuration.requirements, - requirement_files=requirement_configuration.requirement_files, - constraint_files=requirement_configuration.constraint_files, - network_configuration=resolver_configuration.network_configuration, - transitive=resolver_configuration.transitive, - ignore_errors=options.ignore_errors, - ) - else: - with TRACER.timed("Resolving requirements."): - result = resolve( - targets=targets, - requirements=requirement_configuration.requirements, - requirement_files=requirement_configuration.requirement_files, - constraint_files=requirement_configuration.constraint_files, - allow_prereleases=resolver_configuration.allow_prereleases, - transitive=resolver_configuration.transitive, - indexes=resolver_configuration.repos_configuration.indexes, - find_links=resolver_configuration.repos_configuration.find_links, - resolver_version=resolver_configuration.resolver_version, - network_configuration=resolver_configuration.network_configuration, - password_entries=resolver_configuration.repos_configuration.password_entries, - build=resolver_configuration.allow_builds, - use_wheel=resolver_configuration.allow_wheels, - prefer_older_binary=resolver_configuration.prefer_older_binary, - use_pep517=resolver_configuration.use_pep517, - build_isolation=resolver_configuration.build_isolation, - compile=options.compile, - max_parallel_jobs=resolver_configuration.max_jobs, - ignore_errors=options.ignore_errors, - preserve_log=resolver_configuration.preserve_log, - pip_version=resolver_configuration.version, - resolver=ConfiguredResolver(pip_configuration=resolver_configuration), - ) - + result = resolve( + targets=targets, + requirement_configuration=requirement_configuration, + resolver_configuration=resolver_configuration, + compile_pyc=options.compile, + ignore_errors=options.ignore_errors, + ) for installed_dist in result.installed_distributions: pex_builder.add_distribution( installed_dist.distribution, fingerprint=installed_dist.fingerprint diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 680a9978b..83ce23400 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -499,21 +499,9 @@ def _export(self): lockfile_path, lock_file = self._load_lockfile() targets = target_options.configure(self.options).resolve_targets() - resolved_targets = targets.unique_targets() - if len(resolved_targets) > 1: - return Error( - "A lock can only be exported for a single target in the {pip!r} format.\n" - "There were {count} targets selected:\n" - "{targets}".format( - pip=ExportFormat.PIP, - count=len(resolved_targets), - targets="\n".join( - "{index}. {target}".format(index=index, target=target) - for index, target in enumerate(resolved_targets, start=1) - ), - ) - ) - target = next(iter(resolved_targets)) + target = targets.require_unique_target( + purpose="exporting a lock in the {pip!r} format".format(pip=ExportFormat.PIP) + ) network_configuration = resolver_options.create_network_configuration(self.options) with TRACER.timed("Selecting locks for {target}".format(target=target)): diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py new file mode 100644 index 000000000..640fe4f77 --- /dev/null +++ b/pex/resolve/configured_resolve.py @@ -0,0 +1,104 @@ +# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.lock_resolver import resolve_from_lock +from pex.resolve.pex_repository_resolver import resolve_from_pex +from pex.resolve.requirement_configuration import RequirementConfiguration +from pex.resolve.resolver_configuration import ( + LockRepositoryConfiguration, + PexRepositoryConfiguration, +) +from pex.resolve.resolvers import Installed +from pex.resolver import resolve as resolve_via_pip +from pex.result import try_ +from pex.targets import Targets +from pex.tracer import TRACER +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pex.resolve.resolver_options import ResolverConfiguration + + +def resolve( + targets, # type: Targets + requirement_configuration, # type: RequirementConfiguration + resolver_configuration, # type: ResolverConfiguration + compile_pyc=False, # type: bool + ignore_errors=False, # type: bool +): + # type: (...) -> Installed + if isinstance(resolver_configuration, LockRepositoryConfiguration): + lock = try_(resolver_configuration.parse_lock()) + with TRACER.timed( + "Resolving requirements from lock file {lock_file}".format(lock_file=lock.source) + ): + pip_configuration = resolver_configuration.pip_configuration + return try_( + resolve_from_lock( + targets=targets, + lock=lock, + resolver=ConfiguredResolver(pip_configuration=pip_configuration), + requirements=requirement_configuration.requirements, + requirement_files=requirement_configuration.requirement_files, + constraint_files=requirement_configuration.constraint_files, + transitive=pip_configuration.transitive, + indexes=pip_configuration.repos_configuration.indexes, + find_links=pip_configuration.repos_configuration.find_links, + resolver_version=pip_configuration.resolver_version, + network_configuration=pip_configuration.network_configuration, + password_entries=pip_configuration.repos_configuration.password_entries, + build=pip_configuration.allow_builds, + use_wheel=pip_configuration.allow_wheels, + prefer_older_binary=pip_configuration.prefer_older_binary, + use_pep517=pip_configuration.use_pep517, + build_isolation=pip_configuration.build_isolation, + compile=compile_pyc, + max_parallel_jobs=pip_configuration.max_jobs, + pip_version=lock.pip_version, + ) + ) + elif isinstance(resolver_configuration, PexRepositoryConfiguration): + with TRACER.timed( + "Resolving requirements from PEX {pex_repository}.".format( + pex_repository=resolver_configuration.pex_repository + ) + ): + return resolve_from_pex( + targets=targets, + pex=resolver_configuration.pex_repository, + requirements=requirement_configuration.requirements, + requirement_files=requirement_configuration.requirement_files, + constraint_files=requirement_configuration.constraint_files, + network_configuration=resolver_configuration.network_configuration, + transitive=resolver_configuration.transitive, + ignore_errors=ignore_errors, + ) + else: + with TRACER.timed("Resolving requirements."): + return resolve_via_pip( + targets=targets, + requirements=requirement_configuration.requirements, + requirement_files=requirement_configuration.requirement_files, + constraint_files=requirement_configuration.constraint_files, + allow_prereleases=resolver_configuration.allow_prereleases, + transitive=resolver_configuration.transitive, + indexes=resolver_configuration.repos_configuration.indexes, + find_links=resolver_configuration.repos_configuration.find_links, + resolver_version=resolver_configuration.resolver_version, + network_configuration=resolver_configuration.network_configuration, + password_entries=resolver_configuration.repos_configuration.password_entries, + build=resolver_configuration.allow_builds, + use_wheel=resolver_configuration.allow_wheels, + prefer_older_binary=resolver_configuration.prefer_older_binary, + use_pep517=resolver_configuration.use_pep517, + build_isolation=resolver_configuration.build_isolation, + compile=compile_pyc, + max_parallel_jobs=resolver_configuration.max_jobs, + ignore_errors=ignore_errors, + preserve_log=resolver_configuration.preserve_log, + pip_version=resolver_configuration.version, + resolver=ConfiguredResolver(pip_configuration=resolver_configuration), + ) diff --git a/pex/targets.py b/pex/targets.py index b0ed50d2c..53d56ded4 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -12,10 +12,12 @@ from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform from pex.third_party.packaging.specifiers import SpecifierSet + +from pex.result import Error from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from typing import Iterable, Iterator, Optional, Tuple + from typing import Iterable, Iterator, Optional, Tuple, Union import attr # vendor:skip else: @@ -337,3 +339,21 @@ def iter_targets(): yield complete_platform return OrderedSet(iter_targets()) + + def require_unique_target(self, purpose): + # type: (str) -> Union[Target, Error] + resolved_targets = self.unique_targets() + if len(resolved_targets) != 1: + return Error( + "A single target is required for {purpose}.\n" + "There were {count} targets selected:\n" + "{targets}".format( + purpose=purpose, + count=len(resolved_targets), + targets="\n".join( + "{index}. {target}".format(index=index, target=target) + for index, target in enumerate(resolved_targets, start=1) + ), + ) + ) + return cast(Target, next(iter(resolved_targets))) diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index 100a04887..a5d673e9a 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -194,7 +194,12 @@ def create( custom_prompt = prompt interpreter.execute(args=args, env=env) # Modern virtualenv provides a pyvenv.cfg; so we provide one on 16.7.12's behalf - # since users might expect one. + # since users might expect one. To ward off any confusion for readers of the emitted + # pyvenv.cfg file, we add a bespoke created-by field to help make it clear that Pex + # created the pyvenv.cfg file on virtualenv 16.7.12's behalf. + # N.B.: This bespoke created-by "note" field is not related to the + # Virtualenv.created_by property which reflects the underlying venv technology. + # In this case it will report "virtualenv 16.7.12". with open(os.path.join(venv_dir, "pyvenv.cfg"), "w") as fp: fp.write( dedent( @@ -203,7 +208,7 @@ def create( include-system-site-packages = {include_system_site_packages} virtualenv = {virtualenv_version} version = {python_version} - created_by = pex {pex_version} + created-by = pex {pex_version} """ ).format( home=os.path.dirname(interpreter.binary), From 940cc9f4d20114b63be4a8074250c9824f75b40f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 28 Apr 2023 08:07:10 -0700 Subject: [PATCH 2/2] Fix import sort. --- pex/targets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pex/targets.py b/pex/targets.py index 53d56ded4..c2afb858e 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -11,9 +11,8 @@ from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform -from pex.third_party.packaging.specifiers import SpecifierSet - from pex.result import Error +from pex.third_party.packaging.specifiers import SpecifierSet from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: