From 1b53551513641d1803326aaa1d5e4e0a54ed735b Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 25 Nov 2024 11:10:51 -0800 Subject: [PATCH] Probably mostly working locked build system use. --- pex/build_system/__init__.py | 176 +++++++++++- pex/build_system/pep_517.py | 71 +---- pex/build_system/pep_518.py | 113 +------- pex/resolve/lock_resolver.py | 251 +++++++++++++++++- pex/resolve/locked_resolve.py | 5 + pex/resolve/lockfile/subset.py | 77 ++++-- tests/build_system/test_pep_518.py | 3 +- .../integration/build_system/test_pep_518.py | 3 +- 8 files changed, 491 insertions(+), 208 deletions(-) diff --git a/pex/build_system/__init__.py b/pex/build_system/__init__.py index 3cfc24085..029046b60 100644 --- a/pex/build_system/__init__.py +++ b/pex/build_system/__init__.py @@ -3,10 +3,26 @@ from __future__ import absolute_import +import json +import os +import subprocess +from textwrap import dedent + +from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode, safe_mkdtemp +from pex.dist_metadata import Distribution +from pex.interpreter import PythonInterpreter +from pex.jobs import Job, SpawnedJob +from pex.pex import PEX +from pex.pex_bootstrapper import VenvPex, ensure_venv +from pex.pex_builder import PEXBuilder +from pex.result import Error from pex.typing import TYPE_CHECKING +from pex.variables import ENV +from pex.venv.bin_path import BinPath +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Tuple + from typing import Any, Iterable, Mapping, Optional, Tuple, Union import attr # vendor:skip else: @@ -33,3 +49,161 @@ class BuildSystemTable(object): DEFAULT_BUILD_SYSTEM_TABLE = BuildSystemTable( requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND ) + + +# Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h +# this seems an appropriate signal of DNE vs execute and fail. +_HOOK_UNAVAILABLE_EXIT_CODE = 75 + + +@attr.s(frozen=True) +class BuildSystem(object): + @classmethod + def create( + cls, + interpreter, # type: PythonInterpreter + requires, # type: Iterable[str] + resolved, # type: Iterable[Distribution] + build_backend, # type: str + backend_path, # type: Tuple[str, ...] + extra_requirements=None, # type: Optional[Iterable[str]] + use_system_time=False, # type: bool + **extra_env # type: str + ): + # type: (...) -> Union[BuildSystem, Error] + pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK) + pex_builder.info.venv = True + pex_builder.info.venv_site_packages_copies = True + pex_builder.info.venv_bin_path = BinPath.PREPEND + # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect. + pex_builder.info.venv_hermetic_scripts = False + for req in requires: + pex_builder.add_requirement(req) + for dist in resolved: + pex_builder.add_distribution(dist) + pex_builder.freeze(bytecode_compile=False) + venv_pex = ensure_venv(PEX(pex_builder.path(), interpreter=interpreter)) + if extra_requirements: + # N.B.: We install extra requirements separately instead of having them resolved and + # handed in with the `resolved` above because there are cases in the wild where the + # build system requires (PEP-518) and the results of PEP-517 `get_requires_for_*` can + # return overlapping requirements. Pip will error for overlaps complaining of duplicate + # requirements if we attempt to resolve all the requirements at once; so we instead + # resolve and install in two phases. This obviously has problems! That said, it is, in + # fact, how Pip's internal PEP-517 build frontend works; so we emulate that. + virtualenv = Virtualenv(venv_pex.venv_dir) + # Python 3.5 comes with Pip 9.0.1 which is pretty broken: it doesn't work with our test + # cases; so we upgrade. + # For Python 2.7 we use virtualenv (there is no -m venv built into Python) and that + # comes with Pip 22.0.2, Python 3.6 comes with Pip 18.1 and Python 3.7 comes with + # Pip 22.04 and the default Pips only get newer with newer version of Pythons. These all + # work well enough for our test cases and, in general, they should work well enough with + # the Python they come paired with. + upgrade_pip = virtualenv.interpreter.version[:2] == (3, 5) + virtualenv.ensure_pip(upgrade=upgrade_pip) + with open(os.devnull, "wb") as dev_null: + _, process = virtualenv.interpreter.open_process( + args=[ + "-m", + "pip", + "install", + "--ignore-installed", + "--no-user", + "--no-warn-script-location", + ] + + list(extra_requirements), + stdout=dev_null, + stderr=subprocess.PIPE, + ) + _, stderr = process.communicate() + if process.returncode != 0: + return Error( + "Failed to install extra requirement in venv at {venv_dir}: " + "{extra_requirements}\nSTDERR:\n{stderr}".format( + venv_dir=venv_pex.venv_dir, + extra_requirements=", ".join(extra_requirements), + stderr=stderr.decode("utf-8"), + ) + ) + + # Ensure all PEX* env vars are stripped except for PEX_ROOT and PEX_VERBOSE. We want folks + # to be able to steer the location of the cache and the logging verbosity, but nothing else. + # We control the entry-point, etc. of the PEP-518 build backend venv for internal use. + with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE)) as env: + if extra_env: + env.update(extra_env) + if backend_path: + env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path)) + if not use_system_time: + env.update(REPRODUCIBLE_BUILDS_ENV) + return cls( + venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env + ) + + venv_pex = attr.ib() # type: VenvPex + build_backend = attr.ib() # type: str + requires = attr.ib() # type: Tuple[str, ...] + env = attr.ib() # type: Mapping[str, str] + + def invoke_build_hook( + self, + project_directory, # type: str + hook_method, # type: str + hook_args=(), # type: Iterable[Any] + hook_kwargs=None, # type: Optional[Mapping[str, Any]] + ): + # type: (...) -> Union[SpawnedJob[Any], Error] + + # The interfaces are spec'd here: https://peps.python.org/pep-0517 + build_backend_module, _, _ = self.build_backend.partition(":") + build_backend_object = self.build_backend.replace(":", ".") + build_hook_result = os.path.join( + safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json" + ) + args = self.venv_pex.execute_args( + additional_args=( + "-c", + dedent( + """\ + import json + import sys + + import {build_backend_module} + + + if not hasattr({build_backend_object}, {hook_method!r}): + sys.exit({hook_unavailable_exit_code}) + + result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r}) + with open({result_file!r}, "w") as fp: + json.dump(result, fp) + """ + ).format( + build_backend_module=build_backend_module, + build_backend_object=build_backend_object, + hook_method=hook_method, + hook_args=tuple(hook_args), + hook_kwargs=dict(hook_kwargs) if hook_kwargs else {}, + hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE, + result_file=build_hook_result, + ), + ) + ) + process = subprocess.Popen( + args=args, + env=self.env, + cwd=project_directory, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + return SpawnedJob.file( + Job( + command=args, + process=process, + context="PEP-517:{hook_method} at {project_directory}".format( + hook_method=hook_method, project_directory=project_directory + ), + ), + output_file=build_hook_result, + result_func=lambda file_content: json.loads(file_content.decode("utf-8")), + ) diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index e96303d47..99ce8195b 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -3,14 +3,11 @@ from __future__ import absolute_import -import json import os -import subprocess -from textwrap import dedent from pex import third_party -from pex.build_system import DEFAULT_BUILD_BACKEND -from pex.build_system.pep_518 import BuildSystem, load_build_system +from pex.build_system import DEFAULT_BUILD_BACKEND, BuildSystem +from pex.build_system.pep_518 import load_build_system from pex.common import safe_mkdtemp from pex.dist_metadata import DistMetadata, Distribution, MetadataType from pex.jobs import Job, SpawnedJob @@ -134,67 +131,21 @@ def _invoke_build_hook( ) ) - build_system_or_error = _get_build_system( + result = _get_build_system( target, resolver, project_directory, extra_requirements=hook_extra_requirements, pip_version=pip_version, ) - if isinstance(build_system_or_error, Error): - return build_system_or_error - build_system = build_system_or_error - - # The interfaces are spec'd here: https://peps.python.org/pep-0517 - build_backend_module, _, _ = build_system.build_backend.partition(":") - build_backend_object = build_system.build_backend.replace(":", ".") - build_hook_result = os.path.join(safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json") - args = build_system.venv_pex.execute_args( - additional_args=( - "-c", - dedent( - """\ - import json - import sys - - import {build_backend_module} - - - if not hasattr({build_backend_object}, {hook_method!r}): - sys.exit({hook_unavailable_exit_code}) - - result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r}) - with open({result_file!r}, "w") as fp: - json.dump(result, fp) - """ - ).format( - build_backend_module=build_backend_module, - build_backend_object=build_backend_object, - hook_method=hook_method, - hook_args=tuple(hook_args), - hook_kwargs=dict(hook_kwargs) if hook_kwargs else {}, - hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE, - result_file=build_hook_result, - ), - ) - ) - process = subprocess.Popen( - args=args, - env=build_system.env, - cwd=project_directory, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return SpawnedJob.file( - Job( - command=args, - process=process, - context="PEP-517:{hook_method} at {project_directory}".format( - hook_method=hook_method, project_directory=project_directory - ), - ), - output_file=build_hook_result, - result_func=lambda file_content: json.loads(file_content.decode("utf-8")), + if isinstance(result, Error): + return result + + return result.invoke_build_hook( + project_directory=project_directory, + hook_method=hook_method, + hook_args=hook_args, + hook_kwargs=hook_kwargs, ) diff --git a/pex/build_system/pep_518.py b/pex/build_system/pep_518.py index ac5db2424..59a5add8c 100644 --- a/pex/build_system/pep_518.py +++ b/pex/build_system/pep_518.py @@ -4,31 +4,22 @@ from __future__ import absolute_import import os.path -import subprocess from pex import toml -from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_SYSTEM_TABLE, BuildSystemTable -from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode -from pex.dist_metadata import Distribution -from pex.interpreter import PythonInterpreter -from pex.pex import PEX -from pex.pex_bootstrapper import VenvPex, ensure_venv -from pex.pex_builder import PEXBuilder +from pex.build_system import ( + DEFAULT_BUILD_BACKEND, + DEFAULT_BUILD_SYSTEM_TABLE, + BuildSystem, + BuildSystemTable, +) from pex.resolve.resolvers import Resolver from pex.result import Error from pex.targets import LocalInterpreter, Target, Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING -from pex.variables import ENV -from pex.venv.bin_path import BinPath -from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import Iterable, Mapping, Optional, Tuple, Union - - import attr # vendor:skip -else: - from pex.third_party import attr + from typing import Iterable, Optional, Union def _read_build_system_table( @@ -62,96 +53,6 @@ def _read_build_system_table( ) -@attr.s(frozen=True) -class BuildSystem(object): - @classmethod - def create( - cls, - interpreter, # type: PythonInterpreter - requires, # type: Iterable[str] - resolved, # type: Iterable[Distribution] - build_backend, # type: str - backend_path, # type: Tuple[str, ...] - extra_requirements=None, # type: Optional[Iterable[str]] - use_system_time=False, # type: bool - **extra_env # type: str - ): - # type: (...) -> Union[BuildSystem, Error] - pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK) - pex_builder.info.venv = True - pex_builder.info.venv_site_packages_copies = True - pex_builder.info.venv_bin_path = BinPath.PREPEND - # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect. - pex_builder.info.venv_hermetic_scripts = False - for req in requires: - pex_builder.add_requirement(req) - for dist in resolved: - pex_builder.add_distribution(dist) - pex_builder.freeze(bytecode_compile=False) - venv_pex = ensure_venv(PEX(pex_builder.path(), interpreter=interpreter)) - if extra_requirements: - # N.B.: We install extra requirements separately instead of having them resolved and - # handed in with the `resolved` above because there are cases in the wild where the - # build system requires (PEP-518) and the results of PEP-517 `get_requires_for_*` can - # return overlapping requirements. Pip will error for overlaps complaining of duplicate - # requirements if we attempt to resolve all the requirements at once; so we instead - # resolve and install in two phases. This obviously has problems! That said, it is, in - # fact, how Pip's internal PEP-517 build frontend works; so we emulate that. - virtualenv = Virtualenv(venv_pex.venv_dir) - # Python 3.5 comes with Pip 9.0.1 which is pretty broken: it doesn't work with our test - # cases; so we upgrade. - # For Python 2.7 we use virtualenv (there is no -m venv built into Python) and that - # comes with Pip 22.0.2, Python 3.6 comes with Pip 18.1 and Python 3.7 comes with - # Pip 22.04 and the default Pips only get newer with newer version of Pythons. These all - # work well enough for our test cases and, in general, they should work well enough with - # the Python they come paired with. - upgrade_pip = virtualenv.interpreter.version[:2] == (3, 5) - virtualenv.ensure_pip(upgrade=upgrade_pip) - with open(os.devnull, "wb") as dev_null: - _, process = virtualenv.interpreter.open_process( - args=[ - "-m", - "pip", - "install", - "--ignore-installed", - "--no-user", - "--no-warn-script-location", - ] - + list(extra_requirements), - stdout=dev_null, - stderr=subprocess.PIPE, - ) - _, stderr = process.communicate() - if process.returncode != 0: - return Error( - "Failed to install extra requirement in venv at {venv_dir}: " - "{extra_requirements}\nSTDERR:\n{stderr}".format( - venv_dir=venv_pex.venv_dir, - extra_requirements=", ".join(extra_requirements), - stderr=stderr.decode("utf-8"), - ) - ) - - # Ensure all PEX* env vars are stripped except for PEX_ROOT and PEX_VERBOSE. We want folks - # to be able to steer the location of the cache and the logging verbosity, but nothing else. - # We control the entry-point, etc. of the PEP-518 build backend venv for internal use. - with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE)) as env: - if extra_env: - env.update(extra_env) - if backend_path: - env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path)) - if not use_system_time: - env.update(REPRODUCIBLE_BUILDS_ENV) - return cls( - venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env - ) - - venv_pex = attr.ib() # type: VenvPex - build_backend = attr.ib() # type: str - requires = attr.ib() # type: Tuple[str, ...] - env = attr.ib() # type: Mapping[str, str] - - def _maybe_load_build_system_table(project_directory): # type: (str) -> Union[Optional[BuildSystemTable], Error] diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index a7277f57c..47eebf378 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -3,28 +3,222 @@ from __future__ import absolute_import +import hashlib +import os +import tarfile +from collections import OrderedDict, defaultdict + from pex.auth import PasswordDatabase, PasswordEntry +from pex.build_system import BuildSystem, BuildSystemTable +from pex.cache.dirs import CacheDir +from pex.common import open_zip, safe_mkdtemp from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import Requirement, is_wheel +from pex.dist_metadata import Distribution, Requirement, is_sdist, is_tar_sdist, is_wheel +from pex.exceptions import production_assert +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.interpreter import PythonInterpreter from pex.network_configuration import NetworkConfiguration -from pex.pep_427 import InstallableType +from pex.pep_427 import InstallableType, install_wheel_chroot from pex.pip.tool import PackageIndexConfiguration from pex.pip.version import PipVersionValue from pex.resolve.lock_downloader import LockDownloader -from pex.resolve.locked_resolve import LocalProjectArtifact +from pex.resolve.locked_resolve import ( + DownloadableArtifact, + LocalProjectArtifact, + LockedResolve, + Resolved, +) +from pex.resolve.lockfile.download_manager import DownloadedArtifact from pex.resolve.lockfile.model import Lockfile -from pex.resolve.lockfile.subset import subset +from pex.resolve.lockfile.subset import subset, subset_for_target from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import BuildConfiguration, ResolverVersion -from pex.resolve.resolvers import Resolver, ResolveResult +from pex.resolve.resolvers import ResolvedDistribution, Resolver, ResolveResult from pex.resolver import BuildAndInstallRequest, BuildRequest, InstallRequest from pex.result import Error, try_ -from pex.targets import Targets +from pex.sorted_tuple import SortedTuple +from pex.targets import Target, Targets from pex.tracer import TRACER -from pex.typing import TYPE_CHECKING +from pex.typing import TYPE_CHECKING, cast +from pex.util import CacheHelper if TYPE_CHECKING: - from typing import Iterable, Optional, Sequence, Tuple, Union + from typing import DefaultDict, Dict, Iterable, List, Optional, Sequence, Tuple, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +@attr.s(frozen=True) +class LockedSourceDistribution(object): + target = attr.ib() # type: Target + source_artifact = attr.ib() # type: DownloadedArtifact + build_system_table = attr.ib() # type: BuildSystemTable + locked_resolves = attr.ib() # type: Tuple[LockedResolve, ...] + + +def build_locked_source_distribution( + locked_source_distribution, # type: LockedSourceDistribution + install_requests, # type: Iterable[InstallRequest] + result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value +): + # type: (...) -> Union[ResolvedDistribution, Error] + + installed_wheels_dir = CacheDir.INSTALLED_WHEELS.path() + build_system_distributions = [] # type: List[Distribution] + for install_request in install_requests: + install_result = install_request.result(installed_wheels_dir) + installed_wheel = install_wheel_chroot( + wheel_path=install_request.wheel_path, destination=install_result.build_chroot + ) + build_system_distributions.append(Distribution.load(installed_wheel.prefix_dir)) + + result = BuildSystem.create( + interpreter=PythonInterpreter.get(), + requires=locked_source_distribution.build_system_table.requires, + resolved=build_system_distributions, + build_backend=locked_source_distribution.build_system_table.build_backend, + backend_path=locked_source_distribution.build_system_table.backend_path, + ) + if isinstance(result, Error): + return result + + source_artifact_path = locked_source_distribution.source_artifact.path + if is_sdist(source_artifact_path): + chroot = safe_mkdtemp() + if is_tar_sdist(source_artifact_path): + with tarfile.open(source_artifact_path) as tar_fp: + tar_fp.extractall(chroot) + else: + with open_zip(source_artifact_path) as zip_fp: + zip_fp.extractall(chroot) + for root, _, files in os.walk(chroot, topdown=True): + if any(f in ("setup.py", "setup.cfg", "pyproject.toml") for f in files): + project_directory = root + break + else: + return Error("TODO(John Sirois): XXX: Can't happen!") + else: + project_directory = source_artifact_path + + build_dir = os.path.join(safe_mkdtemp(), "build") + os.mkdir(build_dir) + spawned_job = try_( + result.invoke_build_hook( + project_directory, + hook_method="build_wheel", + hook_args=[build_dir], + ) + ) + distribution = spawned_job.map(lambda _: Distribution.load(build_dir)).await_result() + build_wheel_fingerprint = CacheHelper.hash(distribution.location, hasher=hashlib.sha256) + if result_type is InstallableType.INSTALLED_WHEEL_CHROOT: + install_request = InstallRequest( + target=locked_source_distribution.target, + wheel_path=distribution.location, + fingerprint=build_wheel_fingerprint, + ) + install_result = install_request.result(installed_wheels_dir) + installed_wheel = install_wheel_chroot( + wheel_path=install_request.wheel_path, destination=install_result.build_chroot + ) + distribution = Distribution.load(installed_wheel.prefix_dir) + + return ResolvedDistribution( + target=locked_source_distribution.target, + fingerprinted_distribution=FingerprintedDistribution( + distribution=distribution, fingerprint=build_wheel_fingerprint + ), + direct_requirements=SortedTuple(), + ) + + +def build_locked_source_distributions( + locked_source_distributions, # type: Sequence[LockedSourceDistribution] + lock_downloader, # type: LockDownloader + result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + build_configuration=BuildConfiguration(), # type: BuildConfiguration + dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration +): + # type: (...) -> Union[Iterable[ResolvedDistribution], Error] + + downloadable_artifacts_by_locked_source_distribution = ( + {} + ) # type: Dict[LockedSourceDistribution, Tuple[DownloadableArtifact, ...]] + subset_errors = OrderedDict() # type: OrderedDict[LockedSourceDistribution, Tuple[Error, ...]] + for locked_source_distribution in locked_source_distributions: + subset_result = subset_for_target( + target=locked_source_distribution.target, + locked_resolves=locked_source_distribution.locked_resolves, + requirements_to_resolve=tuple( + Requirement.parse(req) + for req in locked_source_distribution.build_system_table.requires + ), + build_configuration=build_configuration, + dependency_configuration=dependency_configuration, + ) + if isinstance(subset_result, Resolved): + downloadable_artifacts_by_locked_source_distribution[ + locked_source_distribution + ] = subset_result.downloadable_artifacts + elif isinstance(subset_result, tuple) and subset_result: + subset_errors[locked_source_distribution] = subset_result + if subset_errors: + return Error("TODO(John Sirois): XXX: build a subset errors message") + + downloaded_artifacts = try_( + lock_downloader.download_artifacts( + tuple( + (downloadable_artifact, locked_source_distribution.target) + for locked_source_distribution, downloadable_artifacts in downloadable_artifacts_by_locked_source_distribution.items() + for downloadable_artifact in downloadable_artifacts + ) + ) + ) + install_requests_by_locked_source_distribution = defaultdict( + list + ) # type: DefaultDict[LockedSourceDistribution, List[InstallRequest]] + resolve_errors = defaultdict( + list + ) # type: DefaultDict[LockedSourceDistribution, List[DownloadedArtifact]] + for ( + locked_source_distribution, + downloadable_artifacts, + ) in downloadable_artifacts_by_locked_source_distribution.items(): + for downloadable_artifact in downloadable_artifacts: + downloaded_artifact = downloaded_artifacts[downloadable_artifact] + if is_wheel(downloaded_artifact.path): + install_requests_by_locked_source_distribution[locked_source_distribution].append( + InstallRequest( + target=locked_source_distribution.target, + wheel_path=downloaded_artifact.path, + fingerprint=downloaded_artifact.fingerprint, + ) + ) + else: + resolve_errors[locked_source_distribution].append(downloaded_artifact) + if resolve_errors: + return Error("TODO(John Sirois): XXX: build a resolve errors message") + + # TODO(John Sirois): now we have a list of install requests needed per each source distribution + # build system, parallelize install + create pip venv + build wheel + built_distributions = [] # type: List[ResolvedDistribution] + build_errors = [] # type: List[Error] + for ( + locked_source_distribution, + install_requests, + ) in install_requests_by_locked_source_distribution.items(): + build_result = build_locked_source_distribution( + locked_source_distribution, install_requests, result_type + ) + if isinstance(build_result, ResolvedDistribution): + built_distributions.append(build_result) + else: + build_errors.append(build_result) + if build_errors: + return Error("TODO(John Sirois): XXX: build a build errors message") + return built_distributions def resolve_from_lock( @@ -102,8 +296,9 @@ def resolve_from_lock( return downloaded_artifacts with TRACER.timed("Categorizing {} downloaded artifacts".format(len(downloaded_artifacts))): - build_requests = [] - install_requests = [] + build_requests = [] # type: List[BuildRequest] + locked_build_requests = [] # type: List[LockedSourceDistribution] + install_requests = [] # type: List[InstallRequest] for resolved_subset in subset_result.subsets: for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts: downloaded_artifact = downloaded_artifacts[downloadable_artifact] @@ -115,6 +310,20 @@ def resolve_from_lock( fingerprint=downloaded_artifact.fingerprint, ) ) + elif lock.lock_build_systems: + production_assert(downloadable_artifact.build_system_table is not None) + build_system_table = cast( + BuildSystemTable, downloadable_artifact.build_system_table + ) + locked_build_system_resolves = lock.build_systems[build_system_table] + locked_build_requests.append( + LockedSourceDistribution( + target=resolved_subset.target, + source_artifact=downloaded_artifact, + build_system_table=build_system_table, + locked_resolves=locked_build_system_resolves, + ) + ) else: build_requests.append( BuildRequest( @@ -123,12 +332,30 @@ def resolve_from_lock( fingerprint=downloaded_artifact.fingerprint, ) ) + build_request_count = len(build_requests) + locked_build_request_count = len(locked_build_requests) + production_assert( + ((build_request_count > 0) ^ (locked_build_request_count > 0)) + or (build_request_count == 0 and locked_build_request_count == 0) + ) with TRACER.timed( "Building {} artifacts and installing {}".format( - len(build_requests), len(build_requests) + len(install_requests) + build_request_count, build_request_count + len(install_requests) ) ): + distributions = list( + try_( + build_locked_source_distributions( + locked_build_requests, + lock_downloader, + result_type=result_type, + build_configuration=build_configuration, + dependency_configuration=dependency_configuration, + ) + ) + ) + build_and_install_request = BuildAndInstallRequest( build_requests=build_requests, install_requests=install_requests, @@ -163,7 +390,7 @@ def resolve_from_lock( # `LockedResolve.resolve` above and need not waste time (~O(100ms)) doing this again. ignore_errors = True - distributions = ( + distributions.extend( build_and_install_request.install_distributions( ignore_errors=ignore_errors, max_parallel_jobs=max_parallel_jobs, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index 785e853cf..61119ad91 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -455,6 +455,11 @@ def create( artifact = attr.ib() # type: Union[FileArtifact, LocalProjectArtifact, VCSArtifact] satisfied_direct_requirements = attr.ib(default=SortedTuple()) # type: SortedTuple[Requirement] + @property + def build_system_table(self): + # type: () -> Optional[BuildSystemTable] + return self.artifact.build_system_table + @attr.s(frozen=True) class Resolved(object): diff --git a/pex/resolve/lockfile/subset.py b/pex/resolve/lockfile/subset.py index 0d8e3a49d..7505a8b68 100644 --- a/pex/resolve/lockfile/subset.py +++ b/pex/resolve/lockfile/subset.py @@ -12,7 +12,7 @@ from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.requirements import LocalProjectRequirement, parse_requirement_strings -from pex.resolve.locked_resolve import Resolved +from pex.resolve.locked_resolve import LockedResolve, Resolved from pex.resolve.lockfile.model import Lockfile from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import BuildConfiguration @@ -43,6 +43,41 @@ class SubsetResult(object): subsets = attr.ib() # type: Tuple[Subset, ...] +def subset_for_target( + target, # type: Target + locked_resolves, # type: Iterable[LockedResolve] + requirements_to_resolve, # type: Iterable[Requirement] + constraints=(), # type: Iterable[Requirement] + source=None, # type: Optional[str] + build_configuration=BuildConfiguration(), # type: BuildConfiguration + transitive=True, # type: bool + include_all_matches=False, # type: bool + dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration +): + # type: (...) -> Union[Resolved, Tuple[Error, ...]] + resolveds = [] + errors = [] + for locked_resolve in locked_resolves: + resolve_result = locked_resolve.resolve( + target, + requirements_to_resolve, + constraints=constraints, + source=source, + build_configuration=build_configuration, + transitive=transitive, + include_all_matches=include_all_matches, + dependency_configuration=dependency_configuration, + # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically + # invalid `pip-legacy-resolver` locks: + # https://github.com/pex-tool/pex/issues/1652 + ) + if isinstance(resolve_result, Resolved): + resolveds.append(resolve_result) + else: + errors.append(resolve_result) + return Resolved.most_specific(resolveds) if resolveds else tuple(errors) + + def subset( targets, # type: Targets lock, # type: Lockfile @@ -105,31 +140,21 @@ def subset( ) ): for target in targets.unique_targets(): - resolveds = [] - errors = [] - for locked_resolve in lock.locked_resolves: - resolve_result = locked_resolve.resolve( - target, - requirements_to_resolve, - constraints=constraints, - source=lock.source, - build_configuration=build_configuration, - transitive=transitive, - include_all_matches=include_all_matches, - dependency_configuration=dependency_configuration, - # TODO(John Sirois): Plumb `--ignore-errors` to support desired but technically - # invalid `pip-legacy-resolver` locks: - # https://github.com/pex-tool/pex/issues/1652 - ) - if isinstance(resolve_result, Resolved): - resolveds.append(resolve_result) - else: - errors.append(resolve_result) - - if resolveds: - resolved_by_target[target] = Resolved.most_specific(resolveds) - elif errors: - errors_by_target[target] = tuple(errors) + result = subset_for_target( + target, + locked_resolves=lock.locked_resolves, + requirements_to_resolve=requirements_to_resolve, + constraints=constraints, + source=lock.source, + build_configuration=build_configuration, + transitive=transitive, + include_all_matches=include_all_matches, + dependency_configuration=dependency_configuration, + ) + if isinstance(result, Resolved): + resolved_by_target[target] = result + elif len(result) > 0: + errors_by_target[target] = result if errors_by_target: return Error( diff --git a/tests/build_system/test_pep_518.py b/tests/build_system/test_pep_518.py index 677b1bc6e..13cf317ad 100644 --- a/tests/build_system/test_pep_518.py +++ b/tests/build_system/test_pep_518.py @@ -5,8 +5,7 @@ import subprocess from textwrap import dedent -from pex.build_system import pep_518 -from pex.build_system.pep_518 import BuildSystem +from pex.build_system import BuildSystem, pep_518 from pex.common import touch from pex.pep_503 import ProjectName from pex.resolve.configured_resolver import ConfiguredResolver diff --git a/tests/integration/build_system/test_pep_518.py b/tests/integration/build_system/test_pep_518.py index fadf61f89..5cdb83188 100644 --- a/tests/integration/build_system/test_pep_518.py +++ b/tests/integration/build_system/test_pep_518.py @@ -4,7 +4,8 @@ import os.path import subprocess -from pex.build_system.pep_518 import BuildSystem, load_build_system +from pex.build_system import BuildSystem +from pex.build_system.pep_518 import load_build_system from pex.pip.version import PipVersion from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.resolver_configuration import PipConfiguration, ReposConfiguration