From 67d5df44166b71db9413731fb1765f462e497f29 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 16 Dec 2023 12:49:32 -0800 Subject: [PATCH 1/4] Switch vendoring from Pip install to Pex install. --- pex/pep_427.py | 19 +-- pex/vendor/__init__.py | 2 +- pex/vendor/__main__.py | 128 ++++++++++-------- tests/integration/venv_ITs/test_issue_1630.py | 18 ++- tests/test_vendor.py | 6 +- 5 files changed, 101 insertions(+), 72 deletions(-) diff --git a/pex/pep_427.py b/pex/pep_427.py index e5422f9d9..8b7661cb3 100644 --- a/pex/pep_427.py +++ b/pex/pep_427.py @@ -394,13 +394,16 @@ def record_files( print("pex", file=fp) installed_files.append(InstalledWheel.create_installed_file(path=fp.name, dest_dir=dest)) - if requested: - requested_path = os.path.join(dest, wheel.metadata_path("REQUESTED")) - touch(requested_path) - installed_files.append( - InstalledWheel.create_installed_file(path=requested_path, dest_dir=dest) - ) + if interpreter: + # Finalize a proper venv install with REQUESTED and a RECORD to support un-installing. + if requested: + requested_path = os.path.join(dest, wheel.metadata_path("REQUESTED")) + touch(requested_path) + installed_files.append( + InstalledWheel.create_installed_file(path=requested_path, dest_dir=dest) + ) + + installed_files.append(InstalledFile(path=record_relpath, hash=None, size=None)) + Record.write(dst=record_abspath, installed_files=installed_files) - installed_files.append(InstalledFile(path=record_relpath, hash=None, size=None)) - Record.write(dst=record_abspath, installed_files=installed_files) return wheel.metadata_files diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py index 0beb2f96f..37508ec77 100644 --- a/pex/vendor/__init__.py +++ b/pex/vendor/__init__.py @@ -86,7 +86,7 @@ def git( rewrite=True, # type: bool constraints=(), # type: Tuple[str, ...] ): - requirement = "git+{repo}@{commit}#egg={project_name}".format( + requirement = "{project_name} @ git+{repo}@{commit}".format( repo=repo, commit=commit, project_name=project_name ) if not prep_command: diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py index 8b5442166..6a480057b 100644 --- a/pex/vendor/__main__.py +++ b/pex/vendor/__main__.py @@ -3,9 +3,10 @@ from __future__ import absolute_import, print_function +import glob import os import pkgutil -import shutil +import re import subprocess import sys from argparse import ArgumentParser @@ -208,6 +209,7 @@ def find_site_packages(prefix_dir): def vendorize(interpreter, root_dir, vendor_specs, prefix, update): + # TODO(John Sirois): XXX: Re-write comment. # There is bootstrapping catch-22 here. In order for `pex.third_party` to work, all 3rdparty # importable code must lie at the top of its vendored chroot. Although # `pex.pep_376.Record.fixup_install` encodes the logic to achieve this layout, we can't run @@ -215,9 +217,9 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): # layout!. We take the tack of saving the current interpreter and then doing the `--prefix` # install off into a temp dir, moving the site-packages importables into the vendor chroot, # importing the code we'll need, then moving the importables back. - moved = {} + pass - prefix_dir_by_vendor_spec = defaultdict(safe_mkdtemp) + unpacked_wheel_chroots_by_vendor_spec = defaultdict(list) for vendor_spec in vendor_specs: # NB: We set --no-build-isolation to prevent pip from installing the requirements listed in # its [build-system] config in its pyproject.toml. @@ -235,24 +237,14 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): # that Pip finds no newer versions of wheel in its cache. As a result, vendoring (at least # via tox) is hermetic. requirement = vendor_spec.prepare() + wheels_dir = safe_mkdtemp() cmd = [ "pip", - "install", + "wheel", "--no-build-isolation", "--no-cache-dir", - "--no-compile", - "--prefix", - prefix_dir_by_vendor_spec[vendor_spec], - # In `--prefix` scheme, Pip warns about installed scripts not being on $PATH. We fix - # this when a PEX is turned into a venv. - "--no-warn-script-location", - # In `--prefix` scheme, Pip normally refuses to install a dependency already in the - # `sys.path` of Pip itself since the requirement is already satisfied. Since `pip`, - # `setuptools` and `wheel` are always in that `sys.path` (Our `pip.pex` venv PEX), we - # force installation so that PEXes with dependencies on those projects get them properly - # installed instead of skipped. - "--force-reinstall", - "--ignore-installed", + "--wheel-dir", + wheels_dir, requirement, ] @@ -274,14 +266,28 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): if result != 0: raise VendorizeError("Failed to vendor {!r}".format(vendor_spec)) - # Temporarily move importable code into the vendor chroot for importing tools later. - site_packages = find_site_packages(prefix_dir=prefix_dir_by_vendor_spec[vendor_spec]) + # Temporarily make importable code available in the vendor chroot for importing Pex code + # later. safe_mkdir(vendor_spec.target_dir) - for item in os.listdir(site_packages): - src = os.path.join(site_packages, item) - dst = os.path.join(vendor_spec.target_dir, item) - shutil.move(src, dst) - moved[dst] = src + for wheel_file in glob.glob(os.path.join(wheels_dir, "*.whl")): + extract_dir = os.path.join(wheels_dir, ".extracted") + output = subprocess.check_output( + ["wheel", "unpack", "--dest", extract_dir, wheel_file] + ).decode("utf-8") + match = re.match(r"^Unpacking to: (?P.+)\.\.\.OK$", output) + assert match is not None, ( + "Failed to determine {wheel_file} unpack dir from wheel unpack output:\n" + "{output}".format(wheel_file=wheel_file, output=output) + ) + unpacked_to_dir = os.path.join(extract_dir, match["unpack_dir"]) + unpacked_wheel_dir = os.path.join(extract_dir, os.path.basename(wheel_file)) + os.rename(unpacked_to_dir, unpacked_wheel_dir) + unpacked_wheel_chroots_by_vendor_spec[vendor_spec].append(unpacked_wheel_dir) + for path in os.listdir(unpacked_wheel_dir): + os.symlink( + os.path.join(unpacked_wheel_dir, path), + os.path.join(vendor_spec.target_dir, path), + ) if vendor_spec.constrain: cmd = ["pip", "freeze", "--all", "--path", vendor_spec.target_dir] @@ -299,7 +305,7 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): rewrite_paths = [os.path.join(root_dir, c) for c in ("pex", "tests")] + vendored_path for rewrite_path in rewrite_paths: - for root, dirs, files in os.walk(rewrite_path): + for root, dirs, files in os.walk(rewrite_path, followlinks=True): if root == os.path.join(root_dir, "pex", "vendor"): dirs[:] = [d for d in dirs if d != "_vendored"] for f in files: @@ -325,47 +331,57 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): # Import all code needed below now before we move any vendored bits it depends on temporarily # back to the prefix site-packages dir. - from pex.dist_metadata import find_distributions - from pex.pep_376 import Record - - dist_by_vendor_spec = OrderedDict() - for vendor_spec in vendor_specs: - for dist in find_distributions(search_path=[vendor_spec.target_dir]): - if dist.project_name == vendor_spec.key: - dist_by_vendor_spec[vendor_spec] = dist - break - if vendor_spec not in dist_by_vendor_spec: - raise RuntimeError("Failed to find a distribution for {}.".format(vendor_spec)) - - # Move the importables back to their temporary dir chroot original locations. - for dst, src in moved.items(): - shutil.move(dst, src) + from pex.dist_metadata import ProjectNameAndVersion, Requirement + from pex.pep_427 import install_wheel_chroot for vendor_spec in vendor_specs: print( bold( green( - "Fixing up scripts and distribution metadata for {requirement}".format( + "Finalizing vendoring of {requirement}".format( requirement=vendor_spec.requirement ) ) ) ) - # Move the `pip install --prefix` temporary dir chroot into the vendor chroot. - prefix_dir = prefix_dir_by_vendor_spec[vendor_spec] - for item in os.listdir(prefix_dir): - shutil.move(os.path.join(prefix_dir, item), os.path.join(vendor_spec.target_dir, item)) - dist = dist_by_vendor_spec[vendor_spec] - - # Finally, let Record fixup the chroot to its final importable form. - record = Record.from_pip_prefix_install( - prefix_dir=vendor_spec.target_dir, - project_name=dist.project_name, - version=dist.version, - ) - record.fixup_install( - exclude=("constraints.txt", "__init__.py", "__pycache__"), interpreter=interpreter - ) + # With Pex code needed for the final vendor installs imported, we can safely clear out the + # vendor install dirs. + for name in os.listdir(vendor_spec.target_dir): + if name in ("__init__.py", "constraints.txt"): + continue + path = os.path.join(vendor_spec.target_dir, name) + assert os.path.islink(path), ( + "Expected {target_dir} to be composed ~purely of top-level symlinks but {path} " + "is not.".format(target_dir=vendor_spec.target_dir, path=path) + ) + os.unlink(path) + + # We want the primary artifact to own any special Pex wheel chroot metadata; so we arrange + # a list of installs that place it last. + primary_project = Requirement.parse(vendor_spec.requirement).project_name + wheel_chroots_by_project_name = { + ProjectNameAndVersion.from_filename( + wheel_chroot + ).canonicalized_project_name: wheel_chroot + for wheel_chroot in unpacked_wheel_chroots_by_vendor_spec[vendor_spec] + } + primary_wheel_chroot = wheel_chroots_by_project_name.pop(primary_project) + wheels_chroots_to_install = list(wheel_chroots_by_project_name.values()) + wheels_chroots_to_install.append(primary_wheel_chroot) + + for wheel_chroot in wheels_chroots_to_install: + dest_dir = safe_mkdtemp() + subprocess.check_call(["wheel", "pack", "--dest-dir", dest_dir, wheel_chroot]) + wheel_files = glob.glob(os.path.join(dest_dir, "*.whl")) + assert len(wheel_files) == 1, ( + "Expected re-packing {wheel_chroot} to produce one `.whl` file but found {count}:\n" + "{wheel_files}" + ).format( + wheel_chroot=wheel_chroot, + count=len(wheel_files), + wheel_files="\n".join(os.path.basename(wheel_file) for wheel_file in wheel_files), + ) + install_wheel_chroot(wheel_path=wheel_files[0], destination=vendor_spec.target_dir) if __name__ == "__main__": diff --git a/tests/integration/venv_ITs/test_issue_1630.py b/tests/integration/venv_ITs/test_issue_1630.py index a0d368b07..2dfe1510c 100644 --- a/tests/integration/venv_ITs/test_issue_1630.py +++ b/tests/integration/venv_ITs/test_issue_1630.py @@ -14,7 +14,7 @@ from testing import PY38, ensure_python_venv, run_pex_command if TYPE_CHECKING: - from typing import Any, List + from typing import Any, Container, List def test_data_files(tmpdir): @@ -63,12 +63,22 @@ def test_data_files(tmpdir): pip_venv = Virtualenv.enclosing(py38) assert pip_venv is not None - def recursive_listing(venv): - # type: (Virtualenv) -> List[str] + def recursive_listing( + venv, # type: Virtualenv + exclude=(), # type: Container[str] + ): + # type: (...) -> List[str] return sorted( os.path.relpath(os.path.join(root, f), venv.venv_dir) for root, _, files in os.walk(venv.venv_dir) for f in files + if f not in exclude ) - assert recursive_listing(pip_venv) == recursive_listing(pex_venv) + # We exclude the REQUESTED .dist-info metadata file which Pip installs, but we currently do not. + # This file is not required as originally spelled out in PEP-376 + # (https://peps.python.org/pep-0376/#one-dist-info-directory-per-installed-distribution): + # "The METADATA, RECORD and INSTALLER files are mandatory, while REQUESTED may be missing." + # This remains true in the modern spec as well. See: + # https://packaging.python.org/en/latest/specifications/recording-installed-packages/#the-dist-info-directory + assert recursive_listing(pip_venv, exclude={"REQUESTED"}) == recursive_listing(pex_venv) diff --git a/tests/test_vendor.py b/tests/test_vendor.py index 7edd9dd85..4fe731aee 100644 --- a/tests/test_vendor.py +++ b/tests/test_vendor.py @@ -26,8 +26,8 @@ def test_git(): repo="https://github.com/foo.git", commit="da39a3ee", project_name="bar" ) assert "bar" == vendor_spec.key - assert "git+https://github.com/foo.git@da39a3ee#egg=bar" == vendor_spec.requirement - assert "git+https://github.com/foo.git@da39a3ee#egg=bar" == vendor_spec.prepare() + assert "bar @ git+https://github.com/foo.git@da39a3ee" == vendor_spec.requirement + assert "bar @ git+https://github.com/foo.git@da39a3ee" == vendor_spec.prepare() def test_git_prep_command(tmpdir): @@ -55,7 +55,7 @@ def test_git_prep_command(tmpdir): ) assert not os.path.exists(prep_file) - assert "git+{repo}@{commit}#egg=bar".format(repo=repo, commit=commit) == vendor_spec.requirement + assert "bar @ git+{repo}@{commit}".format(repo=repo, commit=commit) == vendor_spec.requirement assert not os.path.exists(prep_file) clone = vendor_spec.prepare() From 1f4687b31fbcbc0895701cf0d831957a1c4d4afe Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 16 Dec 2023 12:51:16 -0800 Subject: [PATCH 2/4] Kill now-unused Pip specific code. --- pex/pep_376.py | 252 +------------------------------------------------ 1 file changed, 2 insertions(+), 250 deletions(-) diff --git a/pex/pep_376.py b/pex/pep_376.py index 62238e401..c0b622422 100644 --- a/pex/pep_376.py +++ b/pex/pep_376.py @@ -11,32 +11,16 @@ import json import os import shutil -from contextlib import closing from fileinput import FileInput from pex import hashing -from pex.common import is_pyc_dir, is_pyc_file, is_python_script, safe_mkdir, safe_open -from pex.compatibility import get_stdout_bytes_buffer, urlparse -from pex.dist_metadata import Distribution, EntryPoint, MetadataFiles, MetadataType +from pex.common import is_pyc_dir, is_pyc_file, safe_mkdir, safe_open from pex.interpreter import PythonInterpreter -from pex.pep_440 import Version -from pex.pep_503 import ProjectName from pex.typing import TYPE_CHECKING, cast from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: - from typing import ( - Callable, - Container, - Dict, - Iterable, - Iterator, - Optional, - Protocol, - Text, - Tuple, - Union, - ) + from typing import Callable, Iterable, Iterator, Optional, Protocol, Text, Tuple, Union import attr # vendor:skip @@ -483,240 +467,8 @@ def read( size = int(file_size) if file_size else None yield InstalledFile(path=path, hash=file_hash, size=size) - @staticmethod - def _find_installation( - prefix_dir, # type: str - project_name, # type: str - version, # type: str - ): - # type: (...) -> Optional[MetadataFiles] - - canonical_project_name = ProjectName(project_name) - canonical_version = Version(version) - - # Some distributions in the wild (namely python-certifi-win32 1.6.1, - # see: https://github.com/pantsbuild/pex/issues/1861) create their own directories named - # `site-packages` that are not in-fact located in site-packages (the "purelib" or "platlib" - # sysconfig install paths). Work around these broken packages by just looking for all - # `site-packages` subdirectories of the `prefix_dir` and checking each for the installation - # `RECORD`. There should always be just one such installation `RECORD` resulting from a - # `pip install --prefix --no-deps ` and so this is safe. - site_packages_dirs = [ - os.path.join(root, d) - for root, dirs, _ in os.walk(prefix_dir) - for d in dirs - if d == "site-packages" - ] - for site_packages_dir in site_packages_dirs: - metadata_files = MetadataType.DIST_INFO.load_metadata( - site_packages_dir, project_name=canonical_project_name - ) - if metadata_files and canonical_version == metadata_files.metadata.version: - return metadata_files - return None - - @classmethod - def from_pip_prefix_install( - cls, - prefix_dir, # type: str - project_name, # type: str - version, # type: str - ): - # type: (...) -> Record - metadata_files = cls._find_installation(prefix_dir, project_name, version) - if not metadata_files: - raise RecordNotFoundError( - "Could not find project metadata for {project_name} {version} under " - "{prefix_dir}".format( - project_name=project_name, version=version, prefix_dir=prefix_dir - ) - ) - record_relpath = metadata_files.metadata_file_rel_path("RECORD") - if not record_relpath: - raise RecordNotFoundError( - "Could not find the installation RECORD for {project_name} {version} under " - "{location}".format( - project_name=project_name, - version=version, - location=metadata_files.metadata.location, - ) - ) - - rel_base_dir = os.path.relpath(metadata_files.metadata.location, prefix_dir) - return cls( - project_name=project_name, - version=version, - prefix_dir=prefix_dir, - rel_base_dir=rel_base_dir, - relative_path=record_relpath, - ) - project_name = attr.ib() # type: str version = attr.ib() # type: str prefix_dir = attr.ib() # type: str rel_base_dir = attr.ib() # type: Text relative_path = attr.ib() # type: Text - - def _find_dist_info_file(self, filename): - # type: (str) -> Optional[DistInfoFile] - metadata_files = MetadataType.DIST_INFO.load_metadata( - location=os.path.join(self.prefix_dir, self.rel_base_dir), - project_name=ProjectName(self.project_name), - ) - if metadata_files is None: - return None - - metadata_file_rel_path = metadata_files.metadata_file_rel_path(filename) - if metadata_file_rel_path is None: - return None - - content = metadata_files.read(filename) - if content is None: - return None - - file_path = os.path.join(metadata_files.metadata.location, metadata_file_rel_path) - return DistInfoFile(path=file_path, content=content) - - def fixup_install( - self, - exclude=(), # type: Container[str] - interpreter=None, # type: Optional[PythonInterpreter] - ): - # type: (...) -> InstalledWheel - """Fixes a wheel install to be reproducible and importable. - - After fixed up, this RECORD can be used to re-install the wheel in a venv with `reinstall`. - - :param exclude: Any top-level items to exclude. - :param interpreter: The interpreter used to perform the wheel install. - """ - self._fixup_scripts() - self._fixup_direct_url() - - # The RECORD is unused in PEX zipapp mode and only needed in venv mode. Since it can contain - # relative path entries that differ between interpreters - notably pypy for Python < 3.8 has - # a custom scheme - we just delete the file and create it on-demand for venv re-installs. - os.unlink(os.path.join(self.prefix_dir, self.rel_base_dir, self.relative_path)) - - # An example of the installed wheel chroot we're aiming for: - # .prefix/bin/... # scripts - # .prefix/include/site/pythonX.Y/... # headers - # .prefix/share/... # data files - # greenlet/... # importables - # greenlet-1.1.2.dist-info/... # importables - stash_dir = ".prefix" - prefix_stash = os.path.join(self.prefix_dir, stash_dir) - safe_mkdir(prefix_stash) - - # 1. Move everything into the stash. - for item in os.listdir(self.prefix_dir): - if stash_dir == item or item in exclude: - continue - shutil.move(os.path.join(self.prefix_dir, item), os.path.join(prefix_stash, item)) - # 2. Normalize all `*/{python ver}` paths to `*/pythonX.Y` - for root, dirs, _ in os.walk(prefix_stash): - dirs_to_scan = [] - for d in dirs: - path = os.path.join(root, d) - normalized_path = InstalledFile.normalized_path(path, interpreter=interpreter) - if normalized_path != path: - shutil.move(path, normalized_path) - else: - dirs_to_scan.append(d) - dirs[:] = dirs_to_scan - - # 3. Move `site-packages` content back up to the prefix dir chroot so that content is - # importable when this prefix dir chroot is added to the `sys.path` in PEX zipapp mode. - importable_stash = InstalledFile.normalized_path( - os.path.join(prefix_stash, self.rel_base_dir), interpreter=interpreter - ) - for importable_item in os.listdir(importable_stash): - shutil.move( - os.path.join(importable_stash, importable_item), - os.path.join(self.prefix_dir, importable_item), - ) - os.rmdir(importable_stash) - - return InstalledWheel.save( - prefix_dir=self.prefix_dir, - stash_dir=stash_dir, - record_relpath=self.relative_path, - ) - - def _fixup_scripts(self): - # type: (...) -> None - bin_dir = os.path.join(self.prefix_dir, "bin") - if not os.path.isdir(bin_dir): - return - - console_scripts = {} # type: Dict[Text, EntryPoint] - entry_points_file = self._find_dist_info_file("entry_points.txt") - if entry_points_file: - console_scripts.update( - Distribution.parse_entry_map(entry_points_file.content).get("console_scripts", {}) - ) - - scripts = {} # type: Dict[str, Optional[bytes]] - for script_name in os.listdir(bin_dir): - script_path = os.path.join(bin_dir, script_name) - if is_python_script(script_path): - scripts[script_path] = None - elif script_name in console_scripts: - # When a wheel is installed by Pip and that wheel contains console_scripts, they are - # normally written with a faux-shebang of: - # #!python - # - # Pex relies on this hermetic shebang and only ever reifies it when creating venvs. - # - # If Pip is being run under a Python executable with a path length >127 characters - # on Linux though, it writes a shebang / header of: - # #!/bin/sh - # '''exec' "$0" "$@"' - # ' ''' - # - # That header is immediately followed by the expected console_script shim contents: - # # -*- coding: utf-8 -*- - # import re - # import sys - # from import - # if __name__ == '__main__': - # sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - # sys.exit(main()) - # - # Instead of guessing that 127 characters is the shebang length limit and using - # Pip's safety-hatch `/bin/sh` trick, we forcibly re-write the header to be just the - # expected `#!python` shebang. We detect the end of the header with the known 1st - # line of console_script shim ~code defined in - # pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py on line 41: - # https://github.com/pantsbuild/pex/blob/196b4cd5b8dd4b4af2586460530e9a777262be7d/pex/vendor/_vendored/pip/pip/_vendor/distlib/scripts.py#L41 - scripts[script_path] = b"# -*- coding: utf-8 -*-" - if not scripts: - return - - with closing(FileInput(files=scripts.keys(), inplace=True, mode="rb")) as script_fi: - first_non_shebang_line = None # type: Optional[bytes] - for line in script_fi: - buffer = get_stdout_bytes_buffer() - if script_fi.isfirstline(): - first_non_shebang_line = scripts[script_fi.filename()] - # Ensure python shebangs are reproducible. The only place these can be used is - # in venv mode PEXes where the `#!python` placeholder shebang will be re-written - # to use the venv's python interpreter. - buffer.write(b"#!python\n") - elif ( - not first_non_shebang_line - or cast(bytes, line).strip() == first_non_shebang_line - ): - # N.B.: These lines include the newline already. - buffer.write(cast(bytes, line)) - first_non_shebang_line = None - - def _fixup_direct_url(self): - # type: () -> None - direct_url_file = self._find_dist_info_file("direct_url.json") - if direct_url_file: - if ( - urlparse.urlparse(json.loads(direct_url_file.content.decode("utf-8"))["url"]).scheme - == "file" - ): - os.unlink(direct_url_file.path) From 926f1ccd2dc1b48afb9f0291f079929eefff778f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 16 Dec 2023 13:21:14 -0800 Subject: [PATCH 3/4] Run `tox -evendor`. --- .../attrs/attrs-21.5.0.dev0.dist-info/INSTALLER | 2 +- .../attrs/attrs-21.5.0.dev0.dist-info/REQUESTED | 0 .../attrs-21.5.0.dev0.dist-info/direct_url.json | 1 - .../packaging-20.9.dist-info/INSTALLER | 2 +- .../packaging-20.9.dist-info/REQUESTED | 0 .../pyparsing-2.4.7.dist-info/INSTALLER | 2 +- .../pyparsing-2.4.7.dist-info/RECORD | 7 ------- .../packaging-21.3.dist-info/INSTALLER | 2 +- .../packaging-21.3.dist-info/REQUESTED | 0 .../pyparsing-2.4.7.dist-info/INSTALLER | 2 +- .../pyparsing-2.4.7.dist-info/RECORD | 7 ------- .../packaging-23.1.dist-info/INSTALLER | 2 +- .../packaging-23.1.dist-info/REQUESTED | 0 pex/vendor/_vendored/pip/.prefix/bin/pip | 14 ++++++++++---- pex/vendor/_vendored/pip/.prefix/bin/pip3 | 14 ++++++++++---- pex/vendor/_vendored/pip/.prefix/bin/pip3.8 | 14 ++++++++++---- .../_vendored/pip/pip-20.3.4.dist-info/INSTALLER | 2 +- .../_vendored/pip/pip-20.3.4.dist-info/REQUESTED | 0 .../pip/pip-20.3.4.dist-info/direct_url.json | 1 - .../_vendored/setuptools/.prefix/bin/easy_install | 14 ++++++++++---- .../setuptools/.prefix/bin/easy_install-3.8 | 14 ++++++++++---- .../INSTALLER | 2 +- .../REQUESTED | 0 .../_vendored/toml/toml-0.10.2.dist-info/INSTALLER | 2 +- .../_vendored/toml/toml-0.10.2.dist-info/REQUESTED | 0 25 files changed, 59 insertions(+), 45 deletions(-) delete mode 100644 pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/direct_url.json delete mode 100644 pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD delete mode 100644 pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/RECORD delete mode 100644 pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/pip/pip-20.3.4.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/pip/pip-20.3.4.dist-info/direct_url.json delete mode 100644 pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/REQUESTED delete mode 100644 pex/vendor/_vendored/toml/toml-0.10.2.dist-info/REQUESTED diff --git a/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER b/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER +++ b/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/REQUESTED b/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/direct_url.json b/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/direct_url.json deleted file mode 100644 index 16c9c07af..000000000 --- a/pex/vendor/_vendored/attrs/attrs-21.5.0.dev0.dist-info/direct_url.json +++ /dev/null @@ -1 +0,0 @@ -{"url": "https://github.com/python-attrs/attrs", "vcs_info": {"commit_id": "947bfb542104209a587280701d8cb389c813459d", "requested_revision": "947bfb542104209a587280701d8cb389c813459d", "vcs": "git"}} \ No newline at end of file diff --git a/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER b/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER +++ b/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/REQUESTED b/pex/vendor/_vendored/packaging_20_9/packaging-20.9.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER b/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER +++ b/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD b/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD deleted file mode 100644 index dff93c86c..000000000 --- a/pex/vendor/_vendored/packaging_20_9/pyparsing-2.4.7.dist-info/RECORD +++ /dev/null @@ -1,7 +0,0 @@ -pyparsing-2.4.7.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -pyparsing-2.4.7.dist-info/LICENSE,sha256=ENUSChaAWAT_2otojCIL-06POXQbVzIGBNRVowngGXI,1023 -pyparsing-2.4.7.dist-info/METADATA,sha256=Ry40soZZiZrAkSMQT_KU1_1REe6FKa5UWzbT6YA8Mxs,3636 -pyparsing-2.4.7.dist-info/RECORD,, -pyparsing-2.4.7.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 -pyparsing-2.4.7.dist-info/top_level.txt,sha256=eUOjGzJVhlQ3WS2rFAy2mN3LX_7FKTM5GSJ04jfnLmU,10 -pyparsing.py,sha256=oxX_ZOz8t-eros-UWY7nJgcdUgD-rQ53Ck0qp7_v3Ig,273365 diff --git a/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER b/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER +++ b/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/REQUESTED b/pex/vendor/_vendored/packaging_21_3/packaging-21.3.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/INSTALLER b/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/INSTALLER +++ b/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/RECORD b/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/RECORD deleted file mode 100644 index dff93c86c..000000000 --- a/pex/vendor/_vendored/packaging_21_3/pyparsing-2.4.7.dist-info/RECORD +++ /dev/null @@ -1,7 +0,0 @@ -pyparsing-2.4.7.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -pyparsing-2.4.7.dist-info/LICENSE,sha256=ENUSChaAWAT_2otojCIL-06POXQbVzIGBNRVowngGXI,1023 -pyparsing-2.4.7.dist-info/METADATA,sha256=Ry40soZZiZrAkSMQT_KU1_1REe6FKa5UWzbT6YA8Mxs,3636 -pyparsing-2.4.7.dist-info/RECORD,, -pyparsing-2.4.7.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 -pyparsing-2.4.7.dist-info/top_level.txt,sha256=eUOjGzJVhlQ3WS2rFAy2mN3LX_7FKTM5GSJ04jfnLmU,10 -pyparsing.py,sha256=oxX_ZOz8t-eros-UWY7nJgcdUgD-rQ53Ck0qp7_v3Ig,273365 diff --git a/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/INSTALLER b/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/INSTALLER +++ b/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/REQUESTED b/pex/vendor/_vendored/packaging_23_1/packaging-23.1.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip b/pex/vendor/_vendored/pip/.prefix/bin/pip index 48a49ac5e..85d6e71ae 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip @@ -1,8 +1,14 @@ #!python # -*- coding: utf-8 -*- -import re +import importlib import sys -from pip._internal.cli.main import main + +object_ref = "pip._internal.cli.main:main" +modname, qualname_separator, qualname = object_ref.partition(':') +entry_point = importlib.import_module(modname) +if qualname_separator: + for attr in qualname.split('.'): + entry_point = getattr(entry_point, attr) + if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) + sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip3 b/pex/vendor/_vendored/pip/.prefix/bin/pip3 index 48a49ac5e..85d6e71ae 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip3 +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip3 @@ -1,8 +1,14 @@ #!python # -*- coding: utf-8 -*- -import re +import importlib import sys -from pip._internal.cli.main import main + +object_ref = "pip._internal.cli.main:main" +modname, qualname_separator, qualname = object_ref.partition(':') +entry_point = importlib.import_module(modname) +if qualname_separator: + for attr in qualname.split('.'): + entry_point = getattr(entry_point, attr) + if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) + sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 b/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 index 48a49ac5e..85d6e71ae 100755 --- a/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 +++ b/pex/vendor/_vendored/pip/.prefix/bin/pip3.8 @@ -1,8 +1,14 @@ #!python # -*- coding: utf-8 -*- -import re +import importlib import sys -from pip._internal.cli.main import main + +object_ref = "pip._internal.cli.main:main" +modname, qualname_separator, qualname = object_ref.partition(':') +entry_point = importlib.import_module(modname) +if qualname_separator: + for attr in qualname.split('.'): + entry_point = getattr(entry_point, attr) + if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) + sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER b/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER +++ b/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/REQUESTED b/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/direct_url.json b/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/direct_url.json deleted file mode 100644 index af829810c..000000000 --- a/pex/vendor/_vendored/pip/pip-20.3.4.dist-info/direct_url.json +++ /dev/null @@ -1 +0,0 @@ -{"url": "https://github.com/pantsbuild/pip", "vcs_info": {"commit_id": "386a54f097ece66775d0c7f34fd29bb596c6b0be", "requested_revision": "386a54f097ece66775d0c7f34fd29bb596c6b0be", "vcs": "git"}} \ No newline at end of file diff --git a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install index b6bcc5f50..4f0954ed5 100755 --- a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install +++ b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install @@ -1,8 +1,14 @@ #!python # -*- coding: utf-8 -*- -import re +import importlib import sys -from setuptools.command.easy_install import main + +object_ref = "setuptools.command.easy_install:main" +modname, qualname_separator, qualname = object_ref.partition(':') +entry_point = importlib.import_module(modname) +if qualname_separator: + for attr in qualname.split('.'): + entry_point = getattr(entry_point, attr) + if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) + sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 index b6bcc5f50..4f0954ed5 100755 --- a/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 +++ b/pex/vendor/_vendored/setuptools/.prefix/bin/easy_install-3.8 @@ -1,8 +1,14 @@ #!python # -*- coding: utf-8 -*- -import re +import importlib import sys -from setuptools.command.easy_install import main + +object_ref = "setuptools.command.easy_install:main" +modname, qualname_separator, qualname = object_ref.partition(':') +entry_point = importlib.import_module(modname) +if qualname_separator: + for attr in qualname.split('.'): + entry_point = getattr(entry_point, attr) + if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) + sys.exit(entry_point()) diff --git a/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER b/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER +++ b/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/REQUESTED b/pex/vendor/_vendored/setuptools/setuptools-44.0.0+3acb925dd708430aeaf197ea53ac8a752f7c1863.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER b/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER index a1b589e38..923be60db 100644 --- a/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER +++ b/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/INSTALLER @@ -1 +1 @@ -pip +pex diff --git a/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/REQUESTED b/pex/vendor/_vendored/toml/toml-0.10.2.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 From 42d54e87de3a411179c93a240c036a47027a2867 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 16 Dec 2023 14:00:55 -0800 Subject: [PATCH 4/4] Fixup unfinished business. Kill unused interpreter parameter and fixup comment to reflect new use of pep_427. Fix vendoing finalization excludes to handle bytecode. --- pex/vendor/__main__.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pex/vendor/__main__.py b/pex/vendor/__main__.py index 6a480057b..d68ff546f 100644 --- a/pex/vendor/__main__.py +++ b/pex/vendor/__main__.py @@ -16,7 +16,6 @@ from redbaron import CommentNode, LiteralyEvaluable, NameNode, RedBaron from pex.common import safe_delete, safe_mkdir, safe_mkdtemp, safe_open, safe_rmtree -from pex.interpreter import PythonInterpreter from pex.vendor import VendorSpec, iter_vendor_specs @@ -208,16 +207,13 @@ def find_site_packages(prefix_dir): ) -def vendorize(interpreter, root_dir, vendor_specs, prefix, update): - # TODO(John Sirois): XXX: Re-write comment. +def vendorize(root_dir, vendor_specs, prefix, update): # There is bootstrapping catch-22 here. In order for `pex.third_party` to work, all 3rdparty # importable code must lie at the top of its vendored chroot. Although - # `pex.pep_376.Record.fixup_install` encodes the logic to achieve this layout, we can't run - # that without the current `pex.interpreter.PythonInterpreter` and 1st approximating that - # layout!. We take the tack of saving the current interpreter and then doing the `--prefix` - # install off into a temp dir, moving the site-packages importables into the vendor chroot, - # importing the code we'll need, then moving the importables back. - pass + # `pex.pep_472.install_wheel_chroot` encodes the logic to achieve this layout, we can't run + # that without 1st approximating that layout!. We take the tack of performing an importable + # installation using `pip wheel ...` + `wheel unpack ...`. Although simply un-packing a wheel + # does not make it importable in general, it works for our pure-python vendored code. unpacked_wheel_chroots_by_vendor_spec = defaultdict(list) for vendor_spec in vendor_specs: @@ -344,10 +340,11 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): ) ) ) + # With Pex code needed for the final vendor installs imported, we can safely clear out the # vendor install dirs. for name in os.listdir(vendor_spec.target_dir): - if name in ("__init__.py", "constraints.txt"): + if name.endswith(".pyc") or name in ("__init__.py", "__pycache__", "constraints.txt"): continue path = os.path.join(vendor_spec.target_dir, name) assert os.path.islink(path), ( @@ -395,13 +392,10 @@ def vendorize(interpreter, root_dir, vendor_specs, prefix, update): ) options = parser.parse_args() - # Grab the PythonInterpreter before we nuke the supporting code needed to identify it. - interpreter = PythonInterpreter.get() root_directory = VendorSpec.ROOT try: safe_rmtree(VendorSpec.vendor_root()) vendorize( - interpreter=interpreter, root_dir=root_directory, vendor_specs=list(iter_vendor_specs()), prefix="pex.third_party",