From 9f8c763b3347a6e082ad4e460502693b19a40e69 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 4 Nov 2023 14:39:09 -0700 Subject: [PATCH] Support `.egg-info` dist metadata. (#2264) This will allow inventorying system interpreters and venvs; which will enable strict-checking support for PEX exlucions via `--provided` as well as more general resolving from venvs (i.e.: adding a new option to the `--index`/`--find-links`, `--pex-repository` and `--lock` set of resolve repository options). The new `-d/--distributions` option is added to `pex3 interpreter inspect` to support creating the aforementioned inventories. --- pex/build_system/pep_517.py | 6 +- pex/cli/commands/interpreter.py | 23 + pex/dist_metadata.py | 652 +++++++++++------- pex/environment.py | 15 +- pex/pep_376.py | 101 +-- pex/resolve/target_options.py | 10 +- pex/venv/installer.py | 2 +- pex/venv/virtualenv.py | 6 +- tests/build_system/test_pep_518.py | 4 +- .../cli/commands/test_interpreter_inspect.py | 24 +- .../cli/commands/test_venv_create.py | 8 +- .../integration/test_interpreter_selection.py | 2 - tests/integration/test_issue_940.py | 6 +- tests/integration/venv_ITs/test_virtualenv.py | 2 +- tests/test_dist_metadata.py | 129 ++-- tests/test_environment.py | 1 - 16 files changed, 628 insertions(+), 363 deletions(-) diff --git a/pex/build_system/pep_517.py b/pex/build_system/pep_517.py index 0abed252b..ba8dd9330 100644 --- a/pex/build_system/pep_517.py +++ b/pex/build_system/pep_517.py @@ -13,7 +13,7 @@ from pex.build_system import DEFAULT_BUILD_BACKEND from pex.build_system.pep_518 import BuildSystem, load_build_system from pex.common import safe_mkdtemp -from pex.dist_metadata import DistMetadata, Distribution +from pex.dist_metadata import DistMetadata, Distribution, MetadataType from pex.jobs import Job, SpawnedJob from pex.pip.version import PipVersion, PipVersionValue from pex.resolve.resolvers import Resolver @@ -24,7 +24,7 @@ from pex.util import named_temporary_file if TYPE_CHECKING: - from typing import Any, Dict, Iterable, Mapping, Optional, Text, Tuple, Union + from typing import Any, Dict, Iterable, Mapping, Optional, Text, Union _DEFAULT_BUILD_SYSTEMS = {} # type: Dict[PipVersionValue, BuildSystem] @@ -272,4 +272,4 @@ def spawn_prepare_metadata( pip_version=pip_version, ) ) - return spawned_job.map(lambda _: DistMetadata.load(build_dir)) + return spawned_job.map(lambda _: DistMetadata.load(build_dir, MetadataType.DIST_INFO)) diff --git a/pex/cli/commands/interpreter.py b/pex/cli/commands/interpreter.py index a5c90fccf..d375ec6fd 100644 --- a/pex/cli/commands/interpreter.py +++ b/pex/cli/commands/interpreter.py @@ -8,6 +8,7 @@ from pex.cli.command import BuildTimeCommand from pex.commands.command import JsonMixin, OutputMixin +from pex.dist_metadata import find_distributions from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import InterpreterConstraint from pex.resolve import target_options @@ -61,6 +62,15 @@ def _add_inspect_arguments(cls, parser): "and will contain the path to the interpreter." ), ) + parser.add_argument( + "-d", + "--distributions", + action="store_true", + help=( + "Include information about the distributions installed on the interpreter's" + "`sys.path`." + ), + ) cls.add_json_options(parser, entity="verbose output") interpreter_options_parser = parser.add_argument_group( @@ -125,6 +135,7 @@ def _inspect(self): interpreter_info.update( version=interpreter.identity.version_str, requirement=str(InterpreterConstraint.exact_version(interpreter)), + sys_path=interpreter.sys_path, platform=str(interpreter.platform), venv=interpreter.is_venv, ) @@ -151,6 +162,18 @@ def _inspect(self): interpreter_info[ "marker_environment" ] = interpreter.identity.env_markers.as_dict() + if self.options.distributions: + interpreter_info["distributions"] = { + dist.project_name: { + "version": dist.version, + "location": dist.location, + "requires_python": str(dist.metadata.requires_python), + "requires_dists": [ + str(req) for req in dist.metadata.requires_dists + ], + } + for dist in find_distributions(search_path=interpreter.sys_path) + } self.dump_json(self.options, interpreter_info, out) else: out.write(interpreter.binary) diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index fe0a4ada3..3aa02a4e7 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -4,11 +4,12 @@ from __future__ import absolute_import +import errno import functools import glob import importlib +import itertools import os -import re import sys import tarfile import zipfile @@ -22,6 +23,7 @@ from pex import pex_warnings from pex.common import open_zip, pluralize from pex.compatibility import to_unicode +from pex.enum import Enum from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.third_party.packaging.markers import Marker @@ -69,23 +71,16 @@ class MetadataNotFoundError(MetadataError): """Indicates an expected metadata file could not be found for a given distribution.""" -_PKG_INFO_BY_DIST_LOCATION = {} # type: Dict[Text, Optional[Message]] - - def _strip_sdist_path(sdist_path): - # type: (Text) -> Optional[str] - if not sdist_path.endswith((".sdist", ".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".zip")): + # type: (Text) -> Optional[Text] + if not sdist_path.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".zip")): return None sdist_basename = os.path.basename(sdist_path) filename, _ = os.path.splitext(sdist_basename) if filename.endswith(".tar"): filename, _ = os.path.splitext(filename) - # All PEP paths lead here for the definition of a valid project name which limits things to - # ascii; so this str(...) is Python 2.7 safe: https://peps.python.org/pep-0508/#names - # The version part of the basename is similarly restricted by: - # https://peps.python.org/pep-0440/#summary-of-changes-to-pep-440 - return str(filename) + return filename def _parse_message(message): @@ -93,159 +88,299 @@ def _parse_message(message): return cast(Message, Parser().parse(StringIO(to_unicode(message)))) -def _parse_sdist_package_info(sdist_path): - # type: (Text) -> Optional[Message] - sdist_filename = _strip_sdist_path(sdist_path) - if sdist_filename is None: +@attr.s(frozen=True) +class DistMetadataFile(object): + type = attr.ib() # type: MetadataType.Value + location = attr.ib() # type: Text + rel_path = attr.ib() # type: Text + project_name = attr.ib() # type: ProjectName + version = attr.ib() # type: Version + pkg_info = attr.ib(eq=False) # type: Message + + +@attr.s(frozen=True) +class MetadataFiles(object): + metadata = attr.ib() # type: DistMetadataFile + _additional_metadata_files = attr.ib(default=()) # type: Tuple[Text, ...] + _read_function = attr.ib(default=None) # type: Optional[Callable[[Text], bytes]] + + def metadata_file_rel_path(self, metadata_file_name): + # type: (Text) -> Optional[Text] + for rel_path in self._additional_metadata_files: + if os.path.basename(rel_path) == metadata_file_name: + return rel_path return None - pkg_info_path = os.path.join(sdist_filename, "PKG-INFO") + def read(self, metadata_file_name): + # type: (Text) -> Optional[bytes] + rel_path = self.metadata_file_rel_path(metadata_file_name) + if rel_path is None or self._read_function is None: + return None + return self._read_function(rel_path) + - if zipfile.is_zipfile(sdist_path): - with open_zip(sdist_path) as zip: - try: - return _parse_message(zip.read(pkg_info_path)) - except KeyError as e: - pex_warnings.warn( - "Source distribution {} did not have the expected metadata file {}: {}".format( - sdist_path, pkg_info_path, e - ) - ) - return None - - if tarfile.is_tarfile(sdist_path): - with tarfile.open(sdist_path) as tf: - try: - pkg_info = tf.extractfile(pkg_info_path) - if pkg_info is None: - # N.B.: `extractfile` returns None for directories and special files. - return None - with closing(pkg_info) as fp: - return _parse_message(fp.read()) - except KeyError as e: - pex_warnings.warn( - "Source distribution {} did not have the expected metadata file {}: {}".format( - sdist_path, pkg_info_path, e - ) - ) - return None +class MetadataType(Enum["MetadataType.Value"]): + class Value(Enum.Value): + def load_metadata( + self, + location, # type: Text + project_name=None, # type: Optional[ProjectName] + rescan=False, # type: bool + ): + # type: (...) -> Optional[MetadataFiles] + return load_metadata( + location, project_name=project_name, restrict_types_to=(self,), rescan=rescan + ) - return None + DIST_INFO = Value(".dist-info") + EGG_INFO = Value(".egg-info") + PKG_INFO = Value("PKG-INFO") @attr.s(frozen=True) -class DistMetadataFile(object): - project_name = attr.ib() # type: ProjectName - version = attr.ib() # type: Version - path = attr.ib() # type: Text +class MetadataKey(object): + metadata_type = attr.ib() # type: MetadataType.Value + location = attr.ib() # type: Text -def find_dist_info_files( - filename, # type: Text - listing, # type: Iterable[Text] +def _find_installed_metadata_files( + location, # type: Text + metadata_type, # type: MetadataType.Value + metadata_dir_glob, # type: str + metadata_file_name, # type: Text ): - # type: (...) -> Iterator[DistMetadataFile] - - # N.B. We know the captured project_name and version will not contain `-` even though PEP-503 - # allows for them in project names and PEP-440 allows for them in versions in some - # circumstances. This is since we're limiting ourselves to the products of installs by our - # vendored versions of wheel and pip which turn `-` into `_` as explained in `ProjectName` and - # `Version` docs. - dist_info_metadata_pattern = "^{}$".format( - os.path.join(r"(?P.+)-(?P.+)\.dist-info", re.escape(filename)) - ) - wheel_metadata_re = re.compile(dist_info_metadata_pattern) - for item in listing: - match = wheel_metadata_re.match(item) - if match: - yield DistMetadataFile( - project_name=ProjectName(match.group("project_name")), - version=Version(match.group("version")), - path=item, + # type: (...) -> Iterator[MetadataFiles] + metadata_files = glob.glob(os.path.join(location, metadata_dir_glob, metadata_file_name)) + for path in metadata_files: + with open(path, "rb") as fp: + metadata = _parse_message(fp.read()) + project_name_and_version = ProjectNameAndVersion.from_parsed_pkg_info( + source=path, pkg_info=metadata ) + def read_function(rel_path): + # type: (Text) -> bytes + with open(os.path.join(location, rel_path), "rb") as fp: + return fp.read() + + yield MetadataFiles( + metadata=DistMetadataFile( + type=metadata_type, + location=location, + rel_path=os.path.relpath(path, location), + project_name=project_name_and_version.canonicalized_project_name, + version=project_name_and_version.canonicalized_version, + pkg_info=metadata, + ), + additional_metadata_files=tuple( + os.path.relpath(metadata_path, location) + for metadata_path in glob.glob(os.path.join(os.path.dirname(path), "*")) + if os.path.basename(metadata_path) != metadata_file_name + ), + read_function=read_function, + ) -def find_dist_info_file( - project_name, # type: Union[Text, ProjectName] - filename, # type: Text - listing, # type: Iterable[Text] - version=None, # type: Optional[Union[Text, Version]] -): - # type: (...) -> Optional[Text] - normalized_project_name = ( - project_name if isinstance(project_name, ProjectName) else ProjectName(project_name) - ) +def find_wheel_metadata(location): + # type: (Text) -> Optional[MetadataFiles] + with open_zip(location) as zf: + for name in zf.namelist(): + if name.endswith("/"): + continue + dist_info_dir, metadata_file = os.path.split(name) + if os.path.dirname(dist_info_dir): + continue + if "METADATA" != metadata_file: + continue + + with zf.open(name) as fp: + metadata = _parse_message(fp.read()) + project_name_and_version = ProjectNameAndVersion.from_parsed_pkg_info( + source=os.path.join(location, name), pkg_info=metadata + ) + metadata_file_name = os.path.basename(name) + files = [] # type: List[Text] + for rel_path in zf.namelist(): + head, tail = os.path.split(rel_path) + if dist_info_dir == head and tail != metadata_file_name: + files.append(rel_path) + + def read_function(rel_path): + # type: (Text) -> bytes + with open_zip(location) as zf: + return zf.read(rel_path) + + return MetadataFiles( + metadata=DistMetadataFile( + type=MetadataType.DIST_INFO, + location=location, + rel_path=name, + project_name=project_name_and_version.canonicalized_project_name, + version=project_name_and_version.canonicalized_version, + pkg_info=metadata, + ), + additional_metadata_files=tuple(files), + read_function=read_function, + ) + + return None - if isinstance(version, Version): - normalized_version = version - elif isinstance(version, str): - normalized_version = Version(version) - else: - normalized_version = None - for metadata_file in find_dist_info_files(filename, listing): - if normalized_project_name == metadata_file.project_name: - if normalized_version and normalized_version != metadata_file.version: +def _is_dist_pkg_info_file_path(file_path): + # type: (Text) -> bool + + # N.B.: Should be: -/PKG-INFO + project_dir, metadata_file = os.path.split(file_path) + if os.path.dirname(project_dir): + return False + if not "-" in project_dir: + return False + return "PKG-INFO" == metadata_file + + +def find_zip_sdist_metadata(location): + # type: (Text) -> Optional[DistMetadataFile] + with open_zip(location) as zf: + for name in zf.namelist(): + if name.endswith("/") or not _is_dist_pkg_info_file_path(name): continue - return metadata_file.path + with zf.open(name) as fp: + metadata = _parse_message(fp.read()) + project_name_and_version = ProjectNameAndVersion.from_parsed_pkg_info( + source=os.path.join(location, name), pkg_info=metadata + ) + return DistMetadataFile( + type=MetadataType.PKG_INFO, + location=location, + rel_path=name, + project_name=project_name_and_version.canonicalized_project_name, + version=project_name_and_version.canonicalized_version, + pkg_info=metadata, + ) + return None -def _parse_wheel_package_info(wheel_path): - # type: (Text) -> Optional[Message] - if not wheel_path.endswith(".whl") or not zipfile.is_zipfile(wheel_path): - return None - project_name, version, _ = os.path.basename(wheel_path).split("-", 2) - with open_zip(wheel_path) as whl: - metadata_file = find_dist_info_file( - project_name=project_name, - version=version, - filename="METADATA", - listing=whl.namelist(), - ) - if not metadata_file: - return None - with whl.open(metadata_file) as fp: - return _parse_message(fp.read()) +def find_tar_sdist_metadata(location): + # type: (Text) -> Optional[DistMetadataFile] + with tarfile.open(location) as tf: + for member in tf.getmembers(): + if not member.isreg() or not _is_dist_pkg_info_file_path(member.name): + continue + file_obj = tf.extractfile(member) + if file_obj is None: + raise IOError( + errno.ENOENT, + "Could not find {rel_path} in {location}.".format( + rel_path=member.name, location=location + ), + ) + with closing(file_obj) as fp: + metadata = _parse_message(fp.read()) + project_name_and_version = ProjectNameAndVersion.from_parsed_pkg_info( + source=os.path.join(location, member.name), pkg_info=metadata + ) + return DistMetadataFile( + type=MetadataType.PKG_INFO, + location=location, + rel_path=member.name, + project_name=project_name_and_version.canonicalized_project_name, + version=project_name_and_version.canonicalized_version, + pkg_info=metadata, + ) -def _parse_installed_distribution_info(location): - # type: (Text) -> Optional[Message] + return None - if not os.path.isdir(location): - return None - dist_info_dirs = glob.glob(os.path.join(location, "*.dist-info")) - if not dist_info_dirs: - return None +_METADATA_FILES = {} # type: Dict[MetadataKey, Tuple[MetadataFiles, ...]] + - if len(dist_info_dirs) > 1: +def iter_metadata_files( + location, # type: Text + restrict_types_to=(), # type: Tuple[MetadataType.Value, ...] + rescan=False, # type: bool +): + # type: (...) -> Iterator[MetadataFiles] + + files = [] + for metadata_type in restrict_types_to or MetadataType.values(): + key = MetadataKey(metadata_type=metadata_type, location=location) + if rescan: + _METADATA_FILES.pop(key, None) + if key not in _METADATA_FILES: + listing = [] # type: List[MetadataFiles] + if MetadataType.DIST_INFO is metadata_type: + if os.path.isdir(location): + listing.extend( + _find_installed_metadata_files( + location, MetadataType.DIST_INFO, "*.dist-info", "METADATA" + ) + ) + elif location.endswith(".whl") and zipfile.is_zipfile(location): + metadata_files = find_wheel_metadata(location) + if metadata_files: + listing.append(metadata_files) + elif MetadataType.EGG_INFO is metadata_type and os.path.isdir(location): + listing.extend( + _find_installed_metadata_files( + location, MetadataType.EGG_INFO, "*.egg-info", "PKG-INFO" + ) + ) + elif MetadataType.PKG_INFO is metadata_type: + if location.endswith(".zip") and zipfile.is_zipfile(location): + metadata_file = find_zip_sdist_metadata(location) + if metadata_file: + listing.append(MetadataFiles(metadata=metadata_file)) + elif location.endswith( + (".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") + ) and tarfile.is_tarfile(location): + metadata_file = find_tar_sdist_metadata(location) + if metadata_file: + listing.append(MetadataFiles(metadata=metadata_file)) + _METADATA_FILES[key] = tuple(listing) + files.append(_METADATA_FILES[key]) + return itertools.chain.from_iterable(files) + + +def load_metadata( + location, # type: Text + project_name=None, # type: Optional[ProjectName] + restrict_types_to=(), # type: Tuple[MetadataType.Value, ...] + rescan=False, # type: bool +): + # type: (...) -> Optional[MetadataFiles] + all_metadata_files = [ + metadata_files + for metadata_files in iter_metadata_files( + location, restrict_types_to=restrict_types_to, rescan=rescan + ) + if project_name is None or project_name == metadata_files.metadata.project_name + ] + if len(all_metadata_files) == 1: + return all_metadata_files[0] + if len(all_metadata_files) > 1: raise AmbiguousDistributionError( - "Found more than one distribution at {location}:\n{dist_info_dirs}".format( + "Found more than one distribution inside {location}:\n{metadata_files}".format( location=location, - dist_info_dirs="\n".join( - os.path.relpath(dist_info_dir, location) for dist_info_dir in dist_info_dirs + metadata_files="\n".join( + metadata_file.metadata.rel_path for metadata_file in all_metadata_files ), ) ) + return None - metadata_file = os.path.join(dist_info_dirs[0], "METADATA") - if not os.path.exists(metadata_file): - return None - with open(metadata_file, "rb") as fp: - return _parse_message(fp.read()) +_PKG_INFO_BY_DIST_LOCATION = {} # type: Dict[Text, Optional[Message]] def _parse_pkg_info(location): # type: (Text) -> Optional[Message] if location not in _PKG_INFO_BY_DIST_LOCATION: - pkg_info = _parse_wheel_package_info(location) - if not pkg_info: - pkg_info = _parse_sdist_package_info(location) - if not pkg_info: - pkg_info = _parse_installed_distribution_info(location) + pkg_info = None # type: Optional[Message] + metadata_files = load_metadata(location) + if metadata_files: + pkg_info = metadata_files.metadata.pkg_info _PKG_INFO_BY_DIST_LOCATION[location] = pkg_info return _PKG_INFO_BY_DIST_LOCATION[location] @@ -254,7 +389,7 @@ def _parse_pkg_info(location): class ProjectNameAndVersion(object): @classmethod def from_parsed_pkg_info(cls, source, pkg_info): - # type: (str, Message) -> ProjectNameAndVersion + # type: (Text, Message) -> ProjectNameAndVersion project_name = pkg_info.get("Name", None) version = pkg_info.get("Version", None) if project_name is None or version is None: @@ -316,7 +451,7 @@ def canonicalized_version(self): def project_name_and_version( - location, # type: Union[Text, Distribution, Message] + location, # type: Union[Text, Distribution, Message, MetadataFiles] fallback_to_filename=True, # type: bool ): # type: (...) -> Optional[ProjectNameAndVersion] @@ -328,8 +463,18 @@ def project_name_and_version( """ if isinstance(location, Distribution): return ProjectNameAndVersion(project_name=location.project_name, version=location.version) + if isinstance(location, MetadataFiles): + return ProjectNameAndVersion( + project_name=location.metadata.project_name.raw, version=location.metadata.version.raw + ) - pkg_info = location if isinstance(location, Message) else _parse_pkg_info(location) + pkg_info = None # type: Optional[Message] + if isinstance(location, Message): + pkg_info = location + else: + metadata_files = load_metadata(location) + if metadata_files: + pkg_info = metadata_files.metadata.pkg_info if pkg_info is not None: if isinstance(location, str): source = location @@ -342,7 +487,7 @@ def project_name_and_version( def requires_python(location): - # type: (Union[Text, Distribution, Message]) -> Optional[SpecifierSet] + # type: (Union[Text, Distribution, Message, MetadataFiles]) -> Optional[SpecifierSet] """Examines dist for `Python-Requires` metadata and returns version constraints if any. See: https://www.python.org/dev/peps/pep-0345/#requires-python @@ -353,7 +498,15 @@ def requires_python(location): if isinstance(location, Distribution): return location.metadata.requires_python - pkg_info = location if isinstance(location, Message) else _parse_pkg_info(location) + pkg_info = None # type: Optional[Message] + if isinstance(location, Message): + pkg_info = location + elif isinstance(location, MetadataFiles): + pkg_info = location.metadata.pkg_info + else: + metadata_files = load_metadata(location) + if metadata_files: + pkg_info = metadata_files.metadata.pkg_info if pkg_info is None: return None @@ -363,8 +516,34 @@ def requires_python(location): return SpecifierSet(python_requirement) +def _parse_requires_txt(content): + # type: (bytes) -> Iterator[Requirement] + # See: + # + High level: https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#requires-txt + # + Low level: + # + https://github.com/pypa/setuptools/blob/fbe0d7962822c2a1fdde8dd179f2f8b8c8bf8892/pkg_resources/__init__.py#L3256-L3279 + # + https://github.com/pypa/setuptools/blob/fbe0d7962822c2a1fdde8dd179f2f8b8c8bf8892/pkg_resources/__init__.py#L2792-L2818 + marker = "" + for line in content.decode("utf-8").splitlines(): + line = line.strip() + if not line: + continue + if line.startswith("[") and line.endswith("]"): + section = line[1:-1] + extra, _, mark = section.partition(":") + markers = [] # type: List[Text] + if extra: + markers.append('extra == "{extra}"'.format(extra=extra)) + if mark: + markers.append(mark) + if markers: + marker = "; {markers}".format(markers=" and ".join(markers)) + else: + yield Requirement.parse(line + marker) + + def requires_dists(location): - # type: (Union[Text, Distribution, Message]) -> Iterator[Requirement] + # type: (Union[Text, Distribution, Message, MetadataFiles]) -> Iterator[Requirement] """Examines dist for and returns any declared requirements. Looks for `Requires-Dist` metadata. @@ -385,12 +564,32 @@ def requires_dists(location): yield requirement return - pkg_info = location if isinstance(location, Message) else _parse_pkg_info(location) + pkg_info = None # type: Optional[Message] + if isinstance(location, Message): + pkg_info = location + elif isinstance(location, MetadataFiles): + pkg_info = location.metadata.pkg_info + else: + metadata_files = load_metadata(location) + if metadata_files: + pkg_info = metadata_files.metadata.pkg_info if pkg_info is None: return - for requires_dist in pkg_info.get_all("Requires-Dist", ()): - yield Requirement.parse(requires_dist) + requires_dists = pkg_info.get_all("Requires-Dist", ()) + if ( + not requires_dists + and isinstance(location, MetadataFiles) + and MetadataType.EGG_INFO is location.metadata.type + ): + for metadata_file in "requires.txt", "depends.txt": + content = location.read(metadata_file) + if content: + for requirement in _parse_requires_txt(content): + yield requirement + else: + for requires_dist in requires_dists: + yield Requirement.parse(requires_dist) legacy_requires = pkg_info.get_all("Requires", []) # type: List[str] if legacy_requires: @@ -550,21 +749,30 @@ def __str__(self): @attr.s(frozen=True, cache_hash=True) class DistMetadata(object): @classmethod - def load(cls, location): - # type: (Union[Text, Message]) -> DistMetadata + def from_metadata_files(cls, metadata_files): + # type: (MetadataFiles) -> DistMetadata + return cls( + project_name=metadata_files.metadata.project_name, + version=metadata_files.metadata.version, + requires_dists=tuple(requires_dists(metadata_files)), + requires_python=requires_python(metadata_files), + ) - project_name_and_ver = project_name_and_version(location) - if not project_name_and_ver: + @classmethod + def load( + cls, + location, # type: Text + *restrict_types_to # type: MetadataType.Value + ): + # type: (...) -> DistMetadata + + metadata_files = load_metadata(location, restrict_types_to=restrict_types_to) + if metadata_files is None: raise MetadataError( "Failed to determine project name and version for distribution at " "{location}.".format(location=location) ) - return cls( - project_name=ProjectName(project_name_and_ver.project_name), - version=Version(project_name_and_ver.version), - requires_dists=tuple(requires_dists(location)), - requires_python=requires_python(location), - ) + return cls.from_metadata_files(metadata_files) project_name = attr.ib() # type: ProjectName version = attr.ib() # type: Version @@ -580,26 +788,25 @@ def _realpath(path): @attr.s(frozen=True) class Distribution(object): @staticmethod - def _read_metadata_lines(metadata_path): - # type: (Text) -> Iterator[Text] - with open(os.path.join(metadata_path), "rb") as fp: - for line in fp: - # This is pkg_resources.IMetadataProvider.get_metadata_lines behavior, which our - # code expects. - normalized = line.decode("utf-8").strip() - if normalized and not normalized.startswith("#"): - yield normalized + def _read_metadata_lines(metadata_bytes): + # type: (bytes) -> Iterator[Text] + for line in metadata_bytes.splitlines(): + # This is pkg_resources.IMetadataProvider.get_metadata_lines behavior, which our + # code expects. + normalized = line.decode("utf-8").strip() + if normalized and not normalized.startswith("#"): + yield normalized @classmethod - def parse_entry_map(cls, entry_points_metadata_path): - # type: (Text) -> Dict[Text, Dict[Text, EntryPoint]] + def parse_entry_map(cls, entry_points_contents): + # type: (bytes) -> Dict[Text, Dict[Text, EntryPoint]] # This file format is defined here: # https://packaging.python.org/en/latest/specifications/entry-points/#file-format entry_map = defaultdict(dict) # type: DefaultDict[Text, Dict[Text, EntryPoint]] group = None # type: Optional[Text] - for index, line in enumerate(cls._read_metadata_lines(entry_points_metadata_path), start=1): + for index, line in enumerate(cls._read_metadata_lines(entry_points_contents), start=1): if line.startswith("[") and line.endswith("]"): group = line[1:-1] elif not group: @@ -622,9 +829,6 @@ def load(cls, location): location = attr.ib(converter=_realpath) # type: str metadata = attr.ib() # type: DistMetadata - _metadata_files_cache = attr.ib( - factory=dict, init=False, eq=False, repr=False - ) # type: Dict[Text, Text] @property def key(self): @@ -657,61 +861,33 @@ def requires(self): # type: () -> Tuple[Requirement, ...] return self.metadata.requires_dists - def _get_metadata_file(self, name): - # type: (Text) -> Optional[Text] + def _read_metadata_file(self, name): + # type: (str) -> Optional[bytes] normalized_name = os.path.normpath(name) if os.path.isabs(normalized_name): raise ValueError( - "The metadata file name must be a relative path under the .dist-info/ directory. " - "Given: {name}".format(name=name) + "The metadata file name must be a relative path under the .dist-info/ (or " + ".egg-info/) directory. Given: {name}".format(name=name) ) - metadata_file = self._metadata_files_cache.get(normalized_name) - if metadata_file is None: - metadata_file = find_dist_info_file( - project_name=self.metadata.project_name, - version=self.version, - filename=normalized_name, - listing=[ - os.path.relpath(path, self.location) - for path in glob.glob( - os.path.join( - self.location, "*.dist-info/{name}".format(name=normalized_name) - ) - ) - ], - ) - # N.B.: We store the falsey "" as the sentinel that we've searched already and the - # metadata file did not exist. - self._metadata_files_cache[normalized_name] = metadata_file or "" - return metadata_file or None - - def has_metadata(self, name): - # type: (str) -> bool - return self._get_metadata_file(name) is not None - - def get_metadata_lines(self, name): - # type: (Text) -> Iterator[Text] - relative_path = self._get_metadata_file(name) - if relative_path is None: - raise MetadataNotFoundError( - "The metadata file {name} is not present for {project_name} {version} at " - "{location}".format( - name=name, - project_name=self.project_name, - version=self.version, - location=self.location, - ) - ) - for line in self._read_metadata_lines(os.path.join(self.location, relative_path)): - yield line + metadata_file = load_metadata( + location=self.location, project_name=self.metadata.project_name + ) + return metadata_file.read(name) if metadata_file else None + + def iter_metadata_lines(self, name): + # type: (str) -> Iterator[Text] + contents = self._read_metadata_file(name) + if contents: + for line in self._read_metadata_lines(contents): + yield line def get_entry_map(self): # type: () -> Dict[Text, Dict[Text, EntryPoint]] - entry_points_metadata_relpath = self._get_metadata_file("entry_points.txt") - if entry_points_metadata_relpath is None: + entry_points_metadata_file = self._read_metadata_file("entry_points.txt") + if entry_points_metadata_file is None: return defaultdict(dict) - return self.parse_entry_map(os.path.join(self.location, entry_points_metadata_relpath)) + return self.parse_entry_map(entry_points_metadata_file) def __str__(self): # type: () -> str @@ -784,54 +960,46 @@ def __str__(self): def find_distribution( project_name, # type: Union[str, ProjectName] search_path=None, # type: Optional[Iterable[str]] + rescan=False, # type: bool ): # type: (...) -> Optional[Distribution] + canonicalized_project_name = ( + project_name if isinstance(project_name, ProjectName) else ProjectName(project_name) + ) for location in search_path or sys.path: if not os.path.isdir(location): continue - - metadata_file = find_dist_info_file( - project_name=str(project_name), - filename="METADATA", - listing=[ - os.path.relpath(path, location) - for path in glob.glob(os.path.join(location, "*.dist-info/METADATA")) - ], + metadata_files = load_metadata( + location, + project_name=canonicalized_project_name, + restrict_types_to=(MetadataType.DIST_INFO, MetadataType.EGG_INFO), + rescan=rescan, ) - if not metadata_file: - continue - - metadata_path = os.path.join(location, metadata_file) - with open(metadata_path, "rb") as fp: - pkg_info = _parse_message(fp.read()) - dist = Distribution(location=location, metadata=DistMetadata.load(pkg_info)) - if dist.metadata.project_name == ( - project_name if isinstance(project_name, ProjectName) else ProjectName(project_name) - ): - return dist - + if metadata_files: + return Distribution( + location=location, metadata=DistMetadata.from_metadata_files(metadata_files) + ) return None -def find_distributions(search_path=None): - # type: (Optional[Iterable[str]]) -> Iterator[Distribution] - +def find_distributions( + search_path=None, # type: Optional[Iterable[str]] + rescan=False, # type: bool +): + # type: (...) -> Iterator[Distribution] seen = set() for location in search_path or sys.path: if not os.path.isdir(location): continue - for metadata_file in find_dist_info_files( - filename="METADATA", - listing=[ - os.path.relpath(path, location) - for path in glob.glob(os.path.join(location, "*.dist-info/METADATA")) - ], + for metadata_files in iter_metadata_files( + location, + restrict_types_to=(MetadataType.DIST_INFO, MetadataType.EGG_INFO), + rescan=rescan, ): - metadata_path = os.path.realpath(os.path.join(location, metadata_file.path)) - if metadata_path in seen: + if metadata_files.metadata in seen: continue - seen.add(metadata_path) - with open(metadata_path, "rb") as fp: - pkg_info = _parse_message(fp.read()) - yield Distribution(location=location, metadata=DistMetadata.load(pkg_info)) + seen.add(metadata_files.metadata) + yield Distribution( + location=location, metadata=DistMetadata.from_metadata_files(metadata_files) + ) diff --git a/pex/environment.py b/pex/environment.py index ba4694b77..cb61b09fa 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -20,7 +20,7 @@ from pex.pep_425 import CompatibilityTags, TagRank from pex.pep_503 import ProjectName from pex.pex_info import PexInfo -from pex.targets import LocalInterpreter, Target +from pex.targets import Target from pex.third_party.packaging import specifiers from pex.tracer import TRACER from pex.typing import TYPE_CHECKING @@ -35,6 +35,7 @@ List, MutableMapping, Optional, + Text, Tuple, Union, ) @@ -600,19 +601,17 @@ def record_unresolved(dist_not_found): return OrderedSet(resolved_dists_by_key.values()) - _NAMESPACE_PACKAGE_METADATA_RESOURCE = "namespace_packages.txt" - @classmethod def _get_namespace_packages(cls, dist): - if dist.has_metadata(cls._NAMESPACE_PACKAGE_METADATA_RESOURCE): - return list(dist.get_metadata_lines(cls._NAMESPACE_PACKAGE_METADATA_RESOURCE)) - else: - return [] + # type: (Distribution) -> Tuple[Text, ...] + return tuple(dist.iter_metadata_lines("namespace_packages.txt")) @classmethod def _declare_namespace_packages(cls, resolved_dists): # type: (Iterable[Distribution]) -> None - namespace_packages_by_dist = OrderedDict() + namespace_packages_by_dist = ( + OrderedDict() + ) # type: OrderedDict[Distribution, Tuple[Text, ...]] for dist in resolved_dists: namespace_packages = cls._get_namespace_packages(dist) # NB: Dists can explicitly declare empty namespace packages lists to indicate they have none. diff --git a/pex/pep_376.py b/pex/pep_376.py index d0c5cdfd0..013d3be1c 100644 --- a/pex/pep_376.py +++ b/pex/pep_376.py @@ -17,8 +17,10 @@ from pex import dist_metadata, hashing from pex.common import is_pyc_dir, is_pyc_file, is_python_script, safe_mkdir, safe_open from pex.compatibility import get_stdout_bytes_buffer, urlparse -from pex.dist_metadata import Distribution, EntryPoint +from pex.dist_metadata import Distribution, EntryPoint, MetadataFiles, MetadataType from pex.interpreter import PythonInterpreter +from pex.pep_440 import Version +from pex.pep_503 import ProjectName from pex.typing import TYPE_CHECKING, cast from pex.venv.virtualenv import Virtualenv @@ -29,7 +31,6 @@ Dict, Iterable, Iterator, - List, Optional, Protocol, Text, @@ -432,6 +433,12 @@ class UnrecognizedInstallationSchemeError(RecordError): """Indicates a distribution's RECORD was nested in an unrecognized installation scheme.""" +@attr.s(frozen=True) +class DistInfoFile(object): + path = attr.ib() # type: Text + content = attr.ib() # type: bytes + + @attr.s(frozen=True) class Record(object): """Represents the PEP-376 RECORD of an installed wheel. @@ -465,7 +472,10 @@ def _find_installation( project_name, # type: str version, # type: str ): - # type: (...) -> Optional[Tuple[Text, str, List[str]]] + # type: (...) -> Optional[MetadataFiles] + + canonical_project_name = ProjectName(project_name) + canonical_version = Version(version) # Some distributions in the wild (namely python-certifi-win32 1.6.1, # see: https://github.com/pantsbuild/pex/issues/1861) create their own directories named @@ -481,16 +491,11 @@ def _find_installation( if d == "site-packages" ] for site_packages_dir in site_packages_dirs: - site_packages_listing = [ - os.path.relpath(os.path.join(root, f), site_packages_dir) - for root, _, files in os.walk(site_packages_dir) - for f in files - ] - record_relative_path = dist_metadata.find_dist_info_file( - project_name, version=version, filename="RECORD", listing=site_packages_listing + metadata_files = MetadataType.DIST_INFO.load_metadata( + site_packages_dir, project_name=canonical_project_name ) - if record_relative_path: - return record_relative_path, site_packages_dir, site_packages_listing + if metadata_files and canonical_version == metadata_files.metadata.version: + return metadata_files return None @classmethod @@ -501,27 +506,32 @@ def from_prefix_install( version, # type: str ): # type: (...) -> Record - result = cls._find_installation(prefix_dir, project_name, version) - if not result: + metadata_files = cls._find_installation(prefix_dir, project_name, version) + if not metadata_files: raise RecordNotFoundError( - "Could not find the installation RECORD for {project_name} {version} under " + "Could not find project metadata for {project_name} {version} under " "{prefix_dir}".format( project_name=project_name, version=version, prefix_dir=prefix_dir ) ) + record_relpath = metadata_files.metadata_file_rel_path("RECORD") + if not record_relpath: + raise RecordNotFoundError( + "Could not find the installation RECORD for {project_name} {version} under " + "{location}".format( + project_name=project_name, + version=version, + location=metadata_files.metadata.location, + ) + ) - record_relative_path, site_packages, site_packages_listing = result - metadata_dir = os.path.dirname(record_relative_path) - base_dir = os.path.relpath(site_packages, prefix_dir) + rel_base_dir = os.path.relpath(metadata_files.metadata.location, prefix_dir) return cls( project_name=project_name, version=version, prefix_dir=prefix_dir, - rel_base_dir=base_dir, - relative_path=record_relative_path, - metadata_listing=tuple( - path for path in site_packages_listing if metadata_dir == os.path.dirname(path) - ), + rel_base_dir=rel_base_dir, + relative_path=record_relpath, ) project_name = attr.ib() # type: str @@ -529,19 +539,26 @@ def from_prefix_install( prefix_dir = attr.ib() # type: str rel_base_dir = attr.ib() # type: Text relative_path = attr.ib() # type: Text - _metadata_listing = attr.ib() # type: Tuple[str, ...] def _find_dist_info_file(self, filename): - # type: (str) -> Optional[Text] - metadata_file = dist_metadata.find_dist_info_file( - project_name=self.project_name, - version=self.version, - filename=filename, - listing=self._metadata_listing, + # type: (str) -> Optional[DistInfoFile] + metadata_files = MetadataType.DIST_INFO.load_metadata( + location=os.path.join(self.prefix_dir, self.rel_base_dir), + project_name=ProjectName(self.project_name), ) - if not metadata_file: + if metadata_files is None: + return None + + metadata_file_rel_path = metadata_files.metadata_file_rel_path(filename) + if metadata_file_rel_path is None: return None - return os.path.join(self.rel_base_dir, metadata_file) + + content = metadata_files.read(filename) + if content is None: + return None + + file_path = os.path.join(metadata_files.metadata.location, metadata_file_rel_path) + return DistInfoFile(path=file_path, content=content) def fixup_install( self, @@ -616,11 +633,10 @@ def _fixup_scripts(self): return console_scripts = {} # type: Dict[Text, EntryPoint] - entry_points_relpath = self._find_dist_info_file("entry_points.txt") - if entry_points_relpath: - entry_points_abspath = os.path.join(self.prefix_dir, entry_points_relpath) + entry_points_file = self._find_dist_info_file("entry_points.txt") + if entry_points_file: console_scripts.update( - Distribution.parse_entry_map(entry_points_abspath).get("console_scripts", {}) + Distribution.parse_entry_map(entry_points_file.content).get("console_scripts", {}) ) scripts = {} # type: Dict[str, Optional[bytes]] @@ -680,9 +696,10 @@ def _fixup_scripts(self): def _fixup_direct_url(self): # type: () -> None - direct_url_relpath = self._find_dist_info_file("direct_url.json") - if direct_url_relpath: - direct_url_abspath = os.path.join(self.prefix_dir, direct_url_relpath) - with open(direct_url_abspath) as fp: - if urlparse.urlparse(json.load(fp)["url"]).scheme == "file": - os.unlink(direct_url_abspath) + direct_url_file = self._find_dist_info_file("direct_url.json") + if direct_url_file: + if ( + urlparse.urlparse(json.loads(direct_url_file.content.decode("utf-8"))["url"]).scheme + == "file" + ): + os.unlink(direct_url_file.path) diff --git a/pex/resolve/target_options.py b/pex/resolve/target_options.py index d4150873d..fdb74fe71 100644 --- a/pex/resolve/target_options.py +++ b/pex/resolve/target_options.py @@ -79,11 +79,11 @@ def register( 'e.g. "CPython>=2.7,<3" (A CPython interpreter with version >=2.7 AND version <3), ' '">=2.7,<3" (Any Python interpreter with version >=2.7 AND version <3) or "PyPy" (A ' "PyPy interpreter of any version). This argument may be repeated multiple times to OR " - "the constraints. Try `{singe_interpreter_info_cmd}` to find the exact interpreter " + "the constraints. Try `{single_interpreter_info_cmd}` to find the exact interpreter " "constraints of {current_interpreter} and `{all_interpreters_info_cmd}` to find out " "the interpreter constraints of all Python interpreters on the $PATH.".format( current_interpreter=sys.executable, - singe_interpreter_info_cmd=single_interpreter_info_cmd, + single_interpreter_info_cmd=single_interpreter_info_cmd, all_interpreters_info_cmd=all_interpreters_info_cmd, ) ), @@ -95,7 +95,7 @@ def register( def _register_platform_options( parser, # type: _ActionsContainer - singe_interpreter_info_cmd, # type: str + single_interpreter_info_cmd, # type: str all_interpreters_info_cmd, # type: str ): # type: (...) -> None @@ -115,10 +115,10 @@ def _register_platform_options( "https://www.python.org/dev/peps/pep-0427#file-name-convention and influenced by " "https://www.python.org/dev/peps/pep-0425. To find out more, try " "`{all_interpreters_info_cmd}` to print out the platform for all interpreters on the " - "$PATH or `{singe_interpreter_info_cmd}` to inspect the single interpreter " + "$PATH or `{single_interpreter_info_cmd}` to inspect the single interpreter " "{current_interpreter}.".format( current_interpreter=sys.executable, - singe_interpreter_info_cmd=singe_interpreter_info_cmd, + single_interpreter_info_cmd=single_interpreter_info_cmd, all_interpreters_info_cmd=all_interpreters_info_cmd, ) ), diff --git a/pex/venv/installer.py b/pex/venv/installer.py index e6d38601f..a25b13e4d 100644 --- a/pex/venv/installer.py +++ b/pex/venv/installer.py @@ -87,7 +87,7 @@ def ensure_pip_installed( "The virtual environment was successfully created, but Pip was not " "installed:\n{}".format(e) ) - venv_pip_version = find_dist(_PIP, venv.iter_distributions()) + venv_pip_version = find_dist(_PIP, venv.iter_distributions(rescan=True)) if not venv_pip_version: return Error( "Failed to install pip into venv at {venv_dir}".format(venv_dir=venv.venv_dir) diff --git a/pex/venv/virtualenv.py b/pex/venv/virtualenv.py index f0a1629d2..c2dbc1850 100644 --- a/pex/venv/virtualenv.py +++ b/pex/venv/virtualenv.py @@ -347,9 +347,9 @@ def sys_path(self): self._sys_path = tuple(stdout.strip().splitlines()) return self._sys_path - def iter_distributions(self): - # type: () -> Iterator[Distribution] - for dist in find_distributions(search_path=self._interpreter.site_packages): + def iter_distributions(self, rescan=False): + # type: (bool) -> Iterator[Distribution] + for dist in find_distributions(search_path=self._interpreter.site_packages, rescan=rescan): yield dist def _rewrite_base_scripts(self, real_venv_dir): diff --git a/tests/build_system/test_pep_518.py b/tests/build_system/test_pep_518.py index 94daf142d..601d0ee31 100644 --- a/tests/build_system/test_pep_518.py +++ b/tests/build_system/test_pep_518.py @@ -8,13 +8,13 @@ from pex.build_system import pep_518 from pex.build_system.pep_518 import BuildSystem from pex.common import touch -from pex.environment import PEXEnvironment from pex.pep_503 import ProjectName from pex.resolve.configured_resolver import ConfiguredResolver from pex.result import Error from pex.targets import LocalInterpreter from pex.typing import TYPE_CHECKING from pex.variables import ENV +from pex.venv.virtualenv import Virtualenv if TYPE_CHECKING: from typing import Any, Optional, Union @@ -71,7 +71,7 @@ def test_load_build_system_pyproject( assert "flit_core.buildapi" == build_system.build_backend dists = { dist.metadata.project_name - for dist in PEXEnvironment.mount(build_system.venv_pex.pex).resolve() + for dist in Virtualenv(build_system.venv_pex.venv_dir).iter_distributions() } assert ProjectName("flit_core") in dists subprocess.check_call( diff --git a/tests/integration/cli/commands/test_interpreter_inspect.py b/tests/integration/cli/commands/test_interpreter_inspect.py index 43d6cbb41..e5778757c 100644 --- a/tests/integration/cli/commands/test_interpreter_inspect.py +++ b/tests/integration/cli/commands/test_interpreter_inspect.py @@ -8,6 +8,7 @@ from pex.pep_425 import CompatibilityTags from pex.pep_508 import MarkerEnvironment from pex.typing import TYPE_CHECKING, cast +from pex.venv.virtualenv import Virtualenv from testing import IntegResults from testing.cli import run_pex3 @@ -45,8 +46,18 @@ def assert_default_verbose_data( ): # type: (...) -> Dict[str, Any] + return assert_verbose_data(PythonInterpreter.get(), *args, **popen_kwargs) + + +def assert_verbose_data( + interpreter, # type: PythonInterpreter + *args, # type: str + **popen_kwargs # type: str +): + # type: (...) -> Dict[str, Any] + data = json.loads(assert_inspect(*args, **popen_kwargs)) - assert PythonInterpreter.get().binary == data.pop("path") + assert interpreter.binary == data.pop("path") return cast("Dict[str, Any]", data) @@ -124,3 +135,14 @@ def test_inspect_interpreter_selection( "--python-path", os.pathsep.join([os.path.dirname(py38.binary), py310.binary]), ).splitlines() + + +def test_inspect_distributions(tmpdir): + # type: (Any) -> None + + venv = Virtualenv.create(venv_dir=os.path.join(str(tmpdir), "venv")) + venv.install_pip() + venv.interpreter.execute(args=["-mpip", "install", "ansicolors==1.1.8", "cowsay==5.0"]) + + data = assert_verbose_data(venv.interpreter, "-vd", "--python", venv.interpreter.binary) + assert {"pip", "ansicolors", "cowsay"}.issubset(data["distributions"].keys()) diff --git a/tests/integration/cli/commands/test_venv_create.py b/tests/integration/cli/commands/test_venv_create.py index 4a04ffc57..1bac2099f 100644 --- a/tests/integration/cli/commands/test_venv_create.py +++ b/tests/integration/cli/commands/test_venv_create.py @@ -248,7 +248,8 @@ def test_venv_pip(tmpdir): run_pex3("venv", "create", "-d", dest, "--pip").assert_success() assert "pip" in [os.path.basename(exe) for exe in venv.iter_executables()] distributions = { - dist.metadata.project_name: dist.metadata.version for dist in venv.iter_distributions() + dist.metadata.project_name: dist.metadata.version + for dist in venv.iter_distributions(rescan=True) } pip_version = distributions[ProjectName("pip")] expected_prefix = "pip {version} from {prefix}".format(version=pip_version.raw, prefix=dest) @@ -513,9 +514,10 @@ def test_venv_update_target_mismatch( in result.error.strip() ), result.error - assert [] == list(Virtualenv(dest).iter_distributions()) + venv = Virtualenv(dest) + assert [] == list(venv.iter_distributions()) run_pex3("venv", "create", "ansicolors==1.1.8", "-d", dest).assert_success() assert [(ProjectName("ansicolors"), Version("1.1.8"))] == [ (dist.metadata.project_name, dist.metadata.version) - for dist in Virtualenv(dest).iter_distributions() + for dist in venv.iter_distributions(rescan=True) ] diff --git a/tests/integration/test_interpreter_selection.py b/tests/integration/test_interpreter_selection.py index 862868827..9db453738 100644 --- a/tests/integration/test_interpreter_selection.py +++ b/tests/integration/test_interpreter_selection.py @@ -8,10 +8,8 @@ import pytest from pex.common import safe_open, temporary_dir -from pex.dist_metadata import find_distribution from pex.interpreter import PythonInterpreter from pex.interpreter_constraints import InterpreterConstraints -from pex.pep_503 import ProjectName from pex.pex_info import PexInfo from pex.typing import TYPE_CHECKING from testing import ( diff --git a/tests/integration/test_issue_940.py b/tests/integration/test_issue_940.py index 707367ece..89064a762 100644 --- a/tests/integration/test_issue_940.py +++ b/tests/integration/test_issue_940.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import os +import subprocess import sys from textwrap import dedent @@ -45,8 +46,11 @@ def prepare_project(project_dir): verify=False, python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*", ) as whl: + pex_root = os.path.join(str(tmpdir), "pex_root") pex_file = os.path.join(str(tmpdir), "pex") - results = run_pex_command(args=["-o", pex_file, whl]) + results = run_pex_command( + args=["-o", pex_file, "--pex-root", pex_root, "--runtime-pex-root", pex_root, whl] + ) results.assert_success() output, returncode = run_simple_pex(pex_file, args=["-c", "import foo"]) diff --git a/tests/integration/venv_ITs/test_virtualenv.py b/tests/integration/venv_ITs/test_virtualenv.py index 7034515e4..5b1dae04a 100644 --- a/tests/integration/venv_ITs/test_virtualenv.py +++ b/tests/integration/venv_ITs/test_virtualenv.py @@ -52,7 +52,7 @@ def test_enclosing(tmpdir): def index_distributions(venv): # type: (Virtualenv) -> Dict[ProjectName, Distribution] - return {dist.metadata.project_name: dist for dist in venv.iter_distributions()} + return {dist.metadata.project_name: dist for dist in venv.iter_distributions(rescan=True)} def test_iter_distributions_setuptools_not_leaked(tmpdir): diff --git a/tests/test_dist_metadata.py b/tests/test_dist_metadata.py index 2c7cee805..42848f0bd 100644 --- a/tests/test_dist_metadata.py +++ b/tests/test_dist_metadata.py @@ -1,7 +1,7 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import tarfile @@ -11,22 +11,22 @@ import pytest -from pex.common import open_zip, temporary_dir +from pex.common import open_zip, safe_open, temporary_dir, touch from pex.dist_metadata import ( Distribution, MetadataError, + MetadataType, ProjectNameAndVersion, Requirement, - find_dist_info_file, project_name_and_version, requires_dists, requires_python, ) +from pex.pep_503 import ProjectName from pex.pex_warnings import PEXWarning from pex.pip.installation import get_pip from pex.third_party.packaging.specifiers import SpecifierSet from pex.typing import TYPE_CHECKING -from pex.variables import ENV if TYPE_CHECKING: from typing import Any, Iterator, Tuple @@ -129,7 +129,7 @@ def test_project_name_and_version_from_filename_pep625(): # type: () -> None assert ProjectNameAndVersion( "a-distribution-name", "1.2.3" - ) == ProjectNameAndVersion.from_filename("a-distribution-name-1.2.3.sdist") + ) == ProjectNameAndVersion.from_filename("a-distribution-name-1.2.3.tar.gz") def test_project_name_and_version_from_filename_invalid(): @@ -177,13 +177,7 @@ def tmp_path(relpath): # N.B.: Valid PKG-INFO at an invalid location. tf.add(pkg_info_src, arcname="PKG-INFO") - with ENV.patch(PEX_EMIT_WARNINGS="True"), warnings.catch_warnings(record=True) as events: - assert project_name_and_version(sdist_path, fallback_to_filename=False) is None - assert 1 == len(events) - warning = events[0] - assert PEXWarning == warning.category - assert "bar-baz-4.5.6/PKG-INFO" in str(warning.message) - + assert project_name_and_version(sdist_path, fallback_to_filename=False) is None assert ProjectNameAndVersion("bar-baz", "4.5.6") == project_name_and_version( sdist_path, fallback_to_filename=True ) @@ -192,6 +186,7 @@ def tmp_path(relpath): pkf_info_path = "{}/PKG-INFO".format(name_and_version) def write_sdist_tgz(extension): + # type: (str) -> str sdist_path = tmp_path("{}.{}".format(name_and_version, extension)) with tarfile.open(sdist_path, mode="w:gz") as tf: tf.add(pkg_info_src, arcname=pkf_info_path) @@ -201,7 +196,7 @@ def write_sdist_tgz(extension): write_sdist_tgz("tar.gz"), fallback_to_filename=False ) assert expected_metadata_project_name_and_version == project_name_and_version( - write_sdist_tgz("sdist"), fallback_to_filename=False + write_sdist_tgz("tgz"), fallback_to_filename=False ) zip_sdist_path = tmp_path("{}.zip".format(name_and_version)) @@ -288,48 +283,86 @@ def test_wheel_metadata_project_name_fuzzy_issues_1375(): assert expected == project_name_and_version(dist) -def test_find_dist_info_file(): - # type: () -> None +@pytest.mark.parametrize( + "metadata_type", + [ + pytest.param(metadata_type, id=str(metadata_type)) + for metadata_type in (MetadataType.DIST_INFO, MetadataType.EGG_INFO) + ], +) +def test_find_dist_info_file( + tmpdir, # type: Any + metadata_type, # type: MetadataType.Value +): + # type: (...) -> None assert ( - find_dist_info_file( - project_name="foo", - version="1.0", - filename="bar", - listing=[], - ) - is None + metadata_type.load_metadata(location=str(tmpdir), project_name=ProjectName("foo")) is None ) - assert ( - find_dist_info_file( - project_name="foo", - version="1.0", - filename="bar", - listing=[ - "foo-1.0.dist-info/baz", - ], + def metadata_dir_name(project_name_and_version): + # type: (str) -> str + return "{project_name_and_version}.{metadata_type}".format( + project_name_and_version=project_name_and_version, + metadata_type="dist-info" if metadata_type is MetadataType.DIST_INFO else "egg-info", ) - is None + + metadata_file_name = "METADATA" if metadata_type is MetadataType.DIST_INFO else "PKG-INFO" + + touch(os.path.join(str(tmpdir), metadata_dir_name("foo-1.0"), "baz")) + assert ( + metadata_type.load_metadata(location=str(tmpdir), project_name=ProjectName("foo")) is None ) - assert "Foo-1.0.dist-info/bar" == find_dist_info_file( - project_name="foo", - version="1.0", - filename="bar", - listing=[ - "foo-100.dist-info/bar", - "Foo-1.0.dist-info/bar", - "foo-1.0.dist-info/bar", - ], + touch(os.path.join(str(tmpdir), metadata_dir_name("foo-1.0"), metadata_file_name)) + assert ( + metadata_type.load_metadata(location=str(tmpdir), project_name=ProjectName("foo")) is None ) - assert "stress__-.-__Test-1.0rc0.dist-info/direct_url.json" == find_dist_info_file( - project_name="Stress-.__Test", + def write_pkg_info_file( + location, # type: str + name, # type: str + version, # type: str + ): + # type: (...) -> None + with safe_open( + os.path.join( + location, + metadata_dir_name("{name}-{version}".format(name=name, version=version)), + metadata_file_name, + ), + "w", + ) as fp: + print("Metadata-Version: 1.0", file=fp) + print("Name: {name}".format(name=name), file=fp) + print("Version: {version}".format(version=version), file=fp) + + foo_location = os.path.join(str(tmpdir), "foo_location") + touch(os.path.join(foo_location, metadata_dir_name("foo-100"), "bar")) + expected_metadata_relpath = os.path.join(metadata_dir_name("Foo-1.0"), "bar") + touch(os.path.join(foo_location, expected_metadata_relpath)) + write_pkg_info_file(foo_location, name="Foo", version="1.0") + + metadata_files = metadata_type.load_metadata( + location=foo_location, project_name=ProjectName("foo") + ) + assert metadata_files is not None + assert expected_metadata_relpath == metadata_files.metadata_file_rel_path("bar") + + stress_location = os.path.join(str(tmpdir), "stress_location") + touch(os.path.join(stress_location, "direct_url.json")) + touch(os.path.join(stress_location, metadata_dir_name("foo-1.0rc0"), "direct_url.json")) + expected_metadata_relpath = os.path.join( + metadata_dir_name("stress__-.-__Test-1.0rc0"), "direct_url.json" + ) + touch(os.path.join(stress_location, expected_metadata_relpath)) + write_pkg_info_file( + stress_location, + name="stress__-.-__Test", version="1.0rc0", - filename="direct_url.json", - listing=[ - "direct_url.json", - "foo-1.0rc0.dist-info/direct_url.json", - "stress__-.-__Test-1.0rc0.dist-info/direct_url.json", - ], ) + + metadata_files = metadata_type.load_metadata( + location=stress_location, project_name=ProjectName("Stress-.__Test") + ) + assert metadata_files is not None + assert expected_metadata_relpath == metadata_files.metadata_file_rel_path("direct_url.json") diff --git a/tests/test_environment.py b/tests/test_environment.py index 88735dd7d..5c7d53c47 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -24,7 +24,6 @@ from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.resolve.configured_resolver import ConfiguredResolver -from pex.resolve.resolver_configuration import PipConfiguration from pex.targets import LocalInterpreter, Targets from pex.typing import TYPE_CHECKING from testing import (