From 1f073d06e1934c4bbee46e91c87d51acbb864ec0 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Sat, 4 Nov 2023 15:09:33 -0700 Subject: [PATCH] Add support for Pip 23.3.1. (#2276) This required updating the locker parsing to account for differences in 23.3.x log output. Although 23.3 has significant changes from 23.2.x, I was away during its release and no one clamored for it; so I'm just moving ahead to the small 23.3.1 update. The changelogs are here: + https://pip.pypa.io/en/stable/news/#v23-3-1 + https://pip.pypa.io/en/stable/news/#v23-3 --- .github/workflows/ci.yml | 8 +- pex/pip/version.py | 9 ++ pex/resolve/locker.py | 115 +++++++++--------- pex/resolve/lockfile/create.py | 1 - pex/resolve/resolved_requirement.py | 2 - .../cli/commands/test_issue_1801.py | 25 ++-- tox.ini | 3 +- 7 files changed, 90 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7950b4f2..bd9d0edfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: - py311-pip20 - py311-pip22_3_1 - py311-pip23_1_2 - - py312-pip23_2 + - py312-pip23_3_1 - pypy310-pip20 - pypy310-pip22_3_1 - pypy310-pip23_1_2 @@ -66,7 +66,7 @@ jobs: - py311-pip20-integration - py311-pip22_3_1-integration - py311-pip23_1_2-integration - - py312-pip23_2-integration + - py312-pip23_3_1-integration - pypy310-pip20-integration - pypy310-pip22_3_1-integration - pypy310-pip23_1_2-integration @@ -107,10 +107,10 @@ jobs: matrix: include: - python-version: [ 3, 12 ] - tox-env: py312-pip23_2 + tox-env: py312-pip23_3_1 tox-env-python: python3.11 - python-version: [ 3, 12 ] - tox-env: py312-pip23_2-integration + tox-env: py312-pip23_3_1-integration tox-env-python: python3.11 steps: - name: Calculate Pythons to Expose diff --git a/pex/pip/version.py b/pex/pip/version.py index e31222d1f..454fbdcd6 100644 --- a/pex/pip/version.py +++ b/pex/pip/version.py @@ -213,6 +213,15 @@ def values(cls): requires_python=">=3.7", ) + v23_3_1 = PipVersionValue( + version="23.3.1", + # N.B.: The setuptools 68.2.2 release was available on 10/21/2023 (the Pip 23.3.1 release + # date) but 68.0.0 is the last setuptools version to support 3.7. + setuptools_version="68.0.0", + wheel_version="0.41.2", + requires_python=">=3.7", + ) + VENDORED = v20_3_4_patched LATEST = LatestPipVersion() DEFAULT = DefaultPipVersion(preferred=(VENDORED, v23_2)) diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py index 821060991..b79d848fc 100644 --- a/pex/resolve/locker.py +++ b/pex/resolve/locker.py @@ -38,18 +38,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import ( - DefaultDict, - Dict, - Iterable, - List, - Mapping, - Optional, - Pattern, - Set, - Text, - Tuple, - ) + from typing import DefaultDict, Dict, Iterable, Mapping, Optional, Pattern, Set, Text, Tuple import attr # vendor:skip @@ -212,7 +201,6 @@ class AnalyzeError(Exception): class ArtifactBuildResult(object): url = attr.ib() # type: ArtifactURL pin = attr.ib() # type: Pin - requirement = attr.ib() # type: Requirement @attr.s(frozen=True) @@ -238,11 +226,7 @@ def build_result(self, line): version = Version(match.group("version")) requirement = Requirement.parse(match.group("requirement")) pin = Pin(project_name=requirement.project_name, version=version) - return ArtifactBuildResult( - url=self._artifact_url, - pin=pin, - requirement=requirement, - ) + return ArtifactBuildResult(url=self._artifact_url, pin=pin) class Locker(LogAnalyzer): @@ -269,7 +253,7 @@ def __init__( self._saved = set() # type: Set[Pin] self._selected_path_to_pin = {} # type: Dict[str, Pin] - self._resolved_requirements = [] # type: List[ResolvedRequirement] + self._resolved_requirements = OrderedDict() # type: OrderedDict[Pin, ResolvedRequirement] self._pep_691_endpoints = set() # type: Set[Endpoint] self._links = defaultdict( OrderedDict @@ -309,23 +293,46 @@ def _extract_resolve_data(artifact_url): partial_artifact = PartialArtifact(artifact_url, fingerprint) return pin, partial_artifact + def _maybe_record_wheel(self, url): + # type: (str) -> ArtifactURL + artifact_url = ArtifactURL.parse(url) + if artifact_url.is_wheel: + pin, partial_artifact = self._extract_resolve_data(artifact_url) + + additional_artifacts = self._links[pin] + additional_artifacts.pop(artifact_url, None) + self._resolved_requirements[pin] = ResolvedRequirement( + pin=pin, + artifact=partial_artifact, + additional_artifacts=tuple(additional_artifacts.values()), + ) + self._selected_path_to_pin[os.path.basename(artifact_url.path)] = pin + return artifact_url + def analyze(self, line): # type: (str) -> LogAnalyzer.Continue[None] # The log sequence for processing a resolved requirement is as follows (log lines irrelevant # to our purposes omitted): # - # 1.) "... Found link ..." + # 1.) "... Found link ..." # ... - # 1.) "... Found link ..." - # 2.) "... Added to build tracker ..." - # * 3.) Lines related to extracting metadata from if the selected - # distribution is an sdist in any form (VCS, local directory, source archive). - # * 3.5. ERR) "... WARNING: Discarding . Command errored out with ... - # * 3.5. SUC) "... Source in has version , which satisfies requirement " - # " from ..." - # 4.) "... Removed from ... from build tracker ..." - # 5.) "... Saved / + # 1.) "... Found link ..." + # 1.5. URL) "... Looking up "" in the cache" + # 1.5. PATH) "... Processing ..." + # 2.) "... Added to build tracker ..." + # * 3.) Lines related to extracting metadata from if the selected + # distribution is an sdist in any form (VCS, local directory, source archive). + # * 3.5. ERR) "... WARNING: Discarding . Command errored out with ..." + # * 3.5. SUC) "... Source in has version , which satisfies requirement from ..." + # 4.) "... Removed from ... from build tracker ..." + # 5.) "... Saved / + + # Although section 1.5 is always present in all supported Pip versions, the lines in sections + # 2-4 are optionally present depending on selected artifact type (wheel vs sdist vs ...) and + # Pip version. It is constant; however, that sections 2-4 are present in all supported Pip + # versions when dealing with an artifact that needs to be built (sdist, VCS url or local + # project). # The lines in section 3 can contain this same pattern of lines if the metadata extraction # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to @@ -402,15 +409,12 @@ def analyze(self, line): additional_artifacts = self._links[build_result.pin] additional_artifacts.pop(artifact_url, None) - self._resolved_requirements.append( - ResolvedRequirement( - requirement=build_result.requirement, - pin=build_result.pin, - artifact=PartialArtifact( - url=artifact_url, fingerprint=source_fingerprint, verified=verified - ), - additional_artifacts=tuple(additional_artifacts.values()), - ) + self._resolved_requirements[build_result.pin] = ResolvedRequirement( + pin=build_result.pin, + artifact=PartialArtifact( + url=artifact_url, fingerprint=source_fingerprint, verified=verified + ), + additional_artifacts=tuple(additional_artifacts.values()), ) return self.Continue() @@ -428,29 +432,24 @@ def analyze(self, line): ) return self.Continue() + match = re.search(r"Looking up \"(?P[^\s]+)\" in the cache", line) + if match: + self._maybe_record_wheel(match.group("url")) + + match = re.search(r"Processing (?P.*\.(whl|tar\.(gz|bz2|xz)|tgz|tbz2|txz|zip))", line) + if match: + self._maybe_record_wheel( + "file://{path}".format(path=os.path.abspath(match.group("path"))) + ) + match = re.search( r"Added (?P.+) from (?P[^\s]+) .*to build tracker", line, ) if match: raw_requirement = match.group("requirement") - url = ArtifactURL.parse(match.group("url")) - if url.is_wheel: - requirement = Requirement.parse(raw_requirement) - pin, partial_artifact = self._extract_resolve_data(url) - - additional_artifacts = self._links[pin] - additional_artifacts.pop(url, None) - self._resolved_requirements.append( - ResolvedRequirement( - requirement=requirement, - pin=pin, - artifact=partial_artifact, - additional_artifacts=tuple(additional_artifacts.values()), - ) - ) - self._selected_path_to_pin[os.path.basename(url.path)] = pin - else: + url = self._maybe_record_wheel(match.group("url")) + if not url.is_wheel: self._artifact_build_observer = ArtifactBuildObserver( done_building_patterns=( re.compile( @@ -483,7 +482,9 @@ def analyze(self, line): match = re.search(r"Saved (?P.+)$", line) if match: saved_path = match.group("file_path") - self._saved.add(self._selected_path_to_pin[os.path.basename(saved_path)]) + build_result_pin = self._selected_path_to_pin.get(os.path.basename(saved_path)) + if build_result_pin: + self._saved.add(build_result_pin) return self.Continue() if self.style in (LockStyle.SOURCES, LockStyle.UNIVERSAL): @@ -500,7 +501,7 @@ def analysis_completed(self): # type: () -> None resolved_requirements = [ resolved_requirement - for resolved_requirement in self._resolved_requirements + for resolved_requirement in self._resolved_requirements.values() if resolved_requirement.pin in self._saved ] diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 19d7a1181..ead291673 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -20,7 +20,6 @@ from pex.pep_503 import ProjectName from pex.pip.download_observer import DownloadObserver from pex.pip.tool import PackageIndexConfiguration -from pex.pip.version import PipVersion from pex.resolve import lock_resolver, locker, resolvers from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.downloads import ArtifactDownloader diff --git a/pex/resolve/resolved_requirement.py b/pex/resolve/resolved_requirement.py index a6c7ce01e..ad5e30769 100644 --- a/pex/resolve/resolved_requirement.py +++ b/pex/resolve/resolved_requirement.py @@ -119,9 +119,7 @@ class PartialArtifact(object): class ResolvedRequirement(object): pin = attr.ib() # type: Pin artifact = attr.ib() # type: PartialArtifact - requirement = attr.ib() # type: Requirement additional_artifacts = attr.ib(default=()) # type: Tuple[PartialArtifact, ...] - via = attr.ib(default=()) # type: Tuple[str, ...] def iter_artifacts(self): # type: () -> Iterator[PartialArtifact] diff --git a/tests/integration/cli/commands/test_issue_1801.py b/tests/integration/cli/commands/test_issue_1801.py index 3d8d49edc..77c5804df 100644 --- a/tests/integration/cli/commands/test_issue_1801.py +++ b/tests/integration/cli/commands/test_issue_1801.py @@ -29,16 +29,25 @@ def test_preserve_pip_download_log(): expected_algorithm = "sha256" expected_hash = "00d2dde5a675579325902536738dd27e4fac1fd68f773fe36c21044eb559e187" with open(log_path) as fp: + log_text = fp.read() + + assert re.search( # N.B.: Modern Pip excludes hashes from logged URLs when the index serves up PEP-691 json # responses. - assert re.search( - r"Added ansicolors==1\.1\.8 from https?://\S+/{url_suffix}(?:#{algorithm}={hash})? to build tracker".format( - url_suffix=re.escape(expected_url_suffix), - algorithm=re.escape(expected_algorithm), - hash=re.escape(expected_hash), - ), - fp.read(), - ) + r"Added ansicolors==1\.1\.8 from https?://\S+/{url_suffix}(?:#{algorithm}={hash})? to build tracker".format( + url_suffix=re.escape(expected_url_suffix), + algorithm=re.escape(expected_algorithm), + hash=re.escape(expected_hash), + ), + log_text, + ) or re.search( + # N.B.: Even more modern Pip does not log "Added ... to build tracker" lines for pre-built + # wheels; so we look for an alternate expected log line. + r"Looking up \"https?://\S+/{url_suffix}\" in the cache".format( + url_suffix=re.escape(expected_url_suffix), + ), + log_text, + ) lockfile = json_codec.loads(result.output) assert 1 == len(lockfile.locked_resolves) diff --git a/tox.ini b/tox.ini index 3f015264e..a6b96f7a3 100644 --- a/tox.ini +++ b/tox.ini @@ -58,6 +58,7 @@ setenv = pip23_1_1: _PEX_PIP_VERSION=23.1.1 pip23_1_2: _PEX_PIP_VERSION=23.1.2 pip23_2: _PEX_PIP_VERSION=23.2 + pip23_3_1: _PEX_PIP_VERSION=23.3.1 # Python 3 (until a fix here in 3.9: https://bugs.python.org/issue13601) switched from stderr # being unbuffered to stderr being buffered by default. This can lead to tests checking stderr # failing to see what they expect if the stderr buffer block has not been flushed. Force stderr @@ -69,7 +70,7 @@ whitelist_externals = bash git -[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-}integration] +[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-,pip23_3_1-}integration] deps = pytest-xdist==1.34.0 {[testenv]deps}