Add support for Pip 23.3.1. (#2276)

This required updating the locker parsing to account for differences in 23.3.x log output. Although 23.3 has significant changes from 23.2.x, I was away during its release and no one clamored for it; so I'm just moving ahead to the small 23.3.1 update. The changelogs are here: + https://pip.pypa.io/en/stable/news/#v23-3-1 + https://pip.pypa.io/en/stable/news/#v23-3
pex-tool · Nov 4, 2023 · 1f073d0 · 1f073d0
1 parent 9f8c763
commit 1f073d0
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 73 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -56,7 +56,7 @@ jobs:
           - py311-pip20
           - py311-pip22_3_1
           - py311-pip23_1_2
-          - py312-pip23_2
+          - py312-pip23_3_1
           - pypy310-pip20
           - pypy310-pip22_3_1
           - pypy310-pip23_1_2
@@ -66,7 +66,7 @@ jobs:
           - py311-pip20-integration
           - py311-pip22_3_1-integration
           - py311-pip23_1_2-integration
-          - py312-pip23_2-integration
+          - py312-pip23_3_1-integration
           - pypy310-pip20-integration
           - pypy310-pip22_3_1-integration
           - pypy310-pip23_1_2-integration
@@ -107,10 +107,10 @@ jobs:
       matrix:
         include:
           - python-version: [ 3, 12 ]
-            tox-env: py312-pip23_2
+            tox-env: py312-pip23_3_1
             tox-env-python: python3.11
           - python-version: [ 3, 12 ]
-            tox-env: py312-pip23_2-integration
+            tox-env: py312-pip23_3_1-integration
             tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose

diff --git a/pex/pip/version.py b/pex/pip/version.py
@@ -213,6 +213,15 @@ def values(cls):
         requires_python=">=3.7",
     )
 
+    v23_3_1 = PipVersionValue(
+        version="23.3.1",
+        # N.B.: The setuptools 68.2.2 release was available on 10/21/2023 (the Pip 23.3.1 release
+        # date) but 68.0.0 is the last setuptools version to support 3.7.
+        setuptools_version="68.0.0",
+        wheel_version="0.41.2",
+        requires_python=">=3.7",
+    )
+
     VENDORED = v20_3_4_patched
     LATEST = LatestPipVersion()
     DEFAULT = DefaultPipVersion(preferred=(VENDORED, v23_2))
diff --git a/pex/resolve/locker.py b/pex/resolve/locker.py
@@ -38,18 +38,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import (
-        DefaultDict,
-        Dict,
-        Iterable,
-        List,
-        Mapping,
-        Optional,
-        Pattern,
-        Set,
-        Text,
-        Tuple,
-    )
+    from typing import DefaultDict, Dict, Iterable, Mapping, Optional, Pattern, Set, Text, Tuple
 
     import attr  # vendor:skip
 
@@ -212,7 +201,6 @@ class AnalyzeError(Exception):
 class ArtifactBuildResult(object):
     url = attr.ib()  # type: ArtifactURL
     pin = attr.ib()  # type: Pin
-    requirement = attr.ib()  # type: Requirement
 
 
 @attr.s(frozen=True)
@@ -238,11 +226,7 @@ def build_result(self, line):
         version = Version(match.group("version"))
         requirement = Requirement.parse(match.group("requirement"))
         pin = Pin(project_name=requirement.project_name, version=version)
-        return ArtifactBuildResult(
-            url=self._artifact_url,
-            pin=pin,
-            requirement=requirement,
-        )
+        return ArtifactBuildResult(url=self._artifact_url, pin=pin)
 
 
 class Locker(LogAnalyzer):
@@ -269,7 +253,7 @@ def __init__(
         self._saved = set()  # type: Set[Pin]
         self._selected_path_to_pin = {}  # type: Dict[str, Pin]
 
-        self._resolved_requirements = []  # type: List[ResolvedRequirement]
+        self._resolved_requirements = OrderedDict()  # type: OrderedDict[Pin, ResolvedRequirement]
         self._pep_691_endpoints = set()  # type: Set[Endpoint]
         self._links = defaultdict(
             OrderedDict
@@ -309,23 +293,46 @@ def _extract_resolve_data(artifact_url):
         partial_artifact = PartialArtifact(artifact_url, fingerprint)
         return pin, partial_artifact
 
+    def _maybe_record_wheel(self, url):
+        # type: (str) -> ArtifactURL
+        artifact_url = ArtifactURL.parse(url)
+        if artifact_url.is_wheel:
+            pin, partial_artifact = self._extract_resolve_data(artifact_url)
+
+            additional_artifacts = self._links[pin]
+            additional_artifacts.pop(artifact_url, None)
+            self._resolved_requirements[pin] = ResolvedRequirement(
+                pin=pin,
+                artifact=partial_artifact,
+                additional_artifacts=tuple(additional_artifacts.values()),
+            )
+            self._selected_path_to_pin[os.path.basename(artifact_url.path)] = pin
+        return artifact_url
+
     def analyze(self, line):
         # type: (str) -> LogAnalyzer.Continue[None]
 
         # The log sequence for processing a resolved requirement is as follows (log lines irrelevant
         # to our purposes omitted):
         #
-        #   1.)       "... Found link <url1> ..."
+        #   1.)        "... Found link <url1> ..."
         #   ...
-        #   1.)       "... Found link <urlN> ..."
-        #   2.)       "... Added <varying info ...> to build tracker ..."
-        # * 3.)       Lines related to extracting metadata from <requirement> if the selected
-        #             distribution is an sdist in any form (VCS, local directory, source archive).
-        # * 3.5. ERR) "... WARNING: Discarding <url> <varying info...>. Command errored out with ...
-        # * 3.5. SUC) "... Source in <tmp> has version <version>, which satisfies requirement "
-        #             "<requirement> from <url> ..."
-        #   4.)       "... Removed <requirement> from <url> ... from build tracker ..."
-        #   5.)       "... Saved <download dir>/<artifact file>
+        #   1.)        "... Found link <urlN> ..."
+        #   1.5. URL)  "... Looking up "<url>" in the cache"
+        #   1.5. PATH) "... Processing <path> ..."
+        #   2.)        "... Added <varying info ...> to build tracker ..."
+        # * 3.)        Lines related to extracting metadata from <requirement> if the selected
+        #              distribution is an sdist in any form (VCS, local directory, source archive).
+        # * 3.5. ERR)  "... WARNING: Discarding <url> <varying info...>. Command errored out with ..."
+        # * 3.5. SUC)  "... Source in <tmp> has version <version>, which satisfies requirement <requirement> from <url> ..."
+        #   4.)        "... Removed <requirement> from <url> ... from build tracker ..."
+        #   5.)        "... Saved <download dir>/<artifact file>
+
+        # Although section 1.5 is always present in all supported Pip versions, the lines in sections
+        # 2-4 are optionally present depending on selected artifact type (wheel vs sdist vs ...) and
+        # Pip version. It is constant; however, that sections 2-4 are present in all supported Pip
+        # versions when dealing with an artifact that needs to be built (sdist, VCS url or local
+        # project).
 
         # The lines in section 3 can contain this same pattern of lines if the metadata extraction
         # proceeds via PEP-517 which recursively uses Pip to resolve build dependencies. We want to
@@ -402,15 +409,12 @@ def analyze(self, line):
                 additional_artifacts = self._links[build_result.pin]
                 additional_artifacts.pop(artifact_url, None)
 
-                self._resolved_requirements.append(
-                    ResolvedRequirement(
-                        requirement=build_result.requirement,
-                        pin=build_result.pin,
-                        artifact=PartialArtifact(
-                            url=artifact_url, fingerprint=source_fingerprint, verified=verified
-                        ),
-                        additional_artifacts=tuple(additional_artifacts.values()),
-                    )
+                self._resolved_requirements[build_result.pin] = ResolvedRequirement(
+                    pin=build_result.pin,
+                    artifact=PartialArtifact(
+                        url=artifact_url, fingerprint=source_fingerprint, verified=verified
+                    ),
+                    additional_artifacts=tuple(additional_artifacts.values()),
                 )
             return self.Continue()
 
@@ -428,29 +432,24 @@ def analyze(self, line):
             )
             return self.Continue()
 
+        match = re.search(r"Looking up \"(?P<url>[^\s]+)\" in the cache", line)
+        if match:
+            self._maybe_record_wheel(match.group("url"))
+
+        match = re.search(r"Processing (?P<path>.*\.(whl|tar\.(gz|bz2|xz)|tgz|tbz2|txz|zip))", line)
+        if match:
+            self._maybe_record_wheel(
+                "file://{path}".format(path=os.path.abspath(match.group("path")))
+            )
+
         match = re.search(
             r"Added (?P<requirement>.+) from (?P<url>[^\s]+) .*to build tracker",
             line,
         )
         if match:
             raw_requirement = match.group("requirement")
-            url = ArtifactURL.parse(match.group("url"))
-            if url.is_wheel:
-                requirement = Requirement.parse(raw_requirement)
-                pin, partial_artifact = self._extract_resolve_data(url)
-
-                additional_artifacts = self._links[pin]
-                additional_artifacts.pop(url, None)
-                self._resolved_requirements.append(
-                    ResolvedRequirement(
-                        requirement=requirement,
-                        pin=pin,
-                        artifact=partial_artifact,
-                        additional_artifacts=tuple(additional_artifacts.values()),
-                    )
-                )
-                self._selected_path_to_pin[os.path.basename(url.path)] = pin
-            else:
+            url = self._maybe_record_wheel(match.group("url"))
+            if not url.is_wheel:
                 self._artifact_build_observer = ArtifactBuildObserver(
                     done_building_patterns=(
                         re.compile(
@@ -483,7 +482,9 @@ def analyze(self, line):
         match = re.search(r"Saved (?P<file_path>.+)$", line)
         if match:
             saved_path = match.group("file_path")
-            self._saved.add(self._selected_path_to_pin[os.path.basename(saved_path)])
+            build_result_pin = self._selected_path_to_pin.get(os.path.basename(saved_path))
+            if build_result_pin:
+                self._saved.add(build_result_pin)
             return self.Continue()
 
         if self.style in (LockStyle.SOURCES, LockStyle.UNIVERSAL):
@@ -500,7 +501,7 @@ def analysis_completed(self):
         # type: () -> None
         resolved_requirements = [
             resolved_requirement
-            for resolved_requirement in self._resolved_requirements
+            for resolved_requirement in self._resolved_requirements.values()
             if resolved_requirement.pin in self._saved
         ]
 

diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py
@@ -20,7 +20,6 @@
 from pex.pep_503 import ProjectName
 from pex.pip.download_observer import DownloadObserver
 from pex.pip.tool import PackageIndexConfiguration
-from pex.pip.version import PipVersion
 from pex.resolve import lock_resolver, locker, resolvers
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.downloads import ArtifactDownloader

diff --git a/pex/resolve/resolved_requirement.py b/pex/resolve/resolved_requirement.py
@@ -119,9 +119,7 @@ class PartialArtifact(object):
 class ResolvedRequirement(object):
     pin = attr.ib()  # type: Pin
     artifact = attr.ib()  # type: PartialArtifact
-    requirement = attr.ib()  # type: Requirement
     additional_artifacts = attr.ib(default=())  # type: Tuple[PartialArtifact, ...]
-    via = attr.ib(default=())  # type: Tuple[str, ...]
 
     def iter_artifacts(self):
         # type: () -> Iterator[PartialArtifact]

diff --git a/tests/integration/cli/commands/test_issue_1801.py b/tests/integration/cli/commands/test_issue_1801.py
@@ -29,16 +29,25 @@ def test_preserve_pip_download_log():
     expected_algorithm = "sha256"
     expected_hash = "00d2dde5a675579325902536738dd27e4fac1fd68f773fe36c21044eb559e187"
     with open(log_path) as fp:
+        log_text = fp.read()
+
+    assert re.search(
         # N.B.: Modern Pip excludes hashes from logged URLs when the index serves up PEP-691 json
         # responses.
-        assert re.search(
-            r"Added ansicolors==1\.1\.8 from https?://\S+/{url_suffix}(?:#{algorithm}={hash})? to build tracker".format(
-                url_suffix=re.escape(expected_url_suffix),
-                algorithm=re.escape(expected_algorithm),
-                hash=re.escape(expected_hash),
-            ),
-            fp.read(),
-        )
+        r"Added ansicolors==1\.1\.8 from https?://\S+/{url_suffix}(?:#{algorithm}={hash})? to build tracker".format(
+            url_suffix=re.escape(expected_url_suffix),
+            algorithm=re.escape(expected_algorithm),
+            hash=re.escape(expected_hash),
+        ),
+        log_text,
+    ) or re.search(
+        # N.B.: Even more modern Pip does not log "Added ... to build tracker" lines for pre-built
+        # wheels; so we look for an alternate expected log line.
+        r"Looking up \"https?://\S+/{url_suffix}\" in the cache".format(
+            url_suffix=re.escape(expected_url_suffix),
+        ),
+        log_text,
+    )
 
     lockfile = json_codec.loads(result.output)
     assert 1 == len(lockfile.locked_resolves)

diff --git a/tox.ini b/tox.ini
@@ -58,6 +58,7 @@ setenv =
     pip23_1_1: _PEX_PIP_VERSION=23.1.1
     pip23_1_2: _PEX_PIP_VERSION=23.1.2
     pip23_2: _PEX_PIP_VERSION=23.2
+    pip23_3_1: _PEX_PIP_VERSION=23.3.1
     # Python 3 (until a fix here in 3.9: https://bugs.python.org/issue13601) switched from stderr
     # being unbuffered to stderr being buffered by default. This can lead to tests checking stderr
     # failing to see what they expect if the stderr buffer block has not been flushed. Force stderr
@@ -69,7 +70,7 @@ whitelist_externals =
     bash
     git
 
-[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-}integration]
+[testenv:py{py27-subprocess,py27,py35,py36,py37,py38,py39,py310,27,35,36,37,38,39,310,311,312}-{,pip20-,pip22_2-,pip22_3-,pip22_3_1-,pip23_0-,pip23_0_1-,pip23_1-,pip23_1_1-,pip23_1_2-,pip23_2-,pip23_3_1-}integration]
 deps =
     pytest-xdist==1.34.0
     {[testenv]deps}