Add support for Pip 22.2.2.

Pex can now support multiple Pip versions and be configured to fallback as
needed when newer Pip does not support a specified interpreter.

Fixes pex-tool#1805

pex/bin/pex.py

@@ -33,6 +33,7 @@
 from pex.pex_builder import CopyMode, PEXBuilder
 from pex.pex_info import PexInfo
 from pex.resolve import requirement_options, resolver_options, target_configuration, target_options
+from pex.resolve.config import finalize as finalize_resolve_config
 from pex.resolve.configured_resolver import ConfiguredResolver
 from pex.resolve.lock_resolver import resolve_from_lock
 from pex.resolve.pex_repository_resolver import resolve_from_pex
@@ -658,6 +659,7 @@ def build_pex(
                             build_isolation=pip_configuration.build_isolation,
                             compile=options.compile,
                             max_parallel_jobs=pip_configuration.max_jobs,
+                            pip_version=lock.pip_version,
                         )
                     )
             elif isinstance(resolver_configuration, PexRepositoryConfiguration):
@@ -699,6 +701,8 @@ def build_pex(
                         max_parallel_jobs=resolver_configuration.max_jobs,
                         ignore_errors=options.ignore_errors,
                         preserve_log=resolver_configuration.preserve_log,
+                        pip_version=resolver_configuration.version,
+                        resolver=ConfiguredResolver(pip_configuration=resolver_configuration),
                     )
 
             for installed_dist in result.installed_distributions:
@@ -771,6 +775,10 @@ def main(args=None):
             except target_configuration.InterpreterConstraintsNotSatisfied as e:
                 die(str(e), exit_code=CANNOT_SETUP_INTERPRETER)
 
+            resolver_configuration = finalize_resolve_config(
+                resolver_configuration, targets, context="PEX building"
+            )
+
             sys.exit(
                 catch(
                     do_main,

pex/build_system/pep_517.py

@@ -11,55 +11,75 @@
 from pex import third_party
 from pex.build_system.pep_518 import BuildSystem, load_build_system
 from pex.dist_metadata import Distribution
+from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve.resolvers import Resolver
 from pex.result import Error
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
 from pex.util import named_temporary_file
 
 if TYPE_CHECKING:
-    from typing import Optional, Union
+    from typing import Dict, Union
 
 _DEFAULT_BUILD_BACKEND = "setuptools.build_meta:__legacy__"
-_DEFAULT_REQUIRES = ["setuptools"]
-_DEFAULT_BUILD_SYSTEM = None  # type: Optional[BuildSystem]
+_DEFAULT_BUILD_SYSTEMS = {}  # type: Dict[PipVersionValue, BuildSystem]
 
 
-def _default_build_system():
-    # type: () -> BuildSystem
-    global _DEFAULT_BUILD_SYSTEM
-    if _DEFAULT_BUILD_SYSTEM is None:
+def _default_build_system(
+    pip_version,  # type: PipVersionValue
+    resolver,  # type: Resolver
+):
+    # type: (...) -> BuildSystem
+    global _DEFAULT_BUILD_SYSTEMS
+    build_system = _DEFAULT_BUILD_SYSTEMS.get(pip_version)
+    if build_system is None:
         with TRACER.timed(
             "Building {build_backend} build_backend PEX".format(
                 build_backend=_DEFAULT_BUILD_BACKEND
             )
         ):
-            _DEFAULT_BUILD_SYSTEM = BuildSystem.create(
-                requires=_DEFAULT_REQUIRES,
-                resolved=tuple(
+            extra_env = {}  # type: Dict[str, str]
+            if pip_version is PipVersion.VENDORED:
+                requires = ["setuptools"]
+                resolved = tuple(
                     Distribution.load(dist_location)
-                    for dist_location in third_party.expose(_DEFAULT_REQUIRES)
-                ),
+                    for dist_location in third_party.expose(requires)
+                )
+                extra_env.update(__PEX_UNVENDORED__="1")
+            else:
+                requires = [pip_version.setuptools_requirement]
+                resolved = tuple(
+                    installed_distribution.fingerprinted_distribution.distribution
+                    for installed_distribution in resolver.resolve_requirements(
+                        requires
+                    ).installed_distributions
+                )
+            build_system = BuildSystem.create(
+                requires=requires,
+                resolved=resolved,
                 build_backend=_DEFAULT_BUILD_BACKEND,
-                __PEX_UNVENDORED__="1",
+                **extra_env
             )
-    return _DEFAULT_BUILD_SYSTEM
+            _DEFAULT_BUILD_SYSTEMS[pip_version] = build_system
+    return build_system
 
 
 def _get_build_system(
+    pip_version,  # type: PipVersionValue
     resolver,  # type: Resolver
     project_directory,  # type: str
 ):
     # type: (...) -> Union[BuildSystem, Error]
     custom_build_system_or_error = load_build_system(resolver, project_directory)
     if custom_build_system_or_error:
         return custom_build_system_or_error
-    return _default_build_system()
+    return _default_build_system(pip_version=pip_version, resolver=resolver)
 
 
 def build_sdist(
     project_directory,  # type: str
     dist_dir,  # type: str
+    pip_version,  # type: PipVersionValue
     resolver,  # type: Resolver
 ):
     # type: (...) -> Union[str, Error]
@@ -76,7 +96,7 @@ def build_sdist(
             )
         )
 
-    build_system_or_error = _get_build_system(resolver, project_directory)
+    build_system_or_error = _get_build_system(pip_version, resolver, project_directory)
     if isinstance(build_system_or_error, Error):
         return build_system_or_error
     build_system = build_system_or_error

pex/build_system/testing.py

@@ -11,7 +11,9 @@
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.pip.installation import get_pip
+from pex.pip.version import PipVersion
 from pex.resolve.configured_resolver import ConfiguredResolver
+from pex.resolve.resolver_configuration import PipConfiguration
 from pex.result import Error
 from pex.typing import TYPE_CHECKING
 
@@ -33,7 +35,17 @@ def assert_expected_dist(dist):
         assert Version(version) == dist.metadata.version
 
     sdist_dir = os.path.join(str(tmpdir), "sdist_dir")
-    location = build_sdist(project_dir, sdist_dir, ConfiguredResolver.default())
+
+    # This test utility is used by all versions of Python Pex supports; so we need to use the
+    # vendored Pip which is guaranteed to work with all those Python versions.
+    pip_version = PipVersion.VENDORED
+
+    location = build_sdist(
+        project_dir,
+        sdist_dir,
+        pip_version,
+        ConfiguredResolver(PipConfiguration(version=pip_version)),
+    )
     assert not isinstance(location, Error)
     assert sdist_dir == os.path.dirname(location)
 

pex/cli/commands/lock.py

@@ -18,6 +18,7 @@
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.resolve import requirement_options, resolver_options, target_options
+from pex.resolve.config import finalize as finalize_resolve_config
 from pex.resolve.locked_resolve import LockConfiguration, LockStyle, Resolved, TargetSystem
 from pex.resolve.lockfile import json_codec
 from pex.resolve.lockfile.create import create
@@ -419,7 +420,13 @@ def _create(self):
                 target_configuration=target_configuration,
             )
         )
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = try_(
+            finalize_resolve_config(
+                resolver_configuration=resolver_options.create_pip_configuration(self.options),
+                targets=targets,
+                context="lock creation",
+            )
+        )
         self._dump_lockfile(
             try_(
                 create(
@@ -573,6 +580,19 @@ def _update(self):
                 action="updating", style=lock_file.style, target_configuration=target_configuration
             )
         )
+
+        lock_updater = attr.evolve(
+            lock_updater,
+            pip_configuration=try_(
+                finalize_resolve_config(
+                    resolver_configuration=lock_updater.pip_configuration,
+                    targets=targets,
+                    context="lock updating",
+                    pip_version=lock_file.pip_version,
+                )
+            ),
+        )
+
         with TRACER.timed("Selecting locks to update"):
             subset_result = try_(
                 subset(

pex/pip/local_project.py

@@ -10,6 +10,7 @@
 from pex.build_system import pep_517
 from pex.common import temporary_dir
 from pex.hashing import Fingerprint, Sha256
+from pex.pip.version import PipVersionValue
 from pex.resolve.resolvers import Resolver
 from pex.result import Error, try_
 from pex.tracer import TRACER
@@ -23,17 +24,19 @@
 
 def fingerprint_local_project(
     directory,  # type: str
+    pip_version,  # type: PipVersionValue
     resolver,  # type: Resolver
 ):
     # type: (...) -> Fingerprint
     digest = Sha256()
-    try_(digest_local_project(directory, digest, resolver))
+    try_(digest_local_project(directory, digest, pip_version, resolver))
     return digest.hexdigest()
 
 
 def digest_local_project(
     directory,  # type: str
     digest,  # type: HintedDigest
+    pip_version,  # type: PipVersionValue
     resolver,  # type: Resolver
     dest_dir=None,  # type: Optional[str]
 ):
@@ -43,6 +46,7 @@ def digest_local_project(
             sdist_or_error = pep_517.build_sdist(
                 project_directory=directory,
                 dist_dir=os.path.join(td, "dists"),
+                pip_version=pip_version,
                 resolver=resolver,
             )
             if isinstance(sdist_or_error, Error):

pex/platforms.py

@@ -234,7 +234,9 @@ def parse_tags(output):
                     raise AssertionError("Finished with count {}.".format(count))
 
         job = SpawnedJob.stdout(
-            job=get_pip().spawn_debug(platform=self, manylinux=manylinux), result_func=parse_tags
+            # TODO(John Sirois): XXX: Plumb pip_version and resolver.
+            job=get_pip().spawn_debug(platform=self, manylinux=manylinux),
+            result_func=parse_tags,
         )
         return job.await_result()
 

pex/resolve/config.py

@@ -0,0 +1,81 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from pex.pip.installation import compatible_version, validate_targets
+from pex.pip.version import PipVersionValue
+from pex.resolve.resolver_configuration import (
+    LockRepositoryConfiguration,
+    PexRepositoryConfiguration,
+    PipConfiguration,
+)
+from pex.result import Error, catch, try_
+from pex.targets import Targets
+from pex.typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+    from typing import Optional, TypeVar, Union
+
+    import attr  # vendor:skip
+
+    Configuration = Union[LockRepositoryConfiguration, PexRepositoryConfiguration, PipConfiguration]
+    _C = TypeVar("_C", bound=Configuration)
+
+else:
+    from pex.third_party import attr
+
+
+def _finalize_pip_configuration(
+    pip_configuration,  # type: PipConfiguration
+    targets,  # type: Targets
+    context,  # type: str
+    pip_version=None,  # type: Optional[PipVersionValue]
+):
+    # type: (...) -> Union[PipConfiguration, Error]
+    version = pip_version or pip_configuration.version
+    if pip_configuration.allow_version_fallback:
+        return attr.evolve(pip_configuration, version=compatible_version(targets, version, context))
+
+    result = catch(validate_targets, targets, version, context)
+    if isinstance(result, Error):
+        return result
+    return attr.evolve(pip_configuration, version=version)
+
+
+def finalize(
+    resolver_configuration,  # type: _C
+    targets,  # type: Targets
+    context,  # type: str
+    pip_version=None,  # type: Optional[PipVersionValue]
+):
+    # type: (...) -> Union[_C, Error]
+
+    if isinstance(resolver_configuration, PipConfiguration):
+        return cast(
+            "_C",
+            _finalize_pip_configuration(
+                resolver_configuration, targets, context, pip_version=pip_version
+            ),
+        )
+
+    if isinstance(resolver_configuration, LockRepositoryConfiguration):
+        lock_file = try_(resolver_configuration.parse_lock())
+        pip_configuration = try_(
+            _finalize_pip_configuration(
+                resolver_configuration.pip_configuration,
+                targets,
+                context,
+                pip_version=pip_version or lock_file.pip_version,
+            )
+        )
+        return cast(
+            "_C",
+            attr.evolve(
+                resolver_configuration,
+                parse_lock=lambda: lock_file,
+                pip_configuration=pip_configuration,
+            ),
+        )
+
+    return resolver_configuration

pex/resolve/configured_resolver.py

@@ -62,6 +62,7 @@ def resolve_lock(
                 transitive=self.pip_configuration.transitive,
                 verify_wheels=True,
                 max_parallel_jobs=self.pip_configuration.max_jobs,
+                pip_version=pip_version or self.pip_configuration.version,
             )
         )
 
@@ -91,4 +92,6 @@ def resolve_requirements(
             max_parallel_jobs=self.pip_configuration.max_jobs,
             ignore_errors=False,
             verify_wheels=True,
+            pip_version=pip_version or self.pip_configuration.version,
+            resolver=self,
         )