Upgrade to Pex 2.1.4.

The primary changes to adapt to were:
1. The `pex.resolver.resolve` function signature changed.
2. The results of resolutions are now installed wheel chroots instead
   of zipped wheels.
3. Some packaging code once used by Pex and our plugin tests is now
   deleted.

Fixes pantsbuild#8786

This is built on top of:

Revert "Revert "Upgrade to Pex 2.0.3. (pantsbuild#8704)" (pantsbuild#8787)"

    This reverts commit 91d4af0.

3rdparty/python/requirements.txt

@@ -3,16 +3,14 @@ beautifulsoup4>=4.6.0,<4.7
 cffi==1.13.2
 contextlib2==0.5.5
 coverage>=4.5,<4.6
-# TODO remove this pin once we resolve https://github.com/pantsbuild/pants/issues/8502
-cryptography==2.7
 dataclasses==0.6
 docutils==0.14
 fasteners==0.15.0
 Markdown==2.1.1
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==2.0.3
+pex==2.1.0
 psutil==5.6.3
 Pygments==2.3.1
 pyopenssl==17.3.0
@@ -24,8 +22,8 @@ py_zipkin==0.18.4
 requests[security]>=2.20.1
 responses==0.10.4
 setproctitle==1.1.10
-setuptools==40.6.3
+setuptools==44.0.0
 toml==0.10.0
 typing-extensions==3.7.4
-wheel==0.31.1
+wheel==0.33.6
 www-authenticate==0.9.2

contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py

@@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
   default_version = 'lambdex==0.1.3'
   # TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
   # https://github.com/wickman/lambdex/issues/6 fixed.
-  default_extra_requirements = ['setuptools==40.8.0']
+  default_extra_requirements = ['setuptools==44.0.0']
   default_entry_point = 'lambdex.bin.lambdex'

src/python/pants/backend/awslambda/python/lambdex.py

@@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
   default_version = 'lambdex==0.1.3'
   # TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
   # https://github.com/wickman/lambdex/issues/6 fixed.
-  default_extra_requirements = ['setuptools==40.8.0']
+  default_extra_requirements = ['setuptools==44.0.0']
   default_entry_point = 'lambdex.bin.lambdex'

src/python/pants/backend/python/rules/download_pex_bin.py

@@ -37,13 +37,13 @@ def directory_digest(self) -> Digest:
   class Factory(Script):
     options_scope = 'download-pex-bin'
     name = 'pex'
-    default_version = 'v1.6.12'
+    default_version = 'v2.1.4'
 
     default_versions_and_digests = {
       PlatformConstraint.none: ToolForPlatform(
-        digest=Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1',
-                      1842359),
-        version=ToolVersion('v1.6.12'),
+        digest=Digest('6c5ae1f6b9aa40c97bd26a154849044b49f4d698a6abb9ac58ce006bda9cbd4a',
+                      2614246),
+        version=ToolVersion('v2.1.4'),
       ),
     }
 

src/python/pants/backend/python/rules/pex.py

@@ -104,6 +104,10 @@ async def create_pex(
     argv.extend(["--entry-point", request.entry_point])
   argv.extend(request.interpreter_constraints.generate_pex_arg_list())
   argv.extend(request.additional_args)
+  if python_setup.manylinux:
+    argv.extend(['--manylinux', python_setup.manylinux])
+  else:
+    argv.append('--no-manylinux')
 
   source_dir_name = 'source_files'
   argv.append(f'--sources-directory={source_dir_name}')
@@ -116,8 +120,8 @@ async def create_pex(
   merged_digest = await Get[Digest](DirectoriesToMerge(directories=all_inputs))
 
   # NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
-  # cross-building, we specify that out PEX command be run on the current local platform. When we
-  # support cross-building through CLI flags we can configure requests that build a PEX for out
+  # cross-building, we specify that our PEX command be run on the current local platform. When we
+  # support cross-building through CLI flags we can configure requests that build a PEX for our
   # local platform that are able to execute on a different platform, but for now in order to
   # guarantee correct build we need to restrict this command to execute on the same platform type
   # that the output is intended for. The correct way to interpret the keys

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -10,11 +10,9 @@
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.python.subsystems import pex_build_util
-from pants.backend.python.subsystems.executable_pex_tool import ExecutablePexTool
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.base.exceptions import IncompatiblePlatformsError
 from pants.engine.rules import rule, subsystem_rule
-from pants.python.python_requirement import PythonRequirement
 from pants.python.python_setup import PythonSetup
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property

src/python/pants/backend/python/targets/unpacked_whls.py

@@ -60,7 +60,7 @@ def __init__(self, module_name, libraries=None, include_patterns=None, exclude_p
     """
     deprecated_conditional(
       lambda: type(within_data_subdir) not in (bool, type(None)),
-      removal_version='1.26.0.dev2',
+      removal_version='1.27.0.dev2',
       entity_description='A non-boolean value for `within_data_subdir`',
       hint_message='The location of the .data subdirectory will be inferred from the module name!',
     )

src/python/pants/backend/python/tasks/BUILD

@@ -5,6 +5,7 @@ python_library(
   dependencies=[
     '3rdparty/python:dataclasses',
     '3rdparty/python:pex',
+    '3rdparty/python:wheel',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
     'src/python/pants/backend/native/config',

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -7,8 +7,8 @@
 import shutil
 from pathlib import Path
 
-from pex import pep425tags
 from pex.interpreter import PythonInterpreter
+from wheel import pep425tags
 
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.native.tasks.link_shared_libraries import SharedLibrary

src/python/pants/backend/python/tasks/setup_py.py

@@ -9,6 +9,8 @@
 import textwrap
 from abc import ABC, abstractmethod
 from collections import OrderedDict, defaultdict
+from pathlib import Path
+from typing import Dict
 
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX

src/python/pants/init/BUILD

@@ -7,7 +7,6 @@ python_library(
     '3rdparty/python:dataclasses',
     '3rdparty/python:pex',
     '3rdparty/python:setuptools',
-    '3rdparty/python:wheel',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:build_root',

src/python/pants/python/pex_build_util.py

@@ -6,9 +6,8 @@
 from pathlib import Path
 from typing import Callable, Sequence, Set
 
-from pex.fetcher import Fetcher
 from pex.pex_builder import PEXBuilder
-from pex.resolver import resolve
+from pex.resolver import resolve_multi
 from pex.util import DistributionHelper
 from twitter.common.collections import OrderedSet
 
@@ -20,7 +19,6 @@
 from pants.python.python_requirement import PythonRequirement
 from pants.python.python_setup import PythonSetup
 from pants.subsystem.subsystem import Subsystem
-from pants.util.collections import assert_single_element
 from pants.util.contextutil import temporary_file
 
 
@@ -78,7 +76,7 @@ class Factory(Subsystem):
     @classmethod
     def register_options(cls, register):
       super(PexBuilderWrapper.Factory, cls).register_options(register)
-      register('--setuptools-version', advanced=True, default='40.6.3',
+      register('--setuptools-version', advanced=True, default='44.0.0',
                help='The setuptools version to include in the pex if namespace packages need to be '
                     'injected.')
 
@@ -133,99 +131,56 @@ def add_requirement_libs_from(self, req_libs, platforms=None):
     reqs = [req for req_lib in req_libs for req in req_lib.requirements]
     self.add_resolved_requirements(reqs, platforms=platforms)
 
-  class SingleDistExtractionError(Exception): pass
+  def resolve_distributions(self, reqs, platforms=None):
+    """Multi-platform dependency resolution.
 
-  def extract_single_dist_for_current_platform(self, reqs, dist_key):
-    """Resolve a specific distribution from a set of requirements matching the current platform.
-
-    :param list reqs: A list of :class:`PythonRequirement` to resolve.
-    :param str dist_key: The value of `distribution.key` to match for a `distribution` from the
-                         resolved requirements.
-    :return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
-    :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
-             `dist_key`.
+    :param reqs: A list of :class:`PythonRequirement` to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
+                      Defaults to the platforms specified by PythonSetup.
+    :returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
+              the given platforms.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
-    try:
-      matched_dist = assert_single_element(list(
-        dist
-        for _, dists in distributions.items()
-        for dist in dists
-        if dist.key == dist_key
-      ))
-    except (StopIteration, ValueError) as e:
-      raise self.SingleDistExtractionError(
-        f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
-      )
-    return matched_dist
-
-  def _resolve_distributions_by_platform(self, reqs, platforms):
     deduped_reqs = OrderedSet(reqs)
     find_links = OrderedSet()
     for req in deduped_reqs:
-      self._log.debug(f'  Dumping requirement: {req}')
-      self._builder.add_requirement(str(req.requirement))
       if req.repository:
         find_links.add(req.repository)
 
-    # Resolve the requirements into distributions.
-    distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
-      find_links)
-    return distributions
+    return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)
 
   def add_resolved_requirements(self, reqs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param reqs: A list of :class:`PythonRequirement` to resolve.
-    :param platforms: A list of :class:`Platform`s to resolve requirements for.
+    :param reqs: A list of :class:`PythonRequirement`s to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
-    locations = set()
-    for platform, dists in distributions.items():
-      for dist in dists:
-        if dist.location not in locations:
-          self._log.debug(f'  Dumping distribution: .../{os.path.basename(dist.location)}')
-          self.add_distribution(dist)
-        locations.add(dist.location)
-
-  def _resolve_multi(self, interpreter, requirements, platforms, find_links):
-    """Multi-platform dependency resolution for PEX files.
+    for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
+      requirement = resolved_dist.requirement
+      self._log.debug(f'  Dumping requirement: {requirement}')
+      self._builder.add_requirement(str(requirement))
 
-    Returns a list of distributions that must be included in order to satisfy a set of requirements.
-    That may involve distributions for multiple platforms.
+      distribution = resolved_dist.distribution
+      self._log.debug(f'  Dumping distribution: .../{os.path.basename(distribution.location)}')
+      self.add_distribution(distribution)
 
-    :param interpreter: The :class:`PythonInterpreter` to resolve for.
-    :param requirements: A list of :class:`PythonRequirement` objects to resolve.
-    :param platforms: A list of :class:`Platform`s to resolve for.
-    :param find_links: Additional paths to search for source packages during resolution.
-    :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
-             to satisfy the requirements on that platform.
-    """
+  def _resolve_multi(self, requirements, platforms=None, find_links=None):
     python_setup = self._python_setup_subsystem
     python_repos = self._python_repos_subsystem
     platforms = platforms or python_setup.platforms
-    find_links = find_links or []
-    distributions = {}
-    fetchers = python_repos.get_fetchers()
-    fetchers.extend(Fetcher([path]) for path in find_links)
-
-    for platform in platforms:
-      requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
-        str(interpreter.identity))
-      resolved_dists = resolve(
-        requirements=[str(req.requirement) for req in requirements],
-        interpreter=interpreter,
-        fetchers=fetchers,
-        platform=platform,
-        context=python_repos.get_network_context(),
-        cache=requirements_cache_dir,
-        cache_ttl=python_setup.resolver_cache_ttl,
-        allow_prereleases=python_setup.resolver_allow_prereleases,
-        use_manylinux=python_setup.use_manylinux)
-      distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
-
-    return distributions
+    find_links = list(find_links) if find_links else []
+    find_links.extend(python_repos.repos)
+
+    return resolve_multi(
+      requirements=[str(req.requirement) for req in requirements],
+      interpreters=[self._builder.interpreter],
+      indexes=python_repos.indexes,
+      find_links=find_links,
+      platforms=platforms,
+      cache=python_setup.resolver_cache_dir,
+      allow_prereleases=python_setup.resolver_allow_prereleases,
+      manylinux=python_setup.manylinux,
+      max_parallel_jobs=python_setup.resolver_jobs)
 
   def add_sources_from(self, tgt: Target) -> None:
     dump_source = _create_source_dumper(self._builder, tgt)
@@ -238,7 +193,7 @@ def add_sources_from(self, tgt: Target) -> None:
         raise
 
     if (getattr(tgt, '_resource_target_specs', None) or
-      getattr(tgt, '_synthetic_resources_target', None)):
+        getattr(tgt, '_synthetic_resources_target', None)):
       # No one should be on old-style resources any more.  And if they are,
       # switching to the new python pipeline will be a great opportunity to fix that.
       raise TaskError(

src/python/pants/python/python_repos.py

@@ -1,58 +1,7 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import logging
-
-from pex.fetcher import Fetcher, PyPIFetcher
-from pex.http import RequestsContext, StreamFilelike, requests
-
 from pants.subsystem.subsystem import Subsystem
-from pants.util.memo import memoized_method
-
-
-logger = logging.getLogger(__name__)
-
-
-# TODO: These methods of RequestsContext are monkey-patched out to work around
-# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
-_REQUESTS_TIMEOUTS = (15, 30)
-
-
-def _open_monkey(self, link):
-  # requests does not support file:// -- so we must short-circuit manually
-  if link.local:
-    return open(link.local_path, 'rb')  # noqa: T802
-  for attempt in range(self._max_retries + 1):
-    try:
-      return StreamFilelike(self._session.get(
-          link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
-          timeout=_REQUESTS_TIMEOUTS),
-          link)
-    except requests.exceptions.ReadTimeout:
-      # Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
-      # so we'll retry them ourselves.
-      logger.warning(f'Read timeout trying to fetch {link.url}, retrying. '
-                     f'{self._max_retries - attempt} retries remain.')
-    except requests.exceptions.RequestException as e:
-      raise self.Error(e)
-
-  raise self.Error(
-      requests.packages.urllib3.exceptions.MaxRetryError(
-          None,
-          link,
-          'Exceeded max retries of %d' % self._max_retries))
-
-
-def _resolve_monkey(self, link):
-  return link.wrap(self._session.head(
-      link.url, verify=self._verify, allow_redirects=True,
-      headers={'User-Agent': self.USER_AGENT},
-      timeout=_REQUESTS_TIMEOUTS,
-  ).url)
-
-
-RequestsContext.open = _open_monkey
-RequestsContext.resolve = _resolve_monkey
 
 
 class PythonRepos(Subsystem):
@@ -78,15 +27,3 @@ def repos(self):
   @property
   def indexes(self):
     return self.get_options().indexes
-
-  @memoized_method
-  def get_fetchers(self):
-    fetchers = []
-    fetchers.extend(Fetcher([url]) for url in self.repos)
-    fetchers.extend(PyPIFetcher(url) for url in self.indexes)
-    return fetchers
-
-  @memoized_method
-  def get_network_context(self):
-    # TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
-    return RequestsContext()