Revert "Revert "Upgrade to Pex 2.0.3. (pantsbuild#8704)" (pantsbuild#…

…8787)"

This reverts commit 91d4af0.

3rdparty/python/requirements.txt

@@ -12,7 +12,7 @@ Markdown==2.1.1
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==1.6.12
+pex==2.0.3
 psutil==5.6.3
 Pygments==2.3.1
 pyopenssl==17.3.0

pants.travis-ci.ini

@@ -4,18 +4,22 @@
 # Turn off all nailgun use.
 execution_strategy: subprocess
 
+# If we use typical default process parallelism tied to core count, we see too many cores under
+# travis and either get oomkilled from launching too many processes with too much total memory
+# overhead or else just generally thrash the container and slow things down.
+travis_parallelism: 4
+
 [compile.rsc]
-# If we use the default of 1 worker per core, we see too many cores under travis
-# and get oomkilled from launching too many workers with too much total memory
-# overhead.
-worker_count: 4
+worker_count: %(travis_parallelism)s
+
+[python-setup]
+resolver_jobs: %(travis_parallelism)s
 
 [test.pytest]
 # NB: We set a maximum timeout of 9.8 minutes to fail before hitting Travis' 10 minute timeout (which
 # doesn't give us useful debug info).
 timeout_maximum: 590
 
-
 [test.junit]
 # NB: See `test.pytest`.
 timeout_maximum: 540

src/python/pants/backend/python/rules/pex.py

@@ -97,6 +97,8 @@ async def create_pex(
   interpreter constraints."""
 
   argv = ["--output-file", request.output_filename]
+  if python_setup.resolver_jobs:
+    argv.extend(["--jobs", python_setup.resolver_jobs])
   if request.entry_point is not None:
     argv.extend(["--entry-point", request.entry_point])
   argv.extend(request.interpreter_constraints.generate_pex_arg_list())

src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -7,9 +7,8 @@
 from pathlib import Path
 from typing import Callable, Sequence, Set
 
-from pex.fetcher import Fetcher
 from pex.pex_builder import PEXBuilder
-from pex.resolver import resolve
+from pex.resolver import resolve_multi
 from pex.util import DistributionHelper
 from twitter.common.collections import OrderedSet
 
@@ -26,7 +25,6 @@
 from pants.build_graph.files import Files
 from pants.build_graph.target import Target
 from pants.subsystem.subsystem import Subsystem
-from pants.util.collections import assert_single_element
 from pants.util.contextutil import temporary_file
 
 
@@ -177,112 +175,62 @@ def __init__(self,
   def add_requirement_libs_from(self, req_libs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param builder: Dump the requirements into this builder.
-    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
     :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
-    :param log: Use this logger.
     :param platforms: A list of :class:`Platform`s to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
     reqs = [req for req_lib in req_libs for req in req_lib.requirements]
     self.add_resolved_requirements(reqs, platforms=platforms)
 
-  class SingleDistExtractionError(Exception): pass
+  def resolve_distributions(self, reqs, platforms=None):
+    """Multi-platform dependency resolution.
 
-  def extract_single_dist_for_current_platform(self, reqs, dist_key):
-    """Resolve a specific distribution from a set of requirements matching the current platform.
-
-    :param list reqs: A list of :class:`PythonRequirement` to resolve.
-    :param str dist_key: The value of `distribution.key` to match for a `distribution` from the
-                         resolved requirements.
-    :return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
-    :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
-             `dist_key`.
+    :param reqs: A list of :class:`PythonRequirement` to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
+                      Defaults to the platforms specified by PythonSetup.
+    :returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
+              the given platforms.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
-    try:
-      matched_dist = assert_single_element(list(
-        dist
-        for _, dists in distributions.items()
-        for dist in dists
-        if dist.key == dist_key
-      ))
-    except (StopIteration, ValueError) as e:
-      raise self.SingleDistExtractionError(
-        f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
-      )
-    return matched_dist
-
-  def _resolve_distributions_by_platform(self, reqs, platforms):
     deduped_reqs = OrderedSet(reqs)
     find_links = OrderedSet()
     for req in deduped_reqs:
-      self._log.debug(f'  Dumping requirement: {req}')
-      self._builder.add_requirement(str(req.requirement))
       if req.repository:
         find_links.add(req.repository)
 
-    # Resolve the requirements into distributions.
-    distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
-      find_links)
-    return distributions
+    return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)
 
   def add_resolved_requirements(self, reqs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param builder: Dump the requirements into this builder.
-    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
-    :param reqs: A list of :class:`PythonRequirement` to resolve.
-    :param log: Use this logger.
-    :param platforms: A list of :class:`Platform`s to resolve requirements for.
+    :param reqs: A list of :class:`PythonRequirement`s to resolve.
+    :param platforms: A list of platform strings to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
-    distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
-    locations = set()
-    for platform, dists in distributions.items():
-      for dist in dists:
-        if dist.location not in locations:
-          self._log.debug(f'  Dumping distribution: .../{os.path.basename(dist.location)}')
-          self.add_distribution(dist)
-        locations.add(dist.location)
-
-  def _resolve_multi(self, interpreter, requirements, platforms, find_links):
-    """Multi-platform dependency resolution for PEX files.
+    for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
+      requirement = resolved_dist.requirement
+      self._log.debug(f'  Dumping requirement: {requirement}')
+      self._builder.add_requirement(str(requirement))
 
-    Returns a list of distributions that must be included in order to satisfy a set of requirements.
-    That may involve distributions for multiple platforms.
+      distribution = resolved_dist.distribution
+      self._log.debug(f'  Dumping distribution: .../{os.path.basename(distribution.location)}')
+      self.add_distribution(distribution)
 
-    :param interpreter: The :class:`PythonInterpreter` to resolve for.
-    :param requirements: A list of :class:`PythonRequirement` objects to resolve.
-    :param platforms: A list of :class:`Platform`s to resolve for.
-    :param find_links: Additional paths to search for source packages during resolution.
-    :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
-             to satisfy the requirements on that platform.
-    """
+  def _resolve_multi(self, requirements, platforms=None, find_links=None):
     python_setup = self._python_setup_subsystem
     python_repos = self._python_repos_subsystem
     platforms = platforms or python_setup.platforms
-    find_links = find_links or []
-    distributions = {}
-    fetchers = python_repos.get_fetchers()
-    fetchers.extend(Fetcher([path]) for path in find_links)
-
-    for platform in platforms:
-      requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
-        str(interpreter.identity))
-      resolved_dists = resolve(
-        requirements=[str(req.requirement) for req in requirements],
-        interpreter=interpreter,
-        fetchers=fetchers,
-        platform=platform,
-        context=python_repos.get_network_context(),
-        cache=requirements_cache_dir,
-        cache_ttl=python_setup.resolver_cache_ttl,
-        allow_prereleases=python_setup.resolver_allow_prereleases,
-        use_manylinux=python_setup.use_manylinux)
-      distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
-
-    return distributions
+    find_links = list(find_links) if find_links else []
+    find_links.extend(python_repos.repos)
+
+    return resolve_multi(
+      requirements=[str(req.requirement) for req in requirements],
+      interpreters=[self._builder.interpreter],
+      indexes=python_repos.indexes,
+      find_links=find_links,
+      platforms=platforms,
+      cache=python_setup.resolver_cache_dir,
+      allow_prereleases=python_setup.resolver_allow_prereleases,
+      max_parallel_jobs=python_setup.resolver_jobs)
 
   def add_sources_from(self, tgt: Target) -> None:
     dump_source = _create_source_dumper(self._builder, tgt)

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -9,12 +9,10 @@
 
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
-from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems import pex_build_util
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.base.exceptions import IncompatiblePlatformsError
-from pants.binaries.executable_pex_tool import ExecutablePexTool
 from pants.engine.rules import optionable_rule, rule
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
@@ -125,25 +123,6 @@ def check_build_for_current_platform_only(self, targets):
     ))
 
 
-class BuildSetupRequiresPex(ExecutablePexTool):
-  options_scope = 'build-setup-requires-pex'
-
-  @classmethod
-  def register_options(cls, register):
-    super().register_options(register)
-    register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
-             help='The setuptools version to use when executing `setup.py` scripts.')
-    register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
-             help='The wheel version to use when executing `setup.py` scripts.')
-
-  @property
-  def base_requirements(self):
-    return [
-      PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
-      PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
-    ]
-
-
 @dataclass(frozen=True)
 class PexBuildEnvironment:
   cpp_flags: Tuple[str, ...]

src/python/pants/backend/python/subsystems/python_repos.py

@@ -1,59 +1,7 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-import logging
-
-from pex.fetcher import Fetcher, PyPIFetcher
-from pex.http import RequestsContext, StreamFilelike, requests
-
 from pants.subsystem.subsystem import Subsystem
-from pants.util.memo import memoized_method
-
-
-logger = logging.getLogger(__name__)
-
-
-# TODO: These methods of RequestsContext are monkey-patched out to work around
-# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
-_REQUESTS_TIMEOUTS = (15, 30)
-
-
-def _open_monkey(self, link):
-  # requests does not support file:// -- so we must short-circuit manually
-  if link.local:
-    return open(link.local_path, 'rb')  # noqa: T802
-  for attempt in range(self._max_retries + 1):
-    try:
-      return StreamFilelike(self._session.get(
-          link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
-          timeout=_REQUESTS_TIMEOUTS),
-          link)
-    except requests.exceptions.ReadTimeout:
-      # Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
-      # so we'll retry them ourselves.
-      logger.log('Read timeout trying to fetch %s, retrying. %d retries remain.' % (
-          link.url,
-          self._max_retries - attempt))
-    except requests.exceptions.RequestException as e:
-      raise self.Error(e)
-
-  raise self.Error(
-      requests.packages.urllib3.exceptions.MaxRetryError(
-          None,
-          link,
-          'Exceeded max retries of %d' % self._max_retries))
-
-
-def _resolve_monkey(self, link):
-  return link.wrap(self._session.head(
-      link.url, verify=self._verify, allow_redirects=True,
-      headers={'User-Agent': self.USER_AGENT},
-      timeout=_REQUESTS_TIMEOUTS,
-  ).url)
-
-
-RequestsContext.open = _open_monkey
-RequestsContext.resolve = _resolve_monkey
 
 
 class PythonRepos(Subsystem):
@@ -75,15 +23,3 @@ def repos(self):
   @property
   def indexes(self):
     return self.get_options().indexes
-
-  @memoized_method
-  def get_fetchers(self):
-    fetchers = []
-    fetchers.extend(Fetcher([url]) for url in self.repos)
-    fetchers.extend(PyPIFetcher(url) for url in self.indexes)
-    return fetchers
-
-  @memoized_method
-  def get_network_context(self):
-    # TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
-    return RequestsContext()

src/python/pants/backend/python/subsystems/python_setup.py

@@ -44,10 +44,6 @@ def register_options(cls, register):
     register('--resolver-cache-dir', advanced=True, default=None, metavar='<dir>',
              help='The parent directory for the requirement resolver cache. '
                   'If unspecified, a standard path under the workdir is used.')
-    register('--resolver-cache-ttl', advanced=True, type=int, metavar='<seconds>',
-             default=10 * 365 * 86400,  # 10 years.
-             help='The time in seconds before we consider re-resolving an open-ended requirement, '
-                  'e.g. "flask>=0.2" if a matching distribution is available on disk.')
     register('--resolver-allow-prereleases', advanced=True, type=bool, default=UnsetBool,
              fingerprint=True, help='Whether to include pre-releases when resolving requirements.')
     register('--artifact-cache-dir', advanced=True, default=None, metavar='<dir>',
@@ -60,9 +56,8 @@ def register_options(cls, register):
                   '"<PATH>" (the contents of the PATH env var), '
                   '"<PEXRC>" (paths in the PEX_PYTHON_PATH variable in a pexrc file), '
                   '"<PYENV>" (all python versions under $(pyenv root)/versions).')
-    register('--resolver-use-manylinux', advanced=True, type=bool, default=True, fingerprint=True,
-             help='Whether to consider manylinux wheels when resolving requirements for linux '
-                  'platforms.')
+    register('--resolver-jobs', type=int, default=None, advanced=True, fingerprint=True,
+             help='The maximum number of concurrent jobs to resolve wheels with.')
 
   @property
   def interpreter_constraints(self):
@@ -105,17 +100,13 @@ def resolver_cache_dir(self):
     return (self.get_options().resolver_cache_dir or
             os.path.join(self.scratch_dir, 'resolved_requirements'))
 
-  @property
-  def resolver_cache_ttl(self):
-    return self.get_options().resolver_cache_ttl
-
   @property
   def resolver_allow_prereleases(self):
     return self.get_options().resolver_allow_prereleases
 
   @property
-  def use_manylinux(self):
-    return self.get_options().resolver_use_manylinux
+  def resolver_jobs(self):
+    return self.get_options().resolver_jobs
 
   @property
   def artifact_cache_dir(self):