Revert "Upgrade to Pex 2.0.3. (#8704)"

3rdparty/python/requirements.txt

@@ -12,7 +12,7 @@ Markdown==2.1.1
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==2.0.3
+pex==1.6.12
 psutil==5.6.3
 Pygments==2.3.1
 pyopenssl==17.3.0

pants.travis-ci.ini

@@ -4,22 +4,18 @@
 # Turn off all nailgun use.
 execution_strategy: subprocess
 
-# If we use typical default process parallelism tied to core count, we see too many cores under
-# travis and either get oomkilled from launching too many processes with too much total memory
-# overhead or else just generally thrash the container and slow things down.
-travis_parallelism: 4
-
 [compile.rsc]
-worker_count: %(travis_parallelism)s
-
-[python-setup]
-resolver_jobs: %(travis_parallelism)s
+# If we use the default of 1 worker per core, we see too many cores under travis
+# and get oomkilled from launching too many workers with too much total memory
+# overhead.
+worker_count: 4
 
 [test.pytest]
 # NB: We set a maximum timeout of 9.8 minutes to fail before hitting Travis' 10 minute timeout (which
 # doesn't give us useful debug info).
 timeout_maximum: 590
 
+
 [test.junit]
 # NB: See `test.pytest`.
 timeout_maximum: 540

src/python/pants/backend/python/rules/download_pex_bin.py

@@ -59,8 +59,8 @@ def create_execute_request(self,
 @rule
 async def download_pex_bin() -> DownloadedPexBin:
   # TODO: Inject versions and digests here through some option, rather than hard-coding it.
-  url = 'https://github.com/pantsbuild/pex/releases/download/v2.0.3/pex'
-  digest = Digest('183a14145553186ca1c0f2877e5eb3a1d7504501f711bb7b84b281342ffbd5ce', 2427459)
+  url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
+  digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359)
   snapshot = await Get(Snapshot, UrlToFetch(url, digest))
   return DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)
 

src/python/pants/backend/python/rules/pex.py

@@ -97,8 +97,6 @@ async def create_pex(
   interpreter constraints."""
 
   argv = ["--output-file", request.output_filename]
-  if python_setup.resolver_jobs:
-    argv.extend(["--jobs", python_setup.resolver_jobs])
   if request.entry_point is not None:
     argv.extend(["--entry-point", request.entry_point])
   argv.extend(request.interpreter_constraints.generate_pex_arg_list())

src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -7,8 +7,9 @@
 from pathlib import Path
 from typing import Callable, Sequence, Set
 
+from pex.fetcher import Fetcher
 from pex.pex_builder import PEXBuilder
-from pex.resolver import resolve_multi
+from pex.resolver import resolve
 from pex.util import DistributionHelper
 from twitter.common.collections import OrderedSet
 
@@ -25,6 +26,7 @@
 from pants.build_graph.files import Files
 from pants.build_graph.target import Target
 from pants.subsystem.subsystem import Subsystem
+from pants.util.collections import assert_single_element
 from pants.util.contextutil import temporary_file
 
 
@@ -175,62 +177,112 @@ def __init__(self,
   def add_requirement_libs_from(self, req_libs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
+    :param builder: Dump the requirements into this builder.
+    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
     :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
+    :param log: Use this logger.
     :param platforms: A list of :class:`Platform`s to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
     reqs = [req for req_lib in req_libs for req in req_lib.requirements]
     self.add_resolved_requirements(reqs, platforms=platforms)
 
-  def resolve_distributions(self, reqs, platforms=None):
-    """Multi-platform dependency resolution.
+  class SingleDistExtractionError(Exception): pass
 
-    :param reqs: A list of :class:`PythonRequirement` to resolve.
-    :param platforms: A list of platform strings to resolve requirements for.
-                      Defaults to the platforms specified by PythonSetup.
-    :returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
-              the given platforms.
+  def extract_single_dist_for_current_platform(self, reqs, dist_key):
+    """Resolve a specific distribution from a set of requirements matching the current platform.
+
+    :param list reqs: A list of :class:`PythonRequirement` to resolve.
+    :param str dist_key: The value of `distribution.key` to match for a `distribution` from the
+                         resolved requirements.
+    :return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
+    :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
+             `dist_key`.
     """
+    distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
+    try:
+      matched_dist = assert_single_element(list(
+        dist
+        for _, dists in distributions.items()
+        for dist in dists
+        if dist.key == dist_key
+      ))
+    except (StopIteration, ValueError) as e:
+      raise self.SingleDistExtractionError(
+        f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
+      )
+    return matched_dist
+
+  def _resolve_distributions_by_platform(self, reqs, platforms):
     deduped_reqs = OrderedSet(reqs)
     find_links = OrderedSet()
     for req in deduped_reqs:
+      self._log.debug(f'  Dumping requirement: {req}')
+      self._builder.add_requirement(str(req.requirement))
       if req.repository:
         find_links.add(req.repository)
 
-    return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)
+    # Resolve the requirements into distributions.
+    distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
+      find_links)
+    return distributions
 
   def add_resolved_requirements(self, reqs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
-    :param reqs: A list of :class:`PythonRequirement`s to resolve.
-    :param platforms: A list of platform strings to resolve requirements for.
+    :param builder: Dump the requirements into this builder.
+    :param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
+    :param reqs: A list of :class:`PythonRequirement` to resolve.
+    :param log: Use this logger.
+    :param platforms: A list of :class:`Platform`s to resolve requirements for.
                       Defaults to the platforms specified by PythonSetup.
     """
-    for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
-      requirement = resolved_dist.requirement
-      self._log.debug(f'  Dumping requirement: {requirement}')
-      self._builder.add_requirement(str(requirement))
+    distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
+    locations = set()
+    for platform, dists in distributions.items():
+      for dist in dists:
+        if dist.location not in locations:
+          self._log.debug(f'  Dumping distribution: .../{os.path.basename(dist.location)}')
+          self.add_distribution(dist)
+        locations.add(dist.location)
+
+  def _resolve_multi(self, interpreter, requirements, platforms, find_links):
+    """Multi-platform dependency resolution for PEX files.
 
-      distribution = resolved_dist.distribution
-      self._log.debug(f'  Dumping distribution: .../{os.path.basename(distribution.location)}')
-      self.add_distribution(distribution)
+    Returns a list of distributions that must be included in order to satisfy a set of requirements.
+    That may involve distributions for multiple platforms.
 
-  def _resolve_multi(self, requirements, platforms=None, find_links=None):
+    :param interpreter: The :class:`PythonInterpreter` to resolve for.
+    :param requirements: A list of :class:`PythonRequirement` objects to resolve.
+    :param platforms: A list of :class:`Platform`s to resolve for.
+    :param find_links: Additional paths to search for source packages during resolution.
+    :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
+             to satisfy the requirements on that platform.
+    """
     python_setup = self._python_setup_subsystem
     python_repos = self._python_repos_subsystem
     platforms = platforms or python_setup.platforms
-    find_links = list(find_links) if find_links else []
-    find_links.extend(python_repos.repos)
-
-    return resolve_multi(
-      requirements=[str(req.requirement) for req in requirements],
-      interpreters=[self._builder.interpreter],
-      indexes=python_repos.indexes,
-      find_links=find_links,
-      platforms=platforms,
-      cache=python_setup.resolver_cache_dir,
-      allow_prereleases=python_setup.resolver_allow_prereleases,
-      max_parallel_jobs=python_setup.resolver_jobs)
+    find_links = find_links or []
+    distributions = {}
+    fetchers = python_repos.get_fetchers()
+    fetchers.extend(Fetcher([path]) for path in find_links)
+
+    for platform in platforms:
+      requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
+        str(interpreter.identity))
+      resolved_dists = resolve(
+        requirements=[str(req.requirement) for req in requirements],
+        interpreter=interpreter,
+        fetchers=fetchers,
+        platform=platform,
+        context=python_repos.get_network_context(),
+        cache=requirements_cache_dir,
+        cache_ttl=python_setup.resolver_cache_ttl,
+        allow_prereleases=python_setup.resolver_allow_prereleases,
+        use_manylinux=python_setup.use_manylinux)
+      distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]
+
+    return distributions
 
   def add_sources_from(self, tgt: Target) -> None:
     dump_source = _create_source_dumper(self._builder, tgt)

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -9,10 +9,12 @@
 
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
+from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems import pex_build_util
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.base.exceptions import IncompatiblePlatformsError
+from pants.binaries.executable_pex_tool import ExecutablePexTool
 from pants.engine.rules import optionable_rule, rule
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_property
@@ -123,6 +125,25 @@ def check_build_for_current_platform_only(self, targets):
     ))
 
 
+class BuildSetupRequiresPex(ExecutablePexTool):
+  options_scope = 'build-setup-requires-pex'
+
+  @classmethod
+  def register_options(cls, register):
+    super().register_options(register)
+    register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
+             help='The setuptools version to use when executing `setup.py` scripts.')
+    register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
+             help='The wheel version to use when executing `setup.py` scripts.')
+
+  @property
+  def base_requirements(self):
+    return [
+      PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
+      PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
+    ]
+
+
 @dataclass(frozen=True)
 class PexBuildEnvironment:
   cpp_flags: Tuple[str, ...]

src/python/pants/backend/python/subsystems/python_repos.py

@@ -1,7 +1,59 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import logging
+
+from pex.fetcher import Fetcher, PyPIFetcher
+from pex.http import RequestsContext, StreamFilelike, requests
+
 from pants.subsystem.subsystem import Subsystem
+from pants.util.memo import memoized_method
+
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: These methods of RequestsContext are monkey-patched out to work around
+# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
+_REQUESTS_TIMEOUTS = (15, 30)
+
+
+def _open_monkey(self, link):
+  # requests does not support file:// -- so we must short-circuit manually
+  if link.local:
+    return open(link.local_path, 'rb')  # noqa: T802
+  for attempt in range(self._max_retries + 1):
+    try:
+      return StreamFilelike(self._session.get(
+          link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
+          timeout=_REQUESTS_TIMEOUTS),
+          link)
+    except requests.exceptions.ReadTimeout:
+      # Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
+      # so we'll retry them ourselves.
+      logger.log('Read timeout trying to fetch %s, retrying. %d retries remain.' % (
+          link.url,
+          self._max_retries - attempt))
+    except requests.exceptions.RequestException as e:
+      raise self.Error(e)
+
+  raise self.Error(
+      requests.packages.urllib3.exceptions.MaxRetryError(
+          None,
+          link,
+          'Exceeded max retries of %d' % self._max_retries))
+
+
+def _resolve_monkey(self, link):
+  return link.wrap(self._session.head(
+      link.url, verify=self._verify, allow_redirects=True,
+      headers={'User-Agent': self.USER_AGENT},
+      timeout=_REQUESTS_TIMEOUTS,
+  ).url)
+
+
+RequestsContext.open = _open_monkey
+RequestsContext.resolve = _resolve_monkey
 
 
 class PythonRepos(Subsystem):
@@ -23,3 +75,15 @@ def repos(self):
   @property
   def indexes(self):
     return self.get_options().indexes
+
+  @memoized_method
+  def get_fetchers(self):
+    fetchers = []
+    fetchers.extend(Fetcher([url]) for url in self.repos)
+    fetchers.extend(PyPIFetcher(url) for url in self.indexes)
+    return fetchers
+
+  @memoized_method
+  def get_network_context(self):
+    # TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
+    return RequestsContext()

src/python/pants/backend/python/subsystems/python_setup.py

@@ -44,6 +44,10 @@ def register_options(cls, register):
     register('--resolver-cache-dir', advanced=True, default=None, metavar='<dir>',
              help='The parent directory for the requirement resolver cache. '
                   'If unspecified, a standard path under the workdir is used.')
+    register('--resolver-cache-ttl', advanced=True, type=int, metavar='<seconds>',
+             default=10 * 365 * 86400,  # 10 years.
+             help='The time in seconds before we consider re-resolving an open-ended requirement, '
+                  'e.g. "flask>=0.2" if a matching distribution is available on disk.')
     register('--resolver-allow-prereleases', advanced=True, type=bool, default=UnsetBool,
              fingerprint=True, help='Whether to include pre-releases when resolving requirements.')
     register('--artifact-cache-dir', advanced=True, default=None, metavar='<dir>',
@@ -56,8 +60,9 @@ def register_options(cls, register):
                   '"<PATH>" (the contents of the PATH env var), '
                   '"<PEXRC>" (paths in the PEX_PYTHON_PATH variable in a pexrc file), '
                   '"<PYENV>" (all python versions under $(pyenv root)/versions).')
-    register('--resolver-jobs', type=int, default=None, advanced=True, fingerprint=True,
-             help='The maximum number of concurrent jobs to resolve wheels with.')
+    register('--resolver-use-manylinux', advanced=True, type=bool, default=True, fingerprint=True,
+             help='Whether to consider manylinux wheels when resolving requirements for linux '
+                  'platforms.')
 
   @property
   def interpreter_constraints(self):
@@ -100,13 +105,17 @@ def resolver_cache_dir(self):
     return (self.get_options().resolver_cache_dir or
             os.path.join(self.scratch_dir, 'resolved_requirements'))
 
+  @property
+  def resolver_cache_ttl(self):
+    return self.get_options().resolver_cache_ttl
+
   @property
   def resolver_allow_prereleases(self):
     return self.get_options().resolver_allow_prereleases
 
   @property
-  def resolver_jobs(self):
-    return self.get_options().resolver_jobs
+  def use_manylinux(self):
+    return self.get_options().resolver_use_manylinux
 
   @property
   def artifact_cache_dir(self):