Skip to content

Commit

Permalink
Revert "Upgrade to Pex 2.0.3. (pantsbuild#8704)" (pantsbuild#8787)
Browse files Browse the repository at this point in the history
This reverts commit 879df20.

See: pantsbuild#8786
  • Loading branch information
jsirois authored Dec 10, 2019
1 parent 9107b41 commit 91d4af0
Show file tree
Hide file tree
Showing 25 changed files with 479 additions and 398 deletions.
2 changes: 1 addition & 1 deletion 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Markdown==2.1.1
packaging==16.8
parameterized==0.6.1
pathspec==0.5.9
pex==2.0.3
pex==1.6.12
psutil==5.6.3
Pygments==2.3.1
pyopenssl==17.3.0
Expand Down
14 changes: 5 additions & 9 deletions pants.travis-ci.ini
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,18 @@
# Turn off all nailgun use.
execution_strategy: subprocess

# If we use typical default process parallelism tied to core count, we see too many cores under
# travis and either get oomkilled from launching too many processes with too much total memory
# overhead or else just generally thrash the container and slow things down.
travis_parallelism: 4

[compile.rsc]
worker_count: %(travis_parallelism)s

[python-setup]
resolver_jobs: %(travis_parallelism)s
# If we use the default of 1 worker per core, we see too many cores under travis
# and get oomkilled from launching too many workers with too much total memory
# overhead.
worker_count: 4

[test.pytest]
# NB: We set a maximum timeout of 9.8 minutes to fail before hitting Travis' 10 minute timeout (which
# doesn't give us useful debug info).
timeout_maximum: 590


[test.junit]
# NB: See `test.pytest`.
timeout_maximum: 540
Expand Down
4 changes: 2 additions & 2 deletions src/python/pants/backend/python/rules/download_pex_bin.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ def create_execute_request(self,
@rule
async def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v2.0.3/pex'
digest = Digest('183a14145553186ca1c0f2877e5eb3a1d7504501f711bb7b84b281342ffbd5ce', 2427459)
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
digest = Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1', 1842359)
snapshot = await Get(Snapshot, UrlToFetch(url, digest))
return DownloadedPexBin(executable=snapshot.files[0], directory_digest=snapshot.directory_digest)

Expand Down
2 changes: 0 additions & 2 deletions src/python/pants/backend/python/rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,6 @@ async def create_pex(
interpreter constraints."""

argv = ["--output-file", request.output_filename]
if python_setup.resolver_jobs:
argv.extend(["--jobs", python_setup.resolver_jobs])
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
Expand Down
114 changes: 83 additions & 31 deletions src/python/pants/backend/python/subsystems/pex_build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
from pathlib import Path
from typing import Callable, Sequence, Set

from pex.fetcher import Fetcher
from pex.pex_builder import PEXBuilder
from pex.resolver import resolve_multi
from pex.resolver import resolve
from pex.util import DistributionHelper
from twitter.common.collections import OrderedSet

Expand All @@ -25,6 +26,7 @@
from pants.build_graph.files import Files
from pants.build_graph.target import Target
from pants.subsystem.subsystem import Subsystem
from pants.util.collections import assert_single_element
from pants.util.contextutil import temporary_file


Expand Down Expand Up @@ -175,62 +177,112 @@ def __init__(self,
def add_requirement_libs_from(self, req_libs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
reqs = [req for req_lib in req_libs for req in req_lib.requirements]
self.add_resolved_requirements(reqs, platforms=platforms)

def resolve_distributions(self, reqs, platforms=None):
"""Multi-platform dependency resolution.
class SingleDistExtractionError(Exception): pass

:param reqs: A list of :class:`PythonRequirement` to resolve.
:param platforms: A list of platform strings to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
the given platforms.
def extract_single_dist_for_current_platform(self, reqs, dist_key):
"""Resolve a specific distribution from a set of requirements matching the current platform.
:param list reqs: A list of :class:`PythonRequirement` to resolve.
:param str dist_key: The value of `distribution.key` to match for a `distribution` from the
resolved requirements.
:return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
:raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
`dist_key`.
"""
distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
try:
matched_dist = assert_single_element(list(
dist
for _, dists in distributions.items()
for dist in dists
if dist.key == dist_key
))
except (StopIteration, ValueError) as e:
raise self.SingleDistExtractionError(
f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
)
return matched_dist

def _resolve_distributions_by_platform(self, reqs, platforms):
deduped_reqs = OrderedSet(reqs)
find_links = OrderedSet()
for req in deduped_reqs:
self._log.debug(f' Dumping requirement: {req}')
self._builder.add_requirement(str(req.requirement))
if req.repository:
find_links.add(req.repository)

return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)
# Resolve the requirements into distributions.
distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
find_links)
return distributions

def add_resolved_requirements(self, reqs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
:param reqs: A list of :class:`PythonRequirement`s to resolve.
:param platforms: A list of platform strings to resolve requirements for.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
requirement = resolved_dist.requirement
self._log.debug(f' Dumping requirement: {requirement}')
self._builder.add_requirement(str(requirement))
distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self._log.debug(f' Dumping distribution: .../{os.path.basename(dist.location)}')
self.add_distribution(dist)
locations.add(dist.location)

def _resolve_multi(self, interpreter, requirements, platforms, find_links):
"""Multi-platform dependency resolution for PEX files.
distribution = resolved_dist.distribution
self._log.debug(f' Dumping distribution: .../{os.path.basename(distribution.location)}')
self.add_distribution(distribution)
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
def _resolve_multi(self, requirements, platforms=None, find_links=None):
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = list(find_links) if find_links else []
find_links.extend(python_repos.repos)

return resolve_multi(
requirements=[str(req.requirement) for req in requirements],
interpreters=[self._builder.interpreter],
indexes=python_repos.indexes,
find_links=find_links,
platforms=platforms,
cache=python_setup.resolver_cache_dir,
allow_prereleases=python_setup.resolver_allow_prereleases,
max_parallel_jobs=python_setup.resolver_jobs)
find_links = find_links or []
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)

for platform in platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
allow_prereleases=python_setup.resolver_allow_prereleases,
use_manylinux=python_setup.use_manylinux)
distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]

return distributions

def add_sources_from(self, tgt: Target) -> None:
dump_source = _create_source_dumper(self._builder, tgt)
Expand Down
21 changes: 21 additions & 0 deletions src/python/pants/backend/python/subsystems/python_native_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@

from pants.backend.native.subsystems.native_toolchain import NativeToolchain
from pants.backend.native.targets.native_library import NativeLibrary
from pants.backend.python.python_requirement import PythonRequirement
from pants.backend.python.subsystems import pex_build_util
from pants.backend.python.subsystems.python_setup import PythonSetup
from pants.backend.python.targets.python_distribution import PythonDistribution
from pants.base.exceptions import IncompatiblePlatformsError
from pants.binaries.executable_pex_tool import ExecutablePexTool
from pants.engine.rules import optionable_rule, rule
from pants.subsystem.subsystem import Subsystem
from pants.util.memo import memoized_property
Expand Down Expand Up @@ -123,6 +125,25 @@ def check_build_for_current_platform_only(self, targets):
))


class BuildSetupRequiresPex(ExecutablePexTool):
options_scope = 'build-setup-requires-pex'

@classmethod
def register_options(cls, register):
super().register_options(register)
register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
help='The setuptools version to use when executing `setup.py` scripts.')
register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
help='The wheel version to use when executing `setup.py` scripts.')

@property
def base_requirements(self):
return [
PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
]


@dataclass(frozen=True)
class PexBuildEnvironment:
cpp_flags: Tuple[str, ...]
Expand Down
64 changes: 64 additions & 0 deletions src/python/pants/backend/python/subsystems/python_repos.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,59 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import logging

from pex.fetcher import Fetcher, PyPIFetcher
from pex.http import RequestsContext, StreamFilelike, requests

from pants.subsystem.subsystem import Subsystem
from pants.util.memo import memoized_method


logger = logging.getLogger(__name__)


# TODO: These methods of RequestsContext are monkey-patched out to work around
# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
_REQUESTS_TIMEOUTS = (15, 30)


def _open_monkey(self, link):
# requests does not support file:// -- so we must short-circuit manually
if link.local:
return open(link.local_path, 'rb') # noqa: T802
for attempt in range(self._max_retries + 1):
try:
return StreamFilelike(self._session.get(
link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
timeout=_REQUESTS_TIMEOUTS),
link)
except requests.exceptions.ReadTimeout:
# Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
# so we'll retry them ourselves.
logger.log('Read timeout trying to fetch %s, retrying. %d retries remain.' % (
link.url,
self._max_retries - attempt))
except requests.exceptions.RequestException as e:
raise self.Error(e)

raise self.Error(
requests.packages.urllib3.exceptions.MaxRetryError(
None,
link,
'Exceeded max retries of %d' % self._max_retries))


def _resolve_monkey(self, link):
return link.wrap(self._session.head(
link.url, verify=self._verify, allow_redirects=True,
headers={'User-Agent': self.USER_AGENT},
timeout=_REQUESTS_TIMEOUTS,
).url)


RequestsContext.open = _open_monkey
RequestsContext.resolve = _resolve_monkey


class PythonRepos(Subsystem):
Expand All @@ -23,3 +75,15 @@ def repos(self):
@property
def indexes(self):
return self.get_options().indexes

@memoized_method
def get_fetchers(self):
fetchers = []
fetchers.extend(Fetcher([url]) for url in self.repos)
fetchers.extend(PyPIFetcher(url) for url in self.indexes)
return fetchers

@memoized_method
def get_network_context(self):
# TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
return RequestsContext()
17 changes: 13 additions & 4 deletions src/python/pants/backend/python/subsystems/python_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,10 @@ def register_options(cls, register):
register('--resolver-cache-dir', advanced=True, default=None, metavar='<dir>',
help='The parent directory for the requirement resolver cache. '
'If unspecified, a standard path under the workdir is used.')
register('--resolver-cache-ttl', advanced=True, type=int, metavar='<seconds>',
default=10 * 365 * 86400, # 10 years.
help='The time in seconds before we consider re-resolving an open-ended requirement, '
'e.g. "flask>=0.2" if a matching distribution is available on disk.')
register('--resolver-allow-prereleases', advanced=True, type=bool, default=UnsetBool,
fingerprint=True, help='Whether to include pre-releases when resolving requirements.')
register('--artifact-cache-dir', advanced=True, default=None, metavar='<dir>',
Expand All @@ -56,8 +60,9 @@ def register_options(cls, register):
'"<PATH>" (the contents of the PATH env var), '
'"<PEXRC>" (paths in the PEX_PYTHON_PATH variable in a pexrc file), '
'"<PYENV>" (all python versions under $(pyenv root)/versions).')
register('--resolver-jobs', type=int, default=None, advanced=True, fingerprint=True,
help='The maximum number of concurrent jobs to resolve wheels with.')
register('--resolver-use-manylinux', advanced=True, type=bool, default=True, fingerprint=True,
help='Whether to consider manylinux wheels when resolving requirements for linux '
'platforms.')

@property
def interpreter_constraints(self):
Expand Down Expand Up @@ -100,13 +105,17 @@ def resolver_cache_dir(self):
return (self.get_options().resolver_cache_dir or
os.path.join(self.scratch_dir, 'resolved_requirements'))

@property
def resolver_cache_ttl(self):
return self.get_options().resolver_cache_ttl

@property
def resolver_allow_prereleases(self):
return self.get_options().resolver_allow_prereleases

@property
def resolver_jobs(self):
return self.get_options().resolver_jobs
def use_manylinux(self):
return self.get_options().resolver_use_manylinux

@property
def artifact_cache_dir(self):
Expand Down
Loading

0 comments on commit 91d4af0

Please sign in to comment.