Skip to content

Commit

Permalink
Upgrade to Pex 2.1.4.
Browse files Browse the repository at this point in the history
The primary changes to adapt to were:
1. The `pex.resolver.resolve` function signature changed.
2. The results of resolutions are now installed wheel chroots instead
   of zipped wheels.
3. Some packaging code once used by Pex and our plugin tests is now
   deleted.

Fixes pantsbuild#8786

This is built on top of:

Revert "Revert "Upgrade to Pex 2.0.3. (pantsbuild#8704)" (pantsbuild#8787)"

    This reverts commit 91d4af0.
  • Loading branch information
jsirois committed Feb 21, 2020
1 parent 5416b41 commit 9f82bd6
Show file tree
Hide file tree
Showing 23 changed files with 136 additions and 214 deletions.
8 changes: 3 additions & 5 deletions 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,14 @@ beautifulsoup4>=4.6.0,<4.7
cffi==1.13.2
contextlib2==0.5.5
coverage>=4.5,<4.6
# TODO remove this pin once we resolve https://github.com/pantsbuild/pants/issues/8502
cryptography==2.7
dataclasses==0.6
docutils==0.14
fasteners==0.15.0
Markdown==2.1.1
packaging==16.8
parameterized==0.6.1
pathspec==0.5.9
pex==2.0.3
pex==2.1.0
psutil==5.6.3
Pygments==2.3.1
pyopenssl==17.3.0
Expand All @@ -24,8 +22,8 @@ py_zipkin==0.18.4
requests[security]>=2.20.1
responses==0.10.4
setproctitle==1.1.10
setuptools==40.6.3
setuptools==44.0.0
toml==0.10.0
typing-extensions==3.7.4
wheel==0.31.1
wheel==0.33.6
www-authenticate==0.9.2
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
default_version = 'lambdex==0.1.3'
# TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
# https://github.com/wickman/lambdex/issues/6 fixed.
default_extra_requirements = ['setuptools==40.8.0']
default_extra_requirements = ['setuptools==44.0.0']
default_entry_point = 'lambdex.bin.lambdex'
2 changes: 1 addition & 1 deletion src/python/pants/backend/awslambda/python/lambdex.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@ class Lambdex(PythonToolBase):
default_version = 'lambdex==0.1.3'
# TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
# https://github.com/wickman/lambdex/issues/6 fixed.
default_extra_requirements = ['setuptools==40.8.0']
default_extra_requirements = ['setuptools==44.0.0']
default_entry_point = 'lambdex.bin.lambdex'
8 changes: 4 additions & 4 deletions src/python/pants/backend/python/rules/download_pex_bin.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ def directory_digest(self) -> Digest:
class Factory(Script):
options_scope = 'download-pex-bin'
name = 'pex'
default_version = 'v1.6.12'
default_version = 'v2.1.4'

default_versions_and_digests = {
PlatformConstraint.none: ToolForPlatform(
digest=Digest('ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1',
1842359),
version=ToolVersion('v1.6.12'),
digest=Digest('6c5ae1f6b9aa40c97bd26a154849044b49f4d698a6abb9ac58ce006bda9cbd4a',
2614246),
version=ToolVersion('v2.1.4'),
),
}

Expand Down
8 changes: 6 additions & 2 deletions src/python/pants/backend/python/rules/pex.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@ async def create_pex(
argv.extend(["--entry-point", request.entry_point])
argv.extend(request.interpreter_constraints.generate_pex_arg_list())
argv.extend(request.additional_args)
if python_setup.manylinux:
argv.extend(['--manylinux', python_setup.manylinux])
else:
argv.append('--no-manylinux')

source_dir_name = 'source_files'
argv.append(f'--sources-directory={source_dir_name}')
Expand All @@ -116,8 +120,8 @@ async def create_pex(
merged_digest = await Get[Digest](DirectoriesToMerge(directories=all_inputs))

# NB: PEX outputs are platform dependent so in order to get a PEX that we can use locally, without
# cross-building, we specify that out PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for out
# cross-building, we specify that our PEX command be run on the current local platform. When we
# support cross-building through CLI flags we can configure requests that build a PEX for our
# local platform that are able to execute on a different platform, but for now in order to
# guarantee correct build we need to restrict this command to execute on the same platform type
# that the output is intended for. The correct way to interpret the keys
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,9 @@
from pants.backend.native.subsystems.native_toolchain import NativeToolchain
from pants.backend.native.targets.native_library import NativeLibrary
from pants.backend.python.subsystems import pex_build_util
from pants.backend.python.subsystems.executable_pex_tool import ExecutablePexTool
from pants.backend.python.targets.python_distribution import PythonDistribution
from pants.base.exceptions import IncompatiblePlatformsError
from pants.engine.rules import rule, subsystem_rule
from pants.python.python_requirement import PythonRequirement
from pants.python.python_setup import PythonSetup
from pants.subsystem.subsystem import Subsystem
from pants.util.memo import memoized_property
Expand Down
2 changes: 1 addition & 1 deletion src/python/pants/backend/python/targets/unpacked_whls.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def __init__(self, module_name, libraries=None, include_patterns=None, exclude_p
"""
deprecated_conditional(
lambda: type(within_data_subdir) not in (bool, type(None)),
removal_version='1.26.0.dev2',
removal_version='1.27.0.dev2',
entity_description='A non-boolean value for `within_data_subdir`',
hint_message='The location of the .data subdirectory will be inferred from the module name!',
)
Expand Down
1 change: 1 addition & 0 deletions src/python/pants/backend/python/tasks/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ python_library(
dependencies=[
'3rdparty/python:dataclasses',
'3rdparty/python:pex',
'3rdparty/python:wheel',
'3rdparty/python/twitter/commons:twitter.common.collections',
'3rdparty/python/twitter/commons:twitter.common.dirutil',
'src/python/pants/backend/native/config',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
import shutil
from pathlib import Path

from pex import pep425tags
from pex.interpreter import PythonInterpreter
from wheel import pep425tags

from pants.backend.native.targets.native_library import NativeLibrary
from pants.backend.native.tasks.link_shared_libraries import SharedLibrary
Expand Down
2 changes: 2 additions & 0 deletions src/python/pants/backend/python/tasks/setup_py.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import textwrap
from abc import ABC, abstractmethod
from collections import OrderedDict, defaultdict
from pathlib import Path
from typing import Dict

from pex.interpreter import PythonInterpreter
from pex.pex import PEX
Expand Down
1 change: 0 additions & 1 deletion src/python/pants/init/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ python_library(
'3rdparty/python:dataclasses',
'3rdparty/python:pex',
'3rdparty/python:setuptools',
'3rdparty/python:wheel',
'3rdparty/python/twitter/commons:twitter.common.collections',
'src/python/pants/base:build_environment',
'src/python/pants/base:build_root',
Expand Down
113 changes: 34 additions & 79 deletions src/python/pants/python/pex_build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@
from pathlib import Path
from typing import Callable, Sequence, Set

from pex.fetcher import Fetcher
from pex.pex_builder import PEXBuilder
from pex.resolver import resolve
from pex.resolver import resolve_multi
from pex.util import DistributionHelper
from twitter.common.collections import OrderedSet

Expand All @@ -20,7 +19,6 @@
from pants.python.python_requirement import PythonRequirement
from pants.python.python_setup import PythonSetup
from pants.subsystem.subsystem import Subsystem
from pants.util.collections import assert_single_element
from pants.util.contextutil import temporary_file


Expand Down Expand Up @@ -78,7 +76,7 @@ class Factory(Subsystem):
@classmethod
def register_options(cls, register):
super(PexBuilderWrapper.Factory, cls).register_options(register)
register('--setuptools-version', advanced=True, default='40.6.3',
register('--setuptools-version', advanced=True, default='44.0.0',
help='The setuptools version to include in the pex if namespace packages need to be '
'injected.')

Expand Down Expand Up @@ -133,99 +131,56 @@ def add_requirement_libs_from(self, req_libs, platforms=None):
reqs = [req for req_lib in req_libs for req in req_lib.requirements]
self.add_resolved_requirements(reqs, platforms=platforms)

class SingleDistExtractionError(Exception): pass
def resolve_distributions(self, reqs, platforms=None):
"""Multi-platform dependency resolution.
def extract_single_dist_for_current_platform(self, reqs, dist_key):
"""Resolve a specific distribution from a set of requirements matching the current platform.
:param list reqs: A list of :class:`PythonRequirement` to resolve.
:param str dist_key: The value of `distribution.key` to match for a `distribution` from the
resolved requirements.
:return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
:raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given
`dist_key`.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param platforms: A list of platform strings to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:returns: List of :class:`pex.resolver.ResolvedDistribution` instances meeting requirements for
the given platforms.
"""
distributions = self._resolve_distributions_by_platform(reqs, platforms=['current'])
try:
matched_dist = assert_single_element(list(
dist
for _, dists in distributions.items()
for dist in dists
if dist.key == dist_key
))
except (StopIteration, ValueError) as e:
raise self.SingleDistExtractionError(
f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
)
return matched_dist

def _resolve_distributions_by_platform(self, reqs, platforms):
deduped_reqs = OrderedSet(reqs)
find_links = OrderedSet()
for req in deduped_reqs:
self._log.debug(f' Dumping requirement: {req}')
self._builder.add_requirement(str(req.requirement))
if req.repository:
find_links.add(req.repository)

# Resolve the requirements into distributions.
distributions = self._resolve_multi(self._builder.interpreter, deduped_reqs, platforms,
find_links)
return distributions
return self._resolve_multi(deduped_reqs, platforms=platforms, find_links=find_links)

def add_resolved_requirements(self, reqs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
:param reqs: A list of :class:`PythonRequirement`s to resolve.
:param platforms: A list of platform strings to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
distributions = self._resolve_distributions_by_platform(reqs, platforms=platforms)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self._log.debug(f' Dumping distribution: .../{os.path.basename(dist.location)}')
self.add_distribution(dist)
locations.add(dist.location)

def _resolve_multi(self, interpreter, requirements, platforms, find_links):
"""Multi-platform dependency resolution for PEX files.
for resolved_dist in self.resolve_distributions(reqs, platforms=platforms):
requirement = resolved_dist.requirement
self._log.debug(f' Dumping requirement: {requirement}')
self._builder.add_requirement(str(requirement))

Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
distribution = resolved_dist.distribution
self._log.debug(f' Dumping distribution: .../{os.path.basename(distribution.location)}')
self.add_distribution(distribution)

:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
def _resolve_multi(self, requirements, platforms=None, find_links=None):
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = find_links or []
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)

for platform in platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
allow_prereleases=python_setup.resolver_allow_prereleases,
use_manylinux=python_setup.use_manylinux)
distributions[platform] = [resolved_dist.distribution for resolved_dist in resolved_dists]

return distributions
find_links = list(find_links) if find_links else []
find_links.extend(python_repos.repos)

return resolve_multi(
requirements=[str(req.requirement) for req in requirements],
interpreters=[self._builder.interpreter],
indexes=python_repos.indexes,
find_links=find_links,
platforms=platforms,
cache=python_setup.resolver_cache_dir,
allow_prereleases=python_setup.resolver_allow_prereleases,
manylinux=python_setup.manylinux,
max_parallel_jobs=python_setup.resolver_jobs)

def add_sources_from(self, tgt: Target) -> None:
dump_source = _create_source_dumper(self._builder, tgt)
Expand All @@ -238,7 +193,7 @@ def add_sources_from(self, tgt: Target) -> None:
raise

if (getattr(tgt, '_resource_target_specs', None) or
getattr(tgt, '_synthetic_resources_target', None)):
getattr(tgt, '_synthetic_resources_target', None)):
# No one should be on old-style resources any more. And if they are,
# switching to the new python pipeline will be a great opportunity to fix that.
raise TaskError(
Expand Down
63 changes: 0 additions & 63 deletions src/python/pants/python/python_repos.py
Original file line number Diff line number Diff line change
@@ -1,58 +1,7 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import logging

from pex.fetcher import Fetcher, PyPIFetcher
from pex.http import RequestsContext, StreamFilelike, requests

from pants.subsystem.subsystem import Subsystem
from pants.util.memo import memoized_method


logger = logging.getLogger(__name__)


# TODO: These methods of RequestsContext are monkey-patched out to work around
# https://github.com/pantsbuild/pex/issues/26: we should upstream a fix for this.
_REQUESTS_TIMEOUTS = (15, 30)


def _open_monkey(self, link):
# requests does not support file:// -- so we must short-circuit manually
if link.local:
return open(link.local_path, 'rb') # noqa: T802
for attempt in range(self._max_retries + 1):
try:
return StreamFilelike(self._session.get(
link.url, verify=self._verify, stream=True, headers={'User-Agent': self.USER_AGENT},
timeout=_REQUESTS_TIMEOUTS),
link)
except requests.exceptions.ReadTimeout:
# Connect timeouts are handled by the HTTPAdapter, unfortunately read timeouts are not
# so we'll retry them ourselves.
logger.warning(f'Read timeout trying to fetch {link.url}, retrying. '
f'{self._max_retries - attempt} retries remain.')
except requests.exceptions.RequestException as e:
raise self.Error(e)

raise self.Error(
requests.packages.urllib3.exceptions.MaxRetryError(
None,
link,
'Exceeded max retries of %d' % self._max_retries))


def _resolve_monkey(self, link):
return link.wrap(self._session.head(
link.url, verify=self._verify, allow_redirects=True,
headers={'User-Agent': self.USER_AGENT},
timeout=_REQUESTS_TIMEOUTS,
).url)


RequestsContext.open = _open_monkey
RequestsContext.resolve = _resolve_monkey


class PythonRepos(Subsystem):
Expand All @@ -78,15 +27,3 @@ def repos(self):
@property
def indexes(self):
return self.get_options().indexes

@memoized_method
def get_fetchers(self):
fetchers = []
fetchers.extend(Fetcher([url]) for url in self.repos)
fetchers.extend(PyPIFetcher(url) for url in self.indexes)
return fetchers

@memoized_method
def get_network_context(self):
# TODO(wickman): Add retry, conn_timeout, threads, etc configuration here.
return RequestsContext()
Loading

0 comments on commit 9f82bd6

Please sign in to comment.