Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to pex 1.6.6. #7186

Merged
merged 1 commit into from
Apr 16, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ansicolors==1.0.2
asttokens==1.1.13
beautifulsoup4>=4.6.0,<4.7
cffi==1.11.5
configparser==3.5.0 ; python_version<'3'
configparser==3.7.1 ; python_version<'3'
contextlib2==0.5.5
coverage>=4.5,<4.6
docutils>=0.12,<0.13
Expand All @@ -17,7 +17,7 @@ more-itertools<6.0.0 ; python_version<'3'
packaging==16.8
parameterized==0.6.1
pathspec==0.5.9
pex==1.5.3
pex==1.6.6
psutil==5.4.8
pycodestyle==2.4.0
pyflakes==2.0.0
Expand All @@ -33,7 +33,7 @@ requests[security]>=2.20.1
responses==0.10.4
scandir==1.2
setproctitle==1.1.10
setuptools==40.4.3
setuptools==40.6.3
six>=1.9.0,<2
subprocess32==3.2.7 ; python_version<'3'
wheel==0.31.1
Expand Down
12 changes: 6 additions & 6 deletions build-support/bin/release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,17 +82,13 @@ function run_pex() {
# TODO: Cache this in case we run pex multiple times
(
PEX_VERSION="$(requirement pex | sed -e "s|pex==||")"
PEX_PEX=pex27
if [[ "${python_three:-false}" == "true" ]]; then
PEX_PEX=pex36
fi

pexdir="$(mktemp -d -t build_pex.XXXXX)"
trap "rm -rf ${pexdir}" EXIT

pex="${pexdir}/${PEX_PEX}"
pex="${pexdir}/pex"

curl -sSL "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/${PEX_PEX}" > "${pex}"
curl -sSL "${PEX_DOWNLOAD_PREFIX}/v${PEX_VERSION}/pex" > "${pex}"
chmod +x "${pex}"
"${pex}" "$@"
)
Expand Down Expand Up @@ -658,8 +654,12 @@ function build_pex() {
platform_flags=("${platform_flags[@]}" "--platform=${platform}")
done

# Pants depends on twitter.common libraries that trigger pex warnings for not properly declaring
jsirois marked this conversation as resolved.
Show resolved Hide resolved
# their dependency on setuptools (for namespace package support). To prevent these known warnings
# from polluting stderr we pass `--no-emit-warnings`.
execute_pex \
-o "${dest}" \
--no-emit-warnings \
--script=pants \
--interpreter-constraint="${interpreter_constraint}" \
"${platform_flags[@]}" \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,11 @@

class Lambdex(PythonToolBase):
options_scope = 'lambdex'
default_requirements = ['lambdex==0.1.2']
default_requirements = [
'lambdex==0.1.3',

# TODO(John Sirois): Remove when we can upgrade to a version of lambdex with
# https://github.com/wickman/lambdex/issues/6 fixed.
'setuptools==40.8.0'
]
default_entry_point = 'lambdex.bin.lambdex'
88 changes: 5 additions & 83 deletions src/python/pants/backend/python/interpreter_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,11 @@
import logging
import os
import shutil
from builtins import map, str
from builtins import str
from collections import defaultdict

from pex.interpreter import PythonInterpreter
from pex.package import EggPackage, Package, SourcePackage
from pex.resolver import resolve

from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.backend.python.subsystems.python_setup import PythonSetup
from pants.backend.python.targets.python_target import PythonTarget
from pants.base.exceptions import TaskError
Expand All @@ -27,24 +24,14 @@
logger = logging.getLogger(__name__)


# TODO(wickman) Create a safer version of this and add to twitter.common.dirutil
def _safe_link(src, dst):
try:
os.unlink(dst)
except OSError:
pass
os.symlink(src, dst)


# TODO: Move under subsystems/ .
class PythonInterpreterCache(Subsystem):
"""Finds python interpreters on the local system."""
options_scope = 'python-interpreter-cache'

@classmethod
def subsystem_dependencies(cls):
return (super(PythonInterpreterCache, cls).subsystem_dependencies() +
(PythonSetup, PythonRepos))
return super(PythonInterpreterCache, cls).subsystem_dependencies() + (PythonSetup,)

class UnsatisfiableInterpreterConstraintsError(TaskError):
"""Indicates a python interpreter matching given constraints could not be located."""
Expand All @@ -63,10 +50,6 @@ def _matching(cls, interpreters, filters=()):
def _python_setup(self):
return PythonSetup.global_instance()

@memoized_property
def _python_repos(self):
return PythonRepos.global_instance()

@memoized_property
def _cache_dir(self):
cache_dir = self._python_setup.interpreter_cache_dir
Expand Down Expand Up @@ -124,17 +107,17 @@ def _interpreter_from_relpath(self, path, filters=()):
return None
except OSError:
return None
interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
interpreter = PythonInterpreter.from_binary(executable)
if self._matches(interpreter, filters=filters):
return self._resolve(interpreter)
return interpreter
return None

def _setup_interpreter(self, interpreter, identity_str):
cache_target_path = os.path.join(self._cache_dir, identity_str)
with safe_concurrent_creation(cache_target_path) as safe_path:
os.mkdir(safe_path) # Parent will already have been created by safe_concurrent_creation.
os.symlink(interpreter.binary, os.path.join(safe_path, 'python'))
return self._resolve(interpreter, safe_path)
return interpreter

def _setup_cached(self, filters=()):
"""Find all currently-cached interpreters."""
Expand Down Expand Up @@ -193,67 +176,6 @@ def unsatisfied_filters():
'Initialized Python interpreter cache with {}'.format(', '.join([x.binary for x in matches])))
return matches

def _resolve(self, interpreter, interpreter_dir=None):
"""Resolve and cache an interpreter with a setuptools and wheel capability."""
interpreter = self._resolve_interpreter(interpreter, interpreter_dir,
self._python_setup.setuptools_requirement())
if interpreter:
return self._resolve_interpreter(interpreter, interpreter_dir,
self._python_setup.wheel_requirement())

def _resolve_interpreter(self, interpreter, interpreter_dir, requirement):
"""Given a :class:`PythonInterpreter` and a requirement, return an interpreter with the
capability of resolving that requirement or ``None`` if it's not possible to install a
suitable requirement.

If interpreter_dir is unspecified, operates on the default location.
"""
if interpreter.satisfies([requirement]):
return interpreter

if not interpreter_dir:
interpreter_dir = os.path.join(self._cache_dir, str(interpreter.identity))

target_link = os.path.join(interpreter_dir, requirement.key)
bdist = self._resolve_and_link(interpreter, requirement, target_link)
if bdist:
return interpreter.with_extra(bdist.name, bdist.raw_version, bdist.path)
else:
logger.debug('Failed to resolve requirement {} for {}'.format(requirement, interpreter))

def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist

# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
resolved_dists = resolve(requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
# The local interpreter cache is, by definition, composed of
# interpreters for the 'current' platform.
platform='current',
context=self._python_repos.get_network_context(),
precedence=precedence)
if not resolved_dists:
return None

assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, resolved_dists))))

dist_location = resolved_dists[0].distribution.location
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
logger.debug(' installed {}'.format(target_location))
return Package.from_href(target_location)

def _purge_interpreter(self, interpreter_dir):
try:
logger.info('Detected stale interpreter `{}` in the interpreter cache, purging.'
Expand Down
14 changes: 3 additions & 11 deletions src/python/pants/backend/python/rules/python_test_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,9 @@ def run_python_test(transitive_hydrated_target, pytest):
target_root = transitive_hydrated_target.root

# TODO: Inject versions and digests here through some option, rather than hard-coding it.
interpreter_major, interpreter_minor = sys.version_info[0:2]
pex_name, digest = {
(2, 7): ("pex27", Digest('0ecbf48e3e240a413189194a9f829aec10446705c84db310affe36e23e741dbc', 1812737)),
(3, 6): ("pex36", Digest('ba865e7ce7a840070d58b7ba24e7a67aff058435cfa34202abdd878e7b5d351d', 1812158)),
(3, 7): ("pex37", Digest('51bf8e84d5290fe5ff43d45be78d58eaf88cf2a5e995101c8ff9e6a73a73343d', 1813189))
}.get((interpreter_major, interpreter_minor), (None, None))
if pex_name is None:
raise ValueError("Current interpreter {}.{} is not supported, as there is no corresponding PEX to download.".format(interpreter_major, interpreter_minor))

pex_snapshot = yield Get(Snapshot,
UrlToFetch("https://github.com/pantsbuild/pex/releases/download/v1.6.1/{}".format(pex_name), digest))
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.6/pex'
digest = Digest('61bb79384db0da8c844678440bd368bcbfac17bbdb865721ad3f9cb0ab29b629', 1826945)
pex_snapshot = yield Get(Snapshot, UrlToFetch(url, digest))

all_targets = [target_root] + [dep.root for dep in transitive_hydrated_target.dependencies]

Expand Down
1 change: 0 additions & 1 deletion src/python/pants/backend/python/subsystems/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ python_library(
dependencies = [
'3rdparty/python:future',
'3rdparty/python:pex',
'3rdparty/python:setuptools',
'3rdparty/python/twitter/commons:twitter.common.collections',
'src/python/pants/base:build_environment',
'src/python/pants/base:exceptions',
Expand Down
Loading