Skip to content

Commit

Permalink
Upgrade to pex 1.6.2.
Browse files Browse the repository at this point in the history
This allows us to get rid of resolving setuptools and wheel in our
interpreter cache.

Fixes pantsbuild#6927
  • Loading branch information
jsirois committed Feb 22, 2019
1 parent 4097052 commit aff5194
Show file tree
Hide file tree
Showing 13 changed files with 119 additions and 347 deletions.
4 changes: 2 additions & 2 deletions 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ mock==2.0.0
packaging==16.8
parameterized==0.6.1
pathspec==0.5.9
pex==1.5.3
pex==1.6.2
psutil==5.4.8
pycodestyle==2.4.0
pyflakes==2.0.0
Expand All @@ -30,7 +30,7 @@ requests[security]>=2.20.1
responses==0.10.4
scandir==1.2
setproctitle==1.1.10
setuptools==40.4.3
setuptools==40.6.3
six>=1.9.0,<2
subprocess32==3.2.7 ; python_version<'3'
wheel==0.31.1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,5 @@

class Lambdex(PythonToolBase):
options_scope = 'lambdex'
default_requirements = ['lambdex==0.1.2']
default_requirements = ['lambdex==0.1.3']
default_entry_point = 'lambdex.bin.lambdex'
90 changes: 6 additions & 84 deletions src/python/pants/backend/python/interpreter_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,11 @@
import logging
import os
import shutil
from builtins import map, str
from builtins import str
from collections import defaultdict

from pex.interpreter import PythonInterpreter
from pex.package import EggPackage, Package, SourcePackage
from pex.resolver import resolve

from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.backend.python.subsystems.python_setup import PythonSetup
from pants.backend.python.targets.python_target import PythonTarget
from pants.base.exceptions import TaskError
Expand All @@ -27,24 +24,14 @@
logger = logging.getLogger(__name__)


# TODO(wickman) Create a safer version of this and add to twitter.common.dirutil
def _safe_link(src, dst):
try:
os.unlink(dst)
except OSError:
pass
os.symlink(src, dst)


# TODO: Move under subsystems/ .
class PythonInterpreterCache(Subsystem):
"""Finds python interpreters on the local system."""
options_scope = 'python-interpreter-cache'

@classmethod
def subsystem_dependencies(cls):
return (super(PythonInterpreterCache, cls).subsystem_dependencies() +
(PythonSetup, PythonRepos))
return super(PythonInterpreterCache, cls).subsystem_dependencies() + (PythonSetup,)

class UnsatisfiableInterpreterConstraintsError(TaskError):
"""Indicates a python interpreter matching given constraints could not be located."""
Expand All @@ -63,10 +50,6 @@ def _matching(cls, interpreters, filters=()):
def _python_setup(self):
return PythonSetup.global_instance()

@memoized_property
def _python_repos(self):
return PythonRepos.global_instance()

@memoized_property
def _cache_dir(self):
cache_dir = self._python_setup.interpreter_cache_dir
Expand Down Expand Up @@ -124,17 +107,17 @@ def _interpreter_from_relpath(self, path, filters=()):
return None
except OSError:
return None
interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
interpreter = PythonInterpreter.from_binary(executable)
if self._matches(interpreter, filters=filters):
return self._resolve(interpreter)
return interpreter
return None

def _setup_interpreter(self, interpreter, identity_str):
cache_target_path = os.path.join(self._cache_dir, identity_str)
with safe_concurrent_creation(cache_target_path) as safe_path:
os.mkdir(safe_path) # Parent will already have been created by safe_concurrent_creation.
os.symlink(interpreter.binary, os.path.join(safe_path, 'python'))
return self._resolve(interpreter, safe_path)
return interpreter

def _setup_cached(self, filters=()):
"""Find all currently-cached interpreters."""
Expand Down Expand Up @@ -193,67 +176,6 @@ def unsatisfied_filters():
'Initialized Python interpreter cache with {}'.format(', '.join([x.binary for x in matches])))
return matches

def _resolve(self, interpreter, interpreter_dir=None):
"""Resolve and cache an interpreter with a setuptools and wheel capability."""
interpreter = self._resolve_interpreter(interpreter, interpreter_dir,
self._python_setup.setuptools_requirement())
if interpreter:
return self._resolve_interpreter(interpreter, interpreter_dir,
self._python_setup.wheel_requirement())

def _resolve_interpreter(self, interpreter, interpreter_dir, requirement):
"""Given a :class:`PythonInterpreter` and a requirement, return an interpreter with the
capability of resolving that requirement or ``None`` if it's not possible to install a
suitable requirement.
If interpreter_dir is unspecified, operates on the default location.
"""
if interpreter.satisfies([requirement]):
return interpreter

if not interpreter_dir:
interpreter_dir = os.path.join(self._cache_dir, str(interpreter.identity))

target_link = os.path.join(interpreter_dir, requirement.key)
bdist = self._resolve_and_link(interpreter, requirement, target_link)
if bdist:
return interpreter.with_extra(bdist.name, bdist.raw_version, bdist.path)
else:
logger.debug('Failed to resolve requirement {} for {}'.format(requirement, interpreter))

def _resolve_and_link(self, interpreter, requirement, target_link):
# Short-circuit if there is a local copy.
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
bdist = Package.from_href(os.path.realpath(target_link))
if bdist.satisfies(requirement):
return bdist

# Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
# Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
# wheels.
precedence = (EggPackage, SourcePackage)
resolved_dists = resolve(requirements=[requirement],
fetchers=self._python_repos.get_fetchers(),
interpreter=interpreter,
# The local interpreter cache is, by definition, composed of
# interpreters for the 'current' platform.
platform='current',
context=self._python_repos.get_network_context(),
precedence=precedence)
if not resolved_dists:
return None

assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
'found:\n\t{}'.format(requirement,
'\n\t'.join(map(str, resolved_dists))))

dist_location = resolved_dists[0].distribution.location
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
logger.debug(' installed {}'.format(target_location))
return Package.from_href(target_location)

def _purge_interpreter(self, interpreter_dir):
try:
logger.info('Detected stale interpreter `{}` in the interpreter cache, purging.'
Expand All @@ -262,5 +184,5 @@ def _purge_interpreter(self, interpreter_dir):
except Exception as e:
logger.warn(
'Caught exception {!r} during interpreter purge. Please run `./pants clean-all`!'
.format(e)
.format(e)
)
1 change: 0 additions & 1 deletion src/python/pants/backend/python/subsystems/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ python_library(
dependencies = [
'3rdparty/python:future',
'3rdparty/python:pex',
'3rdparty/python:setuptools',
'3rdparty/python/twitter/commons:twitter.common.collections',
'src/python/pants/base:build_environment',
'src/python/pants/base:exceptions',
Expand Down
79 changes: 73 additions & 6 deletions src/python/pants/backend/python/subsystems/pex_build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@
from pex.fetcher import Fetcher
from pex.pex_builder import PEXBuilder
from pex.resolver import resolve
from pex.util import DistributionHelper
from twitter.common.collections import OrderedSet

from pants.backend.python.python_requirement import PythonRequirement
from pants.backend.python.subsystems.python_repos import PythonRepos
from pants.backend.python.subsystems.python_setup import PythonSetup
from pants.backend.python.targets.python_binary import PythonBinary
Expand All @@ -26,6 +28,7 @@
from pants.base.exceptions import TaskError
from pants.build_graph.files import Files
from pants.subsystem.subsystem import Subsystem
from pants.util.contextutil import temporary_file


def is_python_target(tgt):
Expand Down Expand Up @@ -109,6 +112,13 @@ class PexBuilderWrapper(object):
class Factory(Subsystem):
options_scope = 'pex-builder-wrapper'

@classmethod
def register_options(cls, register):
super(PexBuilderWrapper.Factory, cls).register_options(register)
register('--setuptools-version', advanced=True, default='40.6.3',
help='The setuptools version to include in the pex if namespace packages need to be '
'injected.')

@classmethod
def subsystem_dependencies(cls):
return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + (
Expand All @@ -118,23 +128,38 @@ def subsystem_dependencies(cls):

@classmethod
def create(cls, builder, log=None):
options = cls.global_instance().get_options()
setuptools_requirement = 'setuptools=={}'.format(options.setuptools_version)

log = log or logging.getLogger(__name__)

return PexBuilderWrapper(builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(setuptools_requirement),
log=log)

def __init__(self, builder, python_repos_subsystem, python_setup_subsystem, log):
def __init__(self,
builder,
python_repos_subsystem,
python_setup_subsystem,
setuptools_requirement,
log):
assert isinstance(builder, PEXBuilder)
assert isinstance(python_repos_subsystem, PythonRepos)
assert isinstance(python_setup_subsystem, PythonSetup)
assert isinstance(setuptools_requirement, PythonRequirement)
assert log is not None

self._builder = builder
self._python_repos_subsystem = python_repos_subsystem
self._python_setup_subsystem = python_setup_subsystem
self._setuptools_requirement = setuptools_requirement
self._log = log

self._distributions = {}
self._frozen = False

def add_requirement_libs_from(self, req_libs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
Expand Down Expand Up @@ -162,7 +187,7 @@ def add_resolved_requirements(self, reqs, platforms=None):
find_links = OrderedSet()
for req in deduped_reqs:
self._log.debug(' Dumping requirement: {}'.format(req))
self._builder.add_requirement(req.requirement)
self._builder.add_requirement(str(req.requirement))
if req.repository:
find_links.add(req.repository)

Expand All @@ -174,7 +199,7 @@ def add_resolved_requirements(self, reqs, platforms=None):
for dist in dists:
if dist.location not in locations:
self._log.debug(' Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
self._builder.add_distribution(dist)
self.add_distribution(dist)
locations.add(dist.location)

def _resolve_multi(self, interpreter, requirements, platforms, find_links):
Expand Down Expand Up @@ -202,7 +227,7 @@ def _resolve_multi(self, interpreter, requirements, platforms, find_links):
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
resolved_dists = resolve(
requirements=[req.requirement for req in requirements],
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
Expand Down Expand Up @@ -236,13 +261,49 @@ def add_sources_from(self, tgt):
raise TaskError('Old-style resources not supported for target {}. '
'Depend on resources() targets instead.'.format(tgt.address.spec))

def _prepare_inits(self):
chroot = self._builder.chroot()
sources = chroot.get('source') | chroot.get('resource')

packages = set()
for source in sources:
if source.endswith('.py'):
pkg_dir = os.path.dirname(source)
if pkg_dir and pkg_dir not in packages:
package = ''
for component in pkg_dir.split(os.sep):
package = os.path.join(package, component)
packages.add(package)

missing_pkg_files = []
for package in packages:
pkg_file = os.path.join(package, '__init__.py')
if pkg_file not in sources:
missing_pkg_files.append(pkg_file)

if missing_pkg_files:
with temporary_file() as ns_package:
ns_package.write(b'__import__("pkg_resources").declare_namespace(__name__)')
ns_package.flush()
for missing_pkg_file in missing_pkg_files:
self._builder.add_source(ns_package.name, missing_pkg_file)

return missing_pkg_files

def freeze(self):
self._builder.freeze()
if not self._frozen:
if self._prepare_inits():
dist = self._distributions.get('setuptools')
if not dist:
self.add_resolved_requirements([self._setuptools_requirement])
self._builder.freeze()
self._frozen = True

def set_entry_point(self, entry_point):
self._builder.set_entry_point(entry_point)

def build(self, safe_path):
self.freeze()
self._builder.build(safe_path)

def set_shebang(self, shebang):
Expand All @@ -261,13 +322,19 @@ def add_interpreter_constraints_from(self, constraint_tgts):

def add_direct_requirements(self, reqs):
for req in reqs:
self._builder.add_requirement(req)
self._builder.add_requirement(str(req))

def add_distribution(self, dist):
self._builder.add_distribution(dist)
self._register_distribution(dist)

def add_dist_location(self, location):
self._builder.add_dist_location(location)
dist = DistributionHelper.distribution_from_path(location)
self._register_distribution(dist)

def _register_distribution(self, dist):
self._distributions[dist.key] = dist

def set_script(self, script):
self._builder.set_script(script)
18 changes: 8 additions & 10 deletions src/python/pants/backend/python/subsystems/python_native_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,18 +125,16 @@ class BuildSetupRequiresPex(ExecutablePexTool):
options_scope = 'build-setup-requires-pex'

@classmethod
def subsystem_dependencies(cls):
return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup,)

@memoized_property
def python_setup(self):
return PythonSetup.global_instance()
def register_options(cls, register):
super(BuildSetupRequiresPex, cls).register_options(register)
register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
help='The setuptools version to use when executing `setup.py` scripts.')
register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
help='The wheel version to use when executing `setup.py` scripts.')

@property
def base_requirements(self):
# TODO: would we ever want to configure these requirement versions separately from the global
# PythonSetup values?
return [
PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)),
PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)),
PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
]
Loading

0 comments on commit aff5194

Please sign in to comment.