Skip to content

Commit

Permalink
Port pants to pex 0.8.x
Browse files Browse the repository at this point in the history
SSIA.  The user facing parts of this change:
  - package resolution happens via requests now -- this is way more reliable
    and results in fewer Untranslateable exceptions
  - PEX_VERBOSE now controls all pex verbosity, including better messages around Untranslateable (yay!)
  - fixes a major pex regression with namespace packages introduced somewhere in 0.5.x

The pex 0.7.0 -> 0.8.0 changelog:

* *API change*: Decouple translation from package iteration.  This removes
  the Obtainer construct entirely, which likely means if you're using PEX as
  a library, you will need to change your code if you were doing anything
  nontrivial.  This adds a couple new options to ``resolve`` but simplifies
  the story around how to cache packages.
  [RB #785](https://rbcommons.com/s/twitter/r/785/)
* Refactor http handling in pex to allow for alternate http implementations.  Adds support
  for [requests](https://github.com/kennethreitz/requests),
  improving both performance and security.   For more information, read the commit notes at
  [91c7f32](pex-tool/pex@91c7f32).
  [RB #778](https://rbcommons.com/s/twitter/r/778/)
* Improvements to API documentation throughout.
* Renamed ``Tracer`` to ``TraceLogger`` to prevent nondeterministic isort ordering.
* Refactor tox.ini to increase the number of environment combinations and improve coverage.
* Adds HTTP retry support for the RequestsContext.
  [RB #1303](https://rbcommons.com/s/twitter/r/1303/)
* Make pex --version correct.
  [Issue #19](pex-tool/pex#19)
* Bug fix: Fix over-aggressive sys.modules scrubbing for namespace packages.  Under
  certain circumstances, namespace packages in site-packages could conflict with packages
  within a PEX, causing them to fail importing.
  [RB #1378](https://rbcommons.com/s/twitter/r/1378/)
* Bug fix: Replace uses of ``os.unsetenv(...)`` with ``del os.environ[...]``
  [Pull Request #11](pex-tool/pex#11)
* Bug fix: Scrub sys.path and sys.modules based upon both supplied path and
  realpath of files and directories.  Newer versions of virtualenv on Linux symlink site-packages
  which caused those packages to not be removed from sys.path correctly.
  [Issue #21](pex-tool/pex#21)
* Bug fix: The pex -s option was not correctly pulling in transitive dependencies.
  [Issue #22](pex-tool/pex#22)
* Bug fix: Adds ``content`` method to HTTP contexts that does HTML content decoding, fixing
  an encoding issue only experienced when using Python 3.
  [Issue #10](pex-tool/pex#10)

Testing Done:
build-support/bin/ci.sh

Reviewed at https://rbcommons.com/s/twitter/r/1421/
  • Loading branch information
wickman committed Dec 4, 2014
1 parent df75d99 commit 80b9759
Show file tree
Hide file tree
Showing 14 changed files with 78 additions and 91 deletions.
4 changes: 3 additions & 1 deletion 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ docutils>=0.12,<0.13
Markdown==2.1.1
mock==1.0.1
mox==0.5.3
pex>=0.7.0,<0.8
# Until https://github.com/pantsbuild/pex/issues/28 is addressed, we cannot
# do pex<0.9.0.
pex>=0.8.1,<0.8.999999
psutil==1.1.3
lockfile==0.10.2
Pygments==1.4
Expand Down
2 changes: 2 additions & 0 deletions build-support/pants_venv
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ function activate_pants_venv() {
# Install as an egg so IDEs can grok all deps. Installing as sources may
# run afoul of this bug: http://youtrack.jetbrains.com/issue/PY-6477.
# TODO: Remove the --egg once this is resolved. It won't work in the wheel world anyway.
# Use -f ${REPO_ROOT}/third_party if patching in local dependencies
# pip_extra="--egg -f ${REPO_ROOT}/third_party"
pip_extra="--egg"
else
pip_extra=""
Expand Down
5 changes: 2 additions & 3 deletions src/python/pants/backend/python/binary_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass

def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
def __init__(self, target, run_tracker, interpreter=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
Expand All @@ -45,8 +45,7 @@ def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
interpreter=self.interpreter)

def run(self):
print('Building PythonBinary %s:' % self.target)
Expand Down
27 changes: 17 additions & 10 deletions src/python/pants/backend/python/interpreter_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@
from pkg_resources import Requirement
import shutil


from pex.archiver import Archiver
from pex.crawler import Crawler
from pex.installer import EggInstaller
from pex.interpreter import PythonIdentity, PythonInterpreter
from pex.obtainer import Obtainer
from pex.iterator import Iterator
from pex.package import EggPackage, SourcePackage

from pants.backend.python.python_setup import PythonSetup
from pants.backend.python.resolver import crawler_from_config, fetchers_from_config
from pants.backend.python.resolver import context_from_config, fetchers_from_config
from pants.util.dirutil import safe_mkdir


Expand All @@ -39,33 +40,39 @@ def _resolve_interpreter(config, interpreter, requirement, logger=print):
return interpreter

def installer_provider(sdist):
return EggInstaller(sdist, strict=requirement.key != 'setuptools', interpreter=interpreter)
return EggInstaller(
Archiver.unpack(sdist),
strict=requirement.key != 'setuptools',
interpreter=interpreter)

egg = _resolve_and_link(
config,
requirement,
os.path.join(interpreter_dir, requirement.key),
installer_provider,
logger=logger)

if egg:
return interpreter.with_extra(egg.name, egg.raw_version, egg.url)
return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
else:
logger('Failed to resolve requirement %s for %s' % (requirement, interpreter))


def _resolve_and_link(config, requirement, target_link, installer_provider, logger=print):
# Short-circuit if there is a local copy
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg

fetchers = fetchers_from_config(config)
crawler = crawler_from_config(config)
obtainer = Obtainer(crawler, fetchers, [])
obtainer_iterator = obtainer.iter(requirement)
links = [link for link in obtainer_iterator if isinstance(link, SourcePackage)]
context = context_from_config(config)
iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]

for link in links:
logger(' fetching %s' % link.url)
sdist = link.fetch()
sdist = context.fetch(link)
logger(' installing %s' % sdist)
installer = installer_provider(sdist)
dist_location = installer.bdist()
Expand Down
8 changes: 2 additions & 6 deletions src/python/pants/backend/python/python_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@ class PythonBuilder(object):
def __init__(self, run_tracker):
self._run_tracker = run_tracker

def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=False,
debug=False):

def build(self, targets, args, interpreter=None, fast_tests=False, debug=False):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
Expand All @@ -38,7 +36,6 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
test_targets,
args,
interpreter=interpreter,
conn_timeout=conn_timeout,
fast=fast_tests,
debug=debug).run()
if rv != 0:
Expand All @@ -48,8 +45,7 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
rv = PythonBinaryBuilder(
binary_target,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
interpreter=interpreter).run()
if rv != 0:
return rv

Expand Down
19 changes: 10 additions & 9 deletions src/python/pants/backend/python/python_chroot.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,14 @@ def __init__(self,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
interpreter=None):
self._config = Config.from_cache()
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
self._conn_timeout = conn_timeout

# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
Expand Down Expand Up @@ -127,10 +125,9 @@ def copy_to_chroot(base, path, add_function):
resource=resources_tgt.address.spec))
raise

def _dump_requirement(self, req, dynamic, repo):
self.debug(' Dumping requirement: %s%s%s' % (str(req),
' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else ''))
self._builder.add_requirement(req, dynamic, repo)
def _dump_requirement(self, req):
self.debug(' Dumping requirement: %s' % req)
self._builder.add_requirement(req)

def _dump_distribution(self, dist):
self.debug(' Dumping distribution: .../%s' % os.path.basename(dist.location))
Expand Down Expand Up @@ -194,19 +191,23 @@ def dump(self):
reqs_from_libraries.add(req)

reqs_to_build = OrderedSet()
find_links = []

for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req._requirement, False, req._repository)
self._dump_requirement(req.requirement)
if req.repository:
find_links.append(req.repository)

distributions = resolve_multi(
self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
conn_timeout=self._conn_timeout)
find_links=find_links)

locations = set()
for platform, dist_set in distributions.items():
Expand Down
59 changes: 20 additions & 39 deletions src/python/pants/backend/python/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
print_function, unicode_literals)

from pex.fetcher import Fetcher, PyPIFetcher
from pex.http import Crawler
from pex.crawler import Crawler
from pex.http import Context
from pex.interpreter import PythonInterpreter
from pex.obtainer import CachingObtainer
from pex.iterator import Iterator
from pex.platforms import Platform
from pex.resolver import resolve
from pex.translator import Translator
Expand All @@ -29,33 +30,17 @@ def fetchers_from_config(config):
return fetchers


def crawler_from_config(config, conn_timeout=None):
download_cache = PythonSetup(config).scratch_dir('download_cache', default_name='downloads')
return Crawler(cache=download_cache, conn_timeout=conn_timeout)


class PantsObtainer(CachingObtainer):
def iter(self, requirement):
if hasattr(requirement, 'repository') and requirement.repository:
obtainer = CachingObtainer(
install_cache=self.install_cache,
ttl=self.ttl,
crawler=self._crawler,
fetchers=[Fetcher([requirement.repository])],
translators=self._translator)
for package in obtainer.iter(requirement):
yield package
else:
for package in super(PantsObtainer, self).iter(requirement):
yield package
def context_from_config(config):
# TODO(wickman) Add retry, conn_timeout, threads, etc configuration here.
return Context.get()


def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
ttl=3600,
find_links=None):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
Expand All @@ -68,35 +53,31 @@ def resolve_multi(config,
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))

install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
fetchers = fetchers_from_config(config)
if find_links:
fetchers.extend(Fetcher([path]) for path in find_links)
context = context_from_config(config)

for platform in platforms:
translator = Translator.default(
install_cache=install_cache,
distributions[platform] = resolve(
requirements=requirements,
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
conn_timeout=conn_timeout)

obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)

distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
context=context,
cache=cache,
cache_ttl=ttl)

return distributions
1 change: 0 additions & 1 deletion src/python/pants/backend/python/tasks/pytest_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ def is_python_test(target):
test_builder = PythonTestBuilder(targets=test_targets,
args=args,
interpreter=self.interpreter,
conn_timeout=self.conn_timeout,
fast=self.get_options().fast,
debug=debug)
with self.context.new_workunit(name='run',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,7 @@ def create_binary(self, binary):
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
chroot.dump()
Expand Down
3 changes: 1 addition & 2 deletions src/python/pants/backend/python/tasks/python_repl.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ def execute(self):
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

chroot.dump()
builder.freeze()
Expand Down
3 changes: 1 addition & 2 deletions src/python/pants/backend/python/tasks/python_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ def execute(self):
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
interpreter=interpreter)

chroot.dump()
builder.freeze()
Expand Down
11 changes: 0 additions & 11 deletions src/python/pants/backend/python/tasks/python_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,8 @@


class PythonTask(Task):
@classmethod
def register_options(cls, register):
super(PythonTask, cls).register_options(register)
register('--timeout', type=int, default=0,
help='Number of seconds to wait for http connections.')

def __init__(self, *args, **kwargs):
super(PythonTask, self).__init__(*args, **kwargs)
self.conn_timeout = (self.get_options().timeout or
self.context.config.getdefault('connection_timeout'))

self._compatibilities = self.get_options().interpreter or [b'']
self._interpreter_cache = None
self._interpreter = None
Expand Down Expand Up @@ -96,5 +87,3 @@ def temporary_pex_builder(self, interpreter=None, pex_info=None, parent_dir=None
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
yield builder
builder.chroot().delete()


6 changes: 2 additions & 4 deletions src/python/pants/backend/python/test_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,10 @@ class PythonTestBuilder(object):
PythonRequirement('unittest2py3k', version_filter=lambda py, pl: py.startswith('3'))
]

def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False, debug=False):
def __init__(self, targets, args, interpreter=None, fast=False, debug=False):
self._targets = targets
self._args = args
self._interpreter = interpreter or PythonInterpreter.get()
self._conn_timeout = conn_timeout

# If fast is true, we run all the tests in a single chroot. This is MUCH faster than
# creating a chroot for each test target. However running each test separately is more
Expand Down Expand Up @@ -292,8 +291,7 @@ def _test_runner(self, targets, stdout, stderr):
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter,
conn_timeout=self._conn_timeout)
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
Expand Down
Loading

0 comments on commit 80b9759

Please sign in to comment.