Port pants to pex 0.8.x

SSIA.  The user facing parts of this change:
  - package resolution happens via requests now -- this is way more reliable
    and results in fewer Untranslateable exceptions
  - PEX_VERBOSE now controls all pex verbosity, including better messages around Untranslateable (yay!)
  - fixes a major pex regression with namespace packages introduced somewhere in 0.5.x

The pex 0.7.0 -> 0.8.0 changelog:

* *API change*: Decouple translation from package iteration.  This removes
  the Obtainer construct entirely, which likely means if you're using PEX as
  a library, you will need to change your code if you were doing anything
  nontrivial.  This adds a couple new options to ``resolve`` but simplifies
  the story around how to cache packages.
  [RB #785](https://rbcommons.com/s/twitter/r/785/)
* Refactor http handling in pex to allow for alternate http implementations.  Adds support
  for [requests](https://github.com/kennethreitz/requests),
  improving both performance and security.   For more information, read the commit notes at
  [91c7f32](pex-tool/pex@91c7f32).
  [RB #778](https://rbcommons.com/s/twitter/r/778/)
* Improvements to API documentation throughout.
* Renamed ``Tracer`` to ``TraceLogger`` to prevent nondeterministic isort ordering.
* Refactor tox.ini to increase the number of environment combinations and improve coverage.
* Adds HTTP retry support for the RequestsContext.
  [RB #1303](https://rbcommons.com/s/twitter/r/1303/)
* Make pex --version correct.
  [Issue #19](pex-tool/pex#19)
* Bug fix: Fix over-aggressive sys.modules scrubbing for namespace packages.  Under
  certain circumstances, namespace packages in site-packages could conflict with packages
  within a PEX, causing them to fail importing.
  [RB #1378](https://rbcommons.com/s/twitter/r/1378/)
* Bug fix: Replace uses of ``os.unsetenv(...)`` with ``del os.environ[...]``
  [Pull Request #11](pex-tool/pex#11)
* Bug fix: Scrub sys.path and sys.modules based upon both supplied path and
  realpath of files and directories.  Newer versions of virtualenv on Linux symlink site-packages
  which caused those packages to not be removed from sys.path correctly.
  [Issue #21](pex-tool/pex#21)
* Bug fix: The pex -s option was not correctly pulling in transitive dependencies.
  [Issue #22](pex-tool/pex#22)
* Bug fix: Adds ``content`` method to HTTP contexts that does HTML content decoding, fixing
  an encoding issue only experienced when using Python 3.
  [Issue #10](pex-tool/pex#10)

Testing Done:
build-support/bin/ci.sh

Reviewed at https://rbcommons.com/s/twitter/r/1421/

3rdparty/python/requirements.txt

@@ -9,7 +9,9 @@ docutils>=0.12,<0.13
 Markdown==2.1.1
 mock==1.0.1
 mox==0.5.3
-pex>=0.7.0,<0.8
+# Until https://github.com/pantsbuild/pex/issues/28 is addressed, we cannot
+# do pex<0.9.0.
+pex>=0.8.1,<0.8.999999
 psutil==1.1.3
 lockfile==0.10.2
 Pygments==1.4

build-support/pants_venv

@@ -50,6 +50,8 @@ function activate_pants_venv() {
       # Install as an egg so IDEs can grok all deps. Installing as sources may
       # run afoul of this bug: http://youtrack.jetbrains.com/issue/PY-6477.
       # TODO: Remove the --egg once this is resolved. It won't work in the wheel world anyway.
+      # Use -f ${REPO_ROOT}/third_party if patching in local dependencies
+      # pip_extra="--egg -f ${REPO_ROOT}/third_party"
       pip_extra="--egg"
     else
       pip_extra=""

src/python/pants/backend/python/binary_builder.py

@@ -21,7 +21,7 @@ class PythonBinaryBuilder(object):
   class NotABinaryTargetException(Exception):
     pass
 
-  def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
+  def __init__(self, target, run_tracker, interpreter=None):
     self.target = target
     self.interpreter = interpreter or PythonInterpreter.get()
     if not isinstance(target, PythonBinary):
@@ -45,8 +45,7 @@ def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None):
         targets=[target],
         builder=builder,
         platforms=target.platforms,
-        interpreter=self.interpreter,
-        conn_timeout=conn_timeout)
+        interpreter=self.interpreter)
 
   def run(self):
     print('Building PythonBinary %s:' % self.target)

src/python/pants/backend/python/interpreter_cache.py

@@ -9,14 +9,15 @@
 from pkg_resources import Requirement
 import shutil
 
-
+from pex.archiver import Archiver
+from pex.crawler import Crawler
 from pex.installer import EggInstaller
 from pex.interpreter import PythonIdentity, PythonInterpreter
-from pex.obtainer import Obtainer
+from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage
 
 from pants.backend.python.python_setup import PythonSetup
-from pants.backend.python.resolver import crawler_from_config, fetchers_from_config
+from pants.backend.python.resolver import context_from_config, fetchers_from_config
 from pants.util.dirutil import safe_mkdir
 
 
@@ -39,33 +40,39 @@ def _resolve_interpreter(config, interpreter, requirement, logger=print):
     return interpreter
 
   def installer_provider(sdist):
-    return EggInstaller(sdist, strict=requirement.key != 'setuptools', interpreter=interpreter)
+    return EggInstaller(
+        Archiver.unpack(sdist),
+        strict=requirement.key != 'setuptools',
+        interpreter=interpreter)
 
   egg = _resolve_and_link(
       config,
       requirement,
       os.path.join(interpreter_dir, requirement.key),
       installer_provider,
       logger=logger)
+
   if egg:
-    return interpreter.with_extra(egg.name, egg.raw_version, egg.url)
+    return interpreter.with_extra(egg.name, egg.raw_version, egg.path)
   else:
     logger('Failed to resolve requirement %s for %s' % (requirement, interpreter))
 
 
 def _resolve_and_link(config, requirement, target_link, installer_provider, logger=print):
+  # Short-circuit if there is a local copy
   if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
     egg = EggPackage(os.path.realpath(target_link))
     if egg.satisfies(requirement):
       return egg
+
   fetchers = fetchers_from_config(config)
-  crawler = crawler_from_config(config)
-  obtainer = Obtainer(crawler, fetchers, [])
-  obtainer_iterator = obtainer.iter(requirement)
-  links = [link for link in obtainer_iterator if isinstance(link, SourcePackage)]
+  context = context_from_config(config)
+  iterator = Iterator(fetchers=fetchers, crawler=Crawler(context))
+  links = [link for link in iterator.iter(requirement) if isinstance(link, SourcePackage)]
+
   for link in links:
     logger('    fetching %s' % link.url)
-    sdist = link.fetch()
+    sdist = context.fetch(link)
     logger('    installing %s' % sdist)
     installer = installer_provider(sdist)
     dist_location = installer.bdist()

src/python/pants/backend/python/python_builder.py

@@ -17,9 +17,7 @@ class PythonBuilder(object):
   def __init__(self, run_tracker):
     self._run_tracker = run_tracker
 
-  def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=False,
-            debug=False):
-
+  def build(self, targets, args, interpreter=None, fast_tests=False, debug=False):
     test_targets = []
     binary_targets = []
     interpreter = interpreter or PythonInterpreter.get()
@@ -38,7 +36,6 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
         test_targets,
         args,
         interpreter=interpreter,
-        conn_timeout=conn_timeout,
         fast=fast_tests,
         debug=debug).run()
     if rv != 0:
@@ -48,8 +45,7 @@ def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=F
       rv = PythonBinaryBuilder(
           binary_target,
           self._run_tracker,
-          interpreter=interpreter,
-          conn_timeout=conn_timeout).run()
+          interpreter=interpreter).run()
       if rv != 0:
         return rv
 

src/python/pants/backend/python/python_chroot.py

@@ -60,16 +60,14 @@ def __init__(self,
                extra_requirements=None,
                builder=None,
                platforms=None,
-               interpreter=None,
-               conn_timeout=None):
+               interpreter=None):
     self._config = Config.from_cache()
     self._targets = targets
     self._extra_requirements = list(extra_requirements) if extra_requirements else []
     self._platforms = platforms
     self._interpreter = interpreter or PythonInterpreter.get()
     self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
                                           interpreter=self._interpreter)
-    self._conn_timeout = conn_timeout
 
     # Note: unrelated to the general pants artifact cache.
     self._egg_cache_root = os.path.join(
@@ -127,10 +125,9 @@ def copy_to_chroot(base, path, add_function):
                                resource=resources_tgt.address.spec))
           raise
 
-  def _dump_requirement(self, req, dynamic, repo):
-    self.debug('  Dumping requirement: %s%s%s' % (str(req),
-      ' (dynamic)' if dynamic else '', ' (repo: %s)' % repo if repo else ''))
-    self._builder.add_requirement(req, dynamic, repo)
+  def _dump_requirement(self, req):
+    self.debug('  Dumping requirement: %s' % req)
+    self._builder.add_requirement(req)
 
   def _dump_distribution(self, dist):
     self.debug('  Dumping distribution: .../%s' % os.path.basename(dist.location))
@@ -194,19 +191,23 @@ def dump(self):
         reqs_from_libraries.add(req)
 
     reqs_to_build = OrderedSet()
+    find_links = []
+
     for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
       if not req.should_build(self._interpreter.python, Platform.current()):
         self.debug('Skipping %s based upon version filter' % req)
         continue
       reqs_to_build.add(req)
-      self._dump_requirement(req._requirement, False, req._repository)
+      self._dump_requirement(req.requirement)
+      if req.repository:
+        find_links.append(req.repository)
 
     distributions = resolve_multi(
          self._config,
          reqs_to_build,
          interpreter=self._interpreter,
          platforms=self._platforms,
-         conn_timeout=self._conn_timeout)
+         find_links=find_links)
 
     locations = set()
     for platform, dist_set in distributions.items():

src/python/pants/backend/python/resolver.py

@@ -6,9 +6,10 @@
                         print_function, unicode_literals)
 
 from pex.fetcher import Fetcher, PyPIFetcher
-from pex.http import Crawler
+from pex.crawler import Crawler
+from pex.http import Context
 from pex.interpreter import PythonInterpreter
-from pex.obtainer import CachingObtainer
+from pex.iterator import Iterator
 from pex.platforms import Platform
 from pex.resolver import resolve
 from pex.translator import Translator
@@ -29,33 +30,17 @@ def fetchers_from_config(config):
   return fetchers
 
 
-def crawler_from_config(config, conn_timeout=None):
-  download_cache = PythonSetup(config).scratch_dir('download_cache', default_name='downloads')
-  return Crawler(cache=download_cache, conn_timeout=conn_timeout)
-
-
-class PantsObtainer(CachingObtainer):
-  def iter(self, requirement):
-    if hasattr(requirement, 'repository') and requirement.repository:
-      obtainer = CachingObtainer(
-          install_cache=self.install_cache,
-          ttl=self.ttl,
-          crawler=self._crawler,
-          fetchers=[Fetcher([requirement.repository])],
-          translators=self._translator)
-      for package in obtainer.iter(requirement):
-        yield package
-    else:
-      for package in super(PantsObtainer, self).iter(requirement):
-        yield package
+def context_from_config(config):
+  # TODO(wickman) Add retry, conn_timeout, threads, etc configuration here.
+  return Context.get()
 
 
 def resolve_multi(config,
                   requirements,
                   interpreter=None,
                   platforms=None,
-                  conn_timeout=None,
-                  ttl=3600):
+                  ttl=3600,
+                  find_links=None):
   """Multi-platform dependency resolution for PEX files.
 
      Given a pants configuration and a set of requirements, return a list of distributions
@@ -68,35 +53,31 @@ def resolve_multi(config,
                          If None specified, defaults to current interpreter.
      :param platforms: Optional list of platforms against requirements will be resolved. If
                          None specified, the defaults from `config` will be used.
-     :param conn_timeout: Optional connection timeout for any remote fetching.
      :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
                  "flask>=0.2" if a matching distribution is available on disk.  Defaults
                  to 3600.
+     :param find_links: Additional paths to search for source packages during resolution.
   """
   distributions = dict()
   interpreter = interpreter or PythonInterpreter.get()
   if not isinstance(interpreter, PythonInterpreter):
     raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
 
-  install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
+  cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
   platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
+  fetchers = fetchers_from_config(config)
+  if find_links:
+    fetchers.extend(Fetcher([path]) for path in find_links)
+  context = context_from_config(config)
 
   for platform in platforms:
-    translator = Translator.default(
-        install_cache=install_cache,
+    distributions[platform] = resolve(
+        requirements=requirements,
         interpreter=interpreter,
+        fetchers=fetchers,
         platform=platform,
-        conn_timeout=conn_timeout)
-
-    obtainer = PantsObtainer(
-        install_cache=install_cache,
-        crawler=crawler_from_config(config, conn_timeout=conn_timeout),
-        fetchers=fetchers_from_config(config) or [PyPIFetcher()],
-        translators=translator)
-
-    distributions[platform] = resolve(requirements=requirements,
-                                      obtainer=obtainer,
-                                      interpreter=interpreter,
-                                      platform=platform)
+        context=context,
+        cache=cache,
+        cache_ttl=ttl)
 
   return distributions

src/python/pants/backend/python/tasks/pytest_run.py

@@ -49,7 +49,6 @@ def is_python_test(target):
       test_builder = PythonTestBuilder(targets=test_targets,
                                        args=args,
                                        interpreter=self.interpreter,
-                                       conn_timeout=self.conn_timeout,
                                        fast=self.get_options().fast,
                                        debug=debug)
       with self.context.new_workunit(name='run',

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -54,8 +54,7 @@ def create_binary(self, binary):
         targets=[binary],
         builder=builder,
         platforms=binary.platforms,
-        interpreter=interpreter,
-        conn_timeout=self.conn_timeout)
+        interpreter=interpreter)
 
       pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
       chroot.dump()

src/python/pants/backend/python/tasks/python_repl.py

@@ -48,8 +48,7 @@ def execute(self):
           targets=targets,
           extra_requirements=extra_requirements,
           builder=builder,
-          interpreter=interpreter,
-          conn_timeout=self.conn_timeout)
+          interpreter=interpreter)
 
         chroot.dump()
         builder.freeze()

src/python/pants/backend/python/tasks/python_run.py

@@ -40,8 +40,7 @@ def execute(self):
           targets=[binary],
           builder=builder,
           platforms=binary.platforms,
-          interpreter=interpreter,
-          conn_timeout=self.conn_timeout)
+          interpreter=interpreter)
 
         chroot.dump()
         builder.freeze()

src/python/pants/backend/python/tasks/python_task.py

@@ -17,17 +17,8 @@
 
 
 class PythonTask(Task):
-  @classmethod
-  def register_options(cls, register):
-    super(PythonTask, cls).register_options(register)
-    register('--timeout', type=int, default=0,
-             help='Number of seconds to wait for http connections.')
-
   def __init__(self, *args, **kwargs):
     super(PythonTask, self).__init__(*args, **kwargs)
-    self.conn_timeout = (self.get_options().timeout or
-                         self.context.config.getdefault('connection_timeout'))
-
     self._compatibilities = self.get_options().interpreter or [b'']
     self._interpreter_cache = None
     self._interpreter = None
@@ -96,5 +87,3 @@ def temporary_pex_builder(self, interpreter=None, pex_info=None, parent_dir=None
     builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
     yield builder
     builder.chroot().delete()
-
-

src/python/pants/backend/python/test_builder.py

@@ -65,11 +65,10 @@ class PythonTestBuilder(object):
     PythonRequirement('unittest2py3k', version_filter=lambda py, pl: py.startswith('3'))
   ]
 
-  def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False, debug=False):
+  def __init__(self, targets, args, interpreter=None, fast=False, debug=False):
     self._targets = targets
     self._args = args
     self._interpreter = interpreter or PythonInterpreter.get()
-    self._conn_timeout = conn_timeout
 
     # If fast is true, we run all the tests in a single chroot. This is MUCH faster than
     # creating a chroot for each test target. However running each test separately is more
@@ -292,8 +291,7 @@ def _test_runner(self, targets, stdout, stderr):
       extra_requirements=self._TESTING_TARGETS,
       builder=builder,
       platforms=('current',),
-      interpreter=self._interpreter,
-      conn_timeout=self._conn_timeout)
+      interpreter=self._interpreter)
     try:
       builder = chroot.dump()
       builder.freeze()