Upgrade to pex 1.6.1.

This allows us to get rid of resolving setuptools and wheel in our
interpreter cache.

Fixes pantsbuild#6927

3rdparty/python/requirements.txt

@@ -15,7 +15,7 @@ mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
-pex==1.5.3
+pex==1.6.1
 psutil==5.4.8
 pycodestyle==2.4.0
 pyflakes==2.0.0
@@ -30,7 +30,7 @@ requests[security]>=2.20.1
 responses==0.10.4
 scandir==1.2
 setproctitle==1.1.10
-setuptools==40.4.3
+setuptools==40.6.3
 six>=1.9.0,<2
 subprocess32==3.2.7 ; python_version<'3'
 wheel==0.31.1

contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py

@@ -9,5 +9,5 @@
 
 class Lambdex(PythonToolBase):
   options_scope = 'lambdex'
-  default_requirements = ['lambdex==0.1.2']
+  default_requirements = ['lambdex==0.1.3']
   default_entry_point = 'lambdex.bin.lambdex'

src/python/pants/backend/python/interpreter_cache.py

@@ -6,15 +6,11 @@
 
 import logging
 import os
-import shutil
-from builtins import map, str
+from builtins import str
 from collections import defaultdict
 
 from pex.interpreter import PythonInterpreter
-from pex.package import EggPackage, Package, SourcePackage
-from pex.resolver import resolve
 
-from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.exceptions import TaskError
@@ -27,24 +23,14 @@
 logger = logging.getLogger(__name__)
 
 
-# TODO(wickman) Create a safer version of this and add to twitter.common.dirutil
-def _safe_link(src, dst):
-  try:
-    os.unlink(dst)
-  except OSError:
-    pass
-  os.symlink(src, dst)
-
-
 # TODO: Move under subsystems/ .
 class PythonInterpreterCache(Subsystem):
   """Finds python interpreters on the local system."""
   options_scope = 'python-interpreter-cache'
 
   @classmethod
   def subsystem_dependencies(cls):
-    return (super(PythonInterpreterCache, cls).subsystem_dependencies() +
-            (PythonSetup, PythonRepos))
+    return super(PythonInterpreterCache, cls).subsystem_dependencies() + (PythonSetup,)
 
   class UnsatisfiableInterpreterConstraintsError(TaskError):
     """Indicates a python interpreter matching given constraints could not be located."""
@@ -63,10 +49,6 @@ def _matching(cls, interpreters, filters=()):
   def _python_setup(self):
     return PythonSetup.global_instance()
 
-  @memoized_property
-  def _python_repos(self):
-    return PythonRepos.global_instance()
-
   @memoized_property
   def _cache_dir(self):
     cache_dir = self._python_setup.interpreter_cache_dir
@@ -120,16 +102,16 @@ def _interpreter_from_path(self, path, filters=()):
       executable = os.readlink(os.path.join(path, 'python'))
     except OSError:
       return None
-    interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
+    interpreter = PythonInterpreter.from_binary(executable)
     if self._matches(interpreter, filters=filters):
-      return self._resolve(interpreter)
+      return interpreter
     return None
 
   def _setup_interpreter(self, interpreter, cache_target_path):
     with safe_concurrent_creation(cache_target_path) as safe_path:
       os.mkdir(safe_path)  # Parent will already have been created by safe_concurrent_creation.
       os.symlink(interpreter.binary, os.path.join(safe_path, 'python'))
-      return self._resolve(interpreter, safe_path)
+      return interpreter
 
   def _setup_cached(self, filters=()):
     """Find all currently-cached interpreters."""
@@ -190,64 +172,3 @@ def unsatisfied_filters():
     logger.debug(
       'Initialized Python interpreter cache with {}'.format(', '.join([x.binary for x in matches])))
     return matches
-
-  def _resolve(self, interpreter, interpreter_dir=None):
-    """Resolve and cache an interpreter with a setuptools and wheel capability."""
-    interpreter = self._resolve_interpreter(interpreter, interpreter_dir,
-                                            self._python_setup.setuptools_requirement())
-    if interpreter:
-      return self._resolve_interpreter(interpreter, interpreter_dir,
-                                       self._python_setup.wheel_requirement())
-
-  def _resolve_interpreter(self, interpreter, interpreter_dir, requirement):
-    """Given a :class:`PythonInterpreter` and a requirement, return an interpreter with the
-    capability of resolving that requirement or ``None`` if it's not possible to install a
-    suitable requirement.
-
-    If interpreter_dir is unspecified, operates on the default location.
-    """
-    if interpreter.satisfies([requirement]):
-      return interpreter
-
-    if not interpreter_dir:
-      interpreter_dir = os.path.join(self._cache_dir, str(interpreter.identity))
-
-    target_link = os.path.join(interpreter_dir, requirement.key)
-    bdist = self._resolve_and_link(interpreter, requirement, target_link)
-    if bdist:
-      return interpreter.with_extra(bdist.name, bdist.raw_version, bdist.path)
-    else:
-      logger.debug('Failed to resolve requirement {} for {}'.format(requirement, interpreter))
-
-  def _resolve_and_link(self, interpreter, requirement, target_link):
-    # Short-circuit if there is a local copy.
-    if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
-      bdist = Package.from_href(os.path.realpath(target_link))
-      if bdist.satisfies(requirement):
-        return bdist
-
-    # Since we're resolving to bootstrap a bare interpreter, we won't have wheel available.
-    # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into
-    # wheels.
-    precedence = (EggPackage, SourcePackage)
-    resolved_dists = resolve(requirements=[requirement],
-                             fetchers=self._python_repos.get_fetchers(),
-                             interpreter=interpreter,
-                             # The local interpreter cache is, by definition, composed of
-                             #  interpreters for the 'current' platform.
-                             platform='current',
-                             context=self._python_repos.get_network_context(),
-                             precedence=precedence)
-    if not resolved_dists:
-      return None
-
-    assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, '
-                                     'found:\n\t{}'.format(requirement,
-                                                           '\n\t'.join(map(str, resolved_dists))))
-
-    dist_location = resolved_dists[0].distribution.location
-    target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
-    shutil.move(dist_location, target_location)
-    _safe_link(target_location, target_link)
-    logger.debug('    installed {}'.format(target_location))
-    return Package.from_href(target_location)

src/python/pants/backend/python/subsystems/BUILD

@@ -5,7 +5,6 @@ python_library(
   dependencies = [
     '3rdparty/python:future',
     '3rdparty/python:pex',
-    '3rdparty/python:setuptools',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     'src/python/pants/base:build_environment',
     'src/python/pants/base:exceptions',

src/python/pants/backend/python/subsystems/pex_build_util.py

@@ -12,8 +12,10 @@
 from pex.fetcher import Fetcher
 from pex.pex_builder import PEXBuilder
 from pex.resolver import resolve
+from pex.util import DistributionHelper
 from twitter.common.collections import OrderedSet
 
+from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems.python_repos import PythonRepos
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
@@ -25,6 +27,7 @@
 from pants.base.exceptions import TaskError
 from pants.build_graph.files import Files
 from pants.subsystem.subsystem import Subsystem
+from pants.util.contextutil import temporary_file
 
 
 def is_python_target(tgt):
@@ -95,6 +98,13 @@ class PexBuilderWrapper(object):
   class Factory(Subsystem):
     options_scope = 'pex-builder-wrapper'
 
+    @classmethod
+    def register_options(cls, register):
+      super(PexBuilderWrapper.Factory, cls).register_options(register)
+      register('--setuptools-version', advanced=True, default='40.6.3',
+               help='The setuptools version to include in the pex if namespace packages need to be '
+                    'injected.')
+
     @classmethod
     def subsystem_dependencies(cls):
       return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + (
@@ -104,23 +114,38 @@ def subsystem_dependencies(cls):
 
     @classmethod
     def create(cls, builder, log=None):
+      options = cls.global_instance().get_options()
+      setuptools_requirement = 'setuptools=={}'.format(options.setuptools_version)
+
       log = log or logging.getLogger(__name__)
+
       return PexBuilderWrapper(builder=builder,
                                python_repos_subsystem=PythonRepos.global_instance(),
                                python_setup_subsystem=PythonSetup.global_instance(),
+                               setuptools_requirement=PythonRequirement(setuptools_requirement),
                                log=log)
 
-  def __init__(self, builder, python_repos_subsystem, python_setup_subsystem, log):
+  def __init__(self,
+               builder,
+               python_repos_subsystem,
+               python_setup_subsystem,
+               setuptools_requirement,
+               log):
     assert isinstance(builder, PEXBuilder)
     assert isinstance(python_repos_subsystem, PythonRepos)
     assert isinstance(python_setup_subsystem, PythonSetup)
+    assert isinstance(setuptools_requirement, PythonRequirement)
     assert log is not None
 
     self._builder = builder
     self._python_repos_subsystem = python_repos_subsystem
     self._python_setup_subsystem = python_setup_subsystem
+    self._setuptools_requirement = setuptools_requirement
     self._log = log
 
+    self._distributions = {}
+    self._frozen = False
+
   def add_requirement_libs_from(self, req_libs, platforms=None):
     """Multi-platform dependency resolution for PEX files.
 
@@ -148,7 +173,7 @@ def add_resolved_requirements(self, reqs, platforms=None):
     find_links = OrderedSet()
     for req in deduped_reqs:
       self._log.debug('  Dumping requirement: {}'.format(req))
-      self._builder.add_requirement(req.requirement)
+      self._builder.add_requirement(str(req.requirement))
       if req.repository:
         find_links.add(req.repository)
 
@@ -160,7 +185,7 @@ def add_resolved_requirements(self, reqs, platforms=None):
       for dist in dists:
         if dist.location not in locations:
           self._log.debug('  Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
-          self._builder.add_distribution(dist)
+          self.add_distribution(dist)
         locations.add(dist.location)
 
   def _resolve_multi(self, interpreter, requirements, platforms, find_links):
@@ -188,7 +213,7 @@ def _resolve_multi(self, interpreter, requirements, platforms, find_links):
       requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
         str(interpreter.identity))
       resolved_dists = resolve(
-        requirements=[req.requirement for req in requirements],
+        requirements=[str(req.requirement) for req in requirements],
         interpreter=interpreter,
         fetchers=fetchers,
         platform=platform,
@@ -222,13 +247,49 @@ def add_sources_from(self, tgt):
       raise TaskError('Old-style resources not supported for target {}.  '
                       'Depend on resources() targets instead.'.format(tgt.address.spec))
 
+  def _prepare_inits(self):
+    chroot = self._builder.chroot()
+    sources = chroot.get('source') | chroot.get('resource')
+
+    packages = set()
+    for source in sources:
+      if source.endswith('.py'):
+        pkg_dir = os.path.dirname(source)
+        if pkg_dir and pkg_dir not in packages:
+          package = ''
+          for component in pkg_dir.split(os.sep):
+            package = os.path.join(package, component)
+            packages.add(package)
+
+    missing_pkg_files = []
+    for package in packages:
+      pkg_file = os.path.join(package, '__init__.py')
+      if pkg_file not in sources:
+        missing_pkg_files.append(pkg_file)
+
+    if missing_pkg_files:
+      with temporary_file() as ns_package:
+        ns_package.write(b'__import__("pkg_resources").declare_namespace(__name__)')
+        ns_package.flush()
+        for missing_pkg_file in missing_pkg_files:
+          self._builder.add_source(ns_package.name, missing_pkg_file)
+
+    return missing_pkg_files
+
   def freeze(self):
-    self._builder.freeze()
+    if not self._frozen:
+      if self._prepare_inits():
+        dist = self._distributions.get('setuptools')
+        if not dist:
+          self.add_resolved_requirements([self._setuptools_requirement])
+      self._builder.freeze()
+      self._frozen = True
 
   def set_entry_point(self, entry_point):
     self._builder.set_entry_point(entry_point)
 
   def build(self, safe_path):
+    self.freeze()
     self._builder.build(safe_path)
 
   def set_shebang(self, shebang):
@@ -247,13 +308,19 @@ def add_interpreter_constraints_from(self, constraint_tgts):
 
   def add_direct_requirements(self, reqs):
     for req in reqs:
-      self._builder.add_requirement(req)
+      self._builder.add_requirement(str(req))
 
   def add_distribution(self, dist):
     self._builder.add_distribution(dist)
+    self._register_distribution(dist)
 
   def add_dist_location(self, location):
     self._builder.add_dist_location(location)
+    dist = DistributionHelper.distribution_from_path(location)
+    self._register_distribution(dist)
+
+  def _register_distribution(self, dist):
+    self._distributions[dist.key] = dist
 
   def set_script(self, script):
     self._builder.set_script(script)

src/python/pants/backend/python/subsystems/python_native_code.py

@@ -134,18 +134,16 @@ class BuildSetupRequiresPex(ExecutablePexTool):
   options_scope = 'build-setup-requires-pex'
 
   @classmethod
-  def subsystem_dependencies(cls):
-    return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup,)
-
-  @memoized_property
-  def python_setup(self):
-    return PythonSetup.global_instance()
+  def register_options(cls, register):
+    super(BuildSetupRequiresPex, cls).register_options(register)
+    register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3',
+             help='The setuptools version to use when executing `setup.py` scripts.')
+    register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3',
+             help='The wheel version to use when executing `setup.py` scripts.')
 
   @property
   def base_requirements(self):
-    # TODO: would we ever want to configure these requirement versions separately from the global
-    # PythonSetup values?
     return [
-      PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)),
-      PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)),
+      PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)),
+      PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)),
     ]