Support de-vendoring for installs.

Now that pex vendors setuptools and wheel, we build / install sdists with
potentially fragile import semantics for these two distributions. In
particular, we trip up against this when trying to build cryptography.

Robustify installs by de-vendoring setuptools and wheel when used in an
install context. In order to support de-vendoring make the method of
import conditional on an environment variable instead of attempting to
re-write vendored imports at runtime.

Fixes pex-tool#661

pex/base.py

@@ -5,7 +5,11 @@
 
 from pex.compatibility import Iterable
 from pex.compatibility import string as compatibility_string
-from pex.third_party.pkg_resources import Requirement
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import Requirement  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import Requirement
 
 REQUIRED_ATTRIBUTES = (
     'extras',

pex/environment.py

@@ -19,15 +19,25 @@
 from pex.package import distribution_compatible
 from pex.pex_info import PexInfo
 from pex.platforms import Platform
-from pex.third_party.pkg_resources import (
+from pex.tracer import TRACER
+from pex.util import CacheHelper, DistributionHelper
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import (
     DistributionNotFound,
     Environment,
     Requirement,
     WorkingSet,
     find_distributions
-)
-from pex.tracer import TRACER
-from pex.util import CacheHelper, DistributionHelper
+  )  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import (
+    DistributionNotFound,
+    Environment,
+    Requirement,
+    WorkingSet,
+    find_distributions
+  )
 
 
 def _import_pkg_resources():

pex/finders.py

@@ -22,7 +22,10 @@
 import sys
 import zipimport
 
-import pex.third_party.pkg_resources as pkg_resources
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  import pkg_resources  # vendor:skip
+else:
+  import pex.third_party.pkg_resources as pkg_resources
 
 if sys.version_info >= (3, 3) and sys.implementation.name == "cpython":
   import importlib.machinery as importlib_machinery

pex/installer.py

@@ -36,15 +36,9 @@ def __init__(self, source_dir, interpreter=None, install_dir=None):
     """Create an installer from an unpacked source distribution in source_dir."""
     self._source_dir = source_dir
     self._install_tmp = install_dir or safe_mkdtemp()
+    self._interpreter = interpreter or PythonInterpreter.get()
     self._installed = None
 
-    from pex import vendor
-    self._interpreter = vendor.setup_interpreter(distributions=self.mixins,
-                                                 interpreter=interpreter or PythonInterpreter.get())
-    if not self._interpreter.satisfies(self.mixins):
-      raise self.IncapableInterpreter('Interpreter %s not capable of running %s' % (
-          self._interpreter.binary, self.__class__.__name__))
-
   @property
   def mixins(self):
     """Return a list of requirements to load into the setup script prior to invocation."""
@@ -59,34 +53,41 @@ def _setup_command(self):
     raise NotImplementedError
 
   @property
-  def bootstrap_script(self):
+  def setup_py_wrapper(self):
+    # NB: It would be more direct to just over-write setup.py by pre-pending the setuptools import.
+    # We cannot do this however because we would then run afoul of setup.py file in the wild with
+    # from __future__ imports. This mode of injecting the import works around that issue.
     return """
-import sys
-sys.path.insert(0, {root!r})
-
-# Expose vendored mixin path_items (setuptools, wheel, etc.) directly to the package's setup.py.
-from pex import third_party
-third_party.install(root={root!r}, expose={mixins!r})
+# We need to allow setuptools to monkeypatch distutils in case the underlying setup.py uses
+# distutils; otherwise, we won't have access to distutils commands installed vi the
+# `distutils.commands` `entrypoints` setup metadata (which is only supported by setuptools).
+# The prime example here is `bdist_wheel` offered by the wheel dist.
+import setuptools
 
 # Now execute the package's setup.py such that it sees itself as a setup.py executed via
 # `python setup.py ...`
+import sys
 __file__ = 'setup.py'
 sys.argv[0] = __file__
 with open(__file__, 'rb') as fp:
   exec(fp.read())
-""".format(root=third_party.isolated(), mixins=self.mixins)
+"""
 
   def run(self):
     if self._installed is not None:
       return self._installed
 
     with TRACER.timed('Installing %s' % self._install_tmp, V=2):
-      command = [self._interpreter.binary, '-sE', '-'] + self._setup_command()
+      env = self._interpreter.sanitized_environment()
+      env['PYTHONPATH'] = os.pathsep.join(third_party.expose(self.mixins))
+      env['__PEX_UNVENDORED__'] = '1'
+
+      command = [self._interpreter.binary, '-s', '-'] + self._setup_command()
       try:
         Executor.execute(command,
-                         env=self._interpreter.sanitized_environment(),
+                         env=env,
                          cwd=self._source_dir,
-                         stdin_payload=self.bootstrap_script.encode('ascii'))
+                         stdin_payload=self.setup_py_wrapper.encode('ascii'))
         self._installed = True
       except Executor.NonZeroExit as e:
         self._installed = False

pex/interpreter.py

@@ -22,9 +22,13 @@
     get_impl_ver,
     get_impl_version_info
 )
-from pex.third_party.pkg_resources import Distribution, Requirement
 from pex.tracer import TRACER
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import Distribution, Requirement  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import Distribution, Requirement
+
 try:
   from numbers import Integral
 except ImportError:

pex/package.py

@@ -9,9 +9,13 @@
 from pex.base import maybe_requirement
 from pex.link import Link
 from pex.pep425tags import get_supported
-from pex.third_party.pkg_resources import EGG_NAME, parse_version, safe_name, safe_version
 from pex.util import Memoizer
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import EGG_NAME, parse_version, safe_name, safe_version  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import EGG_NAME, parse_version, safe_name, safe_version
+
 
 class Package(Link):
   """Base class for named Python binary packages (e.g. source, egg, wheel)."""

pex/pex.py

@@ -8,7 +8,6 @@
 from distutils import sysconfig
 from site import USER_SITE
 
-import pex.third_party.pkg_resources as pkg_resources
 from pex import third_party
 from pex.bootstrap import Bootstrap
 from pex.common import die
@@ -22,11 +21,20 @@
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
 from pex.pex_info import PexInfo
-from pex.third_party.pkg_resources import EntryPoint, WorkingSet, find_distributions
 from pex.tracer import TRACER
 from pex.util import iter_pth_paths, merge_split, named_temporary_file
 from pex.variables import ENV
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  import pkg_resources  # vendor:skip
+else:
+  import pex.third_party.pkg_resources as pkg_resources
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import EntryPoint, WorkingSet, find_distributions  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import EntryPoint, WorkingSet, find_distributions
+
 
 class DevNull(object):
   def __init__(self):

pex/pex_builder.py

@@ -12,10 +12,14 @@
 from pex.finders import get_entry_point_from_console_script, get_script_from_distributions
 from pex.interpreter import PythonInterpreter
 from pex.pex_info import PexInfo
-from pex.third_party.pkg_resources import DefaultProvider, ZipProvider, get_provider
 from pex.tracer import TRACER
 from pex.util import CacheHelper, DistributionHelper
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import DefaultProvider, ZipProvider, get_provider  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import DefaultProvider, ZipProvider, get_provider
+
 BOOTSTRAP_DIR = '.bootstrap'
 
 LEGACY_BOOSTRAP_PKG = '_pex'
@@ -293,7 +297,11 @@ def _add_dist_zip(self, path, dist_name):
     # into an importable shape. We can do that by installing it into its own
     # wheel dir.
     if dist_name.endswith("whl"):
-      from pex.third_party.wheel.install import WheelFile
+      if "__PEX_UNVENDORED__" in __import__("os").environ:
+        from wheel.install import WheelFile  # vendor:skip
+      else:
+        from pex.third_party.wheel.install import WheelFile
+
       tmp = safe_mkdtemp()
       whltmp = os.path.join(tmp, dist_name)
       os.mkdir(whltmp)

pex/resolvable.py

@@ -13,7 +13,11 @@
 from pex.installer import InstallerBase, Packager
 from pex.package import Package
 from pex.resolver_options import ResolverOptionsBuilder, ResolverOptionsInterface
-from pex.third_party.pkg_resources import Requirement, safe_extra
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import Requirement, safe_extra  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import Requirement, safe_extra
 
 # Extract extras as specified per "declaring extras":
 # https://pythonhosted.org/setuptools/setuptools.html

pex/resolver.py

@@ -10,7 +10,6 @@
 from collections import namedtuple
 from contextlib import contextmanager
 
-import pex.third_party.pkg_resources as pkg_resources
 from pex.common import safe_mkdir
 from pex.fetcher import Fetcher
 from pex.interpreter import PythonInterpreter
@@ -20,10 +19,19 @@
 from pex.platforms import Platform
 from pex.resolvable import ResolvableRequirement, resolvables_from_iterable
 from pex.resolver_options import ResolverOptionsBuilder
-from pex.third_party.pkg_resources import safe_name
 from pex.tracer import TRACER
 from pex.util import DistributionHelper
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  import pkg_resources  # vendor:skip
+else:
+  import pex.third_party.pkg_resources as pkg_resources
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import safe_name  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import safe_name
+
 
 @contextmanager
 def patched_packing_env(env):

pex/resolver_options.py

@@ -10,9 +10,13 @@
 from pex.iterator import Iterator
 from pex.package import EggPackage, SourcePackage, WheelPackage
 from pex.sorter import Sorter
-from pex.third_party.pkg_resources import safe_name
 from pex.translator import ChainedTranslator, EggTranslator, SourceTranslator, WheelTranslator
 
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import safe_name  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import safe_name
+
 
 class ResolverOptionsInterface(object):
   def get_context(self):

pex/third_party/__init__.py

@@ -208,20 +208,35 @@ def install_vendored(cls, prefix, root=None, expose=None):
 
     if expose:
       # But only expose the bits needed.
-      path_by_key = OrderedDict((spec.key, spec.relpath) for spec in vendor.iter_vendor_specs()
-                                if spec.key in expose)
-      path_by_key['pex'] = root  # The pex distribution itself is trivially available to expose.
-
-      unexposed = set(expose) - set(path_by_key.keys())
-      if unexposed:
-        raise ValueError('The following vendored dists are not available to expose: {}'
-                         .format(', '.join(sorted(unexposed))))
-
-      exposed_paths = path_by_key.values()
-      for exposed_path in exposed_paths:
-        sys.path.insert(0, os.path.join(root, exposed_path))
+      exposed_paths = []
+      for path in cls.expose(expose, root):
+        sys.path.insert(0, path)
+        exposed_paths.append(os.path.relpath(path, root))
+
       vendor_importer._expose(exposed_paths)
 
+  @classmethod
+  def expose(cls, dists, root=None):
+    from pex import vendor
+
+    root = cls._abs_root(root)
+
+    def iter_available():
+      yield 'pex', root  # The pex distribution itself is trivially available to expose.
+      for spec in vendor.iter_vendor_specs():
+        yield spec.key, spec.relpath
+
+    path_by_key = OrderedDict((key, relpath) for key, relpath in iter_available() if key in dists)
+
+    unexposed = set(dists) - set(path_by_key.keys())
+    if unexposed:
+      raise ValueError('The following vendored dists are not available to expose: {}'
+                       .format(', '.join(sorted(unexposed))))
+
+    exposed_paths = path_by_key.values()
+    for exposed_path in exposed_paths:
+      yield os.path.join(root, exposed_path)
+
   @classmethod
   def install(cls, uninstallable, prefix, path_items, root=None, warning=None):
     """Install an importer for modules found under ``path_items`` at the given import ``prefix``.
@@ -396,5 +411,13 @@ def install(root=None, expose=None):
   VendorImporter.install_vendored(prefix=import_prefix(), root=root, expose=expose)
 
 
+def expose(dists):
+  from pex.common import safe_delete
+
+  for path in VendorImporter.expose(dists, root=isolated()):
+    safe_delete(os.path.join(path, '__init__.py'))
+    yield path
+
+
 # Implicitly install an importer for vendored code on the first import of pex.third_party.
 install()

pex/util.py

@@ -15,12 +15,21 @@
 from pex.common import rename_if_empty, safe_mkdir, safe_mkdtemp, safe_open
 from pex.compatibility import exec_function
 from pex.finders import register_finders
-from pex.third_party.pkg_resources import (
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from pkg_resources import (
+    find_distributions,
+    resource_isdir,
+    resource_listdir,
+    resource_string
+  )  # vendor:skip
+else:
+  from pex.third_party.pkg_resources import (
     find_distributions,
     resource_isdir,
     resource_listdir,
     resource_string
-)
+  )
 
 
 class DistributionHelper(object):

pex/vendor/__init__.py

@@ -143,7 +143,11 @@ def vendor_runtime(chroot, dest_basedir, label, root_module_names):
             pkg_file = os.path.join(pkg_path, '__init__.py')
             src = os.path.join(VendorSpec.ROOT, pkg_file)
             dest = os.path.join(dest_basedir, pkg_file)
-            chroot.copy(src, dest, label)
+            if os.path.exists(src):
+              chroot.copy(src, dest, label)
+            else:
+              # We delete `pex/vendor/_vendored/<dist>/__init__.py` when isolating third_party.
+              chroot.touch(dest, label)
           for name in vendored_names:
             vendor_module_names[name] = True
             TRACER.log('Vendoring {} from {} @ {}'.format(name, spec, spec.target_dir), V=3)