remove source copying contextmanager and set clang arch for setup.py

src/python/pants/backend/python/subsystems/python_native_toolchain.py

@@ -5,17 +5,65 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
 from contextlib import contextmanager
 
+from pex.interpreter import PythonInterpreter
+
 from pants.binaries.binary_util import BinaryUtil
+from pants.fs.archive import TGZ
 from pants.subsystem.subsystem import Subsystem
-from pants.util.memo import memoized_property
+from pants.util.contextutil import temporary_dir, environment_as
+from pants.util.memo import memoized_method
+
+
+class SandboxedInterpreter(PythonInterpreter):
+  """???"""
+
+  class BinaryDirectoryError(Exception):
+    def __init__(self, dir_path):
+      msg = "path '{}' does not exist or is not a directory".format(dir_path)
+      super(BinaryDirectoryError, self).__init__(msg)
+
+  class BaseInterpreterError(Exception): pass
+
+  def __init__(self, clang_bin_dir_path, base_interp):
+
+    if not os.path.isdir(clang_bin_dir_path):
+      raise BinaryDirectoryError(clang_bin_dir_path)
+    if not isinstance(base_interp, PythonInterpreter):
+      raise BaseInterpreterError("invalid PythonInterpreter: '{}'".format(repr(base_interp)))
+
+    self._clang_bin_dir_path = clang_bin_dir_path
+
+    super(SandboxedInterpreter, self).__init__(
+      base_interp.binary, base_interp.identity, extras=base_interp.extras)
+
+  # made into an instance method here to use self._clang_bin_dir_path
+  def sanitized_environment(self):
+    pre_sanitized_env = super(SandboxedInterpreter, self).sanitized_environment()
+    pre_sanitized_env['PATH'] = self._clang_bin_dir_path
+    # TODO: see Lib/distutils/sysconfig.py and Lib/_osx_support.py in CPython.
+    # this line tells distutils to only compile for 64-bit archs -- if not, it
+    # will attempt to build a fat binary for 32- and 64-bit archs, which makes
+    # clang invoke "lipo", an osx command which does not appear to be open
+    # source.
+    pre_sanitized_env['ARCHFLAGS'] = '-arch x86_64'
+    for env_var in ['CC', 'CXX']:
+      pre_sanitized_env.pop(env_var, None)
+    return pre_sanitized_env
 
 
 class PythonNativeToolchain(object):
   """Represents a self-boostrapping set of binaries and libraries used to
   compile native code in for python dists."""
 
+  class InvalidToolRequest(Exception):
+
+    def __init__(self, rel_path_requested):
+      msg = "relative path '{}' does not exist in the python native toolchain".format(rel_path_requested)
+      super(InvalidToolRequest, self).__init__(msg)
+
   class Factory(Subsystem):
     options_scope = 'python-native-toolchain'
 
@@ -35,7 +83,7 @@ def register_options(cls, register):
                help='Clang version used to compile python native extensions.  '
                     'Used as part of the path to lookup the distribution '
                     'with --binary-util-baseurls and --pants-bootstrapdir',
-               default='5.0.0')
+               default='5.0.1')
 
     # NB: create() is an instance method to allow the user to choose global or
     # scoped -- It's not unreasonable to imagine different stacks for different
@@ -53,15 +101,22 @@ def __init__(self, binary_util, relpath, clang_version):
     self._relpath = relpath
     self._clang_version = clang_version
 
-  @property
-  def clang_version(self):
-    return self._clang_version
-
-  @memoized_property
-  def cpp_compiler_path(self):
-    return self._binary_util.select_binary(
-      self._relpath, self.clang_version, 'clang')
+  @memoized_method
+  def _clang_llvm_distribution_base(self):
+    clang_archive_path = self._binary_util.select_binary(
+      self._relpath, self._clang_version, 'clang+llvm.tar.gz')
+    distribution_workdir = os.path.dirname(clang_archive_path)
+    outdir = os.path.join(distribution_workdir, 'unpacked')
+    if not os.path.exists(outdir):
+      with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
+        TGZ.extract(clang_archive_path, tmp_dist)
+        os.rename(tmp_dist, outdir)
+    return outdir
 
-  @contextmanager
-  def within_setuppy_compile_sandbox(self):
-    yield
+  @memoized_method
+  def clang_bin_dir_path(self):
+    dist_base = self._clang_llvm_distribution_base()
+    bin_dir_path = os.path.join(dist_base, 'bin')
+    if not os.path.exists(bin_dir_path):
+      raise InvalidToolRequest('bin')
+    return bin_dir_path

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -13,7 +13,7 @@
 
 from pex.interpreter import PythonInterpreter
 
-from pants.backend.python.subsystems.python_native_toolchain import PythonNativeToolchain
+from pants.backend.python.subsystems.python_native_toolchain import PythonNativeToolchain, SandboxedInterpreter
 from pants.backend.python.tasks.pex_build_util import is_local_python_dist
 from pants.backend.python.tasks.setup_py import SetupPyRunner
 from pants.base.build_environment import get_buildroot
@@ -98,8 +98,7 @@ def execute(self):
 
     self.context.products.register_data(self.PYTHON_DISTS, built_dists)
 
-  @contextmanager
-  def _with_copied_sources_gen_pantssetup_pythonpath(self, dist_tgt, dist_target_dir):
+  def _copy_sources(self, dist_tgt, dist_target_dir):
     # Copy sources and setup.py over to vt results directory for packaging.
     # NB: The directory structure of the destination directory needs to match 1:1
     # with the directory structure that setup.py expects.
@@ -112,44 +111,18 @@ def _with_copied_sources_gen_pantssetup_pythonpath(self, dist_tgt, dist_target_d
                                   dist_tgt.address.spec_path,
                                   src_relative_to_target_base)
       shutil.copyfile(abs_src_path, src_rel_to_results_dir)
-    with temporary_dir() as tmpdir:
-      # native_sources_joined = ','.join("'{}'".format(x) for x in native_sources)
-      # pantssetup_import_contents = PANTSSETUP_IMPORT_BOILERPLATE.format(
-      #   setup_target=repr(dist_tgt),
-      #   native_sources_joined=native_sources_joined)
-
-      # pantssetup_module_path = os.path.join(tmpdir, 'pantssetup.py')
-
-      # self.context.log.info(repr(dist_tgt))
-      # self.context.log.info('payload fingerprint: {}'.format(dist_tgt.payload.fingerprint()))
-      # self.context.log.info(pantssetup_import_contents)
-      # self.context.log.info('pantssetup_module_path: {}'.format(pantssetup_module_path))
-
-      # with open(pantssetup_module_path, 'w') as pantssetup_module_fh:
-      #   pantssetup_module_fh.write(pantssetup_import_contents)
-
-      # prev_pypath = os.environ.get('PYTHONPATH')
-      # if prev_pypath is None:
-      #   new_pypath = tmpdir
-      # else:
-      #   scrubbed_pypath = re.sub(':$', '', prev_pypath)
-      #   new_pypath = '{}:{}'.format(scrubbed_pypath, tmpdir)
-
-      # self.context.log.info('new_pypath: {}'.format(repr(new_pypath)))
-
-      # with environment_as(PYTHONPATH=new_pypath):
-      #   yield
-      yield
 
   def _create_dist(self, dist_tgt, dist_target_dir):
     """Create a .whl file for the specified python_distribution target."""
     self.context.log.info('dist_target_dir: {}'.format(dist_target_dir))
     interpreter = self.context.products.get_data(PythonInterpreter)
-    with self._with_copied_sources_gen_pantssetup_pythonpath(dist_tgt, dist_target_dir):
-      with self.python_native_toolchain.within_setuppy_compile_sandbox():
-        # Build a whl using SetupPyRunner and return its absolute path.
-        setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter)
-        setup_runner.run()
+    sandboxed_interpreter = SandboxedInterpreter(
+      self.python_native_toolchain.clang_bin_dir_path(), interpreter)
+    self._copy_sources(dist_tgt, dist_target_dir)
+    # Build a whl using SetupPyRunner and return its absolute path.
+    setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=sandboxed_interpreter)
+    # setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter)
+    setup_runner.run()
     return self._get_whl_from_dir(os.path.join(dist_target_dir, 'dist'))
 
   def _get_whl_from_dir(self, install_dir):