diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index a733da334f9..632843ea411 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -16,6 +16,12 @@ RUN yum install -y \ libffi \ libffi-devel \ openssl-devel +# LLVM release tarballs are xz-compressed. +RUN yum install -y xz +# GCC requires these to compile. +RUN yum install -y \ + m4 \ + wget # By default, execute in an environment with python27 enabled. ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7", "--"] diff --git a/src/python/pants/backend/native/subsystems/BUILD b/src/python/pants/backend/native/subsystems/BUILD index 8db0b45aa03..e6f09bd301f 100644 --- a/src/python/pants/backend/native/subsystems/BUILD +++ b/src/python/pants/backend/native/subsystems/BUILD @@ -4,5 +4,9 @@ python_library( dependencies=[ 'src/python/pants/binaries:binary_util', + 'src/python/pants/binaries:execution_environment_mixin', + 'src/python/pants/subsystem', + 'src/python/pants/util:memo', + 'src/python/pants/util:osutil', ], ) diff --git a/src/python/pants/backend/native/subsystems/clang.py b/src/python/pants/backend/native/subsystems/clang.py new file mode 100644 index 00000000000..36458ad6acf --- /dev/null +++ b/src/python/pants/backend/native/subsystems/clang.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment + + +class Clang(NativeTool, ExecutionPathEnvironment): + options_scope = 'clang' + default_version = '6.0.0' + archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/native/subsystems/llvm.py b/src/python/pants/backend/native/subsystems/gcc.py similarity index 57% rename from src/python/pants/backend/native/subsystems/llvm.py rename to src/python/pants/backend/native/subsystems/gcc.py index 9c97eed26ec..18123e0286a 100644 --- a/src/python/pants/backend/native/subsystems/llvm.py +++ b/src/python/pants/backend/native/subsystems/gcc.py @@ -5,10 +5,16 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment -class LLVM(NativeTool): - options_scope = 'llvm' - default_version = '5.0.1' +class GCC(NativeTool, ExecutionPathEnvironment): + options_scope = 'gcc' + default_version = '7.3.0' archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py new file mode 100644 index 00000000000..c7d04d3b43f --- /dev/null +++ b/src/python/pants/backend/native/subsystems/native_toolchain.py @@ -0,0 +1,47 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from contextlib import contextmanager + +from pants.backend.native.subsystems.clang import Clang +from pants.backend.native.subsystems.gcc import GCC +from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils +from pants.binaries.execution_environment_mixin import ExecutionEnvironmentMixin +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_property +from pants.util.osutil import get_os_name, normalize_os_name + + +class NativeToolchain(Subsystem, ExecutionEnvironmentMixin): + + options_scope = 'native-toolchain' + + PLATFORM_SPECIFIC_TOOLCHAINS = { + # TODO(cosmicexplorer): 'darwin' should have everything here, but there's no + # open-source linker for OSX...yet. + 'darwin': [GCC, Clang], + 'linux': [GCC, Binutils, Clang], + } + + @classmethod + def _get_platform_specific_toolchains(cls): + normed_os_name = normalize_os_name(get_os_name()) + return cls.PLATFORM_SPECIFIC_TOOLCHAINS[normed_os_name] + + @classmethod + def subsystem_dependencies(cls): + prev = super(NativeToolchain, cls).subsystem_dependencies() + cur_platform_subsystems = cls._get_platform_specific_toolchains() + return prev + tuple(sub.scoped(cls) for sub in cur_platform_subsystems) + + @memoized_property + def _toolchain_instances(self): + cur_platform_subsystems = self._get_platform_specific_toolchains() + return [sub.scoped_instance(self) for sub in cur_platform_subsystems] + + def modify_environment(self, env): + return self.apply_successive_env_modifications(env, self._toolchain_instances) diff --git a/src/python/pants/backend/native/subsystems/platform_specific/__init__.py b/src/python/pants/backend/native/subsystems/platform_specific/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD b/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD new file mode 100644 index 00000000000..3fc007c5e77 --- /dev/null +++ b/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD @@ -0,0 +1,9 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies=[ + 'src/python/pants/binaries:binary_util', + 'src/python/pants/binaries:execution_environment_mixin', + ], +) diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/__init__.py b/src/python/pants/backend/native/subsystems/platform_specific/linux/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py b/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py new file mode 100644 index 00000000000..ef6667a8c4d --- /dev/null +++ b/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment + + +class Binutils(NativeTool, ExecutionPathEnvironment): + options_scope = 'binutils' + default_version = '2.30' + archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/python/subsystems/BUILD b/src/python/pants/backend/python/subsystems/BUILD index 67f8417b4b2..bb0f66369af 100644 --- a/src/python/pants/backend/python/subsystems/BUILD +++ b/src/python/pants/backend/python/subsystems/BUILD @@ -4,7 +4,10 @@ python_library( dependencies = [ '3rdparty/python:setuptools', + 'src/python/pants/binaries:execution_environment_mixin', + 'src/python/pants/backend/native', 'src/python/pants/option', 'src/python/pants/subsystem', + 'src/python/pants/util:memo', ], ) diff --git a/src/python/pants/backend/python/subsystems/python_dist_build_environment.py b/src/python/pants/backend/python/subsystems/python_dist_build_environment.py new file mode 100644 index 00000000000..0e8cdd4338d --- /dev/null +++ b/src/python/pants/backend/python/subsystems/python_dist_build_environment.py @@ -0,0 +1,37 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.binaries.execution_environment_mixin import ExecutionEnvironmentMixin +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_property + + +class PythonDistBuildEnvironment(Subsystem, ExecutionEnvironmentMixin): + + options_scope = 'python-dist-build-environment' + + @classmethod + def subsystem_dependencies(cls): + return super(PythonDistBuildEnvironment, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),) + + @memoized_property + def _native_toolchain(self): + return NativeToolchain.scoped_instance(self) + + def modify_environment(self, env): + env = self._native_toolchain.modify_environment(env) + # If we're going to be wrapping setup.py-based projects, we really should be + # doing it through a subclass of a distutils UnixCCompiler (in Lib/distutils + # in the CPython source) instead of hoping setup.py knows what to do. The + # default UnixCCompiler from distutils will build a 32/64-bit "fat binary" + # unless you use their undocumented ARCHFLAGS env var, and there may be more + # dragons later on. + env['CC'] = 'gcc' + env['CXX'] = 'g++' + env['ARCHFLAGS'] = '-arch x86_64' + return env diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index b1a93e12dfe..af7a3f78b0c 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -8,10 +8,12 @@ import glob import os import shutil +from contextlib import contextmanager from pex.interpreter import PythonInterpreter from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_dist_build_environment import PythonDistBuildEnvironment from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.tasks.pex_build_util import is_local_python_dist from pants.backend.python.tasks.setup_py import SetupPyRunner @@ -20,7 +22,9 @@ from pants.base.fingerprint_strategy import DefaultFingerprintStrategy from pants.build_graph.address import Address from pants.task.task import Task +from pants.util.contextutil import environment_as from pants.util.dirutil import safe_mkdir +from pants.util.memo import memoized_property class BuildLocalPythonDistributions(Task): @@ -39,18 +43,34 @@ def product_types(cls): def prepare(cls, options, round_manager): round_manager.require_data(PythonInterpreter) + @classmethod + def implementation_version(cls): + return super(BuildLocalPythonDistributions, cls).implementation_version() + [('BuildLocalPythonDistributions', 1)] + + @classmethod + def subsystem_dependencies(cls): + return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (PythonDistBuildEnvironment.scoped(cls),) + + # TODO: seriously consider subclassing UnixCCompiler as well as the build_ext + # command from distutils to control the compiler and linker invocations + # transparently instead of hoping distutils does the right thing. + @memoized_property + def _python_dist_build_environment(self): + return PythonDistBuildEnvironment.scoped_instance(self) + @property def cache_target_dirs(self): return True def execute(self): dist_targets = self.context.targets(is_local_python_dist) - build_graph = self.context.build_graph if dist_targets: with self.invalidated(dist_targets, fingerprint_strategy=DefaultFingerprintStrategy(), invalidate_dependents=True) as invalidation_check: + interpreter = self.context.products.get_data(PythonInterpreter) + for vt in invalidation_check.invalid_vts: if vt.target.dependencies: raise TargetDefinitionException( @@ -58,7 +78,7 @@ def execute(self): 'List any 3rd party requirements in the install_requirements argument ' 'of your setup function.' ) - self._create_dist(vt.target, vt.results_dir) + self._create_dist(vt.target, vt.results_dir, interpreter) for vt in invalidation_check.all_vts: dist = self._get_whl_from_dir(os.path.join(vt.results_dir, 'dist')) @@ -66,26 +86,30 @@ def execute(self): self._inject_synthetic_dist_requirements(dist, req_lib_addr) # Make any target that depends on the dist depend on the synthetic req_lib, # for downstream consumption. - for dependent in build_graph.dependents_of(vt.target.address): - build_graph.inject_dependency(dependent, req_lib_addr) - - def _create_dist(self, dist_tgt, dist_target_dir): - """Create a .whl file for the specified python_distribution target.""" - interpreter = self.context.products.get_data(PythonInterpreter) + for dependent in self.context.build_graph.dependents_of(vt.target.address): + self.context.build_graph.inject_dependency(dependent, req_lib_addr) + def _copy_sources(self, dist_tgt, dist_target_dir): # Copy sources and setup.py over to vt results directory for packaging. # NB: The directory structure of the destination directory needs to match 1:1 # with the directory structure that setup.py expects. - for src_relative_to_target_base in dist_tgt.sources_relative_to_target_base(): + all_sources = list(dist_tgt.sources_relative_to_target_base()) + for src_relative_to_target_base in all_sources: src_rel_to_results_dir = os.path.join(dist_target_dir, src_relative_to_target_base) safe_mkdir(os.path.dirname(src_rel_to_results_dir)) abs_src_path = os.path.join(get_buildroot(), dist_tgt.address.spec_path, src_relative_to_target_base) shutil.copyfile(abs_src_path, src_rel_to_results_dir) - # Build a whl using SetupPyRunner and return its absolute path. - setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter) - setup_runner.run() + + def _create_dist(self, dist_tgt, dist_target_dir, interpreter): + """Create a .whl file for the specified python_distribution target.""" + self.context.log.debug("dist_target_dir: '{}'".format(dist_target_dir)) + self._copy_sources(dist_tgt, dist_target_dir) + with self._python_dist_build_environment.execution_environment(): + # Build a whl using SetupPyRunner and return its absolute path. + setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter) + setup_runner.run() def _inject_synthetic_dist_requirements(self, dist, req_lib_addr): """Inject a synthetic requirements library that references a local wheel. diff --git a/src/python/pants/binaries/BUILD b/src/python/pants/binaries/BUILD index 6d618604be6..b888c57ed19 100644 --- a/src/python/pants/binaries/BUILD +++ b/src/python/pants/binaries/BUILD @@ -30,3 +30,11 @@ python_library( 'src/python/pants/util:memo', ], ) + +python_library( + name='execution_environment_mixin', + sources=['execution_environment_mixin.py'], + dependencies=[ + 'src/python/pants/util:contextutil', + ], +) diff --git a/src/python/pants/binaries/execution_environment_mixin.py b/src/python/pants/binaries/execution_environment_mixin.py new file mode 100644 index 00000000000..6e0e6f02b35 --- /dev/null +++ b/src/python/pants/binaries/execution_environment_mixin.py @@ -0,0 +1,58 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os +from contextlib import contextmanager + +from abc import abstractmethod + +from pants.util.contextutil import environment_as + + +def prepend_path(prev_env, path_var, new_entries): + prev_path = prev_env.get(path_var, None) + if prev_path is None: + return ':'.join(new_entries) + return ':'.join(new_entries + prev_path.split(':')) + + +class ExecutionEnvironmentMixin(object): + + @classmethod + def apply_successive_env_modifications(cls, env, environment_mixin_instances): + for env_modifier in environment_mixin_instances: + env = env_modifier.modify_environment(env) + return env + + # There's no reason this has to be just about the shell "environment", really + # just has to be a contextmanager. Implementing this method using a literal + # chroot, or VM image, or something might be really interesting to just + # completely sidestep the installation problem. + @contextmanager + def execution_environment(self, prev_env=None): + if prev_env is None: + prev_env = os.environ + env_copy = prev_env.copy() + new_env = self.modify_environment(env_copy) + + with environment_as(**new_env): + yield + + @abstractmethod + def modify_environment(self, env): pass + + +class ExecutionPathEnvironment(ExecutionEnvironmentMixin): + + @abstractmethod + def get_additional_paths(self): pass + + def modify_environment(self, env): + additional_paths = self.get_additional_paths() + new_path = prepend_path(env, 'PATH', additional_paths) + env['PATH'] = new_path + return env