Skip to content

Commit

Permalink
Introduce libc subsystem to find crti.o on linux hosts and unskip the…
Browse files Browse the repository at this point in the history
… native backend subsystem tests (#5943)

### Problem

The native backend subsystems tests introduced in #5490 are still skipped, complaining about `crti.o` on linux, which is part of libc. See #5662 -- `clang`'s driver will find the directory containing that file on travis, but `gcc` won't. We should make a way to find this file (which is necessary for creating executables) so we can unskip the native backend testing.

### Solution

- Fix a mistake in #5780 -- we did not check the correct directory with `os.path.isdir()`, so we never found the `LLVM` BinaryTool when downloading it from the LLVM download page.
- Find libc using the new `LibcDev` subsystem. This uses the option `--libc-dir`, if provided, or finds an installation of libc with `crti.o` by invoking `--host-compiler` on the host once to get its search directories (this is necessary on travis, due to ubuntu's nonstandard installation location).
- Expand the definition of executables, compilers, and linkers in `src/python/pants/backend/native/config/environment.py` to cover everything needed to actually compile, and give them the ability to generate an environment suitable for passing into `subprocess.Popen()`.
- Introduce `GCCCCompiler`, `GCCCppCompiler`, `LLVMCCompiler`, and `LLVMCppCompiler` to differentiate between the two different compilers we have available for each language.
- Expose the libc lib directory to the compilers we create in `native_toolchain.py`.
- Unskip tests in `test_native_toolchain.py` from #5490.
- Make the default `CCompiler` and `CppCompiler` into clang, for no particular reason (it will pass CI with gcc as well).

The different compilers we can use will likely be denoted using variants in the future, but this solution allows us to separate the resources generated from each subsystem (`GCC`, `LLVM`, `Binutils`, `XCodeCLITools`) from a fully-functioning compiler that can be invoked (because actually invoking either clang or gcc requires some resources from the other, e.g. headers and libraries). Now, each binary tool only does the job of wrapping the resources it contains, while `native_toolchain.py` does the job of creating either a clang or a gcc compiler that we can invoke on the current host (with all necessary headers, libs, etc).

### Result

The native toolchain tests from #5490 can be unskipped, and we can invoke our toolchain on almost any linux installation without further setup. The tests in `test_native_toolchain.py` now pass in CI, and as we iterate on the native backend, we will continue to have end-to-end testing for both of our compilers, on osx as well, regardless of whichever one we choose to use for `python_dist()` compilation.
  • Loading branch information
cosmicexplorer authored Jul 2, 2018
1 parent 2870ed4 commit 1d65d4a
Show file tree
Hide file tree
Showing 27 changed files with 1,064 additions and 185 deletions.
2 changes: 1 addition & 1 deletion build-support/bin/ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Usage: $0 (-h|-fxbkmsrjlpuyncia)
-f skip python code formatting checks
-x skip bootstrap clean-all (assume bootstrapping from a
fresh clone)
-b skip bootstraping pants from local sources
-b skip bootstrapping pants from local sources
-k skip bootstrapped pants self compile check
-m skip sanity checks of bootstrapped pants and repo BUILD
files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
import os
from collections import namedtuple

from pants.util.contextutil import get_joined_path
from pants.util.process_handler import subprocess
from pants.util.strutil import create_path_env_var


logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -41,7 +41,7 @@ def _prepare_env(self, kwargs):
"""
kwargs = kwargs.copy()
env = kwargs.pop('env', os.environ).copy()
env['PATH'] = get_joined_path(self.extra_paths, env=env, prepend=True)
env['PATH'] = create_path_env_var(self.extra_paths, env=env, prepend=True)
return env, kwargs

def run(self, **kwargs):
Expand Down
121 changes: 113 additions & 8 deletions src/python/pants/backend/native/config/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

import os
from abc import abstractproperty

from pants.engine.rules import SingletonRule
from pants.util.objects import datatype
from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
from pants.util.strutil import create_path_env_var, safe_shlex_join


class Platform(datatype(['normalized_os_name'])):
Expand Down Expand Up @@ -49,33 +51,136 @@ def path_entries(self):
This may be multiple directories, e.g. if the main executable program invokes any subprocesses.
"""

@abstractproperty
def library_dirs(self):
"""Directories containing shared libraries required for a subprocess to run."""

@abstractproperty
def exe_filename(self):
"""The "entry point" -- which file to invoke when PATH is set to `path_entries()`."""

def get_invocation_environment_dict(self, platform):
lib_env_var = platform.resolve_platform_specific({
'darwin': lambda: 'DYLD_LIBRARY_PATH',
'linux': lambda: 'LD_LIBRARY_PATH',
})
return {
'PATH': create_path_env_var(self.path_entries),
lib_env_var: create_path_env_var(self.library_dirs),
}

class Linker(datatype([

class Assembler(datatype([
'path_entries',
'exe_filename',
('platform', Platform),
'library_dirs',
]), Executable):
pass


class CCompiler(datatype([
class Linker(datatype([
'path_entries',
'exe_filename',
('platform', Platform),
'library_dirs',
]), Executable):
pass

# FIXME(#5951): We need a way to compose executables more hygienically. This could be done
# declaratively -- something like: { 'LIBRARY_PATH': DelimitedPathDirectoryEnvVar(...) }. We
# could also just use safe_shlex_join() and create_path_env_var() and keep all the state in the
# environment -- but then we have to remember to use those each time we specialize.
def get_invocation_environment_dict(self, platform):
ret = super(Linker, self).get_invocation_environment_dict(platform).copy()

# TODO: set all LDFLAGS in here or in further specializations of Linker instead of in individual
# tasks.
all_ldflags_for_platform = platform.resolve_platform_specific({
'darwin': lambda: ['-mmacosx-version-min=10.11'],
'linux': lambda: [],
})
ret.update({
'LDSHARED': self.exe_filename,
# FIXME: this overloads the meaning of 'library_dirs' to also mean "directories containing
# static libraries required for creating an executable" (currently, libc). These concepts
# should be distinct.
'LIBRARY_PATH': create_path_env_var(self.library_dirs),
'LDFLAGS': safe_shlex_join(all_ldflags_for_platform),
})

return ret


class CompilerMixin(Executable):

@abstractproperty
def include_dirs(self):
"""Directories to search for header files to #include during compilation."""

# FIXME: LIBRARY_PATH and (DY)?LD_LIBRARY_PATH are used for entirely different purposes, but are
# both sourced from the same `self.library_dirs`!
def get_invocation_environment_dict(self, platform):
ret = super(CompilerMixin, self).get_invocation_environment_dict(platform).copy()

if self.include_dirs:
ret['CPATH'] = create_path_env_var(self.include_dirs)

all_cflags_for_platform = platform.resolve_platform_specific({
'darwin': lambda: ['-mmacosx-version-min=10.11'],
'linux': lambda: [],
})
ret['CFLAGS'] = safe_shlex_join(all_cflags_for_platform)

return ret


class CCompiler(datatype([
'path_entries',
'exe_filename',
'library_dirs',
'include_dirs',
]), CompilerMixin):

def get_invocation_environment_dict(self, platform):
ret = super(CCompiler, self).get_invocation_environment_dict(platform).copy()

ret['CC'] = self.exe_filename

return ret


class CppCompiler(datatype([
'path_entries',
'exe_filename',
('platform', Platform),
]), Executable):
pass
'library_dirs',
'include_dirs',
]), CompilerMixin):

def get_invocation_environment_dict(self, platform):
ret = super(CppCompiler, self).get_invocation_environment_dict(platform).copy()

ret['CXX'] = self.exe_filename

return ret


# TODO(#4020): These classes are performing the work of variants.
class GCCCCompiler(datatype([('c_compiler', CCompiler)])): pass


class LLVMCCompiler(datatype([('c_compiler', CCompiler)])): pass


class GCCCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass


class LLVMCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass


# FIXME: make this an @rule, after we can automatically produce LibcDev and other subsystems in the
# v2 engine (see #5788).
class HostLibcDev(datatype(['crti_object', 'fingerprint'])):

def get_lib_dir(self):
return os.path.dirname(self.crti_object)


def create_native_environment_rules():
Expand Down
1 change: 1 addition & 0 deletions src/python/pants/backend/native/subsystems/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ python_library(
dependencies=[
'src/python/pants/backend/native/config',
'src/python/pants/backend/native/subsystems/binaries',
'src/python/pants/backend/native/subsystems/utils',
'src/python/pants/engine:rules',
'src/python/pants/engine:selectors',
'src/python/pants/subsystem',
Expand Down
1 change: 1 addition & 0 deletions src/python/pants/backend/native/subsystems/binaries/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
python_library(
dependencies=[
'src/python/pants/backend/native/config',
'src/python/pants/backend/native/subsystems/utils',
'src/python/pants/binaries',
'src/python/pants/engine:rules',
'src/python/pants/engine:selectors',
Expand Down
24 changes: 18 additions & 6 deletions src/python/pants/backend/native/subsystems/binaries/binutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import os

from pants.backend.native.config.environment import Linker, Platform
from pants.backend.native.config.environment import Assembler, Linker
from pants.binaries.binary_tool import NativeTool
from pants.engine.rules import RootRule, rule
from pants.engine.selectors import Select
Expand All @@ -21,20 +21,32 @@ class Binutils(NativeTool):
def path_entries(self):
return [os.path.join(self.select(), 'bin')]

def linker(self, platform):
def assembler(self):
return Assembler(
path_entries=self.path_entries(),
exe_filename='as',
library_dirs=[])

def linker(self):
return Linker(
path_entries=self.path_entries(),
exe_filename='ld',
platform=platform)
library_dirs=[])


@rule(Assembler, [Select(Binutils)])
def get_as(binutils):
return binutils.assembler()


@rule(Linker, [Select(Platform), Select(Binutils)])
def get_ld(platform, binutils):
return binutils.linker(platform)
@rule(Linker, [Select(Binutils)])
def get_ld(binutils):
return binutils.linker()


def create_binutils_rules():
return [
get_as,
get_ld,
RootRule(Binutils),
]
66 changes: 35 additions & 31 deletions src/python/pants/backend/native/subsystems/binaries/gcc.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@

import os

from pants.backend.native.config.environment import CCompiler, CppCompiler, Platform
from pants.backend.native.subsystems.binaries.binutils import Binutils
from pants.backend.native.config.environment import (CCompiler, CppCompiler, GCCCCompiler,
GCCCppCompiler)
from pants.backend.native.subsystems.utils.parse_search_dirs import ParseSearchDirs
from pants.binaries.binary_tool import NativeTool
from pants.engine.rules import RootRule, rule
from pants.engine.selectors import Select
from pants.util.memo import memoized_property
from pants.util.memo import memoized_method, memoized_property
from pants.util.strutil import create_path_env_var


class GCC(NativeTool):
Expand All @@ -22,46 +24,48 @@ class GCC(NativeTool):

@classmethod
def subsystem_dependencies(cls):
return super(GCC, cls).subsystem_dependencies() + (Binutils.scoped(cls),)
return super(GCC, cls).subsystem_dependencies() + (ParseSearchDirs.scoped(cls),)

@memoized_property
def _binutils(self):
return Binutils.scoped_instance(self)

def _path_entries_for_platform(self, platform):
# GCC requires an assembler 'as' to be on the path. We need to provide this on linux, so we pull
# it from our Binutils package.
as_assembler_path_entries = platform.resolve_platform_specific({
'darwin': lambda: [],
'linux': lambda: self._binutils.path_entries(),
})
all_path_entries = self.path_entries() + as_assembler_path_entries
return all_path_entries
def _parse_search_dirs(self):
return ParseSearchDirs.scoped_instance(self)

def _lib_search_dirs(self, compiler_exe, path_entries):
return self._parse_search_dirs.get_compiler_library_dirs(
compiler_exe,
env={'PATH': create_path_env_var(path_entries)})

@memoized_method
def path_entries(self):
return [os.path.join(self.select(), 'bin')]

def c_compiler(self, platform):
def c_compiler(self):
exe_filename = 'gcc'
path_entries = self.path_entries()
return CCompiler(
path_entries=self._path_entries_for_platform(platform),
exe_filename='gcc',
platform=platform)

def cpp_compiler(self, platform):
path_entries=path_entries,
exe_filename=exe_filename,
library_dirs=self._lib_search_dirs(exe_filename, path_entries),
include_dirs=[])

def cpp_compiler(self):
exe_filename = 'g++'
path_entries = self.path_entries()
return CppCompiler(
path_entries=self._path_entries_for_platform(platform),
exe_filename='g++',
platform=platform)
path_entries=self.path_entries(),
exe_filename=exe_filename,
library_dirs=self._lib_search_dirs(exe_filename, path_entries),
include_dirs=[])


@rule(CCompiler, [Select(Platform), Select(GCC)])
def get_gcc(platform, gcc):
yield gcc.c_compiler(platform)
@rule(GCCCCompiler, [Select(GCC)])
def get_gcc(gcc):
yield GCCCCompiler(gcc.c_compiler())


@rule(CppCompiler, [Select(Platform), Select(GCC)])
def get_gplusplus(platform, gcc):
yield gcc.cpp_compiler(platform)
@rule(GCCCppCompiler, [Select(GCC)])
def get_gplusplus(gcc):
yield GCCCppCompiler(gcc.cpp_compiler())


def create_gcc_rules():
Expand Down
Loading

0 comments on commit 1d65d4a

Please sign in to comment.