Introduce libc subsystem to find crti.o on linux hosts and unskip the…

… native backend subsystem tests (#5943)

### Problem

The native backend subsystems tests introduced in #5490 are still skipped, complaining about `crti.o` on linux, which is part of libc. See #5662 -- `clang`'s driver will find the directory containing that file on travis, but `gcc` won't. We should make a way to find this file (which is necessary for creating executables) so we can unskip the native backend testing.

### Solution

- Fix a mistake in #5780 -- we did not check the correct directory with `os.path.isdir()`, so we never found the `LLVM` BinaryTool when downloading it from the LLVM download page.
- Find libc using the new `LibcDev` subsystem. This uses the option `--libc-dir`, if provided, or finds an installation of libc with `crti.o` by invoking `--host-compiler` on the host once to get its search directories (this is necessary on travis, due to ubuntu's nonstandard installation location).
- Expand the definition of executables, compilers, and linkers in `src/python/pants/backend/native/config/environment.py` to cover everything needed to actually compile, and give them the ability to generate an environment suitable for passing into `subprocess.Popen()`.
- Introduce `GCCCCompiler`, `GCCCppCompiler`, `LLVMCCompiler`, and `LLVMCppCompiler` to differentiate between the two different compilers we have available for each language.
- Expose the libc lib directory to the compilers we create in `native_toolchain.py`.
- Unskip tests in `test_native_toolchain.py` from #5490.
- Make the default `CCompiler` and `CppCompiler` into clang, for no particular reason (it will pass CI with gcc as well).

The different compilers we can use will likely be denoted using variants in the future, but this solution allows us to separate the resources generated from each subsystem (`GCC`, `LLVM`, `Binutils`, `XCodeCLITools`) from a fully-functioning compiler that can be invoked (because actually invoking either clang or gcc requires some resources from the other, e.g. headers and libraries). Now, each binary tool only does the job of wrapping the resources it contains, while `native_toolchain.py` does the job of creating either a clang or a gcc compiler that we can invoke on the current host (with all necessary headers, libs, etc).

### Result

The native toolchain tests from #5490 can be unskipped, and we can invoke our toolchain on almost any linux installation without further setup. The tests in `test_native_toolchain.py` now pass in CI, and as we iterate on the native backend, we will continue to have end-to-end testing for both of our compilers, on osx as well, regardless of whichever one we choose to use for `python_dist()` compilation.

build-support/bin/ci.sh

@@ -18,7 +18,7 @@ Usage: $0 (-h|-fxbkmsrjlpuyncia)
  -f           skip python code formatting checks
  -x           skip bootstrap clean-all (assume bootstrapping from a
               fresh clone)
- -b           skip bootstraping pants from local sources
+ -b           skip bootstrapping pants from local sources
  -k           skip bootstrapped pants self compile check
  -m           skip sanity checks of bootstrapped pants and repo BUILD
               files

contrib/node/src/python/pants/contrib/node/subsystems/command.py

@@ -9,8 +9,8 @@
 import os
 from collections import namedtuple
 
-from pants.util.contextutil import get_joined_path
 from pants.util.process_handler import subprocess
+from pants.util.strutil import create_path_env_var
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def _prepare_env(self, kwargs):
     """
     kwargs = kwargs.copy()
     env = kwargs.pop('env', os.environ).copy()
-    env['PATH'] = get_joined_path(self.extra_paths, env=env, prepend=True)
+    env['PATH'] = create_path_env_var(self.extra_paths, env=env, prepend=True)
     return env, kwargs
 
   def run(self, **kwargs):

src/python/pants/backend/native/config/environment.py

@@ -5,11 +5,13 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
 from abc import abstractproperty
 
 from pants.engine.rules import SingletonRule
 from pants.util.objects import datatype
 from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
+from pants.util.strutil import create_path_env_var, safe_shlex_join
 
 
 class Platform(datatype(['normalized_os_name'])):
@@ -49,33 +51,136 @@ def path_entries(self):
     This may be multiple directories, e.g. if the main executable program invokes any subprocesses.
     """
 
+  @abstractproperty
+  def library_dirs(self):
+    """Directories containing shared libraries required for a subprocess to run."""
+
   @abstractproperty
   def exe_filename(self):
     """The "entry point" -- which file to invoke when PATH is set to `path_entries()`."""
 
+  def get_invocation_environment_dict(self, platform):
+    lib_env_var = platform.resolve_platform_specific({
+      'darwin': lambda: 'DYLD_LIBRARY_PATH',
+      'linux': lambda: 'LD_LIBRARY_PATH',
+    })
+    return {
+      'PATH': create_path_env_var(self.path_entries),
+      lib_env_var: create_path_env_var(self.library_dirs),
+    }
 
-class Linker(datatype([
+
+class Assembler(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
+    'library_dirs',
 ]), Executable):
   pass
 
 
-class CCompiler(datatype([
+class Linker(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
+    'library_dirs',
 ]), Executable):
-  pass
+
+  # FIXME(#5951): We need a way to compose executables more hygienically. This could be done
+  # declaratively -- something like: { 'LIBRARY_PATH': DelimitedPathDirectoryEnvVar(...) }.  We
+  # could also just use safe_shlex_join() and create_path_env_var() and keep all the state in the
+  # environment -- but then we have to remember to use those each time we specialize.
+  def get_invocation_environment_dict(self, platform):
+    ret = super(Linker, self).get_invocation_environment_dict(platform).copy()
+
+    # TODO: set all LDFLAGS in here or in further specializations of Linker instead of in individual
+    # tasks.
+    all_ldflags_for_platform = platform.resolve_platform_specific({
+      'darwin': lambda: ['-mmacosx-version-min=10.11'],
+      'linux': lambda: [],
+    })
+    ret.update({
+      'LDSHARED': self.exe_filename,
+      # FIXME: this overloads the meaning of 'library_dirs' to also mean "directories containing
+      # static libraries required for creating an executable" (currently, libc). These concepts
+      # should be distinct.
+      'LIBRARY_PATH': create_path_env_var(self.library_dirs),
+      'LDFLAGS': safe_shlex_join(all_ldflags_for_platform),
+    })
+
+    return ret
+
+
+class CompilerMixin(Executable):
+
+  @abstractproperty
+  def include_dirs(self):
+    """Directories to search for header files to #include during compilation."""
+
+  # FIXME: LIBRARY_PATH and (DY)?LD_LIBRARY_PATH are used for entirely different purposes, but are
+  # both sourced from the same `self.library_dirs`!
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CompilerMixin, self).get_invocation_environment_dict(platform).copy()
+
+    if self.include_dirs:
+      ret['CPATH'] = create_path_env_var(self.include_dirs)
+
+    all_cflags_for_platform = platform.resolve_platform_specific({
+      'darwin': lambda: ['-mmacosx-version-min=10.11'],
+      'linux': lambda: [],
+    })
+    ret['CFLAGS'] = safe_shlex_join(all_cflags_for_platform)
+
+    return ret
+
+
+class CCompiler(datatype([
+    'path_entries',
+    'exe_filename',
+    'library_dirs',
+    'include_dirs',
+]), CompilerMixin):
+
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CCompiler, self).get_invocation_environment_dict(platform).copy()
+
+    ret['CC'] = self.exe_filename
+
+    return ret
 
 
 class CppCompiler(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
-]), Executable):
-  pass
+    'library_dirs',
+    'include_dirs',
+]), CompilerMixin):
+
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CppCompiler, self).get_invocation_environment_dict(platform).copy()
+
+    ret['CXX'] = self.exe_filename
+
+    return ret
+
+
+# TODO(#4020): These classes are performing the work of variants.
+class GCCCCompiler(datatype([('c_compiler', CCompiler)])): pass
+
+
+class LLVMCCompiler(datatype([('c_compiler', CCompiler)])): pass
+
+
+class GCCCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass
+
+
+class LLVMCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass
+
+
+# FIXME: make this an @rule, after we can automatically produce LibcDev and other subsystems in the
+# v2 engine (see #5788).
+class HostLibcDev(datatype(['crti_object', 'fingerprint'])):
+
+  def get_lib_dir(self):
+    return os.path.dirname(self.crti_object)
 
 
 def create_native_environment_rules():

src/python/pants/backend/native/subsystems/BUILD

@@ -6,6 +6,7 @@ python_library(
   dependencies=[
     'src/python/pants/backend/native/config',
     'src/python/pants/backend/native/subsystems/binaries',
+    'src/python/pants/backend/native/subsystems/utils',
     'src/python/pants/engine:rules',
     'src/python/pants/engine:selectors',
     'src/python/pants/subsystem',

src/python/pants/backend/native/subsystems/binaries/BUILD

@@ -5,6 +5,7 @@
 python_library(
   dependencies=[
     'src/python/pants/backend/native/config',
+    'src/python/pants/backend/native/subsystems/utils',
     'src/python/pants/binaries',
     'src/python/pants/engine:rules',
     'src/python/pants/engine:selectors',

src/python/pants/backend/native/subsystems/binaries/binutils.py

@@ -7,7 +7,7 @@
 
 import os
 
-from pants.backend.native.config.environment import Linker, Platform
+from pants.backend.native.config.environment import Assembler, Linker
 from pants.binaries.binary_tool import NativeTool
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
@@ -21,20 +21,32 @@ class Binutils(NativeTool):
   def path_entries(self):
     return [os.path.join(self.select(), 'bin')]
 
-  def linker(self, platform):
+  def assembler(self):
+    return Assembler(
+      path_entries=self.path_entries(),
+      exe_filename='as',
+      library_dirs=[])
+
+  def linker(self):
     return Linker(
       path_entries=self.path_entries(),
       exe_filename='ld',
-      platform=platform)
+      library_dirs=[])
+
+
+@rule(Assembler, [Select(Binutils)])
+def get_as(binutils):
+  return binutils.assembler()
 
 
-@rule(Linker, [Select(Platform), Select(Binutils)])
-def get_ld(platform, binutils):
-  return binutils.linker(platform)
+@rule(Linker, [Select(Binutils)])
+def get_ld(binutils):
+  return binutils.linker()
 
 
 def create_binutils_rules():
   return [
+    get_as,
     get_ld,
     RootRule(Binutils),
   ]

src/python/pants/backend/native/subsystems/binaries/gcc.py

@@ -7,12 +7,14 @@
 
 import os
 
-from pants.backend.native.config.environment import CCompiler, CppCompiler, Platform
-from pants.backend.native.subsystems.binaries.binutils import Binutils
+from pants.backend.native.config.environment import (CCompiler, CppCompiler, GCCCCompiler,
+                                                     GCCCppCompiler)
+from pants.backend.native.subsystems.utils.parse_search_dirs import ParseSearchDirs
 from pants.binaries.binary_tool import NativeTool
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
-from pants.util.memo import memoized_property
+from pants.util.memo import memoized_method, memoized_property
+from pants.util.strutil import create_path_env_var
 
 
 class GCC(NativeTool):
@@ -22,46 +24,48 @@ class GCC(NativeTool):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(GCC, cls).subsystem_dependencies() + (Binutils.scoped(cls),)
+    return super(GCC, cls).subsystem_dependencies() + (ParseSearchDirs.scoped(cls),)
 
   @memoized_property
-  def _binutils(self):
-    return Binutils.scoped_instance(self)
-
-  def _path_entries_for_platform(self, platform):
-    # GCC requires an assembler 'as' to be on the path. We need to provide this on linux, so we pull
-    # it from our Binutils package.
-    as_assembler_path_entries = platform.resolve_platform_specific({
-      'darwin': lambda: [],
-      'linux': lambda: self._binutils.path_entries(),
-    })
-    all_path_entries = self.path_entries() + as_assembler_path_entries
-    return all_path_entries
+  def _parse_search_dirs(self):
+    return ParseSearchDirs.scoped_instance(self)
 
+  def _lib_search_dirs(self, compiler_exe, path_entries):
+    return self._parse_search_dirs.get_compiler_library_dirs(
+      compiler_exe,
+      env={'PATH': create_path_env_var(path_entries)})
+
+  @memoized_method
   def path_entries(self):
     return [os.path.join(self.select(), 'bin')]
 
-  def c_compiler(self, platform):
+  def c_compiler(self):
+    exe_filename = 'gcc'
+    path_entries = self.path_entries()
     return CCompiler(
-      path_entries=self._path_entries_for_platform(platform),
-      exe_filename='gcc',
-      platform=platform)
-
-  def cpp_compiler(self, platform):
+      path_entries=path_entries,
+      exe_filename=exe_filename,
+      library_dirs=self._lib_search_dirs(exe_filename, path_entries),
+      include_dirs=[])
+
+  def cpp_compiler(self):
+    exe_filename = 'g++'
+    path_entries = self.path_entries()
     return CppCompiler(
-      path_entries=self._path_entries_for_platform(platform),
-      exe_filename='g++',
-      platform=platform)
+      path_entries=self.path_entries(),
+      exe_filename=exe_filename,
+      library_dirs=self._lib_search_dirs(exe_filename, path_entries),
+      include_dirs=[])
 
 
-@rule(CCompiler, [Select(Platform), Select(GCC)])
-def get_gcc(platform, gcc):
-  yield gcc.c_compiler(platform)
+@rule(GCCCCompiler, [Select(GCC)])
+def get_gcc(gcc):
+  yield GCCCCompiler(gcc.c_compiler())
 
 
-@rule(CppCompiler, [Select(Platform), Select(GCC)])
-def get_gplusplus(platform, gcc):
-  yield gcc.cpp_compiler(platform)
+@rule(GCCCppCompiler, [Select(GCC)])
+def get_gplusplus(gcc):
+  yield GCCCppCompiler(gcc.cpp_compiler())
 
 
 def create_gcc_rules():