Introduce libc subsystem to find crti.o on linux hosts and unskip the native backend subsystem tests

build-support/bin/ci.sh

@@ -18,7 +18,7 @@ Usage: $0 (-h|-fxbkmsrjlpuyncia)
  -f           skip python code formatting checks
  -x           skip bootstrap clean-all (assume bootstrapping from a
               fresh clone)
- -b           skip bootstraping pants from local sources
+ -b           skip bootstrapping pants from local sources
  -k           skip bootstrapped pants self compile check
  -m           skip sanity checks of bootstrapped pants and repo BUILD
               files

contrib/node/src/python/pants/contrib/node/subsystems/command.py

@@ -9,8 +9,8 @@
 import os
 from collections import namedtuple
 
-from pants.util.contextutil import get_joined_path
 from pants.util.process_handler import subprocess
+from pants.util.strutil import create_path_env_var
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +41,7 @@ def _prepare_env(self, kwargs):
     """
     kwargs = kwargs.copy()
     env = kwargs.pop('env', os.environ).copy()
-    env['PATH'] = get_joined_path(self.extra_paths, env=env, prepend=True)
+    env['PATH'] = create_path_env_var(self.extra_paths, env=env, prepend=True)
     return env, kwargs
 
   def run(self, **kwargs):

src/python/pants/backend/native/config/environment.py

@@ -5,11 +5,13 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import os
 from abc import abstractproperty
 
 from pants.engine.rules import SingletonRule
 from pants.util.objects import datatype
 from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
+from pants.util.strutil import create_path_env_var, safe_shlex_join
 
 
 class Platform(datatype(['normalized_os_name'])):
@@ -49,33 +51,136 @@ def path_entries(self):
     This may be multiple directories, e.g. if the main executable program invokes any subprocesses.
     """
 
+  @abstractproperty
+  def library_dirs(self):
+    """Directories containing shared libraries required for a subprocess to run."""
+
   @abstractproperty
   def exe_filename(self):
     """The "entry point" -- which file to invoke when PATH is set to `path_entries()`."""
 
+  def get_invocation_environment_dict(self, platform):
+    lib_env_var = platform.resolve_platform_specific({
+      'darwin': lambda: 'DYLD_LIBRARY_PATH',
+      'linux': lambda: 'LD_LIBRARY_PATH',
+    })
+    return {
+      'PATH': create_path_env_var(self.path_entries),
+      lib_env_var: create_path_env_var(self.library_dirs),
+    }
 
-class Linker(datatype([
+
+class Assembler(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
+    'library_dirs',
 ]), Executable):
   pass
 
 
-class CCompiler(datatype([
+class Linker(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
+    'library_dirs',
 ]), Executable):
-  pass
+
+  # FIXME(#5951): We need a way to compose executables more hygienically. This could be done
+  # declaratively -- something like: { 'LIBRARY_PATH': DelimitedPathDirectoryEnvVar(...) }.  We
+  # could also just use safe_shlex_join() and create_path_env_var() and keep all the state in the
+  # environment -- but then we have to remember to use those each time we specialize.
+  def get_invocation_environment_dict(self, platform):
+    ret = super(Linker, self).get_invocation_environment_dict(platform).copy()
+
+    # TODO: set all LDFLAGS in here or in further specializations of Linker instead of in individual
+    # tasks.
+    all_ldflags_for_platform = platform.resolve_platform_specific({
+      'darwin': lambda: ['-mmacosx-version-min=10.11'],
+      'linux': lambda: [],
+    })
+    ret.update({
+      'LDSHARED': self.exe_filename,
+      # FIXME: this overloads the meaning of 'library_dirs' to also mean "directories containing
+      # static libraries required for creating an executable" (currently, libc). These concepts
+      # should be distinct.
+      'LIBRARY_PATH': create_path_env_var(self.library_dirs),
+      'LDFLAGS': safe_shlex_join(all_ldflags_for_platform),
+    })
+
+    return ret
+
+
+class CompilerMixin(Executable):
+
+  @abstractproperty
+  def include_dirs(self):
+    """Directories to search for header files to #include during compilation."""
+
+  # FIXME: LIBRARY_PATH and (DY)?LD_LIBRARY_PATH are used for entirely different purposes, but are
+  # both sourced from the same `self.library_dirs`!
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CompilerMixin, self).get_invocation_environment_dict(platform).copy()
+
+    if self.include_dirs:
+      ret['CPATH'] = create_path_env_var(self.include_dirs)
+
+    all_cflags_for_platform = platform.resolve_platform_specific({
+      'darwin': lambda: ['-mmacosx-version-min=10.11'],
+      'linux': lambda: [],
+    })
+    ret['CFLAGS'] = safe_shlex_join(all_cflags_for_platform)
+
+    return ret
+
+
+class CCompiler(datatype([
+    'path_entries',
+    'exe_filename',
+    'library_dirs',
+    'include_dirs',
+]), CompilerMixin):
+
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CCompiler, self).get_invocation_environment_dict(platform).copy()
+
+    ret['CC'] = self.exe_filename
+
+    return ret
 
 
 class CppCompiler(datatype([
     'path_entries',
     'exe_filename',
-    ('platform', Platform),
-]), Executable):
-  pass
+    'library_dirs',
+    'include_dirs',
+]), CompilerMixin):
+
+  def get_invocation_environment_dict(self, platform):
+    ret = super(CppCompiler, self).get_invocation_environment_dict(platform).copy()
+
+    ret['CXX'] = self.exe_filename
+
+    return ret
+
+
+# TODO(#4020): These classes are performing the work of variants.
+class GCCCCompiler(datatype([('c_compiler', CCompiler)])): pass
+
+
+class LLVMCCompiler(datatype([('c_compiler', CCompiler)])): pass
+
+
+class GCCCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass
+
+
+class LLVMCppCompiler(datatype([('cpp_compiler', CppCompiler)])): pass
+
+
+# FIXME: make this an @rule, after we can automatically produce LibcDev and other subsystems in the
+# v2 engine (see #5788).
+class HostLibcDev(datatype(['crti_object', 'fingerprint'])):
+
+  def get_lib_dir(self):
+    return os.path.dirname(self.crti_object)
 
 
 def create_native_environment_rules():

src/python/pants/backend/native/subsystems/BUILD

@@ -6,6 +6,7 @@ python_library(
   dependencies=[
     'src/python/pants/backend/native/config',
     'src/python/pants/backend/native/subsystems/binaries',
+    'src/python/pants/backend/native/subsystems/utils',
     'src/python/pants/engine:rules',
     'src/python/pants/engine:selectors',
     'src/python/pants/subsystem',

src/python/pants/backend/native/subsystems/binaries/BUILD

@@ -5,6 +5,7 @@
 python_library(
   dependencies=[
     'src/python/pants/backend/native/config',
+    'src/python/pants/backend/native/subsystems/utils',
     'src/python/pants/binaries',
     'src/python/pants/engine:rules',
     'src/python/pants/engine:selectors',

src/python/pants/backend/native/subsystems/binaries/binutils.py

@@ -7,7 +7,7 @@
 
 import os
 
-from pants.backend.native.config.environment import Linker, Platform
+from pants.backend.native.config.environment import Assembler, Linker
 from pants.binaries.binary_tool import NativeTool
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
@@ -21,20 +21,32 @@ class Binutils(NativeTool):
   def path_entries(self):
     return [os.path.join(self.select(), 'bin')]
 
-  def linker(self, platform):
+  def assembler(self):
+    return Assembler(
+      path_entries=self.path_entries(),
+      exe_filename='as',
+      library_dirs=[])
+
+  def linker(self):
     return Linker(
       path_entries=self.path_entries(),
       exe_filename='ld',
-      platform=platform)
+      library_dirs=[])
+
+
+@rule(Assembler, [Select(Binutils)])
+def get_as(binutils):
+  return binutils.assembler()
 
 
-@rule(Linker, [Select(Platform), Select(Binutils)])
-def get_ld(platform, binutils):
-  return binutils.linker(platform)
+@rule(Linker, [Select(Binutils)])
+def get_ld(binutils):
+  return binutils.linker()
 
 
 def create_binutils_rules():
   return [
+    get_as,
     get_ld,
     RootRule(Binutils),
   ]

src/python/pants/backend/native/subsystems/binaries/gcc.py

@@ -7,12 +7,14 @@
 
 import os
 
-from pants.backend.native.config.environment import CCompiler, CppCompiler, Platform
-from pants.backend.native.subsystems.binaries.binutils import Binutils
+from pants.backend.native.config.environment import (CCompiler, CppCompiler, GCCCCompiler,
+                                                     GCCCppCompiler)
+from pants.backend.native.subsystems.utils.parse_search_dirs import ParseSearchDirs
 from pants.binaries.binary_tool import NativeTool
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
-from pants.util.memo import memoized_property
+from pants.util.memo import memoized_method, memoized_property
+from pants.util.strutil import create_path_env_var
 
 
 class GCC(NativeTool):
@@ -22,46 +24,48 @@ class GCC(NativeTool):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(GCC, cls).subsystem_dependencies() + (Binutils.scoped(cls),)
+    return super(GCC, cls).subsystem_dependencies() + (ParseSearchDirs.scoped(cls),)
 
   @memoized_property
-  def _binutils(self):
-    return Binutils.scoped_instance(self)
-
-  def _path_entries_for_platform(self, platform):
-    # GCC requires an assembler 'as' to be on the path. We need to provide this on linux, so we pull
-    # it from our Binutils package.
-    as_assembler_path_entries = platform.resolve_platform_specific({
-      'darwin': lambda: [],
-      'linux': lambda: self._binutils.path_entries(),
-    })
-    all_path_entries = self.path_entries() + as_assembler_path_entries
-    return all_path_entries
+  def _parse_search_dirs(self):
+    return ParseSearchDirs.scoped_instance(self)
 
+  def _lib_search_dirs(self, compiler_exe, path_entries):
+    return self._parse_search_dirs.get_compiler_library_dirs(
+      compiler_exe,
+      env={'PATH': create_path_env_var(path_entries)})
+
+  @memoized_method
   def path_entries(self):
     return [os.path.join(self.select(), 'bin')]
 
-  def c_compiler(self, platform):
+  def c_compiler(self):
+    exe_filename = 'gcc'
+    path_entries = self.path_entries()
     return CCompiler(
-      path_entries=self._path_entries_for_platform(platform),
-      exe_filename='gcc',
-      platform=platform)
-
-  def cpp_compiler(self, platform):
+      path_entries=path_entries,
+      exe_filename=exe_filename,
+      library_dirs=self._lib_search_dirs(exe_filename, path_entries),
+      include_dirs=[])
+
+  def cpp_compiler(self):
+    exe_filename = 'g++'
+    path_entries = self.path_entries()
     return CppCompiler(
-      path_entries=self._path_entries_for_platform(platform),
-      exe_filename='g++',
-      platform=platform)
+      path_entries=self.path_entries(),
+      exe_filename=exe_filename,
+      library_dirs=self._lib_search_dirs(exe_filename, path_entries),
+      include_dirs=[])
 
 
-@rule(CCompiler, [Select(Platform), Select(GCC)])
-def get_gcc(platform, gcc):
-  yield gcc.c_compiler(platform)
+@rule(GCCCCompiler, [Select(GCC)])
+def get_gcc(gcc):
+  yield GCCCCompiler(gcc.c_compiler())
 
 
-@rule(CppCompiler, [Select(Platform), Select(GCC)])
-def get_gplusplus(platform, gcc):
-  yield gcc.cpp_compiler(platform)
+@rule(GCCCppCompiler, [Select(GCC)])
+def get_gplusplus(gcc):
+  yield GCCCppCompiler(gcc.cpp_compiler())
 
 
 def create_gcc_rules():