add a --toolchain-variant option to select the compiler for C/C++

src/python/pants/backend/native/config/BUILD

@@ -6,6 +6,7 @@ python_library(
   dependencies=[
     '3rdparty/python:future',
     'src/python/pants/engine:rules',
+    'src/python/pants/util:meta',
     'src/python/pants/util:objects',
     'src/python/pants/util:osutil',
   ],

src/python/pants/backend/native/config/environment.py

@@ -6,9 +6,9 @@
 
 import os
 from abc import abstractproperty
-from builtins import object
 
 from pants.engine.rules import SingletonRule
+from pants.util.meta import AbstractClass
 from pants.util.objects import datatype
 from pants.util.osutil import all_normalized_os_names, get_normalized_os_name
 from pants.util.strutil import create_path_env_var
@@ -42,34 +42,54 @@ def resolve_platform_specific(self, platform_specific_funs):
     return fun_for_platform()
 
 
-class Executable(object):
+# NB: @abstractproperty requires inheriting from AbstractClass to work!
+class Executable(AbstractClass):
 
   @abstractproperty
   def path_entries(self):
     """A list of directory paths containing this executable, to be used in a subprocess's PATH.
 
     This may be multiple directories, e.g. if the main executable program invokes any subprocesses.
+
+    :rtype: list of str
+    """
+
+  @abstractproperty
+  def exe_filename(self):
+    """The "entry point" -- which file to invoke when PATH is set to `path_entries()`.
+
+    :rtype: str
     """
 
+  # TODO(#???): rename this to 'runtime_library_dirs'!
   @abstractproperty
   def library_dirs(self):
     """Directories containing shared libraries that must be on the runtime library search path.
 
     Note: this is for libraries needed for the current Executable to run -- see LinkerMixin below
-    for libraries that are needed at link time."""
+    for libraries that are needed at link time.
 
-  @abstractproperty
-  def exe_filename(self):
-    """The "entry point" -- which file to invoke when PATH is set to `path_entries()`."""
+    :rtype: list of str
+    """
 
   @property
   def extra_args(self):
+    """Additional arguments used when invoking this Executable.
+
+    These are typically placed before the invocation-specific command line arguments.
+
+    :rtype: list of str
+    """
     return []
 
   _platform = Platform.create()
 
   @property
   def as_invocation_environment_dict(self):
+    """A dict to use as this Executable's execution environment.
+
+    :rtype: dict of string -> string
+    """
     lib_env_var = self._platform.resolve_platform_specific({
       'darwin': lambda: 'DYLD_LIBRARY_PATH',
       'linux': lambda: 'LD_LIBRARY_PATH',
@@ -92,13 +112,18 @@ class LinkerMixin(Executable):
 
   @abstractproperty
   def linking_library_dirs(self):
-    """Directories to search for libraries needed at link time."""
+    """Directories to search for libraries needed at link time.
+
+    :rtype: list of str
+    """
 
   @abstractproperty
   def extra_object_files(self):
     """A list of object files required to perform a successful link.
 
     This includes crti.o from libc for gcc on Linux, for example.
+
+    :rtype: list of str
     """
 
   @property
@@ -131,7 +156,10 @@ class CompilerMixin(Executable):
 
   @abstractproperty
   def include_dirs(self):
-    """Directories to search for header files to #include during compilation."""
+    """Directories to search for header files to #include during compilation.
+
+    :rtype: list of str
+    """
 
   @property
   def as_invocation_environment_dict(self):

src/python/pants/backend/native/subsystems/binaries/gcc.py

@@ -75,7 +75,7 @@ def _cpp_include_dirs(self):
       ('include/c++', self.version()),
     ])
 
-    # TODO: determine whether there is any manual explaining when any of these file paths are
+    # TODO(#6143): determine whether there is any manual explaining when any of these file paths are
     # necessary.
     # This file is needed for C++ compilation.
     cpp_config_header_path = self._file_mapper.assert_single_path_by_glob(

src/python/pants/backend/native/subsystems/libc_dev.py

@@ -101,6 +101,4 @@ def _host_libc(self):
 
   @memoized_method
   def get_libc_objects(self):
-    if not self.get_options().enable_libc_search:
-      return []
-    return [self._host_libc.crti_object]
+    return [self._host_libc.crti_object] if self.get_options().enable_libc_search else []

src/python/pants/backend/native/subsystems/native_toolchain.py

@@ -141,10 +141,12 @@ class GCCInstallLocationForLLVM(datatype(['toolchain_dir'])):
   """This class is convertible into a list of command line arguments for clang and clang++.
 
   This is only used on Linux. The option --gcc-toolchain stops clang from searching for another gcc
-  on the host system. The option appears to only exist on Linux clang and clang++."""
+  on the host system. The option appears to only exist on Linux clang and clang++.
+  """
 
   @property
   def as_clang_argv(self):
+    # TODO(#6143): describe exactly what this argument does to the clang/clang++ invocation!
     return ['--gcc-toolchain={}'.format(self.toolchain_dir)]
 
 
@@ -161,6 +163,7 @@ def select_llvm_c_toolchain(platform, native_toolchain):
   llvm_c_compiler_args = [
     '-x', 'c', '-std=c11',
     '-nobuiltininc',
+    '-nostdinc',
   ]
 
   if platform.normalized_os_name == 'darwin':
@@ -182,7 +185,7 @@ def select_llvm_c_toolchain(platform, native_toolchain):
   llvm_linker_wrapper = yield Get(LLVMLinker, NativeToolchain, native_toolchain)
   llvm_linker = llvm_linker_wrapper.linker
 
-  # TODO: create an algebra so these changes are more clear!
+  # TODO(#6855): introduce a more concise way to express these compositions of executables.
   working_linker = llvm_linker.copy(
     path_entries=(llvm_linker.path_entries + working_c_compiler.path_entries),
     exe_filename=working_c_compiler.exe_filename,
@@ -203,6 +206,7 @@ def select_llvm_cpp_toolchain(platform, native_toolchain):
     # implementation, or any from the host system. Instead, we use include dirs from the
     # XCodeCLITools or GCC.
     '-nobuiltininc',
+    '-nostdinc',
     '-nostdinc++',
   ]
 
@@ -226,7 +230,7 @@ def select_llvm_cpp_toolchain(platform, native_toolchain):
       # NB: we use g++'s headers on Linux, and therefore their C++ standard library.
       include_dirs=provided_gpp.include_dirs,
       extra_args=(llvm_cpp_compiler_args + provided_clangpp.extra_args + gcc_install.as_clang_argv))
-    # TODO: why are these necessary? this is very mysterious.
+    # TODO(#6855): why are these necessary? this is very mysterious.
     extra_linking_library_dirs = provided_gpp.library_dirs + provided_clangpp.library_dirs
     # Ensure we use libstdc++, provided by g++, during the linking stage.
     linker_extra_args=['-stdlib=libstdc++']
@@ -253,12 +257,15 @@ def select_gcc_c_toolchain(platform, native_toolchain):
   # GCC needs an assembler, so we provide that (platform-specific) tool here.
   assembler = yield Get(Assembler, NativeToolchain, native_toolchain)
 
+  gcc_c_compiler_args = [
+    '-x', 'c', '-std=c11',
+    '-nostdinc',
+  ]
+
   if platform.normalized_os_name == 'darwin':
     # GCC needs access to some headers that are only provided by the XCode toolchain
     # currently (e.g. "_stdio.h"). These headers are unlikely to change across versions, so this is
     # probably safe.
-    # TODO: we should be providing all of these (so we can eventually phase out XCodeCLITools
-    # entirely).
     xcode_clang = yield Get(CCompiler, XCodeCLITools, native_toolchain._xcode_cli_tools)
     new_include_dirs = provided_gcc.include_dirs + xcode_clang.include_dirs
   else:
@@ -267,7 +274,7 @@ def select_gcc_c_toolchain(platform, native_toolchain):
   working_c_compiler = provided_gcc.copy(
     path_entries=(provided_gcc.path_entries + assembler.path_entries),
     include_dirs=new_include_dirs,
-    extra_args=['-x', 'c', '-std=c11'])
+    extra_args=gcc_c_compiler_args)
 
   gcc_linker_wrapper = yield Get(GCCLinker, NativeToolchain, native_toolchain)
   gcc_linker = gcc_linker_wrapper.linker
@@ -290,6 +297,7 @@ def select_gcc_cpp_toolchain(platform, native_toolchain):
 
   gcc_cpp_compiler_args = [
     '-x', 'c++', '-std=c++11',
+    '-nostdinc',
     '-nostdinc++',
   ]
 

src/python/pants/backend/native/subsystems/xcode_cli_tools.py

@@ -84,7 +84,9 @@ def _all_existing_install_prefixes(self):
   # NB: We use @memoized_method in this file for methods which may raise.
   @memoized_method
   def _get_existing_subdirs(self, subdir_name):
-    # TODO: consume ArchiveFileMapper instead of doing all that logic over again in this file.
+    # TODO(#6143): We should attempt to use ParseSearchDirs here or find some documentation on which
+    # directories we should be adding to the include path, and why. If we do need to manually
+    # specify paths, use ArchiveFileMapper instead of doing all that logic over again in this file.
     maybe_subdirs = [os.path.join(pfx, subdir_name) for pfx in self._all_existing_install_prefixes]
     existing_dirs = [existing_dir for existing_dir in maybe_subdirs if is_readable_dir(existing_dir)]
 
@@ -125,22 +127,7 @@ def lib_dirs(self):
 
   @memoized_method
   def include_dirs(self, include_cpp_inc=False):
-    base_inc_dirs = self._get_existing_subdirs('include')
-
-    all_inc_dirs = base_inc_dirs
-    for d in base_inc_dirs:
-      # TODO: figure out what this directory does and why it's not already found by this compiler.
-      secure_inc_dir = os.path.join(d, 'secure')
-      if is_readable_dir(secure_inc_dir):
-        all_inc_dirs.append(secure_inc_dir)
-
-      # TODO: figure out why we need to specify this directory separately.
-      if include_cpp_inc:
-        cpp_inc_dir = os.path.join(d, 'c++/v1')
-        if is_readable_dir(cpp_inc_dir):
-          all_inc_dirs.append(cpp_inc_dir)
-
-    return all_inc_dirs
+    return self._get_existing_subdirs('include')
 
   @memoized_method
   def assembler(self):

tests/python/pants_test/backend/python/tasks/test_ctypes_integration.py

@@ -77,6 +77,7 @@ def _assert_ctypes_binary(self, toolchain_variant):
 
       # Check that we have selected the appropriate compilers for our selected toolchain variant,
       # for both C and C++ compilation.
+      # TODO(#6866): don't parse info logs for testing!
       for compiler_name in self._compiler_names_for_variant[toolchain_variant]:
         self.assertIn("selected compiler exe name: '{}'".format(compiler_name),
                       pants_run.stdout_data)