split off indexing into BinaryTool archives using the new ArchiveFile…

…Mapper

src/python/pants/backend/native/subsystems/binaries/gcc.py

@@ -4,90 +4,87 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import glob
 import os
 
 from pants.backend.native.config.environment import CCompiler, CppCompiler
+from pants.backend.native.subsystems.utils.archive_file_mapper import ArchiveFileMapper
 from pants.binaries.binary_tool import NativeTool
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
-from pants.util.collections import assert_single_element
-from pants.util.memo import memoized_method, memoized_property
+from pants.util.memo import memoized_property
 
 
 class GCC(NativeTool):
+  """Subsystem wrapping an archive providing a GCC distribution.
+
+  This subsystem provides the gcc and g++ compilers.
+
+  NB: The lib and include dirs provided by this distribution are produced by using known relative
+  paths into the distribution of GCC provided on Pantsbuild S3. If we change how we distribute GCC,
+  these methods may have to change. They should be stable to version upgrades, however.
+  """
   options_scope = 'gcc'
   default_version = '7.3.0'
   archive_type = 'tgz'
 
-  @memoized_method
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(GCC, cls).subsystem_dependencies() + (ArchiveFileMapper.scoped(cls),)
+
+  @memoized_property
+  def _file_mapper(self):
+    return ArchiveFileMapper.scoped_instance(self)
+
+  def _filemap(self, all_components_list):
+    return self._file_mapper.map_files(self.select(), all_components_list)
+
+  @memoized_property
   def path_entries(self):
-    return [os.path.join(self.select(), 'bin')]
-
-  class GCCResourceLocationError(Exception): pass
-
-  def _get_check_single_path_by_glob(self, *components):
-    """Assert that the path components (which are joined into a glob) match exactly one path.
-
-    The matched path may be a file or a directory. This method is used to avoid having to guess
-    platform-specific intermediate directory names, e.g. 'x86_64-linux-gnu' or
-    'x86_64-apple-darwin17.5.0'."""
-    glob_path_string = os.path.join(*components)
-    expanded_glob = glob.glob(glob_path_string)
-
-    try:
-      return assert_single_element(expanded_glob)
-    except StopIteration as e:
-      raise self.GCCResourceLocationError(
-        "No elements for glob '{}' -- expected exactly one."
-        .format(glob_path_string),
-        e)
-    except ValueError as e:
-      raise self.GCCResourceLocationError(
-        "Should have exactly one path matching expansion of glob '{}'."
-        .format(glob_path_string),
-        e)
+    return self._filemap([('bin',)])
 
   @memoized_property
   def _common_lib_dirs(self):
-    return [
-      os.path.join(self.select(), 'lib'),
-      os.path.join(self.select(), 'lib64'),
-      os.path.join(self.select(), 'lib/gcc'),
-      self._get_check_single_path_by_glob(
-        self.select(), 'lib/gcc/*', self.version()),
-    ]
+    return self._filemap([
+      ('lib',),
+      ('lib64',),
+      ('lib/gcc',),
+      ('lib/gcc/*', self.version()),
+    ])
 
   @memoized_property
   def _common_include_dirs(self):
-    return [
-      os.path.join(self.select(), 'include'),
-      self._get_check_single_path_by_glob(
-        self.select(), 'lib/gcc/*', self.version(), 'include'),
-    ]
+    return self._filemap([
+      ('include',),
+      ('lib/gcc/*', self.version(), 'include'),
+    ])
 
   def c_compiler(self):
     return CCompiler(
-      path_entries=self.path_entries(),
+      path_entries=self.path_entries,
       exe_filename='gcc',
       library_dirs=self._common_lib_dirs,
       include_dirs=self._common_include_dirs,
       extra_args=[])
 
   @memoized_property
   def _cpp_include_dirs(self):
-    # FIXME: explain why this is necessary!
-    cpp_config_header_path = self._get_check_single_path_by_glob(
-      self.select(), 'include/c++', self.version(), '*/bits/c++config.h')
-    return [
-      os.path.join(self.select(), 'include/c++', self.version()),
-      # Get the directory that makes `#include <bits/c++config.h>` work.
-      os.path.dirname(os.path.dirname(cpp_config_header_path)),
-    ]
+    most_cpp_include_dirs = self._filemap([
+      ('include/c++', self.version()),
+    ])
+
+    # This file is needed for C++ compilation.
+    cpp_config_header_path = self._file_mapper.assert_single_path_by_glob(
+      # NB: There are multiple paths matching this glob unless we provide the full path to
+      # c++config.h, which is why we bypass self._filemap() here.
+      [self.select(), 'include/c++', self.version(), '*/bits/c++config.h'])
+    # Get the directory that makes `#include <bits/c++config.h>` work.
+    plat_cpp_header_dir =  os.path.dirname(os.path.dirname(cpp_config_header_path))
+
+    return most_cpp_include_dirs + [plat_cpp_header_dir]
 
   def cpp_compiler(self):
     return CppCompiler(
-      path_entries=self.path_entries(),
+      path_entries=self.path_entries,
       exe_filename='g++',
       library_dirs=self._common_lib_dirs,
       include_dirs=(self._common_include_dirs + self._cpp_include_dirs),

src/python/pants/backend/native/subsystems/binaries/llvm.py

@@ -7,6 +7,7 @@
 import os
 
 from pants.backend.native.config.environment import CCompiler, CppCompiler, Linker, Platform
+from pants.backend.native.subsystems.utils.archive_file_mapper import ArchiveFileMapper
 from pants.binaries.binary_tool import NativeTool
 from pants.binaries.binary_util import BinaryToolUrlGenerator
 from pants.engine.rules import RootRule, rule
@@ -35,6 +36,16 @@ def generate_urls(self, version, host_platform):
 
 
 class LLVM(NativeTool):
+  """Subsystem wrapping an archive providing an LLVM distribution.
+
+  This subsystem provides the clang and clang++ compilers. It also provides lld, which is not
+  currently used.
+
+  NB: The lib and include dirs provided by this distribution are produced by using known relative
+  paths into the distribution of LLVM from LLVMReleaseUrlGenerator. If LLVM changes the structure of
+  their release archives, these methods may have to change. They should be stable to version
+  upgrades, however.
+  """
   options_scope = 'llvm'
   default_version = '6.0.0'
   archive_type = 'txz'
@@ -46,16 +57,30 @@ def get_external_url_generator(self):
   def select(self):
     unpacked_path = super(LLVM, self).select()
     # The archive from releases.llvm.org wraps the extracted content into a directory one level
-    # deeper, but the one from our S3 does not.
+    # deeper, but the one from our S3 does not. We account for both here.
     children = os.listdir(unpacked_path)
     if len(children) == 1:
       llvm_base_dir = os.path.join(unpacked_path, children[0])
       assert(is_readable_dir(llvm_base_dir))
       return llvm_base_dir
     return unpacked_path
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(LLVM, cls).subsystem_dependencies() + (ArchiveFileMapper.scoped(cls),)
+
+  @memoized_property
+  def _file_mapper(self):
+    return ArchiveFileMapper.scoped_instance(self)
+
+  def _filemap(self, all_components_list):
+    return self._file_mapper.map_files(self.select(), all_components_list)
+
+  _bin_paths = [('bin',)]
+
+  @memoized_property
   def path_entries(self):
-    return [os.path.join(self.select(), 'bin')]
+    return self._filemap(self._bin_paths)
 
   _PLATFORM_SPECIFIC_LINKER_NAME = {
     'darwin': lambda: 'ld64.lld',
@@ -64,7 +89,7 @@ def path_entries(self):
 
   def linker(self, platform):
     return Linker(
-      path_entries=self.path_entries(),
+      path_entries=self.path_entries,
       exe_filename=platform.resolve_platform_specific(
         self._PLATFORM_SPECIFIC_LINKER_NAME),
       library_dirs=[],
@@ -73,27 +98,27 @@ def linker(self, platform):
 
   @memoized_property
   def _common_include_dirs(self):
-    return [os.path.join(self.select(), 'lib/clang', self.version(), 'include')]
+    return self._filemap([('lib/clang', self.version(), 'include')])
 
   @memoized_property
   def _common_lib_dirs(self):
-    return [os.path.join(self.select(), 'lib')]
+    return self._filemap([('lib',)])
 
   def c_compiler(self):
     return CCompiler(
-      path_entries=self.path_entries(),
+      path_entries=self.path_entries,
       exe_filename='clang',
       library_dirs=self._common_lib_dirs,
       include_dirs=self._common_include_dirs,
       extra_args=[])
 
   @memoized_property
   def _cpp_include_dirs(self):
-    return [os.path.join(self.select(), 'include/c++/v1')]
+    return self._filemap([('include/c++/v1',)])
 
   def cpp_compiler(self):
     return CppCompiler(
-      path_entries=self.path_entries(),
+      path_entries=self.path_entries,
       exe_filename='clang++',
       library_dirs=self._common_lib_dirs,
       include_dirs=(self._cpp_include_dirs + self._common_include_dirs),

src/python/pants/backend/native/subsystems/utils/BUILD

@@ -1,5 +1,6 @@
 python_library(
   dependencies=[
+    'src/python/pants/binaries',
     'src/python/pants/subsystem',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:memo',

src/python/pants/backend/native/subsystems/utils/archive_file_mapper.py

@@ -0,0 +1,63 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import glob
+import os
+
+from pants.subsystem.subsystem import Subsystem
+from pants.util.collections import assert_single_element
+
+
+class ArchiveFileMapper(Subsystem):
+  """Index into known paths relative to a base directory.
+
+  This is used with `NativeTool`s that wrap a compressed archive, which may have slightly different
+  paths across platforms. The helper methods from this class """
+
+  options_scope = 'archive-file-mapper'
+
+  class ArchiveFileMappingError(Exception): pass
+
+  def assert_single_path_by_glob(self, components):
+    """Assert that the path components (which are joined into a glob) match exactly one path.
+
+    The matched path may be a file or a directory. This method is used to avoid having to guess
+    platform-specific intermediate directory names, e.g. 'x86_64-linux-gnu' or
+    'x86_64-apple-darwin17.5.0'."""
+    glob_path_string = os.path.join(*components)
+    expanded_glob = glob.glob(glob_path_string)
+
+    try:
+      return assert_single_element(expanded_glob)
+    except StopIteration as e:
+      raise self.ArchiveFileMappingError(
+        "No elements for glob '{}' -- expected exactly one."
+        .format(glob_path_string),
+        e)
+    except ValueError as e:
+      raise self.ArchiveFileMappingError(
+        "Should have exactly one path matching expansion of glob '{}'."
+        .format(glob_path_string),
+        e)
+
+  def map_files(self, base_dir, all_components_list):
+    """Apply `assert_single_path_by_glob()` to all elements of `all_components_list`.
+
+    Each element of `all_components_list` should be a tuple of path components, including
+    wildcards. The elements of each tuple are joined, and interpreted as a glob expression relative
+    to `base_dir`. The resulting glob should match exactly one path.
+
+    :return: List of matched paths, one per element of `all_components_list`.
+    :raises: :class:`ArchiveFileMapper.ArchiveFileMappingError` if more or less than one path was
+             matched by one of the glob expressions interpreted from `all_components_list`.
+    """
+    mapped_paths = []
+    for components_tupled in all_components_list:
+      with_base = [base_dir] + list(components_tupled)
+      # Results are known to exist, since they match a glob.
+      mapped_paths.append(self.assert_single_path_by_glob(with_base))
+
+    return mapped_paths