enhance GCC easyblock to add support for AMD GPU offloading

easybuild/easyblocks/g/gcc.py

@@ -42,6 +42,7 @@
 from distutils.version import LooseVersion
 
 import easybuild.tools.environment as env
+from easybuild.easyblocks.clang import DEFAULT_TARGETS_MAP as LLVM_ARCH_MAP
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
@@ -51,11 +52,18 @@
 from easybuild.tools.modules import get_software_root
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import check_os_dependency, get_os_name, get_os_type
-from easybuild.tools.systemtools import get_gcc_version, get_shared_lib_ext
+from easybuild.tools.systemtools import get_cpu_architecture, get_gcc_version, get_shared_lib_ext
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 from easybuild.tools.utilities import nub
 
 
+# Offloading stages to build
+AMD_LLVM = 'AMD_GCN_LLVM'
+AMD_NEWLIB = 'AMD_GCN_NEWLIB'
+HOST_COMPILER = 'HOST_COMPILER'
+NVIDIA_NEWLIB = 'NVIDIA_NEWLIB'
+NVPTX_TOOLS = 'NVIDIA_NVPTX_TOOLS'
+# Additional symlinks to create for compiler commands
 COMP_CMD_SYMLINKS = {
  'cc': 'gcc',
  'c++': 'g++',
@@ -88,13 +96,23 @@ def extra_options():
  'withlto': [True, "Enable LTO support", CUSTOM],
  'withppl': [False, "Build GCC with PPL support", CUSTOM],
  'withnvptx': [False, "Build GCC with NVPTX offload support", CUSTOM],
+ 'withamdgcn': [False, "Build GCC with AMD GCN offload support", CUSTOM],
  }
  return ConfigureMake.extra_options(extra_vars)
 
  def __init__(self, *args, **kwargs):
  super(EB_GCC, self).__init__(*args, **kwargs)
 
  self.stagedbuild = False
+ # Each build iteration is related to a specific build, first build host compiler, then potentially build NVPTX
+ # offloading and/or AMD GCN offloading
+ self.build_stages = [HOST_COMPILER]
+ self.current_stage = HOST_COMPILER
+ # Directories for additional tools needed when doing offloading to Nvidia and AMD
+ self.nvptx_tools_dir = None # nvptx-tools necessary for Nvidia
+ self.llvm_dir = None # LLVM is necessary when offloading to AMD
+ self.lld_dir = None # LLD is the only required component of LLVM
+ self.newlib_dir = None # Used by both NVPTX and AMD GCN backend
 
  # need to make sure version is an actual version
  # required because of support in SystemCompiler generic easyblock to specify 'system' as version,
@@ -305,6 +323,44 @@ def run_configure_cmd(self, cmd):
  if unknown_options:
  raise EasyBuildError("Unrecognized options found during configure: %s", unknown_options)
 
+ def prepare_step(self, *args, **kwargs):
+ """
+ Check that optional dependencies have been downloaded (for NVPTX and/or AMD GCN offload support)
+ """
+ super(EB_GCC, self).prepare_step(*args, **kwargs)
+ if self.cfg['withnvptx']:
+ nvptx_tools_dir_pattern = os.path.join(self.builddir, 'nvptx-tools-*')
+ hits = glob.glob(nvptx_tools_dir_pattern)
+ if len(hits) == 1:
+ self.nvptx_tools_dir = hits[0]
+ else:
+ raise EasyBuildError("Build specified 'withnvptx', but 'nvptx-tools' was not downloaded."
+ " Ensure that 'nvptx-tools' are among the sources in the EasyConfig")
+ if self.cfg['withamdgcn']:
+ lld_dir_pattern = os.path.join(self.builddir, 'lld-*')
+ hits = glob.glob(lld_dir_pattern)
+ if len(hits) == 1:
+ self.lld_dir = hits[0]
+ else:
+ raise EasyBuildError("Build specified 'withamdgcn', but 'LLD' was not downloaded."
+ " Ensure that 'LLD' are among the sources in the EasyConfig")
+ llvm_dir_pattern = os.path.join(self.builddir, 'llvm-*')
+ hits = glob.glob(llvm_dir_pattern)
+ if len(hits) == 1:
+ self.llvm_dir = hits[0]
+ else:
+ raise EasyBuildError("Build specified 'withamdgcn', but 'LLVM' was not downloaded."
+ " Ensure that 'LLVM' are among the sources in the EasyConfig")
+ if self.cfg['withnvptx'] or self.cfg['withamdgcn']:
+ newlib_dir_pattern = os.path.join(self.builddir, 'newlib-*')
+ hits = glob.glob(newlib_dir_pattern)
+ if len(hits) == 1:
+ self.newlib_dir = hits[0]
+ else:
+ raise EasyBuildError("Could not find 'newlib' among the downloaded sources")
+ # Set the current build stage to the specified stage based on the iteration index
+ self.current_stage = self.build_stages[self.iter_idx]
+
  def configure_step(self):
  """
  Configure for GCC build:
@@ -354,10 +410,15 @@ def configure_step(self):
  if self.cfg['languages']:
  self.configopts += " --enable-languages=%s" % ','.join(self.cfg['languages'])
 
- if self.cfg['withnvptx']:
- if self.iter_idx == 0:
+ if self.cfg['withnvptx'] or self.cfg['withamdgcn']:
+ if self.current_stage == HOST_COMPILER:
  self.configopts += " --without-cuda-driver"
- self.configopts += " --enable-offload-targets=nvptx-none"
+ offload_target = []
+ if self.cfg['withnvptx']:
+ offload_target += ['nvptx-none']
+ if self.cfg['withamdgcn']:
+ offload_target += ['amdgcn-amdhsa']
+ self.configopts += " --enable-offload-targets=%s" % ','.join(offload_target)
  else:
  # register installed GCC as compiler to use nvptx
  path = "%s/bin:%s" % (self.installdir, os.getenv('PATH'))
@@ -368,16 +429,29 @@ def configure_step(self):
  'val': os.getenv('LD_LIBRARY_PATH')
  }
  env.setvar('LD_LIBRARY_PATH', ld_lib_path)
- extra_source = {1: "nvptx-tools", 2: "newlib"}[self.iter_idx]
- extra_source_dirs = glob.glob(os.path.join(self.builddir, '%s-*' % extra_source))
- if len(extra_source_dirs) != 1:
- raise EasyBuildError("Failed to isolate %s source dir" % extra_source)
- if self.iter_idx == 1:
- # compile nvptx-tools
- change_dir(extra_source_dirs[0])
- else: # self.iter_idx == 2
+ if self.current_stage == NVPTX_TOOLS:
+ # Configure NVPTX tools and build
+ change_dir(self.nvptx_tools_dir)
+ return super(EB_GCC, self).configure_step()
+ elif self.current_stage == AMD_LLVM:
+ # Setup symlink from LLD source to 'builddir/lld' so that LLVM can find it
+ symlink(self.lld_dir, '%s/lld' % self.builddir)
+ # Build LLD tools from LLVM needed for AMD offloading
+ # Build LLVM in separate directory
+ self.create_dir('build_llvm_amdgcn')
+ cpu_arch = LLVM_ARCH_MAP[get_cpu_architecture()][0]
+ cmd = ("cmake -DLLVM_TARGETS_TO_BUILD='%s;AMDGPU' -DLLVM_ENABLE_PROJECTS=lld "
+ "-DCMAKE_BUILD_TYPE=Release %s"
+ % (cpu_arch, self.llvm_dir))
+ run_cmd(cmd, log_all=True, simple=True)
+ # Need to terminate the current configuration step, but we can't run 'configure' since LLVM uses
+ # CMake, we therefore run 'CMake' manually and then return nothing.
+ # The normal make stage will build LLVM for us as expected, note that we override the install step
+ # further down in this file to avoid installing LLVM
+ return
+ elif self.current_stage == NVIDIA_NEWLIB:
  # compile nvptx target compiler
- symlink(os.path.join(extra_source_dirs[0], 'newlib'), 'newlib')
+ symlink(os.path.join(self.newlib_dir, 'newlib'), 'newlib')
  self.create_dir("build-nvptx-gcc")
  self.cfg.update('configopts', self.configopts)
  self.cfg.update('configopts', "--with-build-time-tools=%s/nvptx-none/bin" % self.installdir)
@@ -387,7 +461,24 @@ def configure_step(self):
  self.cfg.update('configopts', "--disable-sjlj-exceptions")
  self.cfg.update('configopts', "--enable-newlib-io-long-long")
  self.cfg['configure_cmd_prefix'] = '../'
- return super(EB_GCC, self).configure_step()
+ return super(EB_GCC, self).configure_step()
+ elif self.current_stage == AMD_NEWLIB:
+ # compile AMD GCN target compiler
+ symlink(os.path.join(self.newlib_dir, 'newlib'), 'newlib')
+ self.create_dir("build-amdgcn-gcc")
+ self.cfg.update('configopts', self.configopts)
+ self.cfg.update('configopts', "--with-build-time-tools=%s/amdgcn-amdhsa/bin" % self.installdir)
+ self.cfg.update('configopts', "--target=amdgcn-amdhsa")
+ self.cfg.update('configopts', "--with-newlib")
+ host_type = self.determine_build_and_host_type()[1]
+ self.cfg.update('configopts', "--enable-as-accelerator-for=%s" % host_type)
+ self.cfg.update('configopts', "--disable-sjlj-exceptions")
+ self.cfg.update('configopts', "--disable-libquadmath")
+ self.cfg['configure_cmd_prefix'] = '../'
+ return super(EB_GCC, self).configure_step()
+ else:
+ raise EasyBuildError("Unknown offload configure step: %s, available: %s"
+ % (self.current_stage, self.build_stages))
 
  # enable building of libiberty, if desired
  if self.cfg['withlibiberty']:
@@ -688,7 +779,23 @@ def build_step(self):
  # call standard build_step
  super(EB_GCC, self).build_step()
 
- # make install is just standard install_step, nothing special there
+ def install_step(self, *args, **kwargs):
+ """Custom install step: avoid installing LLVM when building with AMD GCN offloading support"""
+ # When building AMD offloading do not install the LLVM source files with 'make install', instead move a few
+ # binaries that GCC expects when building 'newlib' so that it can reuse the LLVM GCN support
+ # https://gcc.gnu.org/wiki/Offloading#For_AMD_GCN:
+ if self.current_stage == AMD_LLVM:
+ llvm_binaries = [('ar', 'llvm-ar'), ('ranlib', 'llvm-ar'), ('as', 'llvm-mc'), ('nm', 'llvm-nm'),
+ ('ld', 'lld')]
+ from_path_start = glob.glob(os.path.join(self.builddir, 'gcc-*'))
+ if not from_path_start:
+ raise EasyBuildError("Could not find GCC 'build_llvm_amdgcn' build directory")
+ for gcc_tool, llvm_tool in llvm_binaries:
+ from_path = '%s/build_llvm_amdgcn/bin/%s' % (from_path_start[0], llvm_tool)
+ to_path = '%s/amdgcn-amdhsa/bin/%s' % (self.installdir, gcc_tool)
+ copy_file(from_path, to_path)
+ else:
+ super(EB_GCC, self).install_step(*args, **kwargs)
 
  def post_install_step(self, *args, **kwargs):
  """
@@ -770,14 +877,29 @@ def post_install_step(self, *args, **kwargs):
 
  def run_all_steps(self, *args, **kwargs):
  """
- If withnvptx is set, use iterated build:
+ If 'withnvptx' and 'withamdgcn' is set, use iterated build:
  iteration 0 builds the regular host compiler
  iteration 1 builds nvptx-tools
  iteration 2 builds the nvptx target compiler
+ iteration 3 builds LLD for AMD GCN offloading
+ iteration 4 builds the AMD GCN target compiler using LLD from above
  """
+ self.cfg['configopts'] = ['']
+ self.cfg['buildopts'] = ['']
  if self.cfg['withnvptx']:
- self.cfg['configopts'] = [self.cfg['configopts']] * 3
- self.cfg['buildopts'] = [self.cfg['buildopts']] * 3
+ # Add two additional configure and build stages when enabling NVPTX
+ # this is required to first build NVPTX tools and then newlib
+ self.cfg['configopts'] += ['', '']
+ self.cfg['buildopts'] += ['', '']
+ self.build_stages.append(NVPTX_TOOLS)
+ self.build_stages.append(NVIDIA_NEWLIB)
+ if self.cfg['withamdgcn']:
+ # Add two additional stages when enabling AMD GCN
+ # this is required to first build LLVM tools and then newlib
+ self.cfg['configopts'] += ['', '']
+ self.cfg['buildopts'] += ['', '']
+ self.build_stages.append(AMD_LLVM)
+ self.build_stages.append(AMD_NEWLIB)
  return super(EB_GCC, self).run_all_steps(*args, **kwargs)
 
  def sanity_check_step(self):
@@ -849,6 +971,9 @@ def sanity_check_step(self):
  bin_files.extend(['nvptx-none-as', 'nvptx-none-ld'])
  lib_files.append('libgomp-plugin-nvptx.%s' % sharedlib_ext)
 
+ if self.cfg['withamdgcn']:
+ lib_files.append('libgomp-plugin-gcn.%s' % sharedlib_ext)
+
  bin_files = ["bin/%s" % x for x in bin_files]
  libdirs64 = ['lib64']
  libdirs32 = ['lib', 'lib32']