Incline CPU Testing + CI

.gitlab/pnnl/.gitlab-ci.yml

@@ -5,6 +5,13 @@ trigger_deception:
       - local: .gitlab/pnnl/base.gitlab-ci.yml
       - local: .gitlab/pnnl/deception.gitlab-ci.yml
 
+trigger_incline:
+  needs: []
+  trigger:
+    include:
+      - local: .gitlab/pnnl/base.gitlab-ci.yml
+      - local: .gitlab/pnnl/incline.gitlab-ci.yml
+
 SVC-Account-Cleanup:
   image: kfox1111/slurm:deception2
   tags:

.gitlab/pnnl/base.gitlab-ci.yml

@@ -20,6 +20,14 @@ include:
     - &rule_yes_deception_test
       if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
       when: always
+    # Only run when the commit **DOES NOT** contains "[incline-test]"
+    - &rule_no_incline_test
+      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
+      when: never
+    # Only run when the commit **DOES** contains "[incline-test]"
+    - &rule_yes_incline_test
+      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
+      when: always
 
 
 variables:
@@ -246,8 +254,19 @@ stages:
 .deception:
   rules:
     - *rule_yes_deception_test
+    - *rule_no_incline_test
     - *default
   variables:
     MY_CLUSTER: "deception"
     WORKDIR_SUFFIX: "x86_64-build"
     SLURM_ARGS: " --gres=gpu:1 --ntasks=3 "
+
+.incline:
+  rules:
+    - *rule_no_deception_test
+    - *rule_yes_incline_test
+    - *default
+  variables:
+    WORKDIR_SUFFIX: "x86_64-clang-hip-build"
+    MY_CLUSTER: "incline"
+    SLURM_ARGS: " --exclusive --ntasks=3 "

.gitlab/pnnl/incline.gitlab-ci.yml

@@ -0,0 +1,39 @@
+Incline Build:
+  extends:
+    - .cluster_build
+    - .incline
+  variables:
+    SCRIPT_ARGS: " --build-only " #--job=clang-hip "
+
+Incline Test:
+  extends:
+    - .cluster_test
+    - .incline
+  variables:
+    SCRIPT_ARGS: " --test-only " #--job=clang-hip "
+    CTESTARGS: " --timeout 240 --output-on-failure -LE incline-skip "
+  needs: ['Incline Build']
+
+pending:
+  variables:
+    MY_CLUSTER: "Incline"
+  extends:
+    - .report-pending
+  stage: .pre
+
+success:
+  variables:
+    MY_CLUSTER: "Incline"
+  extends:
+    - .report-status
+  stage: .post
+
+failure:
+  stage: .post
+  variables:
+    MY_CLUSTER: "Incline"
+  extends:
+    - .report-status
+  rules:
+    - when: on_failure
+

CMakePresets.json

@@ -14,22 +14,40 @@
             "installDir": "${sourceDir}/install",
             "generator": "Unix Makefiles"
         },
-	{
-	      "name": "ascent",
-	      "inherits": "cuda",
-	      "displayName": "Ascent Build",
-	      "description": "Custom changes specific for Ascent",
-	      "cacheVariables": {
-	            "CMAKE_C_COMPILER": "$env{OLCF_GCC_ROOT}/bin/gcc",
-	            "CMAKE_CXX_COMPILER": "$env{OLCF_GCC_ROOT}/bin/g++"
-	      }
-	},
-	{
-	      "name": "deception",
-	      "inherits": "cuda",
-	      "displayName": "Deception Build",
-	      "description": "Custom changes specific for Deception"
-	}
+        {
+            "name": "cpu",
+            "displayName": "CPU only build",
+            "description": "Base config to build without GPUs",
+            "binaryDir": "${sourceDir}/build",
+            "installDir": "${sourceDir}/install",
+            "generator": "Unix Makefiles",
+            "cacheVariables": {
+                "RESOLVE_USE_CUDA": "OFF",
+                "RESOLVE_USE_GPU": "OFF"
+            }
+        },
+	      {
+            "name": "ascent",
+            "inherits": "cuda",
+            "displayName": "Ascent Build",
+            "description": "Custom changes specific for Ascent",
+            "cacheVariables": {
+                  "CMAKE_C_COMPILER": "$env{OLCF_GCC_ROOT}/bin/gcc",
+                  "CMAKE_CXX_COMPILER": "$env{OLCF_GCC_ROOT}/bin/g++"
+	          }
+        },
+        {
+            "name": "deception",
+            "inherits": "cuda",
+            "displayName": "Deception Build",
+            "description": "Custom changes specific for Deception"
+        },
+        {
+            "name": "incline",
+            "inherits": "cpu",
+            "displayName": "Incline CPU only Build",
+            "description": "Custom changes specific for Incline"
+        }
 
     ]
 }

buildsystem/build.sh

@@ -64,6 +64,7 @@ Clusters:
 
     - deception
     - ascent 
+    - incline
 
   Run \`export MY_CLUSTER=deception\` or invoke the build script with:
 
@@ -136,6 +137,27 @@ fi
 
 module purge
 
+if [[ ! -v MY_CLUSTER ]]
+then
+  export MY_CLUSTER=`uname -n | sed -e 's/[0-9]//g' -e 's/\..*//'`
+fi
+
+# Correctly identify clusters based on hostname
+case $MY_CLUSTER in
+  incline*|dmi*)
+    export MY_CLUSTER=incline
+    ;;
+  dl*|deception|*fat*)
+    export MY_CLUSTER=deception
+    ;;
+  ascent*)
+    export MY_CLUSTER=ascent
+    ;;
+  *)
+    echo "Cluster $MY_CLUSTER not identified - you'll have to set relevant variables manually."
+    ;;
+esac
+
 varfile="${SRCDIR}/buildsystem/${MY_CLUSTER}-env.sh"
 
 if [[ ! -v MY_CLUSTER ]]
@@ -147,7 +169,7 @@ if [[ -f "$varfile" ]]; then
   source $varfile || { echo "Could not source $varfile"; exit 1; }
 else
   echo "No cluster variable file configured for ${MY_CLUSTER}. Try one of:\n"
-  echo "deception, ascent." && exit 1
+  echo "deception, incline, ascent." && exit 1
 fi
 
 echo "Paths:"

buildsystem/configure-modules.sh

@@ -19,6 +19,10 @@ spack module tcl refresh -y && \
 mkdir -p $base/modules && \
 # Note we redirect and destroy old file
 arch=$(spack arch) && \
+if [ $MY_CLUSTER == "incline" ]; then
+  # Trim the last character (see #464 on GitLab)
+  arch=${arch::-1} 
+fi && \
 echo Arch for module path being used is $arch && \
 echo module use -a $SPACK_INSTALL/modules/$arch &> $base/modules/dependencies.sh && \
 spack module tcl loads -r -x resolve resolve &>> $base/modules/dependencies.sh

buildsystem/incline-env.sh

@@ -0,0 +1,5 @@
+source /etc/profile.d/modules.sh
+module purge
+module load gcc/8.4.0
+module load rocm/5.3.0
+source ./buildsystem/spack/incline/modules/dependencies.sh

buildsystem/spack/incline/env.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+source /etc/profile.d/modules.sh
+module purge
+
+# Load system python
+module load python/miniconda4.12
+source /share/apps/python/miniconda4.12/etc/profile.d/conda.sh
+
+# Define environment variables for where spack stores key files
+# For now, SPACK_INSTALL is the path where everything spack related is installed
+# If you want to modify the module install path, edit the spack.yaml manually
+BASE=/qfs/projects/exasgd/resolve/spack-ci
+export SPACK_INSTALL=$BASE/install
+export SPACK_CACHE=$BASE/../$(whoami)/spack-cache
+export SPACK_DISABLE_LOCAL_CONFIG=1
+export SPACK_PYTHON=$(which python)
+
+export tempdir=$SPACK_CACHE
+export TMP=$SPACK_CACHE
+export TMPDIR=$SPACK_CACHE

buildsystem/spack/incline/install.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH -A exasgd
+#SBATCH -p incline
+#SBATCH -N 1
+#SBATCH -n 3
+#SBATCH --gres=gpu:1
+#SBATCH -J resolve_spack
+#SBATCH -o spack_install.%J.output
+#SBATCH -e spack_install.%J.output
+#SBTACH -t 240
+
+export MY_CLUSTER=incline
+. buildsystem/load-spack.sh &&
+spack develop --no-clone --path=$(pwd) resolve@develop &&
+./buildsystem/configure-modules.sh 64
+

buildsystem/spack/incline/modules/dependencies.sh

@@ -0,0 +1,65 @@
+module use -a /qfs/projects/exasgd/resolve/spack-ci/install/modules/linux-centos7-zen
+# pkgconf@=1.9.5%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load pkgconf/1.9.5-gcc-8.4.0-kl4sdjo
+# nghttp2@=1.52.0%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load nghttp2/1.52.0-gcc-8.4.0-pqmjl5g
+# ca-certificates-mozilla@=2023-05-30%gcc@=8.4.0 build_system=generic arch=linux-centos7-zen
+module load ca-certificates-mozilla/2023-05-30-gcc-8.4.0-txgcsig
+# perl@=5.26.0%gcc@=8.4.0+cpanm+opcode+open+shared+threads build_system=generic patches=0eac10e,8cf4302 arch=linux-centos7-zen
+module load perl/5.26.0-gcc-8.4.0-h324qox
+# zlib-ng@=2.1.3%gcc@=8.4.0+compat+opt build_system=autotools patches=299b958,ae9077a,b692621 arch=linux-centos7-zen
+module load zlib-ng/2.1.3-gcc-8.4.0-44tydhr
+# openssl@=3.1.3%gcc@=8.4.0~docs+shared build_system=generic certs=mozilla arch=linux-centos7-zen
+module load openssl/3.1.3-gcc-8.4.0-46yttzm
+# curl@=8.4.0%gcc@=8.4.0~gssapi~ldap~libidn2~librtmp~libssh~libssh2+nghttp2 build_system=autotools libs=shared,static tls=openssl arch=linux-centos7-zen
+module load curl/8.4.0-gcc-8.4.0-g2rrs23
+# ncurses@=6.4%gcc@=8.4.0~symlinks+termlib abi=none build_system=autotools arch=linux-centos7-zen
+module load ncurses/6.4-gcc-8.4.0-jt7rpqq
+# cmake@=3.27.7%gcc@=8.4.0~doc+ncurses+ownlibs build_system=generic build_type=Release arch=linux-centos7-zen
+module load cmake/3.27.7-gcc-8.4.0-tu2rruq
+# gmake@=4.4.1%gcc@=8.4.0~guile build_system=autotools arch=linux-centos7-zen
+module load gmake/4.4.1-gcc-8.4.0-f23wik2
+# libiconv@=1.17%gcc@=8.4.0 build_system=autotools libs=shared,static arch=linux-centos7-zen
+module load libiconv/1.17-gcc-8.4.0-wfdnlg6
+# diffutils@=3.9%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load diffutils/3.9-gcc-8.4.0-qh566r6
+# libsigsegv@=2.14%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load libsigsegv/2.14-gcc-8.4.0-iutj4de
+# m4@=1.4.19%gcc@=8.4.0+sigsegv build_system=autotools patches=9dc5fbd,bfdffa7 arch=linux-centos7-zen
+module load m4/1.4.19-gcc-8.4.0-x7ktvaf
+# autoconf@=2.69%gcc@=8.4.0 build_system=autotools patches=35c4492,7793209,a49dd5b arch=linux-centos7-zen
+module load autoconf/2.69-gcc-8.4.0-npluk5j
+# automake@=1.16.5%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load automake/1.16.5-gcc-8.4.0-tgloywk
+# libtool@=2.4.7%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load libtool/2.4.7-gcc-8.4.0-gs6gyy3
+# gmp@=6.2.1%gcc@=8.4.0+cxx build_system=autotools libs=shared,static patches=69ad2e2 arch=linux-centos7-zen
+module load gmp/6.2.1-gcc-8.4.0-ythx4o2
+# metis@=5.1.0%gcc@=8.4.0~gdb~int64~ipo~real64+shared build_system=cmake build_type=Release generator=make patches=4991da9,93a7903,b1225da arch=linux-centos7-zen
+module load metis/5.1.0-gcc-8.4.0-gsllf6a
+# autoconf-archive@=2023.02.20%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load autoconf-archive/2023.02.20-gcc-8.4.0-ox4hxoe
+# bzip2@=1.0.8%gcc@=8.4.0~debug~pic+shared build_system=generic arch=linux-centos7-zen
+module load bzip2/1.0.8-gcc-8.4.0-3uzyl47
+# xz@=5.4.1%gcc@=8.4.0~pic build_system=autotools libs=shared,static arch=linux-centos7-zen
+module load xz/5.4.1-gcc-8.4.0-dwmuagy
+# libxml2@=2.10.3%gcc@=8.4.0+pic~python+shared build_system=autotools arch=linux-centos7-zen
+module load libxml2/2.10.3-gcc-8.4.0-2hu4ayt
+# pigz@=2.7%gcc@=8.4.0 build_system=makefile arch=linux-centos7-zen
+module load pigz/2.7-gcc-8.4.0-lu7bjb6
+# zstd@=1.5.5%gcc@=8.4.0+programs build_system=makefile compression=none libs=shared,static arch=linux-centos7-zen
+module load zstd/1.5.5-gcc-8.4.0-z7jmyvw
+# tar@=1.34%gcc@=8.4.0 build_system=autotools zip=pigz arch=linux-centos7-zen
+module load tar/1.34-gcc-8.4.0-wcgempy
+# gettext@=0.22.3%gcc@=8.4.0+bzip2+curses+git~libunistring+libxml2+pic+shared+tar+xz build_system=autotools arch=linux-centos7-zen
+module load gettext/0.22.3-gcc-8.4.0-f7dl6un
+# texinfo@=7.0.3%gcc@=8.4.0 build_system=autotools arch=linux-centos7-zen
+module load texinfo/7.0.3-gcc-8.4.0-jma4obj
+# mpfr@=4.2.0%gcc@=8.4.0 build_system=autotools libs=shared,static arch=linux-centos7-zen
+module load mpfr/4.2.0-gcc-8.4.0-cjhi2el
+# openblas@=0.3.24%gcc@=8.4.0~bignuma~consistent_fpcsr+fortran~ilp64+locking+pic+shared build_system=makefile symbol_suffix=none threads=none arch=linux-centos7-zen
+module load openblas/0.3.24-gcc-8.4.0-4ei4hpg
+# suite-sparse@=5.13.0%gcc@=8.4.0~cuda~graphblas~openmp+pic build_system=generic arch=linux-centos7-zen
+module load suite-sparse/5.13.0-gcc-8.4.0-ivey23b
+# resolve@=develop%gcc@=8.4.0~cuda~ipo+klu build_system=cmake build_type=Release dev_path=/people/ruth521/projects/resolve generator=make arch=linux-centos7-zen
+## module load resolve/develop-gcc-8.4.0-ugoj3p3

buildsystem/spack/incline/spack.yaml

@@ -0,0 +1,42 @@
+spack:
+  specs:
+    - resolve~cuda%gcc@8.4.0
+  view: false
+  concretizer:
+    unify: when_possible
+    reuse: true
+  config:
+    concretizer: clingo
+    install_tree:
+      root: $SPACK_INSTALL
+    source_cache: $SPACK_CACHE/source
+    misc_cache: $SPACK_CACHE/misc
+    build_stage: $SPACK_CACHE/build
+    url_fetch_method: curl
+  modules:
+    default:
+      roots:
+        tcl: $SPACK_INSTALL/modules
+  packages:
+    all:
+      permissions:
+        write: group
+        read: world
+        group: exasgd
+    perl:
+      externals:
+      - spec: perl@5.26.0
+        modules:
+        - perl/5.26.0
+      buildable: false
+  compilers:
+  - compiler:
+      spec: gcc@8.4.0
+      paths:
+        cc: /share/apps/gcc/8.4.0/bin/gcc
+        cxx: /share/apps/gcc/8.4.0/bin/g++
+        f77: /share/apps/gcc/8.4.0/bin/gfortran
+        fc: /share/apps/gcc/8.4.0/bin/gfortran
+      operating_system: centos7
+      target: x86_64
+      modules: [gcc/8.4.0]