ci,olcf,crusher: enable Crusher CI

ornladios · May 18, 2023 · b2f0215 · b2f0215
1 parent 189f384
commit b2f0215
Show file tree

Hide file tree

Showing 7 changed files with 325 additions and 6 deletions.
diff --git a/.gitlab/ccache.cmake b/.gitlab/ccache.cmake
@@ -0,0 +1,61 @@
+##============================================================================
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##============================================================================
+
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+
+set(version 4.6.1)
+set(arch x86_64)
+
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
+  set(sha256sum da1e1781bc1c4b019216fa16391af3e1daaee7e7f49a8ec9b0cdc8a1d05c50e2)
+  set(base_url https://github.com/ccache/ccache/releases/download)
+  set(platform linux)
+  set(extension tar.xz)
+elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+  set(sha256sum 3e36ba8c80fbf7f2b95fe0227b9dd1ca6143d721aab052caf0d5729769138059)
+  set(full_url https://gitlab.kitware.com/utils/ci-utilities/-/package_files/534/download)
+  set(filename ccache)
+  set(extension tar.gz)
+elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
+  set(sha256sum a6c6311973aa3d2aae22424895f2f968e5d661be003b25f1bd854a5c0cd57563)
+  set(base_url https://github.com/ccache/ccache/releases/download)
+  set(platform windows)
+  set(extension zip)
+else()
+  message(FATAL_ERROR "Unrecognized platform ${CMAKE_HOST_SYSTEM_NAME}")
+endif()
+
+if(NOT DEFINED filename)
+  set(filename "ccache-${version}-${platform}-${arch}")
+endif()
+
+set(tarball "${filename}.${extension}")
+
+if(NOT DEFINED full_url)
+  set(full_url "${base_url}/v${version}/${tarball}")
+endif()
+
+file(DOWNLOAD
+  "${full_url}" $ENV{CCACHE_INSTALL_DIR}/${tarball}
+  EXPECTED_HASH SHA256=${sha256sum}
+  SHOW_PROGRESS
+  )
+
+execute_process(
+  COMMAND ${CMAKE_COMMAND} -E tar xf ${tarball}
+  WORKING_DIRECTORY $ENV{CCACHE_INSTALL_DIR}
+  RESULT_VARIABLE extract_results
+  )
+
+if(extract_results)
+  message(FATAL_ERROR "Extracting `${tarball}` failed: ${extract_results}.")
+endif()
+
+file(RENAME $ENV{CCACHE_INSTALL_DIR}/${filename} $ENV{CCACHE_INSTALL_DIR}/ccache)
diff --git a/.gitlab/gitlab-ci-ascent.yml b/.gitlab/gitlab-ci-ascent.yml
@@ -68,12 +68,14 @@ ascent-kokkos-cuda:
       libffi
       hdf5
       cmake
+    KOKKOS_VER: 3.7.01
     KOKKOS_OPTS: >-
       -DKokkos_ARCH_POWER9=ON
       -DKokkos_ARCH_VOLTA70=ON
       -DKokkos_ENABLE_CUDA=ON
       -DKokkos_ENABLE_CUDA_LAMBDA=ON
       -DCMAKE_INSTALL_PREFIX:PATH=$CI_BUILDS_DIR/kokkos_install
+      -DCMAKE_CXX_COMPILER:STRING=$CI_BUILDS_DIR/kokkos-$KOKKOS_VER/bin/nvcc_wrapper
 
     Kokkos_DIR: $CI_BUILDS_DIR/kokkos_install
   before_script:

diff --git a/.gitlab/gitlab-ci-crusher.yml b/.gitlab/gitlab-ci-crusher.yml
@@ -0,0 +1,187 @@
+# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Crusher.
+
+stages:
+  - pre
+  - setup
+  - build
+  - post
+
+.setup_env_ecpci: &setup_env_ecpci |
+  module purge
+  module load ${JOB_MODULES}
+  module list
+  export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH"
+
+.install_ccache: &install_ccache |
+  mkdir -p "$CCACHE_INSTALL_DIR"
+  cmake --version
+  cmake -VV -P .gitlab/ccache.cmake
+  ccache -z
+  ccache -s
+
+.crusher-common:
+  except:
+    - schedules
+  interruptible: true
+  variables:
+    CCACHE_BASEDIR:       "/lustre/orion/csc303/scratch/"
+    CCACHE_DIR:           "/lustre/orion/csc303/scratch/vbolea/ci/adios2/ccache"
+    CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime"
+
+    # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it
+    CCACHE_IGNOREOPTIONS: "-isystem=*"
+    CCACHE_NOHASHDIR:     "true"
+    CCACHE_INSTALL_DIR:   "$CI_PROJECT_DIR/deps/ccache_install"
+
+    CMAKE_BUILD_TYPE:     "RelWithDebInfo"
+    CMAKE_GENERATOR:      "Ninja"
+    CMAKE_PREFIX_PATH:    "$CI_PROJECT_DIR/deps/kokkos_install"
+
+    # We do not want to use the user's ~/.gitconfig
+    GIT_CONFIG_GLOBAL:    "true"
+    GITLAB_SITE:          "OLCF Crusher"
+    CI_BIN_DIR:           "$CI_PROJECT_DIR/build"
+
+.setup-common:
+  stage: setup
+  tags: [crusher, shell]
+  before_script:
+    - *setup_env_ecpci
+    - *install_ccache
+  script:
+    - bash scripts/ci/gitlab-ci/run.sh update
+  artifacts:
+    expire_in: 24 hours
+    when: always
+    paths:
+      - deps/*install/
+      - build/
+
+.build-common:
+  stage: build
+  tags: [crusher, slurm]
+  variables:
+    SCHEDULER_PARAMETERS: "-ACSC303_crusher -t30 --nice=0 -c32 --gpus=4 -N 1"
+  before_script:
+    - *setup_env_ecpci
+  script:
+    - bash scripts/ci/gitlab-ci/run.sh configure
+    - bash scripts/ci/gitlab-ci/run.sh build
+    - bash scripts/ci/gitlab-ci/run.sh test
+  after_script:
+    - *setup_env_ecpci
+    - bash scripts/ci/gitlab-ci/run.sh submit
+    - ccache -s
+
+.kokkos-hip-common:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      craype-accel-amd-gfx90a
+      gcc/12
+      cmake
+      rocm/5.4.3
+      git
+      ninja
+      libffi
+      hdf5
+      zstd
+
+setup:crusher-kokkos-hip:
+  variables:
+    KOKKOS_OPTS: >-
+      -DCMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/deps/kokkos_install
+      -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.4.3/hip/bin/hipcc
+      -DKokkos_ARCH_VEGA90A:BOOL=ON
+      -DKokkos_ENABLE_HIP:BOOL=ON
+      -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE:BOOL=OFF
+      -DKokkos_ENABLE_SERIAL:BOOL=ON
+  extends:
+    - .crusher-common
+    - .setup-common
+    - .kokkos-hip-common
+  before_script:
+    - *setup_env_ecpci
+    - *install_ccache
+    - .gitlab/kokkos.sh "$CI_PROJECT_DIR/deps" "3.7.01" $KOKKOS_OPTS
+
+build:crusher-kokkos-hip:
+  extends:
+    - .crusher-common
+    - .build-common
+    - .kokkos-hip-common
+  before_script:
+    - *setup_env_ecpci
+  needs:
+    - setup:crusher-kokkos-hip
+  dependencies:
+    - setup:crusher-kokkos-hip
+
+.cray-common:
+  variables:
+    # Order matters
+    JOB_MODULES: >-
+      PrgEnv-cray
+      cmake
+      git
+      ninja
+      libffi
+      zstd
+      DefApps
+  extends:
+    - .crusher-common
+
+setup:crusher-cray:
+  extends:
+    - .setup-common
+    - .cray-common
+
+build:crusher-cray:
+  extends:
+    - .build-common
+    - .cray-common
+  needs:
+    - setup:crusher-cray
+  dependencies:
+    - setup:crusher-cray
+
+.report-status:
+  tags: [crusher, shell]
+  variables:
+    STATUS_PROJECT: ornladios/ADIOS2
+    STATUS_NAME: OLCF Crusher (Frontier)
+  script: >
+    curl -X POST -H @${GITHUB_CURL_HEADERS}
+    "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}"
+    -d "{\"state\":\"${CI_JOB_NAME}\", \"context\":\"${STATUS_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_DESC}\"}"
+  environment:
+    name: reporting-github
+
+pending:
+  stage: pre
+  variables:
+    STATUS_DESC: Pipeline is running
+  extends:
+    - .report-status
+success:
+  stage: post
+  variables:
+    STATUS_DESC: Pipeline failed
+  extends:
+    - .report-status
+  rules:
+    - when: on_success
+  dependencies:
+    - build:crusher-kokkos-hip
+    - build:crusher-cray
+failure:
+  stage: post
+  variables:
+    STATUS_DESC: Pipeline succeeded
+  extends:
+    - .report-status
+  rules:
+    - when: on_failure
+  dependencies:
+    - build:crusher-kokkos-hip
+    - build:crusher-cray
diff --git a/.gitlab/kokkos.sh b/.gitlab/kokkos.sh
@@ -23,7 +23,6 @@ cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \
   "-DCMAKE_CXX_STANDARD:STRING=17" \
   "-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \
   "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \
-  "-DCMAKE_CXX_COMPILER:STRING=$WORKDIR/kokkos-$VERSION/bin/nvcc_wrapper" \
   $*
 
 cmake --build "$WORKDIR/kokkos_build"

diff --git a/scripts/ci/cmake-v2/ci-crusher-cray.cmake b/scripts/ci/cmake-v2/ci-crusher-cray.cmake
@@ -0,0 +1,32 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(ENV{CC}  craycc)
+set(ENV{CXX} craycxx)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=OFF
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_HDF5:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SST:BOOL=ON
+ADIOS2_USE_ZeroMQ:STRING=OFF
+ADIOS2_USE_ZFP:BOOL=OFF
+ADIOS2_USE_SZ:BOOL=OFF
+ADIOS2_USE_Blosc:BOOL=OFF
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
+")
+
+set(CTEST_TEST_ARGS
+  PARALLEL_LEVEL 8
+  EXCLUDE "Install.*|Engine.Staging.TestThreads.*"
+  )
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
diff --git a/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake b/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake
@@ -0,0 +1,31 @@
+# Client maintainer: vicente.bolea@kitware.com
+
+set(ENV{CC}  gcc)
+set(ENV{CXX} g++)
+set(ENV{FC}  gfortran)
+
+set(dashboard_cache "
+ADIOS2_USE_BZip2:BOOL=OFF
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=OFF
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_HDF5:BOOL=OFF
+ADIOS2_USE_PNG:BOOL=OFF
+ADIOS2_USE_Python:BOOL=OFF
+ADIOS2_USE_SST:BOOL=ON
+ADIOS2_USE_Kokkos=ON
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_CUDA_COMPILER_LAUNCHER=ccache
+CMAKE_DISABLE_FIND_PACKAGE_BISON=ON
+CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON
+")
+
+set(CTEST_TEST_ARGS
+  PARALLEL_LEVEL 8
+  EXCLUDE "Install.*|Engine.Staging.TestThreads.*"
+  )
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)
diff --git a/scripts/ci/gitlab-ci/run.sh b/scripts/ci/gitlab-ci/run.sh
@@ -1,13 +1,19 @@
 #!/bin/bash --login
 set -e
 
+# Strip the job name prefix
+export CI_JOB_NAME="${CI_JOB_NAME#*:}"
 export CI_BUILD_NAME="${CI_COMMIT_BRANCH#github/}_${CI_JOB_NAME}"
 export CI_COMMIT_REF=${CI_COMMIT_SHA}
 export CI_ROOT_DIR="${CI_PROJECT_DIR}/.."
 export CI_SITE_NAME="${GITLAB_SITE}"
 export CI_SOURCE_DIR="${CI_PROJECT_DIR}"
 
-export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}"
+# In OLCF Crusher we must fix the build directory in the yml.
+if [ -z "$CI_BIN_DIR" ]
+then
+  export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}"
+fi
 
 readonly CTEST_SCRIPT=scripts/ci/cmake-v2/ci-${CI_JOB_NAME}.cmake
 if [ ! -f "$CTEST_SCRIPT" ]
@@ -36,10 +42,11 @@ fi
 
 declare -a CTEST_STEP_ARGS=("-Ddashboard_full=OFF")
 case ${STEP} in
-  update) CTEST_STEP_ARGS+=("${UPDATE_ARGS}") ;;
-  build)  CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;;
-  test)   CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;;
-  submit) CTEST_STEP_ARGS+=("-Ddashboard_do_submit_only=ON" "-Ddashboard_do_build=ON" "-Ddashboard_do_test=ON") ;;
+  update)    CTEST_STEP_ARGS+=("${UPDATE_ARGS}") ;;
+  configure) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;;
+  build)     CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;;
+  test)      CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;;
+  submit)    CTEST_STEP_ARGS+=("-Ddashboard_do_submit_only=ON" "-Ddashboard_do_configure=ON" "-Ddashboard_do_build=ON" "-Ddashboard_do_test=ON") ;;
 esac
 CTEST_STEP_ARGS+=("-Ddashboard_do_${STEP}=ON")