From b2f021534626aa4955b9a3a496ab6e24d8acc071 Mon Sep 17 00:00:00 2001 From: Vicente Adolfo Bolea Sanchez Date: Fri, 12 May 2023 12:45:24 -0400 Subject: [PATCH] ci,olcf,crusher: enable Crusher CI --- .gitlab/ccache.cmake | 61 ++++++ .gitlab/gitlab-ci-ascent.yml | 2 + .gitlab/gitlab-ci-crusher.yml | 187 ++++++++++++++++++ .gitlab/kokkos.sh | 1 - scripts/ci/cmake-v2/ci-crusher-cray.cmake | 32 +++ .../ci/cmake-v2/ci-crusher-kokkos-hip.cmake | 31 +++ scripts/ci/gitlab-ci/run.sh | 17 +- 7 files changed, 325 insertions(+), 6 deletions(-) create mode 100644 .gitlab/ccache.cmake create mode 100644 .gitlab/gitlab-ci-crusher.yml create mode 100644 scripts/ci/cmake-v2/ci-crusher-cray.cmake create mode 100644 scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake diff --git a/.gitlab/ccache.cmake b/.gitlab/ccache.cmake new file mode 100644 index 0000000000..5e5f90c66b --- /dev/null +++ b/.gitlab/ccache.cmake @@ -0,0 +1,61 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) + +set(version 4.6.1) +set(arch x86_64) + +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(sha256sum da1e1781bc1c4b019216fa16391af3e1daaee7e7f49a8ec9b0cdc8a1d05c50e2) + set(base_url https://github.com/ccache/ccache/releases/download) + set(platform linux) + set(extension tar.xz) +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + set(sha256sum 3e36ba8c80fbf7f2b95fe0227b9dd1ca6143d721aab052caf0d5729769138059) + set(full_url https://gitlab.kitware.com/utils/ci-utilities/-/package_files/534/download) + set(filename ccache) + set(extension tar.gz) +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + set(sha256sum a6c6311973aa3d2aae22424895f2f968e5d661be003b25f1bd854a5c0cd57563) + set(base_url https://github.com/ccache/ccache/releases/download) + set(platform windows) + set(extension zip) +else() + message(FATAL_ERROR "Unrecognized platform ${CMAKE_HOST_SYSTEM_NAME}") +endif() + +if(NOT DEFINED filename) + set(filename "ccache-${version}-${platform}-${arch}") +endif() + +set(tarball "${filename}.${extension}") + +if(NOT DEFINED full_url) + set(full_url "${base_url}/v${version}/${tarball}") +endif() + +file(DOWNLOAD + "${full_url}" $ENV{CCACHE_INSTALL_DIR}/${tarball} + EXPECTED_HASH SHA256=${sha256sum} + SHOW_PROGRESS + ) + +execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xf ${tarball} + WORKING_DIRECTORY $ENV{CCACHE_INSTALL_DIR} + RESULT_VARIABLE extract_results + ) + +if(extract_results) + message(FATAL_ERROR "Extracting `${tarball}` failed: ${extract_results}.") +endif() + +file(RENAME $ENV{CCACHE_INSTALL_DIR}/${filename} $ENV{CCACHE_INSTALL_DIR}/ccache) diff --git a/.gitlab/gitlab-ci-ascent.yml b/.gitlab/gitlab-ci-ascent.yml index 1d1e5c0a95..5d63e30a38 100644 --- a/.gitlab/gitlab-ci-ascent.yml +++ b/.gitlab/gitlab-ci-ascent.yml @@ -68,12 +68,14 @@ ascent-kokkos-cuda: libffi hdf5 cmake + KOKKOS_VER: 3.7.01 KOKKOS_OPTS: >- -DKokkos_ARCH_POWER9=ON -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ENABLE_CUDA_LAMBDA=ON -DCMAKE_INSTALL_PREFIX:PATH=$CI_BUILDS_DIR/kokkos_install + -DCMAKE_CXX_COMPILER:STRING=$CI_BUILDS_DIR/kokkos-$KOKKOS_VER/bin/nvcc_wrapper Kokkos_DIR: $CI_BUILDS_DIR/kokkos_install before_script: diff --git a/.gitlab/gitlab-ci-crusher.yml b/.gitlab/gitlab-ci-crusher.yml new file mode 100644 index 0000000000..da490a17ad --- /dev/null +++ b/.gitlab/gitlab-ci-crusher.yml @@ -0,0 +1,187 @@ +# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Crusher. + +stages: + - pre + - setup + - build + - post + +.setup_env_ecpci: &setup_env_ecpci | + module purge + module load ${JOB_MODULES} + module list + export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH" + +.install_ccache: &install_ccache | + mkdir -p "$CCACHE_INSTALL_DIR" + cmake --version + cmake -VV -P .gitlab/ccache.cmake + ccache -z + ccache -s + +.crusher-common: + except: + - schedules + interruptible: true + variables: + CCACHE_BASEDIR: "/lustre/orion/csc303/scratch/" + CCACHE_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/ccache" + CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime" + + # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it + CCACHE_IGNOREOPTIONS: "-isystem=*" + CCACHE_NOHASHDIR: "true" + CCACHE_INSTALL_DIR: "$CI_PROJECT_DIR/deps/ccache_install" + + CMAKE_BUILD_TYPE: "RelWithDebInfo" + CMAKE_GENERATOR: "Ninja" + CMAKE_PREFIX_PATH: "$CI_PROJECT_DIR/deps/kokkos_install" + + # We do not want to use the user's ~/.gitconfig + GIT_CONFIG_GLOBAL: "true" + GITLAB_SITE: "OLCF Crusher" + CI_BIN_DIR: "$CI_PROJECT_DIR/build" + +.setup-common: + stage: setup + tags: [crusher, shell] + before_script: + - *setup_env_ecpci + - *install_ccache + script: + - bash scripts/ci/gitlab-ci/run.sh update + artifacts: + expire_in: 24 hours + when: always + paths: + - deps/*install/ + - build/ + +.build-common: + stage: build + tags: [crusher, slurm] + variables: + SCHEDULER_PARAMETERS: "-ACSC303_crusher -t30 --nice=0 -c32 --gpus=4 -N 1" + before_script: + - *setup_env_ecpci + script: + - bash scripts/ci/gitlab-ci/run.sh configure + - bash scripts/ci/gitlab-ci/run.sh build + - bash scripts/ci/gitlab-ci/run.sh test + after_script: + - *setup_env_ecpci + - bash scripts/ci/gitlab-ci/run.sh submit + - ccache -s + +.kokkos-hip-common: + variables: + # Order matters + JOB_MODULES: >- + craype-accel-amd-gfx90a + gcc/12 + cmake + rocm/5.4.3 + git + ninja + libffi + hdf5 + zstd + +setup:crusher-kokkos-hip: + variables: + KOKKOS_OPTS: >- + -DCMAKE_INSTALL_PREFIX:PATH=$CI_PROJECT_DIR/deps/kokkos_install + -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.4.3/hip/bin/hipcc + -DKokkos_ARCH_VEGA90A:BOOL=ON + -DKokkos_ENABLE_HIP:BOOL=ON + -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE:BOOL=OFF + -DKokkos_ENABLE_SERIAL:BOOL=ON + extends: + - .crusher-common + - .setup-common + - .kokkos-hip-common + before_script: + - *setup_env_ecpci + - *install_ccache + - .gitlab/kokkos.sh "$CI_PROJECT_DIR/deps" "3.7.01" $KOKKOS_OPTS + +build:crusher-kokkos-hip: + extends: + - .crusher-common + - .build-common + - .kokkos-hip-common + before_script: + - *setup_env_ecpci + needs: + - setup:crusher-kokkos-hip + dependencies: + - setup:crusher-kokkos-hip + +.cray-common: + variables: + # Order matters + JOB_MODULES: >- + PrgEnv-cray + cmake + git + ninja + libffi + zstd + DefApps + extends: + - .crusher-common + +setup:crusher-cray: + extends: + - .setup-common + - .cray-common + +build:crusher-cray: + extends: + - .build-common + - .cray-common + needs: + - setup:crusher-cray + dependencies: + - setup:crusher-cray + +.report-status: + tags: [crusher, shell] + variables: + STATUS_PROJECT: ornladios/ADIOS2 + STATUS_NAME: OLCF Crusher (Frontier) + script: > + curl -X POST -H @${GITHUB_CURL_HEADERS} + "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA}" + -d "{\"state\":\"${CI_JOB_NAME}\", \"context\":\"${STATUS_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_DESC}\"}" + environment: + name: reporting-github + +pending: + stage: pre + variables: + STATUS_DESC: Pipeline is running + extends: + - .report-status +success: + stage: post + variables: + STATUS_DESC: Pipeline failed + extends: + - .report-status + rules: + - when: on_success + dependencies: + - build:crusher-kokkos-hip + - build:crusher-cray +failure: + stage: post + variables: + STATUS_DESC: Pipeline succeeded + extends: + - .report-status + rules: + - when: on_failure + dependencies: + - build:crusher-kokkos-hip + - build:crusher-cray diff --git a/.gitlab/kokkos.sh b/.gitlab/kokkos.sh index 698fe84ac7..2504b90637 100755 --- a/.gitlab/kokkos.sh +++ b/.gitlab/kokkos.sh @@ -23,7 +23,6 @@ cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \ "-DCMAKE_CXX_STANDARD:STRING=17" \ "-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \ "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \ - "-DCMAKE_CXX_COMPILER:STRING=$WORKDIR/kokkos-$VERSION/bin/nvcc_wrapper" \ $* cmake --build "$WORKDIR/kokkos_build" diff --git a/scripts/ci/cmake-v2/ci-crusher-cray.cmake b/scripts/ci/cmake-v2/ci-crusher-cray.cmake new file mode 100644 index 0000000000..d74802deea --- /dev/null +++ b/scripts/ci/cmake-v2/ci-crusher-cray.cmake @@ -0,0 +1,32 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} craycc) +set(ENV{CXX} craycxx) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_HDF5:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=OFF +ADIOS2_USE_ZFP:BOOL=OFF +ADIOS2_USE_SZ:BOOL=OFF +ADIOS2_USE_Blosc:BOOL=OFF + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE "Install.*|Engine.Staging.TestThreads.*" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake b/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake new file mode 100644 index 0000000000..46c0c147f7 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake @@ -0,0 +1,31 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_HDF5:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON +ADIOS2_USE_Kokkos=ON + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_CUDA_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE "Install.*|Engine.Staging.TestThreads.*" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/gitlab-ci/run.sh b/scripts/ci/gitlab-ci/run.sh index 02ee8c630f..0496b7d14a 100755 --- a/scripts/ci/gitlab-ci/run.sh +++ b/scripts/ci/gitlab-ci/run.sh @@ -1,13 +1,19 @@ #!/bin/bash --login set -e +# Strip the job name prefix +export CI_JOB_NAME="${CI_JOB_NAME#*:}" export CI_BUILD_NAME="${CI_COMMIT_BRANCH#github/}_${CI_JOB_NAME}" export CI_COMMIT_REF=${CI_COMMIT_SHA} export CI_ROOT_DIR="${CI_PROJECT_DIR}/.." export CI_SITE_NAME="${GITLAB_SITE}" export CI_SOURCE_DIR="${CI_PROJECT_DIR}" -export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}" +# In OLCF Crusher we must fix the build directory in the yml. +if [ -z "$CI_BIN_DIR" ] +then + export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}" +fi readonly CTEST_SCRIPT=scripts/ci/cmake-v2/ci-${CI_JOB_NAME}.cmake if [ ! -f "$CTEST_SCRIPT" ] @@ -36,10 +42,11 @@ fi declare -a CTEST_STEP_ARGS=("-Ddashboard_full=OFF") case ${STEP} in - update) CTEST_STEP_ARGS+=("${UPDATE_ARGS}") ;; - build) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; - test) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; - submit) CTEST_STEP_ARGS+=("-Ddashboard_do_submit_only=ON" "-Ddashboard_do_build=ON" "-Ddashboard_do_test=ON") ;; + update) CTEST_STEP_ARGS+=("${UPDATE_ARGS}") ;; + configure) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + build) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + test) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + submit) CTEST_STEP_ARGS+=("-Ddashboard_do_submit_only=ON" "-Ddashboard_do_configure=ON" "-Ddashboard_do_build=ON" "-Ddashboard_do_test=ON") ;; esac CTEST_STEP_ARGS+=("-Ddashboard_do_${STEP}=ON")