[Woptim] Update shared ci to v2023.08.0 (#840)

*    Updates rocmcc to 5.6.0 on Tioga and Corona
*    Updates cce to 16.0.0 on Tioga
*    Updates gcc to 10.3.1 when using intel 19.1.2
*    Remove unused xl 2022 combined with cuda 11.7
*    Switch to IBM clang 12 in one of the lassen jobs

.gitlab-ci.yml

@@ -11,8 +11,10 @@
 # at Lawrence Livermore National Laboratory (LLNL).
 # This entire pipeline is LLNL-specific
 #
-# Important note: This file is a copy of the template provided by
-# llnl/radiuss-shared-ci. It should not require any change from the project.
+# Important note: This file is a template provided by llnl/radiuss-shared-ci.
+# Remains to set variable values, change the reference to the radiuss-shared-ci
+# repo, opt-in and out optional features. The project can then extend it with
+# additional stages.
 #
 # Instead, each project should provide:
 # - .gitlab/custom-jobs-and-variables.yml
@@ -59,7 +61,7 @@ stages:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2023.06.0
+        ref: v2023.08.0
         file: '${CI_MACHINE}-build-and-test.yml'
       - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
     strategy: depend
@@ -69,7 +71,7 @@ stages:
 include:
   # [Optional] checks preliminary to running the actual CI test
   #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: v2023.06.0
+  #  ref: v2023.08.0
   #  file: 'preliminary-ignore-draft-pr.yml'
   # pipelines subscribed by the project
   - local: .gitlab/subscribed-pipelines.yml

.gitlab/corona-build-and-test-extra.yml

@@ -25,8 +25,8 @@
 # This job intentionally tests our umpire package.py because although this job does not
 # explicitly have the ~tools, the package.py should still disable tools from being built.
 ###
-rocmcc_5_5_0_hip_openmp_device_alloc:
+rocmcc_5_6_0_hip_openmp_device_alloc:
   variables:
-    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.5.0 ^hip@5.5.0"
+    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.6.0 ^hip@5.6.0"
   extends: .build_and_test_on_corona
 

.gitlab/custom-jobs-and-variables.yml

@@ -34,9 +34,9 @@ variables:
 
 # Tioga
 # Arguments for top level allocation
-  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
+  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=20m --nodes=1"
 # Arguments for job level allocation
-  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=15m --nodes=1 --begin-time=+5s"
 # Project specific variants for tioga
   PROJECT_TIOGA_VARIANTS: "~shared +fortran +device_alloc tests=basic "
 # Project specific deps for tioga

.gitlab/lassen-build-and-test-extra.yml

@@ -88,7 +88,7 @@ ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_tpls:
     MODULE_LIST: "cuda/11.7.0"
   extends: .build_and_test_on_lassen
 
-xl_2022_08_19_gcc_8_3_1_cuda_11_7_tpls:
+xl_2022_08_19_gcc_8_3_1_cuda_11_2_tpls:
   variables:
     SPEC: "~shared +fortran +cuda +tools tests=basic %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
     MODULE_LIST: "cuda/11.7.0"

.gitlab/ruby-build-and-test-extra.yml

@@ -12,7 +12,12 @@
 # We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
 # the comparison with the original job is easier.
 
-# No overridden jobs so far.
+# Allow failure due to compiler internal error building wrapfumpire.f
+intel_2022_1_0:
+  variables:
+    SPEC: "~shared +fortran +tools tests=basic %intel@2022.1.0"
+  extends: .build_and_test_on_ruby
+  allow_failure: true
 
 ############
 # Extra jobs
@@ -44,16 +49,8 @@ gcc_10_3_1_tpls:
 
 # Oneapi is not available on ruby@toss4 (rhel8)
 ## We deactivate this job as it is known to fail with Umpire: needs gcc toolchain.
-#intel_oneapi_2022_2:
+#intel_2022_1_0:
 #  variables:
 #    ON_RUBY: "OFF"
-#    SPEC: "${PROJECT_RUBY_VARIANTS} %oneapi@2022.2 ${PROJECT_RUBY_DEPS}"
+#    SPEC: "${PROJECT_RUBY_VARIANTS} %intel@2022.1.0 ${PROJECT_RUBY_DEPS}"
 #  extends: .build_and_test_on_ruby
-#
-#intel_oneapi_2022_2_gcc_8_3_1:
-#  variables:
-#    SPEC: "~shared +fortran +tools tests=basic %oneapi@2022.2.gcc.8.3.1"
-#  extends: .build_and_test_on_ruby
-#  allow_failure: true
-
-

.gitlab/tioga-build-and-test-extra.yml

@@ -13,9 +13,9 @@
 # the comparison with the original job is easier.
 
 # We override the cce job because we can’t use +device-alloc with it
-cce_15_0_1:
+cce_16_0_0:
   variables:
-    SPEC: "~shared +fortran tests=basic %cce@15.0.1"
+    SPEC: "~shared +fortran tests=basic %cce@16.0.0"
   extends: .build_and_test_on_tioga
 
 ############
@@ -29,8 +29,8 @@ cce_15_0_1:
 # This job intentionally tests our umpire package.py because although this job does not
 # explicitly have the ~tools, the package.py should still disable tools from being built.
 ###
-rocmcc_5_5_1_hip_openmp_device_alloc:
+rocmcc_5_6_0_hip_openmp_device_alloc:
   variables:
-    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.5.1 ^hip@5.5.1"
+    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.6.0 ^hip@5.6.0"
   extends: .build_and_test_on_tioga
 

.uberenv_config.json

@@ -4,7 +4,7 @@
 "package_final_phase" : "initconfig",
 "package_source_dir" : "../..",
 "spack_url": "https://github.com/spack/spack.git",
-"spack_branch": "e4s-23.02",
+"spack_branch": "v0.20.1",
 "spack_activate" : {},
 "spack_configs_path": "scripts/radiuss-spack-configs",
 "spack_packages_path": "scripts/radiuss-spack-configs/packages",

scripts/gitlab/build_and_test.sh

@@ -21,11 +21,11 @@ hostname="$(hostname)"
 truehostname=${hostname//[0-9]/}
 project_dir="$(pwd)"
 
-build_root=${BUILD_ROOT:-""}
 hostconfig=${HOST_CONFIG:-""}
 spec=${SPEC:-""}
 module_list=${MODULE_LIST:-""}
 job_unique_id=${CI_JOB_ID:-""}
+use_dev_shm=${USE_DEV_SHM:-true}
 
 if [[ -n ${module_list} ]]
 then
@@ -37,7 +37,7 @@ fi
 
 prefix=""
 
-if [[ -d /dev/shm ]]
+if [[ -d /dev/shm && ${use_dev_shm} == true ]]
 then
     prefix="/dev/shm/${hostname}"
     if [[ -z ${job_unique_id} ]]; then
@@ -50,6 +50,10 @@ then
 
     prefix="${prefix}-${job_unique_id}"
     mkdir -p ${prefix}
+else
+    # We set the prefix in the parent directory so that spack dependencies are not installed inside the source tree.
+    prefix="$(pwd)/../spack-and-build-root"
+    mkdir -p ${prefix}
 fi
 
 # Dependencies
@@ -69,20 +73,15 @@ then
         exit 1
     fi
 
-    prefix_opt=""
+    prefix_opt="--prefix=${prefix}"
 
-    if [[ -d /dev/shm ]]
-    then
-        prefix_opt="--prefix=${prefix}"
-
-        # We force Spack to put all generated files (cache and configuration of
-        # all sorts) in a unique location so that there can be no collision
-        # with existing or concurrent Spack.
-        spack_user_cache="${prefix}/spack-user-cache"
-        export SPACK_DISABLE_LOCAL_CONFIG=""
-        export SPACK_USER_CACHE_PATH="${spack_user_cache}"
-        mkdir -p ${spack_user_cache}
-    fi
+    # We force Spack to put all generated files (cache and configuration of
+    # all sorts) in a unique location so that there can be no collision
+    # with existing or concurrent Spack.
+    spack_user_cache="${prefix}/spack-user-cache"
+    export SPACK_DISABLE_LOCAL_CONFIG=""
+    export SPACK_USER_CACHE_PATH="${spack_user_cache}"
+    mkdir -p ${spack_user_cache}
 
     ./scripts/uberenv/uberenv.py --spec="${spec}" ${prefix_opt}
 
@@ -121,17 +120,8 @@ fi
 hostconfig=$(basename ${hostconfig_path})
 
 # Build Directory
-if [[ -z ${build_root} ]]
-then
-    if [[ -d /dev/shm ]]
-    then
-        build_root="${prefix}"
-    else
-        build_root="$(pwd)"
-    fi
-else
-    build_root="${build_root}"
-fi
+# When using /dev/shm, we use prefix for both spack builds and source build, unless BUILD_ROOT was defined
+build_root=${BUILD_ROOT:-"${prefix}"}
 
 build_dir="${build_root}/build_${hostconfig//.cmake/}"
 install_dir="${build_root}/install_${hostconfig//.cmake/}"
@@ -157,7 +147,8 @@ then
     rm -rf ${build_dir} 2>/dev/null
     mkdir -p ${build_dir} && cd ${build_dir}
 
-    if [[ "${truehostname}" == "corona" ]]
+    date
+    if [[ "${truehostname}" == "corona" || "${truehostname}" == "tioga" ]]
     then
         module unload rocm
     fi
@@ -174,11 +165,11 @@ then
       # todo this should use cmake --install once we use CMake 3.15+ everywhere
       make install
     fi
+    date
 
     echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
     echo "~~~~~ Umpire Built"
     echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    date
 fi
 
 # Test
@@ -196,7 +187,9 @@ then
 
     cd ${build_dir}
 
+    date
     ctest --output-on-failure --no-compress-output -T test -VV 2>&1 | tee tests_output.txt
+    date
 
     # If Developer benchmarks enabled, run the no-op benchmark and show output
     if [[ "${option}" != "--build-only" ]] && grep -q -i "UMPIRE_ENABLE_DEVELOPER_BENCHMARKS.*ON" ${hostconfig_path}
@@ -222,19 +215,19 @@ then
         echo "ERROR: failure(s) while running CTest" && exit 1
     fi
 
-    if [[ ! -d ${install_dir} ]]
-    then
-        echo "ERROR: install directory not found : ${install_dir}" && exit 1
-    fi
-
     if grep -q -i "ENABLE_HIP.*ON" ${hostconfig_path}
     then
         echo "WARNING: No install test with HIP"
     else
+        if [[ ! -d ${install_dir} ]]
+        then
+            echo "ERROR: install directory not found : ${install_dir}" && exit 1
+        fi
+
         cd ${install_dir}/examples/umpire/using-with-cmake
         mkdir build && cd build
         if ! $cmake_exe -C ../host-config.cmake ..; then
-        echo "ERROR: running cmake for using-with-cmake test" && exit 1
+        echo "ERROR: running $cmake_exe for using-with-cmake test" && exit 1
         fi
 
         if ! make; then
@@ -246,14 +239,15 @@ then
     echo "~~~~~ Umpire Tests Complete"
     echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
     date
-
-    # echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    # echo "~~~~~ CLEAN UP"
-    # echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
-    # make clean
 fi
 
+#echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+#echo "~~~~~ CLEAN UP"
+#echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+#make clean
+
 cd ${project_dir}
+
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo "~~~~~ Build and test completed"
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"

tests/CMakeLists.txt

@@ -4,6 +4,14 @@
 #
 # SPDX-License-Identifier: (MIT)
 ##############################################################################
+
+# With intel oneapi (as of version 2022.[1|2]) we see errors building gtest
+# due to the default flag "-ffp-model=fast" (relates to comparing to NaN).
+if(C_COMPILER_FAMILY_IS_INTELLLVM)
+  target_compile_options(gtest PRIVATE -ffp-model=precise)
+  target_compile_options(gtest_main PRIVATE -ffp-model=precise)
+endif()
+
 add_subdirectory(tpl)
 add_subdirectory(unit)
 add_subdirectory(integration)