Merge branch 'woptim/mpi-rocm' into woptim/push-packages

.github/workflows/generate-shroud.yml

@@ -2,6 +2,7 @@ on:
   pull_request:
     paths:
       - '**umpire_shroud.yaml'
+      - '**genfumpiresplicer.f'
 
 name: Generate C/FORTRAN Interface
 jobs:

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -16,17 +16,18 @@
 # repo, opt-in and out optional features. The project can then extend it with
 # additional stages.
 #
-# Instead, each project should provide:
+# In addition, each project should copy over and complete:
 # - .gitlab/custom-jobs-and-variables.yml
 # - .gitlab/subscribed-pipelines.yml
-# - .gitlab/${MACHINE}-build-and-test-extra.yml
+#
+# The jobs should be specified in a file local to the project,
+# - .gitlab/jobs/${CI_MACHINE}.yml
+# or generated (see LLNL/Umpire for an example).
 ###############################################################################
 
 # We define the following GitLab pipeline variables:
 variables:
-# Required information about GitHub repository
-  GITHUB_PROJECT_NAME: "Umpire"
-  GITHUB_PROJECT_ORG: "LLNL"
+##### LC GITLAB CONFIGURATION
 # Use the umdev LLNL service user to run CI. This prevents from running
 # pipelines as an actual user.
   LLNL_SERVICE_USER: umdev
@@ -35,43 +36,53 @@ variables:
   CUSTOM_CI_BUILDS_DIR: /usr/workspace/umdev/gitlab-runner
 # Tells Gitlab to recursively update the submodules when cloning the project.
   GIT_SUBMODULE_STRATEGY: recursive
+
+##### PROJECT VARIABLES
 # We build the projects in the CI clone directory.
-# TODO: add a clean-up mechanism
+# Used in script/gitlab/build_and_test.sh script.
+# TODO: add a clean-up mechanism.
   BUILD_ROOT: ${CI_PROJECT_DIR}
+
+##### SHARED_CI CONFIGURATION
+# Required information about GitHub repository
+  GITHUB_PROJECT_NAME: "Umpire"
+  GITHUB_PROJECT_ORG: "LLNL"
 # Set the build-and-test command.
-  BUILD_AND_TEST_CMD: "./scripts/gitlab/build_and_test.sh"
-# Override the list of branch that will skip the "draft PR test".
-# Add protected branches here. Defaults to "develop main master".
-#  ALWAYS_RUN_LIST: ""
+  JOB_CMD: "./scripts/gitlab/build_and_test.sh"
+# Override the pattern describing branches that will skip the "draft PR filter
+# test".  Add protected branches here. See default value in
+# preliminary-ignore-draft-pr.yml.
+#  ALWAYS_RUN_PATTERN: ""
 
-# We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
+# We organize the build-and-test stage with sub-pipelines. Each sub-pipeline
 # corresponds to a test batch on a given machine.
 
 # High level stages
 stages:
-  - machine-checks
+  - prerequisites
   - build-and-test
   - multi-project #TODO unused so far
   - radiuss-spack-testing #TODO unused so far
 
-# Template for jobs triggering a build-and-test sub-pipelines:
+# Template for jobs triggering a build-and-test sub-pipeline:
 .build-and-test:
   stage: build-and-test
   trigger:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2023.08.0
-        file: '${CI_MACHINE}-build-and-test.yml'
-      - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
+        ref: 'v2023.09.0'
+        file: 'pipelines/${CI_MACHINE}.yml'
+      - artifact: '${CI_MACHINE}-jobs.yml'
+        job: 'generate-job-lists'
     strategy: depend
     forward:
       pipeline_variables: true
 
 include:
   # [Optional] checks preliminary to running the actual CI test
   #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: v2023.08.0
+  #  ref: 'v2023.09.0'
   #  file: 'preliminary-ignore-draft-pr.yml'
   # pipelines subscribed by the project
-  - local: .gitlab/subscribed-pipelines.yml
+  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/custom-jobs-and-variables.yml

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2022, Lawrence Livermore National Security, LLC and RADIUSS
+# Copyright (c) 2022-23, Lawrence Livermore National Security, LLC and RADIUSS
 # project contributors. See the COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -14,29 +14,29 @@ variables:
 
 # Ruby
 # Arguments for top level allocation
-  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=10 --nodes=1"
+  RUBY_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=10 --nodes=1"
 # Arguments for job level allocation
-  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --reservation=ci --qos=ci_ruby --time=10 --nodes=1"
+  RUBY_JOB_ALLOC: "--overlap --reservation=ci --qos=ci_ruby --time=10 --nodes=1"
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "~shared +fortran +tools tests=basic "
 # Project specific deps for ruby
   PROJECT_RUBY_DEPS: ""
 
 # Corona
 # Arguments for top level allocation
-  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
+  CORONA_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
 # Arguments for job level allocation
-  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
+  CORONA_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
 # Project specific variants for corona
   PROJECT_CORONA_VARIANTS: "~shared +fortran +device_alloc tests=basic "
 # Project specific deps for corona
   PROJECT_CORONA_DEPS: ""
 
 # Tioga
 # Arguments for top level allocation
-  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=20m --nodes=1"
+  TIOGA_SHARED_ALLOC: "--exclusive --time-limit=25m --nodes=1"
 # Arguments for job level allocation
-  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=15m --nodes=1 --begin-time=+5s"
+  TIOGA_JOB_ALLOC: "--time-limit=15m --nodes=1 --begin-time=+5s"
 # Project specific variants for tioga
   PROJECT_TIOGA_VARIANTS: "~shared +fortran +device_alloc tests=basic "
 # Project specific deps for tioga
@@ -45,17 +45,17 @@ variables:
 # Lassen and Butte use a different job scheduler (spectrum lsf) that does not
 # allow pre-allocation the same way slurm does.
 # Arguments for job level allocation
-  LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 18"
+  LASSEN_JOB_ALLOC: "1 -W 18"
 # Project specific variants for lassen
   PROJECT_LASSEN_VARIANTS: "~shared +fortran +tools tests=basic "
 # Project specific deps for lassen
   PROJECT_LASSEN_DEPS: ""
 
-
 # Configuration shared by build and test jobs specific to this project.
 # Not all configuration can be shared. Here projects can fine tune the
 # CI behavior.
-.custom_build_and_test:
+# See Umpire for an example (export junit test reports).
+.custom_job:
   artifacts:
     reports:
       junit: junit.xml

.gitlab/corona-build-and-test-extra.yml → .gitlab/jobs/corona.yml

@@ -28,5 +28,9 @@
 rocmcc_5_6_0_hip_openmp_device_alloc:
   variables:
     SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.6.0 ^hip@5.6.0"
-  extends: .build_and_test_on_corona
+  extends: .job_on_corona
 
+rocmcc_5_6_0_hip_mpi:
+  variables:
+    SPEC: "~shared +fortran +mpi +rocm tests=basic amdgpu_target=gfx906 %rocmcc@5.6.0 ^hip@5.6.0"
+  extends: .job_on_corona

.gitlab/lassen-build-and-test-extra.yml → .gitlab/jobs/lassen.yml

@@ -23,73 +23,73 @@
 ibm_clang_14_0_5_mpi_shmem:
   variables:
     SPEC: "~shared +tools tests=basic +ipc_shmem +mpi %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 ibm_clang_14_0_5_mpi:
   variables:
     SPEC: "~shared +fortran +tools +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_mpi_shmem:
   variables:
     SPEC: "~shared +tools +cuda +ipc_shmem +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi ^cuda@11.7.0+allow-unsupported-compilers"
     MODULE_LIST: "cuda/11.7.0"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 clang_12_0_1_libcpp:
   variables:
     SPEC: "~shared +tools tests=basic %clang@12.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\""
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 clang_12_0_1_gcc_8_3_1_memleak:
   variables:
     SPEC: "~shared +asan +sanitizer_tests +tools tests=basic %clang@12.0.1.gcc.8.3.1 cxxflags==-fsanitize=address"
     ASAN_OPTIONS: "detect_leaks=1"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 # clang_9_0_0_datarace (build and test on lassen):
 #   variables:
 #     SPEC: "%clang@9.0.0 cxxflags==\"-fsanitize=thread\""
 #   allow_failure: true
-#   extends: .build_and_test_on_lassen
+#   extends: .job_on_lassen
 
 gcc_8_3_1_dev_benchmarks:
   variables:
     SPEC: "~shared +dev_benchmarks +tools build_type=Release %gcc@8.3.1"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 xl_2022_08_19_default_omp_target:
   variables:
     SPEC: "~shared +tools +openmp +openmp_target tests=basic %xl@16.1.1.12"
   allow_failure: true
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 gcc_8_3_1_numa:
   variables:
     SPEC: "~shared +fortran +numa +tools tests=basic %gcc@8.3.1"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 gcc_8_3_1_cuda_11_7_0:
   variables:
     SPEC: "+fortran +cuda +tools +backtrace tests=basic %gcc@8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
     MODULE_LIST: "cuda/11.7.0"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 # Develop/main builds against specific tpl versions.
 
 gcc_8_3_1_tpls:
   variables:
     SPEC: "~shared +fortran +tools tests=basic %gcc@8.3.1"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_tpls:
   variables:
     SPEC: "~shared +fortran +cuda +tools tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
     MODULE_LIST: "cuda/11.7.0"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen
 
 xl_2022_08_19_gcc_8_3_1_cuda_11_2_tpls:
   variables:
     SPEC: "~shared +fortran +cuda +tools tests=basic %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
     MODULE_LIST: "cuda/11.7.0"
-  extends: .build_and_test_on_lassen
+  extends: .job_on_lassen

.gitlab/ruby-build-and-test-extra.yml → .gitlab/jobs/ruby.yml

@@ -16,7 +16,7 @@
 intel_2022_1_0:
   variables:
     SPEC: "~shared +fortran +tools tests=basic %intel@2022.1.0"
-  extends: .build_and_test_on_ruby
+  extends: .job_on_ruby
   allow_failure: true
 
 ############
@@ -29,28 +29,33 @@ intel_2022_1_0:
 gcc_10_3_1_numa:
   variables:
     SPEC: "~shared +fortran +numa +tools tests=basic %gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .job_on_ruby
 
 clang_14_0_6_gcc_10_3_1_sqlite_experimental:
   variables:
     SPEC: "~shared +sqlite_experimental +tools tests=basic %clang@14.0.6.gcc.10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .job_on_ruby
 
 # Develop builds against specific tpl version.
 clang_14_0_6_gcc_10_3_1_tpls:
   variables:
     SPEC: "~shared +fortran +tools tests=basic %clang@14.0.6.gcc.10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .job_on_ruby
 
 gcc_10_3_1_tpls:
   variables:
     SPEC: "~shared +fortran +tools tests=basic %gcc@10.3.1"
-  extends: .build_and_test_on_ruby
+  extends: .job_on_ruby
+
+gcc_10_3_1_ipc_no_mpi:
+  variables:
+    SPEC: "~shared +ipc_shmem tests=basic %gcc@10.3.1"
+  extends: .job_on_ruby
 
 # Oneapi is not available on ruby@toss4 (rhel8)
 ## We deactivate this job as it is known to fail with Umpire: needs gcc toolchain.
 #intel_2022_1_0:
 #  variables:
 #    ON_RUBY: "OFF"
 #    SPEC: "${PROJECT_RUBY_VARIANTS} %intel@2022.1.0 ${PROJECT_RUBY_DEPS}"
-#  extends: .build_and_test_on_ruby
+#  extends: .job_on_ruby

.gitlab/tioga-build-and-test-extra.yml → .gitlab/jobs/tioga.yml

@@ -16,7 +16,7 @@
 cce_16_0_0:
   variables:
     SPEC: "~shared +fortran tests=basic %cce@16.0.0"
-  extends: .build_and_test_on_tioga
+  extends: .job_on_tioga
 
 ############
 # Extra jobs
@@ -32,5 +32,9 @@ cce_16_0_0:
 rocmcc_5_6_0_hip_openmp_device_alloc:
   variables:
     SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.6.0 ^hip@5.6.0"
-  extends: .build_and_test_on_tioga
+  extends: .job_on_tioga
 
+rocmcc_5_6_0_hip_mpi:
+  variables:
+    SPEC: "~shared +fortran +mpi +rocm tests=basic amdgpu_target=gfx90a %rocmcc@5.6.0 ^hip@5.6.0"
+  extends: .job_on_tioga