ci: added rocm gha build

.github/workflows/everything.yml

@@ -161,6 +161,10 @@ jobs:
             compiler: gcc8
             shared: static
             parallel: serial
+          - os: ubuntu22.04
+            compiler: rocm
+            parallel: serial
+            constrains: build_only
     steps:
       - uses: actions/checkout@v4
         with:

scripts/ci/cmake/ci-ubuntu22.04-rocm-serial.cmake

@@ -0,0 +1,44 @@
+set(ENV{CC}  gcc)
+set(ENV{CXX} g++)
+set(ENV{FC}  gfortran)
+set(ENV{FC}  )
+
+execute_process(
+  COMMAND "python3-config" "--prefix"
+  OUTPUT_VARIABLE PY_ROOT
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+set(dashboard_cache "
+BUILD_TESTING:BOOL=OFF
+ADIOS2_BUILD_EXAMPLES:BOOL=ON
+
+ADIOS2_USE_BZip2:BOOL=ON
+ADIOS2_USE_Blosc2:BOOL=ON
+ADIOS2_USE_DataMan:BOOL=ON
+ADIOS2_USE_Fortran:BOOL=ON
+ADIOS2_USE_HDF5:BOOL=ON
+ADIOS2_USE_MPI:BOOL=OFF
+ADIOS2_USE_Python:BOOL=ON
+ADIOS2_USE_SZ:BOOL=ON
+ADIOS2_USE_ZeroMQ:STRING=ON
+ADIOS2_USE_ZFP:BOOL=ON
+ADIOS2_USE_Kokkos:BOOL=ON
+ADIOS2_USE_Sodium:BOOL=OFF
+ADIOS2_USE_Catalyst:BOOL=OFF
+
+Python_ROOT_DIR:PATH=${PY_ROOT}
+Python_FIND_STRATEGY:STRING=LOCATION
+Python_FIND_FRAMEWORK:STRING=FIRST
+
+CMAKE_C_COMPILER_LAUNCHER=ccache
+CMAKE_CXX_COMPILER_LAUNCHER=ccache
+CMAKE_C_FLAGS:STRING=-Wall
+CMAKE_CXX_FLAGS:STRING=-Wall
+CMAKE_Fortran_FLAGS:STRING=-Wall
+OpenMP_gomp_LIBRARY:FILEPATH=/spack/var/spack/environments/adios2-ci-serial/.spack-env/view/lib/libgomp.so.1
+AMDGPU_TARGETS=gfx906
+")
+
+set(CTEST_CMAKE_GENERATOR "Ninja")
+list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}")
+include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake)

scripts/ci/images/Dockerfile.ci-spack-ubuntu20.04-base

@@ -1,51 +1,68 @@
 # vim: ft=dockerfile
-FROM ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01
+ARG BASE_IMAGE=ecpe4s/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01
+FROM $BASE_IMAGE
 
 ARG E4S_VERSION=23.08
+ARG ENABLED_ENVS="serial,ompi,mpich"
+ARG EXTERNAL_PACKAGES
+ARG EXTRA_VARIANTS
+
 
 # Clone and patch spack
-RUN cd / && \
-    git clone --depth 1 --single-branch --branch e4s-${E4S_VERSION} https://github.com/spack/spack
+WORKDIR /
+RUN if ! [ -d /spack ]; then \
+      git clone --depth 1 --single-branch --branch e4s-${E4S_VERSION} https://github.com/spack/spack; \
+    else \
+      git fetch --all && git checkout -t origin/e4s-${E4S_VERSION}; \
+    fi
 
+COPY 0001-mpich-support-ch3-sock.patch /spack
 COPY packages.yaml /etc/spack/packages.yaml
+
+WORKDIR /spack
 # TODO: Patch only needed until we merge https://github.com/spack/spack/pull/40964
-COPY 0001-mpich-support-ch3-sock.patch /spack
+RUN if [ ${E4S_VERSION%.*} -le 23 ]; then git apply 0001-mpich-support-ch3-sock.patch; fi
 
 # Install base specs
-RUN cd spack && \
-    git apply 0001-mpich-support-ch3-sock.patch && \
-    . /spack/share/spack/setup-env.sh && \
+RUN . /spack/share/spack/setup-env.sh && \
+    if test -n "${EXTERNAL_PACKAGES}"; then spack external find ${EXTERNAL_PACKAGES}; fi && \
     spack mirror add E4S "https://cache.e4s.io/${E4S_VERSION}" && \
-    export ADIOS_SPEC_NO_MPI="adios2~mpi" && \
-    export ADIOS_SPEC_OPENMPI="adios2+mpi^openmpi" && \
-    export ADIOS_SPEC_MPICH="adios2+mpi^mpich" && \
+    export ADIOS_SPEC_NO_MPI="adios2~mpi${EXTRA_VARIANTS}" && \
+    export ADIOS_SPEC_OPENMPI="adios2+mpi${EXTRA_VARIANTS} ^openmpi" && \
+    export ADIOS_SPEC_MPICH="adios2+mpi${EXTRA_VARIANTS} ^mpich" && \
     spack config add "config:checksum:false" && \
     spack config add "config:build_jobs:$(nproc)" && \
     spack config add "concretizer:unify:true" && \
+    if echo "$ENABLED_ENVS" | grep -q serial; then \
     spack env create adios2-ci-serial && \
     spack -e adios2-ci-serial add ${ADIOS_SPEC_NO_MPI} && \
     spack -e adios2-ci-serial concretize && \
     spack -e adios2-ci-serial install \
       --include-build-deps \
       --no-check-signature \
       --fail-fast \
-      --only dependencies && \
+      --only dependencies; \
+    fi && \
+    if echo "$ENABLED_ENVS" | grep -q ompi ; then \
     spack env create adios2-ci-ompi && \
     spack -e adios2-ci-ompi add ${ADIOS_SPEC_OPENMPI} && \
     spack -e adios2-ci-ompi concretize && \
     spack -e adios2-ci-ompi install \
       --include-build-deps \
       --no-check-signature \
       --fail-fast \
-      --only dependencies && \
+      --only dependencies; \
+    fi && \
+    if echo "$ENABLED_ENVS" | grep -q mpich; then \
     spack env create adios2-ci-mpich && \
     spack -e adios2-ci-mpich add ${ADIOS_SPEC_MPICH} && \
     spack -e adios2-ci-mpich concretize && \
     spack -e adios2-ci-mpich install \
       --include-build-deps \
       --no-check-signature \
       --fail-fast \
-      --only dependencies && \
+      --only dependencies; \
+    fi && \
     spack clean -a && \
     echo "source /spack/share/spack/setup-env.sh" >> /etc/profile.d/zz-spack.sh
 

scripts/ci/images/Dockerfile.ci-spack-ubuntu22.04-rocm-base

@@ -0,0 +1,23 @@
+ARG BASE_IMAGE=rocm/dev-ubuntu-22.04
+FROM $BASE_IMAGE
+
+# Base dependencies for building VTK-m projects
+RUN apt update && \
+    DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
+    ccache \
+    cmake \
+    curl \
+    g++ \
+    gfortran-11 \
+    git \
+    git-lfs \
+    git-lfs \
+    libcurl4-gnutls-dev \
+    make \
+    mpich \
+    ninja-build \
+    pkg-config \
+    rocthrust-dev \
+    \
+    && \
+    apt clean

scripts/ci/images/build-rocm.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# Build the base image
+docker build \
+  --progress=plain \
+  --rm \
+  --build-arg BASE_IMAGE="rocm/dev-ubuntu-22.04" \
+  -f ./Dockerfile.ci-spack-ubuntu22.04-rocm-base \
+  -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-rocm-base \
+  .
+
+# Build the base image
+docker build \
+  --progress=plain \
+  --rm \
+  --build-arg BASE_IMAGE="ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-rocm-base" \
+  --build-arg ENABLED_ENVS="serial" \
+  --build-arg EXTERNAL_PACKAGES="hip rocprim" \
+  --build-arg E4S_VERSION="24.05" \
+  --build-arg EXTRA_VARIANTS="+blosc2+kokkos+rocm amdgpu_target=gfx906 ^hip@6" \
+  -f ./Dockerfile.ci-spack-ubuntu20.04-base \
+  -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-rocm \
+  .
+
+# Push images to github container registry
+docker push ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-rocm

scripts/ci/images/build-ubuntu.sh

@@ -3,7 +3,7 @@
 set -ex
 
 # Build the base image
-docker build --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base .
+docker build --progress=plain --build-arg EXTRA_VARIANTS="+blosc+ssc" --rm -f ./Dockerfile.ci-spack-ubuntu20.04-base -t ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base .
 
 # Which is also the gcc11 image
 docker tag ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-base ghcr.io/ornladios/adios2:ci-spack-ubuntu20.04-gcc11

scripts/ci/images/packages.yaml

@@ -2,7 +2,7 @@ packages:
   all:
     target: [haswell]
   adios2:
-    require: '+blosc+bzip2+zfp+sz+png+sst+dataman+ssc+hdf5+python+fortran'
+    require: '+bzip2+zfp+sz+png+sst+dataman+hdf5+python+fortran'
   mpich:
     require: 'device=ch3:sock'
   openmpi:

scripts/ci/setup-run/ci-ubuntu22.04-rocm.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+export CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}:/opt/rocm/"
+
+update-alternatives --set gcc "$(update-alternatives --list gcc | grep gcc-11)"
+update-alternatives --set g++ "$(update-alternatives --list g++ | grep g++-11)"
+update-alternatives --set gfortran "$(update-alternatives --list gfortran | grep gfortran-11)"
+
+spack env activate "adios2-ci-${GH_YML_MATRIX_PARALLEL}"

source/adios2/helper/kokkos/adiosKokkos.cpp

@@ -85,7 +85,11 @@ bool IsGPUbuffer(const void *ptr)
     hipError_t ret;
     hipPointerAttribute_t attr;
     ret = hipPointerGetAttributes(&attr, ptr);
+#if defined(ROCM_VERSION_MAJOR) && ROCM_VERSION_MAJOR < 6
     if (ret == hipSuccess && attr.memoryType == hipMemoryTypeDevice)
+#else
+    if (ret == hipSuccess && attr.type == hipMemoryTypeDevice)
+#endif
     {
         return true;
     }