From 6bd91b23c5e1f3bc788626ab969053481663565d Mon Sep 17 00:00:00 2001
From: Bruno Turcksin <bruno.turcksin@gmail.com>
Date: Wed, 13 Sep 2023 14:44:10 +0000
Subject: [PATCH] Add nvshmem to jenkins ci

---
 .jenkins/continuous.groovy        | 22 ++++++++++
 scripts/docker/Dockerfile.nvshmem | 67 +++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 scripts/docker/Dockerfile.nvshmem

diff --git a/.jenkins/continuous.groovy b/.jenkins/continuous.groovy
index 1e2283c7..f34dd155 100644
--- a/.jenkins/continuous.groovy
+++ b/.jenkins/continuous.groovy
@@ -42,6 +42,28 @@ pipeline {
                               make -j8 && cd unit_tests && mpirun -np 2 ./KokkosRemoteSpaces_TestAll'''
                     }
                 }
+                stage('nvshmem') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.nvshmem'
+                            dir 'scripts/docker'
+                            label 'nvidia-docker && large_images'
+                        }
+                    }
+                    steps {
+                        sh '''rm -rf build && mkdir -p build && cd build && \
+                              cmake \
+                                -DCMAKE_BUILD_TYPE=Release \
+                                -DKokkos_DIR=${KOKKOS_ROOT} \
+                                -DCMAKE_CXX_COMPILER=${KOKKOS_ROOT}/bin/nvcc_wrapper \
+                                -DKRS_ENABLE_NVSHMEMSPACE=ON \
+                                -DNVSHMEM_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/nvshmem \
+                                -DKRS_ENABLE_TESTS=ON \
+                                -DCMAKE_CXX_FLAGS=-Werror \
+                              .. && \
+                              make -j8 && cd unit_tests && mpirun -np 2 ./KokkosRemoteSpaces_TestAll'''
+                    }
+                }
             }
         }
     }
diff --git a/scripts/docker/Dockerfile.nvshmem b/scripts/docker/Dockerfile.nvshmem
new file mode 100644
index 00000000..03a5290f
--- /dev/null
+++ b/scripts/docker/Dockerfile.nvshmem
@@ -0,0 +1,67 @@
+FROM nvcr.io/nvidia/nvhpc:23.7-devel-cuda12.2-ubuntu20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+    build-essential \
+    wget \
+    git \
+    bc \
+    ninja-build \
+    git \
+    libev-dev \
+    libevent-dev \
+    libhwloc-dev \
+    pkg-config \
+    clang-format-8 \
+    && \
+    apt-get clean && rm -rf /var/lib/apt/list
+
+ENV PREFIX=/scratch
+ENV ARCHIVE_DIR=${PREFIX}/archive
+ENV SOURCE_DIR=${PREFIX}/source
+ENV BUILD_DIR=${PREFIX}/build
+ENV INSTALL_DIR=/opt
+
+RUN mkdir -p ${PREFIX} && \
+    cd ${PREFIX} && \
+    mkdir archive && \
+    mkdir source && \
+    mkdir build
+
+# Install CMake
+RUN export CMAKE_VERSION=3.22.2 && \
+    export CMAKE_SHA256=38b3befdee8fd2bac06954e2a77cb3072e6833c69d8cc013c0a3b26f1cfdfe37 && \
+    export CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \
+    export CMAKE_ARCHIVE=${ARCHIVE_DIR}/cmake.tar.gz && \
+    export CMAKE_BUILD_DIR=${BUILD_DIR}/cmake && \
+    wget --quiet ${CMAKE_URL} --output-document=${CMAKE_ARCHIVE} && \
+    echo "${CMAKE_SHA256} ${CMAKE_ARCHIVE}" | sha256sum -c && \
+    mkdir -p ${CMAKE_BUILD_DIR} && \
+    tar xf ${CMAKE_ARCHIVE} -C ${CMAKE_BUILD_DIR} --strip-components=1 && \
+    mv ${CMAKE_BUILD_DIR} ${INSTALL_DIR} && \
+    rm -rf ${CMAKE_ARCHIVE} && \
+    rm -rf ${CMAKE_BUILD_DIR}
+ENV PATH=${INSTALL_DIR}/cmake/bin:$PATH
+
+## Install Kokkos
+RUN export KOKKOS_SOURCE_DIR=${SOURCE_DIR}/kokkos && \
+    export KOKKOS_BUILD_DIR=${BUILD_DIR}/kokkos && \
+    export KOKKOS_INSTALL_DIR=${INSTALL_DIR}/kokkos && \
+    cd ${SOURCE_DIR} && git clone https://github.com/kokkos/kokkos && \
+    cd kokkos && \
+    git checkout 4.1.00 && \
+    mkdir -p ${KOKKOS_BUILD_DIR} && \
+    cd ${KOKKOS_BUILD_DIR} && \
+    cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_CXX_COMPILER=${KOKKOS_SOURCE_DIR}/bin/nvcc_wrapper \
+        -DBUILD_SHARED_LIBS=ON \
+        -DCUDAToolkit_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda \
+        -DKokkos_ENABLE_CUDA=ON \
+        -DKokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE=ON \
+        -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_DIR} \
+        ${KOKKOS_SOURCE_DIR} && \
+    make -j${N_PROCS} install && \
+    rm -rf ${KOKKOS_ARCHIVE} && \
+    rm -rf ${KOKKOS_BUILD_DIR} && \
+    rm -rf ${KOKKOS_SOURCE_DIR}
+ENV KOKKOS_ROOT=${INSTALL_DIR}/kokkos