add CUDA support through Kokkos

candi.cfg

@@ -39,6 +39,14 @@ USE_64_BIT_INDICES=OFF
 # Option {ON|OFF}: Enable building of dealii examples?
 BUILD_EXAMPLES=ON
 
+# Option {ON|OFF}: Enable using CUDA/nvidia support using Kokkos?
+USE_KOKKOS_WITH_CUDA=OFF
+SET_CUDA_ARCH=89
+# see https://developer.nvidia.com/cuda-gpus to find compute capability:
+SET_CUDA_ARCH=86
+# Remember to add the /bin of the following path to your PATH:
+#CUDA_HOME=/usr/local/cuda-12.6
+
 # Option {ON|OFF}: Unset CXX and set the compiler as MPI_CXX_COMPILER when configuring deal.II
 #
 # This is recommended for very recent CMake versions but it currently is not reliable enough
@@ -85,7 +93,9 @@ PACKAGES="${PACKAGES} once:sundials"
 PACKAGES="${PACKAGES} once:hdf5"
 #PACKAGES="${PACKAGES} once:netcdf"
 PACKAGES="${PACKAGES} once:p4est"
+#PACKAGES="${PACKAGES} once:kokkos once:kokkoskernels"
 PACKAGES="${PACKAGES} once:trilinos"
+#PACKAGES="${PACKAGES} once:hypre"
 PACKAGES="${PACKAGES} once:petsc"
 PACKAGES="${PACKAGES} once:slepc"
 PACKAGES="${PACKAGES} once:symengine"

deal.II-toolchain/packages/dealii.package

@@ -59,6 +59,16 @@ if [ ${USE_DEAL_II_CMAKE_MPI_COMPILER} = ON ]; then
     unset CC
 fi
 
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_CXX_COMPILER=${KOKKOS_DIR}/bin/nvcc_wrapper \
+      -D DEAL_II_WITH_CUDA=OFF \
+      -D DEAL_II_WITH_KOKKOS=ON \
+      -D KOKKOS_DIR=${KOKKOS_DIR} \
+      -D DEAL_II_MPI_WITH_DEVICE_SUPPORT=OFF"
+    # you need a special MPI version to enable DEAL_II_MPI_WITH_DEVICE_SUPPORT
+fi
+
 # add the user-specified flags at the end (so things can be overriden):
 CONFOPTS="${CONFOPTS} \
 ${DEAL_II_CONFOPTS}"

deal.II-toolchain/packages/hypre.package

@@ -0,0 +1,29 @@
+VERSION=2.32.0;CHECKSUM=321557481d18242a771096999a253ad8
+#VERSION=2.31.0;CHECKSUM=3e6a9cea4e87d5d87301c95200d28242
+
+NAME=v${VERSION}
+SOURCE=https://github.com/hypre-space/hypre/archive/refs/tags/
+EXTRACTSTO=hypre-${VERSION}/src
+PACKING=.tar.gz
+BUILDDIR=${UNPACK_PATH}/${EXTRACTSTO}
+BUILDCHAIN=autotools
+INSTALL_PATH=${INSTALL_PATH}/hypre-${NAME}
+
+CONFOPTS="--enable-shared"
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+  CONFOPTS="${CONFOPTS} --with-gpu-arch=${USE_CUDA_ARCH} --with-cuda --enable-unified-memory"
+fi
+
+
+package_specific_register () {
+    export HYPRE_DIR=${INSTALL_PATH}
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/hypre-${NAME}
+    rm -f ${CONFIG_FILE}
+    echo "
+export HYPRE_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}

deal.II-toolchain/packages/kokkos.package

@@ -0,0 +1,32 @@
+VERSION=4.4.00;CHECKSUM=95af2e2d4b10a67a63cce09715fba127
+
+NAME=${VERSION}
+PACKING=.tar.gz
+SOURCE=https://github.com/kokkos/kokkos/archive/refs/tags/
+EXTRACTSTO=kokkos-${VERSION}
+INSTALL_PATH=${INSTALL_PATH}/kokkos-${NAME}
+BUILDDIR=${BUILD_PATH}/kokkos-${VERSION}
+
+BUILDCHAIN=cmake
+
+CONFOPTS="-D BUILD_SHARED_LIBS=ON"
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+  -D Kokkos_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA_LAMBDA=ON \
+  -D Kokkos_ENABLE_CUDA_CONSTEXPR=ON"
+fi
+
+package_specific_register () {
+    export KOKKOS_DIR=${INSTALL_PATH}
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/kokkos-${NAME}
+    rm -f $CONFIG_FILE
+    echo "
+export KOKKOS_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}

deal.II-toolchain/packages/kokkoskernels.package

@@ -0,0 +1,28 @@
+VERSION=4.4.00;CHECKSUM=a1e596b2153ff260448984c43dacb814
+
+NAME=${VERSION}
+PACKING=.tar.gz
+SOURCE=https://github.com/kokkos/kokkos-kernels/archive/refs/tags/
+EXTRACTSTO=kokkos-kernels-${VERSION}
+BUILDDIR=${BUILD_PATH}/kokkos-kernels-${VERSION}
+
+# we just install into the Kokkos directory:
+INSTALL_PATH=${INSTALL_PATH}/kokkos-${VERSION}
+
+BUILDCHAIN=cmake
+
+CONFOPTS="-D BUILD_SHARED_LIBS=ON"
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS}"
+fi
+
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/kokkos-kernels-${NAME}
+    rm -f $CONFIG_FILE
+    echo "
+#export KOKKOS_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}

deal.II-toolchain/packages/petsc.package

@@ -2,6 +2,9 @@
 ## PETSc                                                                      ##
 ################################################################################
 
+VERSION=3.21.4
+CHECKSUM=e8d76467696fcff970bd56c93dd83476
+
 
 VERSION=3.22.2;CHECKSUM=d18caed83d1bf351c4ccb574b0b376e5
 
@@ -54,6 +57,16 @@ CONFOPTS="\
   --with-x=0 \
   --with-64-bit-indices=${with64bit}"
 
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+CONFOPTS="\
+  ${CONFOPTS} \
+  --with-kokkos=1 \
+  --with-kokkos-kernels=1 \
+  --with-cuda=1 \
+  --with-kokkos-dir=${KOKKOS_DIR}"
+fi
+
 # It would seem simpler to put the {C|CXX|F}OPTFLAGS directly into the
 # CONFOPTS variable, but there is no easy way to quote the
 # multiarguments and retain the quotes correctly when passing to
@@ -66,9 +79,11 @@ else
     OPTFLAGS="-g -O"
 fi
 
-for external_pkg in hypre; do
-    CONFOPTS="${CONFOPTS} --download-${external_pkg}=1"
-done
+if [ ! -z "${HYPRE_DIR}" ]; then
+  CONFOPTS="${CONFOPTS} --with-hypre=1 --with-hypre-dir=${HYPRE_DIR}"
+else
+  CONFOPTS="${CONFOPTS} --download-hypre=1"
+fi
 
 if [ ! -z "${CC}" ]; then
     CONFOPTS="${CONFOPTS} CC=${CC}"

deal.II-toolchain/packages/trilinos.package

@@ -282,6 +282,16 @@ CONFOPTS="\
   -D BUILD_SHARED_LIBS:BOOL=ON \
   ${CONFOPTS}"
 
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+  -D TPL_ENABLE_MPI=ON \
+  -D TPL_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA_LAMBDA=ON \
+  -D Kokkos_ENABLE_CUDA_CONSTEXPR=ON \
+  -D Tpetra_INST_CUDA=ON"
+fi
+
 if [ ${TRILINOS_WITH_COMPLEX} = ON ]; then
     CONFOPTS="\
     -D Trilinos_ENABLE_COMPLEX=ON \
@@ -292,6 +302,20 @@ fi
 CONFOPTS="${CONFOPTS} \
   ${TRILINOS_CONFOPTS}"
 
+package_specific_setup () {
+    if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+        export OMPI_CXX=${UNPACK_PATH}/${EXTRACTSTO}/packages/kokkos/bin/nvcc_wrapper
+        echo "SETTING OMPI_CXX to ${OMPI_CXX}"
+    fi
+}
+
+package_specific_install () {
+    if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+        echo "UNSETTING OMPI_CXX"
+        unset OMPI_CXX
+    fi
+}
+
 package_specific_register () {
     export TRILINOS_DIR=${INSTALL_PATH}
 }