From 76dd6360e4305338f18b9f18a284f7672ec2a0aa Mon Sep 17 00:00:00 2001
From: Timo Heister <timo.heister@gmail.com>
Date: Wed, 31 Jul 2024 08:58:44 -0400
Subject: [PATCH 1/4] cuda support

---
 candi.cfg                                     |  6 ++++
 deal.II-toolchain/packages/dealii.package     | 10 ++++++
 deal.II-toolchain/packages/hypre.package      | 28 ++++++++++++++++
 deal.II-toolchain/packages/kokkos.package     | 32 +++++++++++++++++++
 .../packages/kokkoskernels.package            | 28 ++++++++++++++++
 deal.II-toolchain/packages/petsc.package      | 21 ++++++++++--
 deal.II-toolchain/packages/trilinos.package   | 24 ++++++++++++++
 7 files changed, 146 insertions(+), 3 deletions(-)
 create mode 100644 deal.II-toolchain/packages/hypre.package
 create mode 100644 deal.II-toolchain/packages/kokkos.package
 create mode 100644 deal.II-toolchain/packages/kokkoskernels.package

diff --git a/candi.cfg b/candi.cfg
index 724c3eaa..8faec007 100644
--- a/candi.cfg
+++ b/candi.cfg
@@ -39,6 +39,10 @@ USE_64_BIT_INDICES=OFF
 # Option {ON|OFF}: Enable building of dealii examples?
 BUILD_EXAMPLES=ON
 
+# Option {ON|OFF}: Enable using CUDA/nvidia support using Kokkos?
+USE_KOKKOS_WITH_CUDA=OFF
+SET_CUDA_ARCH=89
+
 # Option {ON|OFF}: Unset CXX and set the compiler as MPI_CXX_COMPILER when configuring deal.II
 #
 # This is recommended for very recent CMake versions but it currently is not reliable enough
@@ -85,7 +89,9 @@ PACKAGES="${PACKAGES} once:sundials"
 PACKAGES="${PACKAGES} once:hdf5"
 #PACKAGES="${PACKAGES} once:netcdf"
 PACKAGES="${PACKAGES} once:p4est"
+#PACKAGES="${PACKAGES} once:kokkos once:kokkoskernels"
 PACKAGES="${PACKAGES} once:trilinos"
+#PACKAGES="${PACKAGES} once:hypre"
 PACKAGES="${PACKAGES} once:petsc"
 PACKAGES="${PACKAGES} once:slepc"
 PACKAGES="${PACKAGES} once:symengine"
diff --git a/deal.II-toolchain/packages/dealii.package b/deal.II-toolchain/packages/dealii.package
index 9be7da7b..ffffaee3 100644
--- a/deal.II-toolchain/packages/dealii.package
+++ b/deal.II-toolchain/packages/dealii.package
@@ -59,6 +59,16 @@ if [ ${USE_DEAL_II_CMAKE_MPI_COMPILER} = ON ]; then
     unset CC
 fi
 
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+      -D CMAKE_CXX_COMPILER=${TRILINOS_DIR}/bin/nvcc_wrapper \
+      -D DEAL_II_WITH_CUDA=OFF \
+      -D DEAL_II_WITH_KOKKOS=ON \
+      -D KOKKOS_DIR=${TRILINOS_DIR} \
+      -D DEAL_II_MPI_WITH_DEVICE_SUPPORT=OFF"
+    # you need a special MPI version to enable DEAL_II_MPI_WITH_DEVICE_SUPPORT
+fi
+
 # add the user-specified flags at the end (so things can be overriden):
 CONFOPTS="${CONFOPTS} \
 ${DEAL_II_CONFOPTS}"
diff --git a/deal.II-toolchain/packages/hypre.package b/deal.II-toolchain/packages/hypre.package
new file mode 100644
index 00000000..cb442747
--- /dev/null
+++ b/deal.II-toolchain/packages/hypre.package
@@ -0,0 +1,28 @@
+VERSION=2.31.0;CHECKSUM=3e6a9cea4e87d5d87301c95200d28242
+
+NAME=v${VERSION}
+SOURCE=https://github.com/hypre-space/hypre/archive/refs/tags/
+EXTRACTSTO=hypre-${VERSION}/src
+PACKING=.tar.gz
+BUILDDIR=${UNPACK_PATH}/${EXTRACTSTO}
+BUILDCHAIN=autotools
+INSTALL_PATH=${INSTALL_PATH}/hypre-${NAME}
+
+CONFOPTS="--enable-shared"
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+  CONFOPTS="${CONFOPTS} --with-gpu-arch=${USE_CUDA_ARCH} --with-cuda --enable-unified-memory"
+fi
+
+
+package_specific_register () {
+    export HYPRE_DIR=${INSTALL_PATH}
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/hypre-${NAME}
+    rm -f ${CONFIG_FILE}
+    echo "
+export HYPRE_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}
diff --git a/deal.II-toolchain/packages/kokkos.package b/deal.II-toolchain/packages/kokkos.package
new file mode 100644
index 00000000..900c6095
--- /dev/null
+++ b/deal.II-toolchain/packages/kokkos.package
@@ -0,0 +1,32 @@
+VERSION=4.4.00;CHECKSUM=95af2e2d4b10a67a63cce09715fba127
+
+NAME=${VERSION}
+PACKING=.tar.gz
+SOURCE=https://github.com/kokkos/kokkos/archive/refs/tags/
+EXTRACTSTO=kokkos-${VERSION}
+INSTALL_PATH=${INSTALL_PATH}/kokkos-${NAME}
+BUILDDIR=${BUILD_PATH}/kokkos-${VERSION}
+
+BUILDCHAIN=cmake
+
+CONFOPTS="-D BUILD_SHARED_LIBS=ON"
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+  -D Kokkos_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA_LAMBDA=ON \
+  -D Kokkos_ENABLE_CUDA_CONSTEXPR=ON"
+fi
+
+package_specific_register () {
+    export KOKKOS_DIR=${INSTALL_PATH}
+}
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/kokkos-${NAME}
+    rm -f $CONFIG_FILE
+    echo "
+export KOKKOS_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}
diff --git a/deal.II-toolchain/packages/kokkoskernels.package b/deal.II-toolchain/packages/kokkoskernels.package
new file mode 100644
index 00000000..85347435
--- /dev/null
+++ b/deal.II-toolchain/packages/kokkoskernels.package
@@ -0,0 +1,28 @@
+VERSION=4.4.00;CHECKSUM=a1e596b2153ff260448984c43dacb814
+
+NAME=${VERSION}
+PACKING=.tar.gz
+SOURCE=https://github.com/kokkos/kokkos-kernels/archive/refs/tags/
+EXTRACTSTO=kokkos-kernels-${VERSION}
+BUILDDIR=${BUILD_PATH}/kokkos-kernels-${VERSION}
+
+# we just install into the Kokkos directory:
+INSTALL_PATH=${INSTALL_PATH}/kokkos-${VERSION}
+
+BUILDCHAIN=cmake
+
+CONFOPTS="-D BUILD_SHARED_LIBS=ON"
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS}"
+fi
+
+
+package_specific_conf () {
+    # Generate configuration file
+    CONFIG_FILE=${CONFIGURATION_PATH}/kokkos-kernels-${NAME}
+    rm -f $CONFIG_FILE
+    echo "
+#export KOKKOS_DIR=${INSTALL_PATH}
+" >> $CONFIG_FILE
+}
diff --git a/deal.II-toolchain/packages/petsc.package b/deal.II-toolchain/packages/petsc.package
index 2ae7c426..29d2c7e7 100644
--- a/deal.II-toolchain/packages/petsc.package
+++ b/deal.II-toolchain/packages/petsc.package
@@ -2,6 +2,9 @@
 ## PETSc                                                                      ##
 ################################################################################
 
+VERSION=3.21.4
+CHECKSUM=e8d76467696fcff970bd56c93dd83476
+
 
 VERSION=3.22.2;CHECKSUM=d18caed83d1bf351c4ccb574b0b376e5
 
@@ -54,6 +57,16 @@ CONFOPTS="\
   --with-x=0 \
   --with-64-bit-indices=${with64bit}"
 
+
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+CONFOPTS="\
+  ${CONFOPTS} \
+  --with-kokkos=1 \
+  --with-kokkos-kernels=1 \
+  --with-cuda=1 \
+  --with-kokkos-dir=${KOKKOS_DIR}"
+fi
+
 # It would seem simpler to put the {C|CXX|F}OPTFLAGS directly into the
 # CONFOPTS variable, but there is no easy way to quote the
 # multiarguments and retain the quotes correctly when passing to
@@ -66,9 +79,11 @@ else
     OPTFLAGS="-g -O"
 fi
 
-for external_pkg in hypre; do
-    CONFOPTS="${CONFOPTS} --download-${external_pkg}=1"
-done
+if [ ! -z "${HYPRE_DIR}" ]; then
+  CONFOPTS="${CONFOPTS} --with-hypre=1 --with-hypre-dir=${HYPRE_DIR}"
+else
+  CONFOPTS="${CONFOPTS} --download-hypre=1"
+fi
 
 if [ ! -z "${CC}" ]; then
     CONFOPTS="${CONFOPTS} CC=${CC}"
diff --git a/deal.II-toolchain/packages/trilinos.package b/deal.II-toolchain/packages/trilinos.package
index 979527f1..704f81ca 100644
--- a/deal.II-toolchain/packages/trilinos.package
+++ b/deal.II-toolchain/packages/trilinos.package
@@ -282,6 +282,16 @@ CONFOPTS="\
   -D BUILD_SHARED_LIBS:BOOL=ON \
   ${CONFOPTS}"
 
+if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+    CONFOPTS="${CONFOPTS} \
+  -D TPL_ENABLE_MPI=ON \
+  -D TPL_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA=ON \
+  -D Kokkos_ENABLE_CUDA_LAMBDA=ON \
+  -D Kokkos_ENABLE_CUDA_CONSTEXPR=ON \
+  -D Tpetra_INST_CUDA=ON"
+fi
+
 if [ ${TRILINOS_WITH_COMPLEX} = ON ]; then
     CONFOPTS="\
     -D Trilinos_ENABLE_COMPLEX=ON \
@@ -292,6 +302,20 @@ fi
 CONFOPTS="${CONFOPTS} \
   ${TRILINOS_CONFOPTS}"
 
+package_specific_setup () {
+    if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+        export OMPI_CXX=${UNPACK_PATH}/${EXTRACTSTO}/packages/kokkos/bin/nvcc_wrapper
+        echo "SETTING OMPI_CXX to ${OMPI_CXX}"
+    fi
+}
+
+package_specific_install () {
+    if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
+        echo "UNSETTING OMPI_CXX"
+        unset OMPI_CXX
+    fi
+}
+
 package_specific_register () {
     export TRILINOS_DIR=${INSTALL_PATH}
 }

From e1e78408169b75e1ba21dbef363f2a69042e17a0 Mon Sep 17 00:00:00 2001
From: Timo Heister <timo.heister@gmail.com>
Date: Tue, 21 Jan 2025 11:16:28 -0500
Subject: [PATCH 2/4] update hypre

---
 deal.II-toolchain/packages/hypre.package | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/deal.II-toolchain/packages/hypre.package b/deal.II-toolchain/packages/hypre.package
index cb442747..ca387a08 100644
--- a/deal.II-toolchain/packages/hypre.package
+++ b/deal.II-toolchain/packages/hypre.package
@@ -1,4 +1,5 @@
-VERSION=2.31.0;CHECKSUM=3e6a9cea4e87d5d87301c95200d28242
+VERSION=2.32.0;CHECKSUM=321557481d18242a771096999a253ad8
+#VERSION=2.31.0;CHECKSUM=3e6a9cea4e87d5d87301c95200d28242
 
 NAME=v${VERSION}
 SOURCE=https://github.com/hypre-space/hypre/archive/refs/tags/

From 9754fc5b428b77a20ea92afae57c8a73e5ebdc0e Mon Sep 17 00:00:00 2001
From: Timo Heister <timo.heister@gmail.com>
Date: Tue, 21 Jan 2025 15:11:51 -0500
Subject: [PATCH 3/4] fix kokkos_dir without trilinos

---
 deal.II-toolchain/packages/dealii.package | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deal.II-toolchain/packages/dealii.package b/deal.II-toolchain/packages/dealii.package
index ffffaee3..29b93548 100644
--- a/deal.II-toolchain/packages/dealii.package
+++ b/deal.II-toolchain/packages/dealii.package
@@ -61,10 +61,10 @@ fi
 
 if [ ${USE_KOKKOS_WITH_CUDA} = ON ]; then
     CONFOPTS="${CONFOPTS} \
-      -D CMAKE_CXX_COMPILER=${TRILINOS_DIR}/bin/nvcc_wrapper \
+      -D CMAKE_CXX_COMPILER=${KOKKOS_DIR}/bin/nvcc_wrapper \
       -D DEAL_II_WITH_CUDA=OFF \
       -D DEAL_II_WITH_KOKKOS=ON \
-      -D KOKKOS_DIR=${TRILINOS_DIR} \
+      -D KOKKOS_DIR=${KOKKOS_DIR} \
       -D DEAL_II_MPI_WITH_DEVICE_SUPPORT=OFF"
     # you need a special MPI version to enable DEAL_II_MPI_WITH_DEVICE_SUPPORT
 fi

From aaae7963edb468902cbec8b661330cdb7d9ce4c4 Mon Sep 17 00:00:00 2001
From: Timo Heister <timo.heister@gmail.com>
Date: Tue, 21 Jan 2025 15:12:44 -0500
Subject: [PATCH 4/4] fix

---
 candi.cfg | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/candi.cfg b/candi.cfg
index 8faec007..d166fd84 100644
--- a/candi.cfg
+++ b/candi.cfg
@@ -42,6 +42,10 @@ BUILD_EXAMPLES=ON
 # Option {ON|OFF}: Enable using CUDA/nvidia support using Kokkos?
 USE_KOKKOS_WITH_CUDA=OFF
 SET_CUDA_ARCH=89
+# see https://developer.nvidia.com/cuda-gpus to find compute capability:
+SET_CUDA_ARCH=86
+# Remember to add the /bin of the following path to your PATH:
+#CUDA_HOME=/usr/local/cuda-12.6
 
 # Option {ON|OFF}: Unset CXX and set the compiler as MPI_CXX_COMPILER when configuring deal.II
 #