feature: add RAJA kernel launches and basic CUDA support

.gitignore

@@ -13,3 +13,4 @@
 *.orig
 __pycache__/
 view
+*.cache*

CMakeLists.txt

@@ -22,8 +22,17 @@ project(serac LANGUAGES CXX C)
 # MPI is required in Serac.
 set(ENABLE_MPI ON CACHE BOOL "")
 
+
+if (ENABLE_CUDA)
+    set(SERAC_USE_CUDA ON CACHE BOOL "")
+    set(RAJA_ENABLE_CUDA ON CACHE BOOL "")
+    set(AXOM_USE_CUDA ON CACHE BOOL "")
+    set(MFEM_USE_CUDA ON CACHE BOOL "")
+endif()
+
+
 if (NOT MPI_C_COMPILER OR NOT MPI_CXX_COMPILER)
-    message(FATAL_ERROR 
+    message(FATAL_ERROR
             "Serac requires MPI.  It is required to provide the MPI C/C++ "
             "compiler wrappers via the CMake variables, "
             "MPI_C_COMPILER and MPI_CXX_COMPILER.")
@@ -131,6 +140,7 @@ include(${PROJECT_SOURCE_DIR}/cmake/thirdparty/SetupSeracThirdParty.cmake)
 
 set(CMAKE_C_FLAGS   ${SERAC_CMAKE_C_FLAGS})
 set(CMAKE_CXX_FLAGS ${SERAC_CMAKE_CXX_FLAGS})
+string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
 include(${PROJECT_SOURCE_DIR}/cmake/SeracConfigHeader.cmake)
 
@@ -196,7 +206,7 @@ if (SERAC_ENABLE_CODEVELOP)
             )
 endif()
 
-install(EXPORT serac-targets 
+install(EXPORT serac-targets
         NAMESPACE serac::
         DESTINATION lib/cmake
         )

cmake/SeracMacros.cmake

@@ -8,7 +8,7 @@
 #------------------------------------------------------------------------------
 # Adds code checks for all cpp/hpp files recursively under the current directory
 # that regex match INCLUDES and excludes any files that regex match EXCLUDES
-# 
+#
 # This creates the following parent build targets:
 #  check - Runs a non file changing style check and CppCheck
 #  style - In-place code formatting
@@ -71,7 +71,7 @@ macro(serac_add_code_checks)
     blt_add_clang_tidy_target(NAME              ${arg_PREFIX}_guidelines_check_tests
                               CHECKS            "clang-analyzer-*,clang-analyzer-cplusplus*,cppcoreguidelines-*,-cppcoreguidelines-avoid-magic-numbers"
                               SRC_FILES         ${_test_sources})
-                                  
+
     if (ENABLE_COVERAGE)
         blt_add_code_coverage_target(NAME   ${arg_PREFIX}_coverage
                                      RUNNER ${CMAKE_MAKE_PROGRAM} test
@@ -121,17 +121,20 @@ endmacro(serac_convert_to_native_escaped_file_path)
 
 ##------------------------------------------------------------------------------
 ## serac_add_tests( SOURCES         [source1 [source2 ...]]
+##                  USE_CUDA        [use CUDA if set]
 ##                  DEPENDS_ON      [dep1 [dep2 ...]]
 ##                  NUM_MPI_TASKS   [num tasks]
 ##                  NUM_OMP_THREADS [num threads])
 ##
 ## Creates an executable per given source and then adds the test to CTest
+## If USE_CUDA is set, this macro will compile a CUDA enabled version of
+## of each unit test.
 ##------------------------------------------------------------------------------
 
 macro(serac_add_tests)
 
     set(options )
-    set(singleValueArgs NUM_MPI_TASKS NUM_OMP_THREADS)
+    set(singleValueArgs NUM_MPI_TASKS NUM_OMP_THREADS USE_CUDA)
     set(multiValueArgs SOURCES DEPENDS_ON)
 
     # Parse the arguments to the macro
@@ -148,13 +151,21 @@ macro(serac_add_tests)
 
     foreach(filename ${arg_SOURCES})
         get_filename_component(test_name ${filename} NAME_WE)
+        if (DEFINED arg_USE_CUDA)
+            set(test_name "${test_name}_cuda")
+        endif()
 
         blt_add_executable(NAME        ${test_name}
                            SOURCES     ${filename}
                            OUTPUT_DIR  ${TEST_OUTPUT_DIRECTORY}
                            DEPENDS_ON  ${arg_DEPENDS_ON}
                            FOLDER      serac/tests )
 
+        if (DEFINED arg_USE_CUDA)
+            target_compile_definitions(${test_name} PUBLIC ENABLE_CUDA)
+            target_compile_definitions(${test_name} PUBLIC USE_CUDA)
+        endif()
+
         blt_add_test(NAME            ${test_name}
                      COMMAND         ${test_name}
                      NUM_MPI_TASKS   ${arg_NUM_MPI_TASKS}

cmake/thirdparty/SetupSeracThirdParty.cmake

@@ -15,7 +15,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         # Manually set includes as system includes
         foreach(_target cuda_runtime cuda)
             get_target_property(_dirs ${_target} INTERFACE_INCLUDE_DIRECTORIES)
-            set_property(TARGET ${_target} 
+            set_property(TARGET ${_target}
                          APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
                          "${_dirs}")
         endforeach()
@@ -59,7 +59,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
     #------------------------------------------------------------------------------
     if(UMPIRE_DIR)
         serac_assert_is_directory(VARIABLE_NAME UMPIRE_DIR)
-        find_package(umpire REQUIRED NO_DEFAULT_PATH 
+        find_package(umpire REQUIRED NO_DEFAULT_PATH
                      PATHS ${UMPIRE_DIR})
         message(STATUS "Umpire support is ON")
         set(UMPIRE_FOUND TRUE)
@@ -73,7 +73,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
     #------------------------------------------------------------------------------
     if(RAJA_DIR)
         serac_assert_is_directory(VARIABLE_NAME RAJA_DIR)
-        find_package(RAJA REQUIRED NO_DEFAULT_PATH 
+        find_package(RAJA REQUIRED NO_DEFAULT_PATH
                      PATHS ${RAJA_DIR})
         message(STATUS "RAJA support is ON")
         set(RAJA_FOUND TRUE)
@@ -100,7 +100,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
 
     # Manually set includes as system includes
     get_target_property(_dirs conduit::conduit INTERFACE_INCLUDE_DIRECTORIES)
-    set_property(TARGET conduit::conduit 
+    set_property(TARGET conduit::conduit
                  APPEND PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
                  "${_dirs}")
 
@@ -154,10 +154,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
                 message(FATAL_ERROR "Serac+Caliper+CUDA requires CMake > 3.17.")
             else()
                 find_package(CUDAToolkit REQUIRED)
-            endif() 
+            endif()
         endif()
 
-        find_package(caliper REQUIRED NO_DEFAULT_PATH 
+        find_package(caliper REQUIRED NO_DEFAULT_PATH
                      PATHS ${CALIPER_DIR})
         message(STATUS "Caliper support is ON")
         set(CALIPER_FOUND TRUE)
@@ -173,7 +173,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         # Note: Sundials is currently only used via MFEM and MFEM's target contains it's information
         serac_assert_is_directory(VARIABLE_NAME SUNDIALS_DIR)
         set(SERAC_USE_SUNDIALS ON CACHE BOOL "")
-        
+
         # Note: MFEM sets SUNDIALS_FOUND itself
         if (NOT SERAC_ENABLE_CODEVELOP)
             set(SUNDIALS_FOUND TRUE)
@@ -195,7 +195,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
 
         #### Store Data that MFEM clears
         set(tpls_to_save AMGX AXOM CALIPER CAMP CONDUIT HDF5
-                         HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA 
+                         HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA
                          SUPERLU_DIST STRUMPACK SUNDIALS TRIBOL UMPIRE)
         foreach(_tpl ${tpls_to_save})
             set(${_tpl}_DIR_SAVE "${${_tpl}_DIR}")
@@ -285,7 +285,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         else()
             add_subdirectory(${PROJECT_SOURCE_DIR}/mfem  ${CMAKE_BINARY_DIR}/mfem)
         endif()
- 
+
         set(MFEM_FOUND TRUE CACHE BOOL "" FORCE)
 
         # Patch the mfem target with the correct include directories
@@ -310,7 +310,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         serac_assert_is_directory(VARIABLE_NAME AXOM_DIR)
 
         find_package(axom REQUIRED
-                        NO_DEFAULT_PATH 
+                        NO_DEFAULT_PATH
                         PATHS ${AXOM_DIR}/lib/cmake)
 
         message(STATUS "Axom support is ON")
@@ -431,7 +431,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
             serac_assert_is_directory(VARIABLE_NAME TRIBOL_DIR)
 
             find_package(tribol REQUIRED
-                                NO_DEFAULT_PATH 
+                                NO_DEFAULT_PATH
                                 PATHS ${TRIBOL_DIR}/lib/cmake)
 
             if(TARGET tribol)
@@ -449,7 +449,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         else()
             set(TRIBOL_FOUND OFF)
         endif()
-        
+
         message(STATUS "Tribol support is " ${TRIBOL_FOUND})
     else()
         set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE)
@@ -468,7 +468,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         endif()
 
         add_subdirectory(${tribol_repo_dir}  ${CMAKE_BINARY_DIR}/tribol)
-        
+
         target_include_directories(redecomp PUBLIC
             $<BUILD_INTERFACE:${tribol_repo_dir}/src>
         )
@@ -477,7 +477,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
             $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/tribol/include>
             $<INSTALL_INTERFACE:include>
         )
-        
+
         set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE)
         set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE)
     endif()
@@ -509,7 +509,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
         # This flag is empty due to us not enabling fortran but we need to strip it
         # so it doesn't propagate in our project
         if("${OpenMP_Fortran_FLAGS}" STREQUAL "")
-            set(OpenMP_Fortran_FLAGS "$<$<NOT:$<COMPILE_LANGUAGE:Fortran>>:-fopenmp=libomp>;$<$<COMPILE_LANGUAGE:Fortran>:-fopenmp>")
+            set(OpenMP_Fortran_FLAGS "-fopenmp")#"$<$<NOT:$<COMPILE_LANGUAGE:Fortran>>:-fopenmp=libomp>;$<$<COMPILE_LANGUAGE:Fortran>:-fopenmp>")
         endif()
 
         foreach(_target axom)

host-configs/lassen-blueos_3_ppc64le_ib_p9-clang@10.0.1.cmake

@@ -30,13 +30,15 @@ if(DEFINED ENV{SPACK_CC})
 else()
 
   set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-ibm-10.0.1-gcc-8.3.1/bin/clang" CACHE PATH "")
+  #set(CMAKE_C_COMPILER "/usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang" CACHE PATH "")
 
   set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-ibm-10.0.1-gcc-8.3.1/bin/clang++" CACHE PATH "")
+  #set(CMAKE_CXX_COMPILER "/usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++" CACHE PATH "")
 
   set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran" CACHE PATH "")
 
 endif()
-
+#-fopenmp -gdwarf-4 -fgpu-rdc
 set(CMAKE_C_STANDARD_LIBRARIES "-lgfortran" CACHE STRING "")
 
 set(CMAKE_CXX_STANDARD_LIBRARIES "-lgfortran" CACHE STRING "")
@@ -69,29 +71,34 @@ set(BLT_MPI_COMMAND_APPEND "mpibind" CACHE STRING "")
 # Cuda
 #------------------------------------------------
 
-set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-11.2.0" CACHE PATH "")
+set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-12.0.0" CACHE PATH "")
+#set(CUDAToolkit_ROOT "/usr/tce/packages/cuda/cuda-10.1.105" CACHE PATH "")
 
 set(CMAKE_CUDA_COMPILER "${CUDAToolkit_ROOT}/bin/nvcc" CACHE PATH "")
 
 set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}" CACHE PATH "")
 
-set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-11.2.0" CACHE PATH "")
+set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-12.0.0" CACHE PATH "")
+#set(CUDA_TOOLKIT_ROOT_DIR "/usr/tce/packages/cuda/cuda-10.1.105" CACHE PATH "")
 
 set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
 
 set(ENABLE_OPENMP ON CACHE BOOL "")
 
 set(ENABLE_CUDA ON CACHE BOOL "")
 
-set(CMAKE_CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "")
+set(ENABLE_CLANG_CUDA OFF CACHE BOOL "")
 
 set(CMAKE_CUDA_FLAGS " --expt-extended-lambda --expt-relaxed-constexpr " CACHE STRING "")
 
-set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "")
+set(CMAKE_CUDA_SEPARABLE_COMPILATION ON CACHE BOOL "")
+
+#set(CMAKE_CUDA_FLAGS " --expt-extended-lambda --expt-relaxed-constexpr " CACHE STRING "")
+#set(CMAKE_CUDA_FLAGS "-fopenmp" CACHE STRING "")
 
 # nvcc does not like gtest's 'pthreads' flag
 
-set(gtest_disable_pthreads ON CACHE BOOL "")
+set(gtest_disable_pthreads OFF CACHE BOOL "")
 
 set(BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3" CACHE STRING "")