From c1e34a79b475287305f4e77e1e1a525be7f5b937 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 20 Dec 2023 15:59:10 -0700 Subject: [PATCH 1/3] Tpetra: Adding support for making Tpetra allocate in shared space by default --- packages/tpetra/CMakeLists.txt | 18 ++++++++++++++++++ .../tpetra/core/cmake/TpetraCore_config.h.in | 2 ++ ...tra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/packages/tpetra/CMakeLists.txt b/packages/tpetra/CMakeLists.txt index f3bb7b9c7abc..9094682a5705 100644 --- a/packages/tpetra/CMakeLists.txt +++ b/packages/tpetra/CMakeLists.txt @@ -339,6 +339,24 @@ ELSE () # NOT Tpetra_INST_SYCL ENDIF () ENDIF () # Tpetra_INST_SYCL +############################################################ +# Shared/Managed Memory Default Allocation +############################################################ +# Normally we allocate Tpetra objects to the basic space by default, +# but some apps might want to allocate memory to the "shared" host/device +# space. This might be CudaUVMSpace, HIPManagedSpace or SYCLSharedUSMSpace. +# These options allow us to do that sort of thing, changing +# core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp +TRIBITS_ADD_OPTION_AND_DEFINE( + Tpetra_ALLOCATE_IN_SHARED_SPACE + HAVE_TPETRA_SHARED_ALLOCS + "Have Tpetra allocate memory in shared/managed space by default" + OFF + ) +IF(Tpetra_ALLOCATE_IN_SHARED_SPACE) + MESSAGE(STATUS "- Tpetra: Allocating the shared/managed space by default.") +ENDIF() + ############################################################ # Device-aware MPI Support diff --git a/packages/tpetra/core/cmake/TpetraCore_config.h.in b/packages/tpetra/core/cmake/TpetraCore_config.h.in index 36be28096997..2202098bfab6 100644 --- a/packages/tpetra/core/cmake/TpetraCore_config.h.in +++ b/packages/tpetra/core/cmake/TpetraCore_config.h.in @@ -173,6 +173,8 @@ #endif +#cmakedefine HAVE_TPETRA_SHARED_ALLOCS + #cmakedefine HAVE_TPETRA_EXPERIMENTAL @TPETRA_DEPRECATED_DECLARATIONS@ diff --git a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp index da79d6bf3123..e49b44e2051e 100644 --- a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp +++ b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp @@ -69,16 +69,26 @@ class KokkosDeviceWrapperNode { }; #ifdef KOKKOS_ENABLE_SYCL +#ifdef HAVE_TPETRA_SHARED_ALLOCS + typedef KokkosDeviceWrapperNode<::Kokkos::Experimental::SYCL, ::Kokkos::Experimental::SYCLSharedUSMSpace> KokkosSYCLWrapperNode; +#else typedef KokkosDeviceWrapperNode<::Kokkos::Experimental::SYCL, ::Kokkos::Experimental::SYCLDeviceUSMSpace> KokkosSYCLWrapperNode; #endif +#endif #ifdef KOKKOS_ENABLE_HIP + // NOTE: KokkosKernels does not currently suport HIPManagedSpace, but + // if it did, we'd add HAVE_TPETRA_MANAGED_ALLOCS here typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPSpace> KokkosHIPWrapperNode; #endif #ifdef KOKKOS_ENABLE_CUDA +#ifdef HAVE_TPETRA_SHARED_ALLOCS + typedef KokkosDeviceWrapperNode<::Kokkos::Cuda,::Kokkos::CudaUVMSpace> KokkosCudaWrapperNode; +#else typedef KokkosDeviceWrapperNode<::Kokkos::Cuda> KokkosCudaWrapperNode; #endif +#endif #ifdef KOKKOS_ENABLE_OPENMP typedef KokkosDeviceWrapperNode<::Kokkos::OpenMP> KokkosOpenMPWrapperNode; From 022446c13139fe7b267b99049173dde98fb69f5c Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 20 Dec 2023 16:27:29 -0700 Subject: [PATCH 2/3] Update Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp Spelling --- .../core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp index e49b44e2051e..68f1a2787a6b 100644 --- a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp +++ b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp @@ -77,7 +77,7 @@ class KokkosDeviceWrapperNode { #endif #ifdef KOKKOS_ENABLE_HIP - // NOTE: KokkosKernels does not currently suport HIPManagedSpace, but + // NOTE: KokkosKernels does not currently support HIPManagedSpace, but // if it did, we'd add HAVE_TPETRA_MANAGED_ALLOCS here typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPSpace> KokkosHIPWrapperNode; #endif From 71d07d7370583c071c3e0fa98cb00b4794be6cb2 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 20 Dec 2023 17:16:02 -0700 Subject: [PATCH 3/3] Update Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp --- .../compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp index 68f1a2787a6b..8d72ea6cbca3 100644 --- a/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp +++ b/packages/tpetra/core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp @@ -77,10 +77,12 @@ class KokkosDeviceWrapperNode { #endif #ifdef KOKKOS_ENABLE_HIP - // NOTE: KokkosKernels does not currently support HIPManagedSpace, but - // if it did, we'd add HAVE_TPETRA_MANAGED_ALLOCS here +#ifdef HAVE_TPETRA_SHARED_ALLOCS + typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPManagedSpace> KokkosHIPWrapperNode; +#else typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPSpace> KokkosHIPWrapperNode; #endif +#endif #ifdef KOKKOS_ENABLE_CUDA #ifdef HAVE_TPETRA_SHARED_ALLOCS