Merge pull request #3621 from anagainaru/sycl-backend

Adding SYCL support to the Kokkos backend

CMakeLists.txt

@@ -237,7 +237,7 @@ endif()
 set(ADIOS2_CONFIG_OPTS
     BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2
     BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ 
-    Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP
+    Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL
 )
 
 GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS})

bindings/CXX11/adios2/cxx11/KokkosView.h

@@ -9,55 +9,16 @@ namespace detail
 {
 
 template <typename T>
-struct memspace_kokkos_to_adios2;
-
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::HostSpace>
-{
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host;
-};
-
-#if defined(KOKKOS_ENABLE_CUDA) && defined(ADIOS2_HAVE_GPU_SUPPORT)
-
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::CudaSpace>
+struct memspace_kokkos_to_adios2
 {
     static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
 };
 
 template <>
-struct memspace_kokkos_to_adios2<Kokkos::CudaUVMSpace>
-{
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
-};
-
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::CudaHostPinnedSpace>
-{
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
-};
-
-#endif
-
-#if defined(KOKKOS_ENABLE_HIP) && defined(ADIOS2_HAVE_GPU_SUPPORT)
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPSpace>
-{
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
-};
-
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPHostPinnedSpace>
-{
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
-};
-
-template <>
-struct memspace_kokkos_to_adios2<Kokkos::Experimental::HIPManagedSpace>
+struct memspace_kokkos_to_adios2<Kokkos::HostSpace>
 {
-    static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU;
+    static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host;
 };
-#endif
 
 } // namespace detail
 

cmake/DetectOptions.cmake

@@ -192,6 +192,9 @@ if(ADIOS2_USE_Kokkos)
         set(ADIOS2_HAVE_Kokkos_HIP TRUE)
         enable_language(HIP)
       endif()
+      if(Kokkos_ENABLE_SYCL)
+	    set(ADIOS2_HAVE_Kokkos_SYCL TRUE)
+      endif()
       set(ADIOS2_HAVE_GPU_Support TRUE)
     endif()
   endif()

scripts/build_scripts/build-adios2-sycl-polaris.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# shellcheck disable=SC2191
+
+module load oneapi
+module load cmake/3.23.2
+module refresh
+
+######## User Configurations ########
+Kokkos_HOME=$HOME/kokkos/kokkos
+ADIOS2_HOME=$(pwd)
+BUILD_DIR=${ADIOS2_HOME}/build-kokkos-polaris
+INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-polaris
+
+num_build_procs=4
+
+######## Kokkos ########
+mkdir -p "${BUILD_DIR}/kokkos"
+rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt"
+rm -rf "${BUILD_DIR}/kokkos/CMakeFiles"
+
+ARGS=(
+    -D CMAKE_BUILD_TYPE=RelWithDebInfo
+    -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
+    -D CMAKE_CXX_COMPILER=clang++
+
+    -D Kokkos_ENABLE_SERIAL=ON
+    -D Kokkos_ARCH_ZEN3=ON
+    -D Kokkos_ENABLE_SYCL=ON
+    -D Kokkos_ARCH_AMPERE80=ON
+    -D Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON
+
+    -D CMAKE_CXX_STANDARD=17
+    -D CMAKE_CXX_EXTENSIONS=OFF
+    -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE
+    -D BUILD_SHARED_LIBS=ON
+)
+cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos"
+cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs}
+cmake --install "${BUILD_DIR}/kokkos"
+
+######## ADIOS2 ########
+mkdir -p "${BUILD_DIR}/adios2"
+rm -f "${BUILD_DIR}/adios2/CMakeCache.txt"
+rm -rf "${BUILD_DIR}/adios2/CMakeFiles"
+
+ARGS_ADIOS=(
+    -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
+    -D BUILD_TESTING=OFF
+    #-D ADIOS2_BUILD_EXAMPLES=OFF
+    -D CMAKE_CXX_COMPILER=g++
+    -D CMAKE_C_COMPILER=gcc
+
+    -D ADIOS2_USE_SST=OFF
+    -D ADIOS2_USE_Kokkos=ON
+    -D Kokkos_ROOT="${INSTALL_DIR}"
+
+    -D CMAKE_CXX_STANDARD=17
+    -D CMAKE_CXX_EXTENSIONS=OFF
+    -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE
+    -D BUILD_SHARED_LIBS=ON
+    -D ADIOS2_USE_Fortran=OFF
+)
+cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2
+cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs}
+cmake --install "${BUILD_DIR}/adios2"

source/adios2/helper/kokkos/adiosKokkos.cpp

@@ -13,10 +13,10 @@
 
 namespace
 {
-template <class MemSpace>
 void KokkosDeepCopy(const char *src, char *dst, size_t byteCount)
 {
-    Kokkos::View<const char *, MemSpace,
+    using mem_space = Kokkos::DefaultExecutionSpace::memory_space;
+    Kokkos::View<const char *, mem_space,
                  Kokkos::MemoryTraits<Kokkos::Unmanaged>>
         srcView(src, byteCount);
     Kokkos::View<char *, Kokkos::HostSpace,
@@ -43,6 +43,10 @@ void KokkosMinMaxImpl(const char * /*values*/, const size_t /*size*/,
                       char & /*min*/, char & /*max*/)
 {
 }
+void KokkosMinMaxImpl(const long double * /*values*/, const size_t /*size*/,
+                      long double & /*min*/, long double & /*max*/)
+{
+}
 void KokkosMinMaxImpl(const std::complex<float> * /*values*/,
                       const size_t /*size*/, std::complex<float> & /*min*/,
                       std::complex<float> & /*max*/)
@@ -62,22 +66,12 @@ namespace helper
 {
 void MemcpyGPUToBuffer(char *dst, const char *GPUbuffer, size_t byteCount)
 {
-#ifdef ADIOS2_HAVE_KOKKOS_CUDA
-    KokkosDeepCopy<Kokkos::CudaSpace>(GPUbuffer, dst, byteCount);
-#endif
-#ifdef ADIOS2_HAVE_KOKKOS_HIP
-    KokkosDeepCopy<Kokkos::Experimental::HIPSpace>(GPUbuffer, dst, byteCount);
-#endif
+    KokkosDeepCopy(GPUbuffer, dst, byteCount);
 }
 
 void MemcpyBufferToGPU(char *GPUbuffer, const char *src, size_t byteCount)
 {
-#ifdef ADIOS2_HAVE_KOKKOS_CUDA
-    KokkosDeepCopy<Kokkos::CudaSpace>(src, GPUbuffer, byteCount);
-#endif
-#ifdef ADIOS2_HAVE_KOKKOS_HIP
-    KokkosDeepCopy<Kokkos::Experimental::HIPSpace>(src, GPUbuffer, byteCount);
-#endif
+    KokkosDeepCopy(src, GPUbuffer, byteCount);
 }
 
 bool IsGPUbuffer(const void *ptr)
@@ -98,6 +92,15 @@ bool IsGPUbuffer(const void *ptr)
     {
         return true;
     }
+#endif
+#ifdef ADIOS2_HAVE_KOKKOS_SYCL
+    auto ret =
+        sycl::address_space_cast<sycl::access::address_space::global_space,
+                                 sycl::access::decorated::no>(ptr);
+    if (ret != nullptr)
+    {
+        return true;
+    }
 #endif
     return false;
 }
@@ -121,6 +124,7 @@ void KokkosInit()
         settings.set_device_id(device_id);
     }
 #endif
+    // GetDevice not supported for SYCL, use the default device
     Kokkos::initialize(settings);
 }