Additional SYCL USM (device pointer explicit copy) and CUDA tuning for DOT

CMakeLists.txt

@@ -84,6 +84,7 @@ register_model(hip HIP HIPStream.cpp)
 register_model(cuda CUDA CUDAStream.cu)
 register_model(kokkos KOKKOS KokkosStream.cpp)
 register_model(sycl SYCL SYCLStream.cpp)
+register_model(syclusm SYCLUSM SYCLStream.cpp)
 register_model(sycl2020 SYCL2020 SYCLStream2020.cpp)
 register_model(acc ACC ACCStream.cpp)
 # defining RAJA collides with the RAJA namespace so USE_RAJA

src/cuda/CUDAStream.cu

@@ -8,6 +8,8 @@
 
 #include "CUDAStream.h"
 
+int DOT_NUM_BLOCKS;
+
 void check_error(void)
 {
   cudaError_t err = cudaGetLastError();
@@ -38,10 +40,16 @@ CUDAStream<T>::CUDAStream(const int ARRAY_SIZE, const int device_index)
     throw std::runtime_error("Invalid device index");
   cudaSetDevice(device_index);
   check_error();
+  cudaDeviceProp prop;
+  cudaGetDeviceProperties(&prop, device_index);
+  check_error();
 
   // Print out device information
   std::cout << "Using CUDA device " << getDeviceName(device_index) << std::endl;
   std::cout << "Driver: " << getDeviceDriver(device_index) << std::endl;
+  DOT_NUM_BLOCKS = 4 * prop.multiProcessorCount;
+  std::cout << "dot " << DOT_NUM_BLOCKS << " " << TBSIZE << " " << array_size
+    << "\n";
 
   array_size = ARRAY_SIZE;
 

src/cuda/CUDAStream.h

@@ -22,7 +22,7 @@
 #endif
 
 #define TBSIZE 1024
-#define DOT_NUM_BLOCKS 256
+extern int DOT_NUM_BLOCKS;
 
 template <class T>
 class CUDAStream : public Stream<T>

src/hip/HIPStream.cpp

@@ -8,8 +8,8 @@
 #include "HIPStream.h"
 #include "hip/hip_runtime.h"
 
-#define TBSIZE 1024
-#define DOT_NUM_BLOCKS 256
+#define TBSIZE 256
+int DOT_NUM_BLOCKS;
 
 void check_error(void)
 {
@@ -42,19 +42,24 @@ HIPStream<T>::HIPStream(const int ARRAY_SIZE, const int device_index)
   hipSetDevice(device_index);
   check_error();
 
+  // get properties
+  hipDeviceProp_t props;
+  hipGetDeviceProperties(&props, 0);
+  DOT_NUM_BLOCKS = props.multiProcessorCount * 4;
+
   // Print out device information
   std::cout << "Using HIP device " << getDeviceName(device_index) << std::endl;
   std::cout << "Driver: " << getDeviceDriver(device_index) << std::endl;
+  std::cout << "DOT_NUM_BLOCKS: " << DOT_NUM_BLOCKS << " TBSIZE " <<
+	  TBSIZE << "\n";
 
   array_size = ARRAY_SIZE;
 
   // Allocate the host array for partial sums for dot kernels
   sums = (T*)malloc(sizeof(T) * DOT_NUM_BLOCKS);
 
   // Check buffers fit on the device
-  hipDeviceProp_t props;
-  hipGetDeviceProperties(&props, 0);
-  if (props.totalGlobalMem < std::size_t{3}*ARRAY_SIZE*sizeof(T))
+  if (props.totalGlobalMem < 3*ARRAY_SIZE*sizeof(T))
     throw std::runtime_error("Device does not have enough memory for all 3 buffers");
 
   // Create device buffers

src/main.cpp

@@ -47,6 +47,8 @@
 #include "SYCLStream.h"
 #elif defined(SYCL2020)
 #include "SYCLStream2020.h"
+#elif defined(SYCLUSM)
+#include "SYCLStreamUSM.h"
 #elif defined(OMP)
 #include "OMPStream.h"
 #endif
@@ -290,7 +292,7 @@ void run()
   // Use the OpenACC implementation
   stream = new ACCStream<T>(ARRAY_SIZE, deviceIndex);
 
-#elif defined(SYCL) || defined(SYCL2020)
+#elif defined(SYCL) || defined(SYCL2020) || defined(SYCLUSM)
   // Use the SYCL implementation
   stream = new SYCLStream<T>(ARRAY_SIZE, deviceIndex);
 

src/sycl/SYCLStream.cpp

@@ -45,7 +45,7 @@ SYCLStream<T>::SYCLStream(const int ARRAY_SIZE, const int device_index)
   else
   {
     dot_num_groups = dev.get_info<info::device::max_compute_units>() * 4;
-    dot_wgsize     = dev.get_info<info::device::max_work_group_size>();
+    dot_wgsize     = 256;       // good for AMD, doesn't hurt elsewhere
   }
 
   // Print out device information