AMD GPU Fixes (#1333)

* With this commit, Nalu now uses the correct APIs for setting the amount of stack per thread in HIP.

* HIP is no longer experimental, thus removing this code.

* Manual formatting

---------

Co-authored-by: Paul Mullowney <pmullown@TheraS02.thera.amd.com>

include/KokkosInterface.h

@@ -46,7 +46,7 @@ namespace nalu {
 #if defined(KOKKOS_ENABLE_CUDA)
 typedef Kokkos::CudaSpace MemSpace;
 #elif defined(KOKKOS_ENABLE_HIP)
-typedef Kokkos::Experimental::HIPSpace MemSpace;
+typedef Kokkos::HIPSpace MemSpace;
 #elif defined(KOKKOS_HAVE_OPENMP)
 typedef Kokkos::OpenMP MemSpace;
 #else

include/aero/actuator/ActuatorTypes.h

@@ -24,7 +24,7 @@ namespace nalu {
 using ActuatorMemSpace = Kokkos::CudaSpace;
 using ActuatorExecutionSpace = Kokkos::DefaultExecutionSpace;
 #elif defined(KOKKOS_ENABLE_HIP)
-using ActuatorMemSpace = Kokkos::Experimental::HIPSpace;
+using ActuatorMemSpace = Kokkos::HIPSpace;
 using ActuatorExecutionSpace = Kokkos::DefaultExecutionSpace;
 #else
 using ActuatorMemSpace = Kokkos::HostSpace;

include/matrix_free/KokkosFramework.h

@@ -62,12 +62,12 @@ struct ExecTraits<Kokkos::Cuda>
 
 #if defined(KOKKOS_ENABLE_HIP)
 template <>
-struct ExecTraits<Kokkos::Experimental::HIP>
+struct ExecTraits<Kokkos::HIP>
 {
   using data_type = double;
   using memory_traits =
     Kokkos::MemoryTraits<Kokkos::Restrict | Kokkos::Aligned>;
-  using memory_space = typename Kokkos::Experimental::HIP::memory_space;
+  using memory_space = typename Kokkos::HIP::memory_space;
   using layout = Kokkos::LayoutLeft;
   static constexpr int alignment = alignof(data_type);
   static constexpr int simd_len = 1;

src/Simulation.C

@@ -50,8 +50,7 @@ Simulation::Simulation(const YAML::Node& root_node)
   cudaDeviceGetLimit(&default_stack_size, cudaLimitStackSize);
   cudaDeviceSetLimit(cudaLimitStackSize, nalu_stack_size);
 #elif defined(KOKKOS_ENABLE_HIP)
-  hipError_t err =
-    hipDeviceGetLimit(&default_stack_size, hipLimitMallocHeapSize);
+  hipError_t err = hipDeviceGetLimit(&default_stack_size, hipLimitStackSize);
   if (err != hipSuccess) {
     /*
      This might be useful at some point so keeping it and commenting out.
@@ -61,7 +60,7 @@ Simulation::Simulation(const YAML::Node& root_node)
     */
   }
 
-  err = hipDeviceSetLimit(hipLimitMallocHeapSize, nalu_stack_size);
+  err = hipDeviceSetLimit(hipLimitStackSize, nalu_stack_size);
   if (err != hipSuccess) {
     /*
      This might be useful at some point so keeping it and commenting out.
@@ -82,8 +81,7 @@ Simulation::~Simulation()
 #if defined(KOKKOS_ENABLE_CUDA)
   cudaDeviceSetLimit(cudaLimitStackSize, default_stack_size);
 #elif defined(KOKKOS_ENABLE_HIP)
-  hipError_t err =
-    hipDeviceSetLimit(hipLimitMallocHeapSize, default_stack_size);
+  hipError_t err = hipDeviceSetLimit(hipLimitStackSize, default_stack_size);
   if (err != hipSuccess) {
     /*
      This might be useful at some point so keeping it and commenting out.

unit_tests.C

@@ -30,7 +30,7 @@ main(int argc, char** argv)
   cudaDeviceSetLimit(cudaLimitStackSize, nalu_stack_size);
 #elif defined(KOKKOS_ENABLE_HIP)
   const size_t nalu_stack_size = 16384;
-  hipError_t err = hipDeviceSetLimit(hipLimitMallocHeapSize, nalu_stack_size);
+  hipError_t err = hipDeviceSetLimit(hipLimitStackSize, nalu_stack_size);
   if (err != hipSuccess) {
     /*
      This might be useful at some point so keeping it and commenting out.

unit_tests/UnitTestBasicKokkos.C

@@ -28,7 +28,7 @@ TEST(BasicKokkos, discover_execution_space)
 #endif
 
 #if defined(KOKKOS_ENABLE_HIP)
-    std::cout << "Kokkos::Experimental::HIP is available." << std::endl;
+    std::cout << "Kokkos::HIP is available." << std::endl;
 #endif
 
     std::cout << "Default execution space info: ";