Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Disable opportunistic reuse in async mr when cuda driver < 11.5 #993

Merged
merged 3 commits into from
Mar 16, 2022
Merged

Disable opportunistic reuse in async mr when cuda driver < 11.5 #993

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2e4bd03
Require CUDA driver 11.5+ for the async mr
rongou Mar 14, 2022
bc6c5d7
also check device attribute
rongou Mar 14, 2022
96acc78
disable reuse for driver < 11.5
rongou Mar 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions include/rmm/mr/device/cuda_async_memory_resource.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ class cuda_async_memory_resource final : public device_memory_resource {
RMM_ASSERT_CUDA_SUCCESS(cudaMemPoolDestroy(pool_handle()));
#endif
}
cuda_async_memory_resource(cuda_async_memory_resource const&) = delete;
cuda_async_memory_resource(cuda_async_memory_resource&&) = delete;
cuda_async_memory_resource(cuda_async_memory_resource const&) = delete;
cuda_async_memory_resource(cuda_async_memory_resource&&) = delete;
cuda_async_memory_resource& operator=(cuda_async_memory_resource const&) = delete;
cuda_async_memory_resource& operator=(cuda_async_memory_resource&&) = delete;
cuda_async_memory_resource& operator=(cuda_async_memory_resource&&) = delete;

/**
* @brief Is cudaMallocAsync supported with this cuda runtime/driver version?
Expand All @@ -129,7 +129,15 @@ class cuda_async_memory_resource final : public device_memory_resource {
auto result = cudaDeviceGetAttribute(&cuda_pool_supported,
cudaDevAttrMemoryPoolsSupported,
rmm::detail::current_device().value());
return result == cudaSuccess and cuda_pool_supported == 1;

int driver_version{};
RMM_CUDA_TRY(cudaDriverGetVersion(&driver_version));
// CUDA drivers before 11.5 have known incompatibilities with the async allocator.
// See https://github.com/NVIDIA/spark-rapids/issues/4710.
constexpr auto min_async_version{11050};

return result == cudaSuccess and cuda_pool_supported == 1 and
driver_version >= min_async_version;
}()};
return runtime_supports_pool and driver_supports_pool;
#else
Expand Down