Implemented readonly-/ramdb preloading

src/CMakeLists.txt

@@ -342,7 +342,8 @@ set( MIOpen_Source
     tensor_api.cpp
     transformers_adam_w_api.cpp
     seq_tensor.cpp
-)
+    handle_common.cpp
+    db_preload.cpp)
 
 if(MIOPEN_ENABLE_AI_KERNEL_TUNING OR MIOPEN_ENABLE_AI_IMMED_MODE_FALLBACK)
     list(APPEND MIOpen_Source conv/heuristics/ai_heuristics.cpp)

src/db_preload.cpp

@@ -0,0 +1,177 @@
+/*******************************************************************************
+ *
+ * MIT License
+ *
+ * Copyright (c) 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ *******************************************************************************/
+
+#include <miopen/execution_context.hpp>
+#include <miopen/ramdb.hpp>
+#include <miopen/readonlyramdb.hpp>
+#include <miopen/type_name.hpp>
+#include <miopen/db_preload.hpp>
+
+#include <miopen/config.h>
+
+#include <atomic>
+#include <chrono>
+#include <execution>
+#include <future>
+#include <mutex>
+
+namespace miopen {
+MIOPEN_INTERNALS_EXPORT DbPreloadStates::~DbPreloadStates()
+{
+    std::unique_lock<std::mutex> lock(mutex);
+
+    if(preload_thread && preload_thread->joinable())
+    {
+        preload_stoper.request_stop();
+        preload_thread->join();
+    }
+}
+
+auto GetDbPreloadStates() -> std::shared_ptr<DbPreloadStates>
+{
+    static auto db_preload_states = std::make_shared<DbPreloadStates>();
+    return db_preload_states;
+}
+
+template <class Db>
+auto DbPreloadStates::GetPreloadedDb(const fs::path& path) -> std::unique_ptr<Db>
+{
+    std::unique_lock<std::mutex> lock{mutex, std::defer_lock};
+
+    // Mutex is need to ensure states.futures is not updated while we work
+    // so we skip locking if it no more writes can happen
+    const auto needs_lock = !started_loading.load(std::memory_order_relaxed);
+
+    if(needs_lock)
+        lock.lock();
+
+    auto it = futures.find(path);
+
+    if(it == futures.end())
+        return nullptr;
+
+    auto future = std::move(it->second);
+
+    if(needs_lock)
+        lock.unlock();
+
+    if(!future.valid())
+        MIOPEN_THROW(miopenStatusInternalError,
+                     "Attempt to reload " + path.string() + " as " + get_type_name<Db>());
+
+    const auto start = std::chrono::high_resolution_clock::now();
+    auto ret         = future.get();
+    const auto end   = std::chrono::high_resolution_clock::now();
+    const auto time  = end - start;
+    MIOPEN_LOG_I("GetPreloadedDb time waiting for the db: " << time.count() * .000001f << " ms");
+    return std::get<std::unique_ptr<Db>>(std::move(ret));
+}
+
+template <class Db>
+auto MakeDbPreloader(DbKinds db_kind, bool is_system) -> DbPreloader
+{
+    if constexpr(std::is_same_v<Db, RamDb>)
+    {
+        return [=](const stop_token& stop, const fs::path& path) -> PreloadedDb {
+            auto db   = std::make_unique<RamDb>(db_kind, path, is_system);
+            auto lock = std::unique_lock<LockFile>(db->GetLockFile(), GetDbLockTimeout());
+            if(!lock)
+                MIOPEN_THROW("Db lock has failed to lock.");
+            db->Prefetch(stop);
+            return {std::move(db)};
+        };
+    }
+    else
+    {
+        std::ignore = is_system;
+
+        return [=](const stop_token& stop, const fs::path& path) -> PreloadedDb {
+            auto db = std::make_unique<Db>(db_kind, path);
+            db->Prefetch(true, stop);
+            return {std::move(db)};
+        };
+    }
+}
+
+template auto MakeDbPreloader<RamDb>(DbKinds db_kind, bool is_system) -> DbPreloader;
+template auto MakeDbPreloader<ReadonlyRamDb>(DbKinds db_kind, bool is_system) -> DbPreloader;
+
+MIOPEN_INTERNALS_EXPORT void DbPreloadStates::StartPreloadingDb(const fs::path& path,
+                                                                DbPreloader&& preloader)
+{
+    if(path.empty())
+        return;
+
+    auto& task =
+        preload_tasks.emplace_back(std::bind(std::move(preloader), std::placeholders::_1, path));
+    futures.emplace(path, task.get_future());
+}
+
+MIOPEN_INTERNALS_EXPORT void
+DbPreloadStates::TryStartPreloadingDbs(const std::function<void()>& preload)
+{
+    if(started_loading.load(std::memory_order_relaxed))
+        return;
+
+    std::unique_lock<std::mutex> lock(mutex);
+
+    if(started_loading.load(std::memory_order_relaxed))
+        return;
+
+    preload();
+
+    started_loading.store(true, std::memory_order_relaxed);
+    // We have finished updating the map and can allow short-cutting the mutex
+
+    if(!preload_tasks.empty())
+    {
+        preload_stoper = stop_source();
+
+        preload_thread = std::thread(
+            [tasks = std::move(preload_tasks), token = preload_stoper.get_token()]() mutable {
+                MIOPEN_LOG_I("DB preload thread started");
+                std::for_each(std::execution::par_unseq,
+                              tasks.begin(),
+                              tasks.end(),
+                              [token](auto&& task) { task(token); });
+                MIOPEN_LOG_I("DB preload thread finished");
+            });
+    }
+}
+
+MIOPEN_INTERNALS_EXPORT auto DbPreloadStates::GetPreloadedRamDb(const fs::path& path)
+    -> std::unique_ptr<RamDb>
+{
+    return GetPreloadedDb<RamDb>(path);
+}
+
+MIOPEN_INTERNALS_EXPORT auto DbPreloadStates::GetPreloadedReadonlyRamDb(const fs::path& path)
+    -> std::unique_ptr<ReadonlyRamDb>
+{
+    return GetPreloadedDb<ReadonlyRamDb>(path);
+}
+
+} // namespace miopen

src/handle_api.cpp

@@ -71,10 +71,10 @@ extern "C" miopenStatus_t miopenGetVersion(size_t* major, size_t* minor, size_t*
 
 extern "C" miopenStatus_t miopenCreate(miopenHandle_t* handle)
 {
-
     return miopen::try_([&] {
-        auto& h = miopen::deref(handle);
-        h       = new miopen::Handle();
+        auto& h  = miopen::deref(handle);
+        auto ptr = std::make_unique<miopen::Handle>();
+        h        = ptr.release();
     });
 }
 
@@ -83,8 +83,9 @@ extern "C" miopenStatus_t miopenCreateWithStream(miopenHandle_t* handle,
 {
 
     return miopen::try_([&] {
-        auto& h = miopen::deref(handle);
-        h       = new miopen::Handle(stream);
+        auto& h  = miopen::deref(handle);
+        auto ptr = std::make_unique<miopen::Handle>(stream);
+        h        = ptr.release();
     });
 }
 

src/handle_common.cpp

@@ -0,0 +1,82 @@
+/*******************************************************************************
+ *
+ * MIT License
+ *
+ * Copyright (c) 2024 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ *******************************************************************************/
+
+#include <miopen/handle.hpp>
+
+#include <miopen/db_preload.hpp>
+#include <miopen/errors.hpp>
+#include <miopen/find_db.hpp>
+#include <miopen/mlo_internal.hpp>
+#include <miopen/ramdb.hpp>
+#include <miopen/readonlyramdb.hpp>
+
+namespace miopen {
+namespace {
+void PreloadDbPair(DbPreloadStates* states, DbKinds kind, fs::path&& system, fs::path&& user)
+{
+#if !MIOPEN_DISABLE_SYSDB
+    states->StartPreloadingDb(system, MakeDbPreloader<ReadonlyRamDb>(kind, true));
+#endif
+#if !MIOPEN_DISABLE_USERDB
+    states->StartPreloadingDb(user, MakeDbPreloader<RamDb>(kind, false));
+#endif
+}
+} // namespace
+
+void Handle::TryStartPreloadingDbs()
+{
+    auto const states = GetDbPreloadStates();
+
+    states->TryStartPreloadingDbs([&]() {
+        ExecutionContext ctx{this};
+
+        MIOPEN_LOG_I("Preloading dbs");
+
+        // conv find-db
+        PreloadDbPair(states.get(),
+                      DbKinds::FindDb,
+                      FindDbRecord::GetInstalledPath(*this, ""),
+                      FindDbRecord::GetUserPath(*this, ""));
+
+        // fusion find-db
+        // it uses perf-db from convolution
+        PreloadDbPair(states.get(),
+                      DbKinds::FindDb,
+                      FindDbRecord::GetInstalledPath(*this, "fusion"),
+                      FindDbRecord::GetUserPath(*this, "fusion"));
+
+        // conv perf-db
+        PreloadDbPair(states.get(), DbKinds::PerfDb, ctx.GetPerfDbPath(), ctx.GetUserPerfDbPath());
+
+        // batchnorm perf-db
+        // it doesn't use find-db
+        PreloadDbPair(states.get(),
+                      DbKinds::PerfDb,
+                      ctx.GetPerfDbPath("batchnorm"),
+                      ctx.GetUserPerfDbPath("batchnorm"));
+    });
+}
+} // namespace miopen

src/hip/handlehip.cpp

@@ -313,6 +313,9 @@ Handle::Handle(miopenAcceleratorQueue_t stream) : impl(std::make_unique<HandleIm
     this->impl->hip_blasLt_handle = CreateHipblasLtHandle();
 #endif
     this->impl->target_properties.Init(this);
+
+    TryStartPreloadingDbs();
+
     MIOPEN_LOG_NQI(*this);
 }
 
@@ -340,6 +343,9 @@ Handle::Handle() : impl(std::make_unique<HandleImpl>())
     this->impl->hip_blasLt_handle = CreateHipblasLtHandle();
 #endif
     this->impl->target_properties.Init(this);
+
+    TryStartPreloadingDbs();
+
     MIOPEN_LOG_NQI(*this);
 }