From 126b277bd03ec58a50fc9b78bc20211ea58a7da9 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 12 May 2025 23:16:25 +0000 Subject: [PATCH] Partially revert "[SYCL][UR][L0] Use leak checker and API logging (for v2) from loader (#17536)" This reverts commit b0b48a409ec5b6efde1f774dfa19f43307aa2a72 (except for test changes). The internal testing failed because the L0 loader used, was older than 1.21.2 (and didn't have leak checking implemented). I did not revert changes in the tests as that doesn't make any difference and it will be easier to reapply the patch in the future. --- .../ur_win_proxy_loader.cpp | 24 +--- .../source/adapters/level_zero/adapter.cpp | 125 +++++++++++++++--- .../source/adapters/level_zero/common.cpp | 12 +- .../source/adapters/level_zero/common.hpp | 1 - .../source/adapters/level_zero/v2/common.hpp | 40 ++++-- 5 files changed, 136 insertions(+), 66 deletions(-) diff --git a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp index 236eb17a31420..3798d81f6a544 100644 --- a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp +++ b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp @@ -107,27 +107,6 @@ void *&getDllHandle() { return dllHandle; } -static bool shouldLoadL0V2adapter() { - auto SyclEnv = std::getenv("SYCL_UR_USE_LEVEL_ZERO_V2"); - auto UREvn = std::getenv("UR_LOADER_USE_LEVEL_ZERO_V2"); - - try { - if (SyclEnv && std::stoi(SyclEnv) == 1) { - return true; - } - } catch (...) { - } - - try { - if (UREvn && std::atoi(UREvn) == 1) { - return true; - } - } catch (...) { - } - - return false; -} - /// Load the adapter libraries void preloadLibraries() { // Suppress system errors. @@ -163,8 +142,7 @@ void preloadLibraries() { getDllHandle() = loadAdapter(UR_LIBRARY_NAME(loader)); loadAdapter(UR_LIBRARY_NAME(adapter_opencl)); loadAdapter(UR_LIBRARY_NAME(adapter_level_zero)); - if (shouldLoadL0V2adapter()) - loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2)); + loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2)); loadAdapter(UR_LIBRARY_NAME(adapter_cuda)); loadAdapter(UR_LIBRARY_NAME(adapter_hip)); loadAdapter(UR_LIBRARY_NAME(adapter_native_cpu)); diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 559f9069c2b01..ac17f35099a01 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -309,12 +309,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() if (UrL0Debug & UR_L0_DEBUG_BASIC) { logger.setLegacySink(std::make_unique()); -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1"); - setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace"); - setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1"); - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); -#endif }; if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { @@ -322,12 +316,19 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); } - if (UrL0LeaksDebug) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); - } - PlatformCache.Compute = [](Result &result) { + static std::once_flag ZeCallCountInitialized; + try { + std::call_once(ZeCallCountInitialized, []() { + if (UrL0LeaksDebug) { + ZeCallCount = new std::map; + } + }); + } catch (...) { + result = exceptionToResult(std::current_exception()); + return; + } + uint32_t UserForcedSysManInit = 0; // Check if the user has disabled the default L0 Env initialization. const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { @@ -418,17 +419,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() loader_version.patch >= 2)) { useInitDrivers = true; } - -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 - if ((loader_version.major == 1 && loader_version.minor < 21) || - (loader_version.major == 1 && loader_version.minor == 21 && - loader_version.patch < 2)) { - UR_LOG( - WARN, - "WARNING: Level Zero Loader version is older than 1.21.2. " - "Please update to the latest version for API logging support.\n"); - } -#endif } if (useInitDrivers) { @@ -545,6 +535,97 @@ void globalAdapterOnDemandCleanup() { } ur_result_t adapterStateTeardown() { + // Print the balance of various create/destroy native calls. + // The idea is to verify if the number of create(+) and destroy(-) calls are + // matched. + if (ZeCallCount && (UrL0LeaksDebug) != 0) { + bool LeakFound = false; + // clang-format off + // + // The format of this table is such that each row accounts for a + // specific type of objects, and all elements in the raw except the last + // one are allocating objects of that type, while the last element is known + // to deallocate objects of that type. + // + std::vector> CreateDestroySet = { + {"zeContextCreate", "zeContextDestroy"}, + {"zeCommandQueueCreate", "zeCommandQueueDestroy"}, + {"zeModuleCreate", "zeModuleDestroy"}, + {"zeKernelCreate", "zeKernelDestroy"}, + {"zeEventPoolCreate", "zeEventPoolDestroy"}, + {"zeCommandListCreateImmediate", "zeCommandListCreate", "zeCommandListDestroy"}, + {"zeEventCreate", "zeEventDestroy"}, + {"zeFenceCreate", "zeFenceDestroy"}, + {"zeImageCreate","zeImageViewCreateExt", "zeImageDestroy"}, + {"zeSamplerCreate", "zeSamplerDestroy"}, + {"zeMemAllocDevice", "zeMemAllocHost", "zeMemAllocShared", "zeMemFree"}, + }; + + // A sample output aimed below is this: + // ------------------------------------------------------------------------ + // zeContextCreate = 1 \---> zeContextDestroy = 1 + // zeCommandQueueCreate = 1 \---> zeCommandQueueDestroy = 1 + // zeModuleCreate = 1 \---> zeModuleDestroy = 1 + // zeKernelCreate = 1 \---> zeKernelDestroy = 1 + // zeEventPoolCreate = 1 \---> zeEventPoolDestroy = 1 + // zeCommandListCreateImmediate = 1 | + // zeCommandListCreate = 1 \---> zeCommandListDestroy = 1 ---> LEAK = 1 + // zeEventCreate = 2 \---> zeEventDestroy = 2 + // zeFenceCreate = 1 \---> zeFenceDestroy = 1 + // zeImageCreate = 0 \---> zeImageDestroy = 0 + // zeSamplerCreate = 0 \---> zeSamplerDestroy = 0 + // zeMemAllocDevice = 0 | + // zeMemAllocHost = 1 | + // zeMemAllocShared = 0 \---> zeMemFree = 1 + // + // clang-format on + // TODO: use logger to print this messages + std::cerr << "Check balance of create/destroy calls\n"; + std::cerr << "----------------------------------------------------------\n"; + std::stringstream ss; + for (const auto &Row : CreateDestroySet) { + int diff = 0; + for (auto I = Row.begin(); I != Row.end();) { + const char *ZeName = (*I).c_str(); + const auto &ZeCount = (*ZeCallCount)[*I]; + + bool First = (I == Row.begin()); + bool Last = (++I == Row.end()); + + if (Last) { + ss << " \\--->"; + diff -= ZeCount; + } else { + diff += ZeCount; + if (!First) { + ss << " | "; + std::cerr << ss.str() << "\n"; + ss.str(""); + ss.clear(); + } + } + ss << std::setw(30) << std::right << ZeName; + ss << " = "; + ss << std::setw(5) << std::left << ZeCount; + } + + if (diff) { + LeakFound = true; + ss << " ---> LEAK = " << diff; + } + + std::cerr << ss.str() << '\n'; + ss.str(""); + ss.clear(); + } + + ZeCallCount->clear(); + delete ZeCallCount; + ZeCallCount = nullptr; + if (LeakFound) + return UR_RESULT_ERROR_INVALID_MEM_OBJECT; + } + // Due to multiple DLLMain definitions with SYCL, register to cleanup the // Global Adapter after refcnt is 0 #if defined(_WIN32) diff --git a/unified-runtime/source/adapters/level_zero/common.cpp b/unified-runtime/source/adapters/level_zero/common.cpp index 0d49bc087496c..397032767f4bd 100644 --- a/unified-runtime/source/adapters/level_zero/common.cpp +++ b/unified-runtime/source/adapters/level_zero/common.cpp @@ -137,18 +137,15 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString) { } // switch } -#ifdef UR_ADAPTER_LEVEL_ZERO_V2 -ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *, const char *, - bool) { - return ZeResult; -} -#else ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName, const char *ZeArgs, bool TraceError) { UR_LOG(DEBUG, "ZE ---> {}{}", ZeName, ZeArgs); if (ZeResult == ZE_RESULT_SUCCESS) { - return ZeResult; + if (UrL0LeaksDebug) { + ++(*ZeCallCount)[ZeName]; + } + return ZE_RESULT_SUCCESS; } if (TraceError) { @@ -158,7 +155,6 @@ ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName, } return ZeResult; } -#endif // Specializations for various L0 structures template <> ze_structure_type_t getZeStructureType() { diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index b464159ce184c..cdbdb8f190b19 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -74,7 +74,6 @@ const int UrL0LeaksDebug = [] { const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG"); if (!UrRet) return 0; - return std::atoi(UrRet); }(); diff --git a/unified-runtime/source/adapters/level_zero/v2/common.hpp b/unified-runtime/source/adapters/level_zero/v2/common.hpp index fc3dcd93ca67b..5f26108207fef 100644 --- a/unified-runtime/source/adapters/level_zero/v2/common.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/common.hpp @@ -15,8 +15,24 @@ #include "../common.hpp" #include "logger/ur_logger.hpp" +namespace { +#define DECLARE_DESTROY_FUNCTION(name) \ + template ze_result_t name##_wrapped(ZeHandleT handle) { \ + return ZE_CALL_NOCHECK_NAME(name, (handle), #name); \ + } + +#define HANDLE_WRAPPER_TYPE(handle, destroy) \ + ze_handle_wrapper> +} // namespace namespace v2 { + +DECLARE_DESTROY_FUNCTION(zeKernelDestroy) +DECLARE_DESTROY_FUNCTION(zeEventDestroy) +DECLARE_DESTROY_FUNCTION(zeEventPoolDestroy) +DECLARE_DESTROY_FUNCTION(zeContextDestroy) +DECLARE_DESTROY_FUNCTION(zeCommandListDestroy) +DECLARE_DESTROY_FUNCTION(zeImageDestroy) namespace raii { template @@ -92,23 +108,23 @@ struct ze_handle_wrapper { bool ownZeHandle; }; -using ze_kernel_handle_t = - ze_handle_wrapper<::ze_kernel_handle_t, zeKernelDestroy>; +using ze_kernel_handle_t = HANDLE_WRAPPER_TYPE(::ze_kernel_handle_t, + zeKernelDestroy); -using ze_event_handle_t = - ze_handle_wrapper<::ze_event_handle_t, zeEventDestroy>; +using ze_event_handle_t = HANDLE_WRAPPER_TYPE(::ze_event_handle_t, + zeEventDestroy); -using ze_event_pool_handle_t = - ze_handle_wrapper<::ze_event_pool_handle_t, zeEventPoolDestroy>; +using ze_event_pool_handle_t = HANDLE_WRAPPER_TYPE(::ze_event_pool_handle_t, + zeEventPoolDestroy); -using ze_context_handle_t = - ze_handle_wrapper<::ze_context_handle_t, zeContextDestroy>; +using ze_context_handle_t = HANDLE_WRAPPER_TYPE(::ze_context_handle_t, + zeContextDestroy); -using ze_command_list_handle_t = - ze_handle_wrapper<::ze_command_list_handle_t, zeCommandListDestroy>; +using ze_command_list_handle_t = HANDLE_WRAPPER_TYPE(::ze_command_list_handle_t, + zeCommandListDestroy); -using ze_image_handle_t = - ze_handle_wrapper<::ze_image_handle_t, zeImageDestroy>; +using ze_image_handle_t = HANDLE_WRAPPER_TYPE(::ze_image_handle_t, + zeImageDestroy); } // namespace raii } // namespace v2