diff --git a/.github/intel-llvm-mirror-base-commit b/.github/intel-llvm-mirror-base-commit index 86da2f756c..7e96fad563 100644 --- a/.github/intel-llvm-mirror-base-commit +++ b/.github/intel-llvm-mirror-base-commit @@ -1 +1 @@ -7ab256b26005c9b3ff6fa6d2d1e8ad714468d055 +58f4da62522087e9fe6b90e6ee5f78f550d3ebe5 diff --git a/scripts/templates/ldrddi.cpp.mako b/scripts/templates/ldrddi.cpp.mako index d70586aa77..8678cc0e18 100644 --- a/scripts/templates/ldrddi.cpp.mako +++ b/scripts/templates/ldrddi.cpp.mako @@ -51,24 +51,21 @@ namespace ur_loader %if func_basename == "AdapterGet": auto context = getContext(); - size_t adapterIndex = 0; - if( nullptr != ${obj['params'][1]['name']} && ${obj['params'][0]['name']} !=0) - { - for( auto& platform : context->platforms ) - { - if(platform.initStatus != ${X}_RESULT_SUCCESS) - continue; - platform.dditable.${th.get_table_name(n, tags, obj)}.${th.make_pfn_name(n, tags, obj)}( 1, &${obj['params'][1]['name']}[adapterIndex], nullptr ); - adapterIndex++; - if (adapterIndex == NumEntries) { - break; - } - } + uint32_t numAdapters = 0; + for (auto &platform : context->platforms) { + if (platform.initStatus != ${X}_RESULT_SUCCESS) + continue; + + uint32_t adapter; + ur_adapter_handle_t *adapterHandle = numAdapters < NumEntries ? &${obj['params'][1]['name']}[numAdapters] : nullptr; + platform.dditable.${th.get_table_name(n, tags, obj)}.${th.make_pfn_name(n, tags, obj)}( 1, adapterHandle, &adapter ); + + numAdapters += adapter; } if( ${obj['params'][2]['name']} != nullptr ) { - *${obj['params'][2]['name']} = static_cast(context->platforms.size()); + *${obj['params'][2]['name']} = numAdapters; } return ${X}_RESULT_SUCCESS; diff --git a/source/adapters/level_zero/adapter.cpp b/source/adapters/level_zero/adapter.cpp index 0809ec97ed..1768f532af 100644 --- a/source/adapters/level_zero/adapter.cpp +++ b/source/adapters/level_zero/adapter.cpp @@ -54,21 +54,21 @@ class ur_legacy_sink : public logger::Sink { }; // Find the corresponding ZesDevice Handle for a given ZeDevice -ur_result_t getZesDeviceHandle(zes_uuid_t coreDeviceUuid, +ur_result_t getZesDeviceHandle(ur_adapter_handle_t_ *adapter, + zes_uuid_t coreDeviceUuid, zes_device_handle_t *ZesDevice, uint32_t *SubDeviceId, ze_bool_t *SubDevice) { uint32_t ZesDriverCount = 0; std::vector ZesDrivers; std::vector ZesDevices; ze_result_t ZesResult = ZE_RESULT_ERROR_INVALID_ARGUMENT; - ZE2UR_CALL(GlobalAdapter->getSysManDriversFunctionPtr, - (&ZesDriverCount, nullptr)); + ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, (&ZesDriverCount, nullptr)); ZesDrivers.resize(ZesDriverCount); - ZE2UR_CALL(GlobalAdapter->getSysManDriversFunctionPtr, + ZE2UR_CALL(adapter->getSysManDriversFunctionPtr, (&ZesDriverCount, ZesDrivers.data())); for (uint32_t I = 0; I < ZesDriverCount; ++I) { ZesResult = ZE_CALL_NOCHECK( - GlobalAdapter->getDeviceByUUIdFunctionPtr, + adapter->getDeviceByUUIdFunctionPtr, (ZesDrivers[I], coreDeviceUuid, ZesDevice, SubDevice, SubDeviceId)); if (ZesResult == ZE_RESULT_SUCCESS) { return UR_RESULT_SUCCESS; @@ -147,7 +147,7 @@ ur_result_t checkDeviceIntelGPUIpVersionOrNewer(uint32_t ipVersion) { * for the devices into the platform. * 10. The function handles exceptions and returns the appropriate result. */ -ur_result_t initPlatforms(PlatformVec &platforms, +ur_result_t initPlatforms(ur_adapter_handle_t_ *adapter, PlatformVec &platforms, ze_result_t ZesResult) noexcept try { std::vector ZeDrivers; std::vector ZeDriverGetHandles; @@ -162,20 +162,20 @@ ur_result_t initPlatforms(PlatformVec &platforms, ZeDriverGetHandles.resize(ZeDriverGetCount); ZE2UR_CALL(zeDriverGet, (&ZeDriverGetCount, ZeDriverGetHandles.data())); } - if (ZeDriverGetCount == 0 && GlobalAdapter->ZeInitDriversCount == 0) { + if (ZeDriverGetCount == 0 && adapter->ZeInitDriversCount == 0) { UR_LOG(ERR, "\nNo Valid L0 Drivers found.\n"); return UR_RESULT_SUCCESS; } - if (GlobalAdapter->InitDriversSupported) { - ZeInitDriversHandles.resize(GlobalAdapter->ZeInitDriversCount); - ZeDrivers.resize(GlobalAdapter->ZeInitDriversCount); - ZE2UR_CALL(GlobalAdapter->initDriversFunctionPtr, - (&GlobalAdapter->ZeInitDriversCount, ZeInitDriversHandles.data(), - &GlobalAdapter->InitDriversDesc)); + if (adapter->InitDriversSupported) { + ZeInitDriversHandles.resize(adapter->ZeInitDriversCount); + ZeDrivers.resize(adapter->ZeInitDriversCount); + ZE2UR_CALL(adapter->initDriversFunctionPtr, + (&adapter->ZeInitDriversCount, ZeInitDriversHandles.data(), + &adapter->InitDriversDesc)); ZeDrivers.assign(ZeInitDriversHandles.begin(), ZeInitDriversHandles.end()); - if (ZeDriverGetCount > 0 && GlobalAdapter->ZeInitDriversCount > 0) { - for (uint32_t X = 0; X < GlobalAdapter->ZeInitDriversCount; ++X) { + if (ZeDriverGetCount > 0 && adapter->ZeInitDriversCount > 0) { + for (uint32_t X = 0; X < adapter->ZeInitDriversCount; ++X) { for (uint32_t Y = 0; Y < ZeDriverGetCount; ++Y) { ZeStruct ZeDriverGetProperties; ZeStruct ZeInitDriverProperties; @@ -234,9 +234,10 @@ ur_result_t initPlatforms(PlatformVec &platforms, ur_zes_device_handle_data_t ZesDeviceData; zes_uuid_t ZesUUID; std::memcpy(&ZesUUID, &device_properties.uuid, sizeof(zes_uuid_t)); - if (getZesDeviceHandle( - ZesUUID, &ZesDeviceData.ZesDevice, &ZesDeviceData.SubDeviceId, - &ZesDeviceData.SubDevice) == UR_RESULT_SUCCESS) { + if (getZesDeviceHandle(adapter, ZesUUID, &ZesDeviceData.ZesDevice, + &ZesDeviceData.SubDeviceId, + &ZesDeviceData.SubDevice) == + UR_RESULT_SUCCESS) { platforms.back()->ZedeviceToZesDeviceMap.insert( std::make_pair(ZeDevices[D], std::move(ZesDeviceData))); } @@ -258,6 +259,51 @@ ur_result_t adapterStateInit() { return UR_RESULT_SUCCESS; } +static bool isBMGorNewer() { + auto urResult = checkDeviceIntelGPUIpVersionOrNewer(0x05004000); + if (urResult != UR_RESULT_SUCCESS && + urResult != UR_RESULT_ERROR_UNSUPPORTED_VERSION) { + UR_LOG(ERR, "Intel GPU IP Version check failed: {}\n", urResult); + throw urResult; + } + + return urResult == UR_RESULT_SUCCESS; +} + +// returns a pair indicating whether to use the V1 adapter and a string +// indicating the reason for the decision. +static std::pair shouldUseV1Adapter() { + auto specificAdapterVersionRequested = + ur_getenv("UR_LOADER_USE_LEVEL_ZERO_V2").has_value() || + ur_getenv("SYCL_UR_USE_LEVEL_ZERO_V2").has_value(); + + auto v2Requested = getenv_tobool("UR_LOADER_USE_LEVEL_ZERO_V2", false); + v2Requested |= getenv_tobool("SYCL_UR_USE_LEVEL_ZERO_V2", false); + + std::string reason = + specificAdapterVersionRequested + ? "Specific adapter version requested by UR_LOADER_USE_LEVEL_ZERO_V2 " + "or SYCL_UR_USE_LEVEL_ZERO_V2" + : "Using default adapter version based on device IP version"; + + if (v2Requested) { + return {false, reason}; + } + + if (!v2Requested && specificAdapterVersionRequested) { + // v1 specifically requested + return {true, reason}; + } + + // default: only enable for devices older than BMG + return {!isBMGorNewer(), reason}; +} + +[[maybe_unused]] static std::pair shouldUseV2Adapter() { + auto [useV1, reason] = shouldUseV1Adapter(); + return {!useV1, reason}; +} + /* This constructor initializes the `ur_adapter_handle_t_` object and sets up the environment for Level Zero (L0) initialization. @@ -297,9 +343,9 @@ Behavior Summary: */ ur_adapter_handle_t_::ur_adapter_handle_t_() : handle_base(), logger(logger::get_logger("level_zero")), RefCount(0) { - ZeInitDriversResult = ZE_RESULT_ERROR_UNINITIALIZED; - ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; - ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; + auto ZeInitDriversResult = ZE_RESULT_ERROR_UNINITIALIZED; + auto ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; + auto ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -325,7 +371,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); } - PlatformCache.Compute = [](Result &result) { uint32_t UserForcedSysManInit = 0; // Check if the user has disabled the default L0 Env initialization. const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { @@ -341,14 +386,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() // not exist in older loader runtimes. #ifndef UR_STATIC_LEVEL_ZERO #ifdef _WIN32 - GlobalAdapter->processHandle = GetModuleHandle(NULL); + processHandle = GetModuleHandle(NULL); #else - GlobalAdapter->processHandle = nullptr; + processHandle = nullptr; #endif #endif - // initialize level zero only once. - if (GlobalAdapter->ZeResult == std::nullopt) { // Setting these environment variables before running zeInit will enable // the validation layer in the Level Zero loader. if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { @@ -382,10 +425,10 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() } UR_LOG(DEBUG, "\nzeInit with flags value of {}\n", static_cast(L0InitFlags)); - GlobalAdapter->ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); - if (GlobalAdapter->ZeInitResult != ZE_RESULT_SUCCESS) { + ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags)); + if (ZeInitResult != ZE_RESULT_SUCCESS) { const char *ErrorString = "Unknown"; - zeParseError(GlobalAdapter->ZeInitResult, ErrorString); + zeParseError(ZeInitResult, ErrorString); UR_LOG(ERR, "\nzeInit failed with {}\n", ErrorString); } @@ -429,109 +472,103 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() if (useInitDrivers) { #ifdef UR_STATIC_LEVEL_ZERO - GlobalAdapter->initDriversFunctionPtr = zeInitDrivers; + initDriversFunctionPtr = zeInitDrivers; #else - GlobalAdapter->initDriversFunctionPtr = + initDriversFunctionPtr = (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( - GlobalAdapter->processHandle, "zeInitDrivers"); + processHandle, "zeInitDrivers"); #endif - if (GlobalAdapter->initDriversFunctionPtr) { + if (initDriversFunctionPtr) { UR_LOG(DEBUG, "\nzeInitDrivers with flags value of {}\n", - static_cast(GlobalAdapter->InitDriversDesc.flags)); - GlobalAdapter->ZeInitDriversResult = - ZE_CALL_NOCHECK(GlobalAdapter->initDriversFunctionPtr, - (&GlobalAdapter->ZeInitDriversCount, nullptr, - &GlobalAdapter->InitDriversDesc)); - if (GlobalAdapter->ZeInitDriversResult == ZE_RESULT_SUCCESS) { - GlobalAdapter->InitDriversSupported = true; + static_cast(InitDriversDesc.flags)); + ZeInitDriversResult = + ZE_CALL_NOCHECK(initDriversFunctionPtr, + (&ZeInitDriversCount, nullptr, &InitDriversDesc)); + if (ZeInitDriversResult == ZE_RESULT_SUCCESS) { + InitDriversSupported = true; } else { const char *ErrorString = "Unknown"; - zeParseError(GlobalAdapter->ZeInitDriversResult, ErrorString); + zeParseError(ZeInitDriversResult, ErrorString); UR_LOG(ERR, "\nzeInitDrivers failed with {}\n", ErrorString); } } } - if (GlobalAdapter->ZeInitResult == ZE_RESULT_SUCCESS || - GlobalAdapter->ZeInitDriversResult == ZE_RESULT_SUCCESS) { - GlobalAdapter->ZeResult = ZE_RESULT_SUCCESS; - } else { - GlobalAdapter->ZeResult = ZE_RESULT_ERROR_UNINITIALIZED; + if (ZeInitResult != ZE_RESULT_SUCCESS && + ZeInitDriversResult != ZE_RESULT_SUCCESS) { + // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. + UR_LOG(ERR, "Level Zero Uninitialized\n"); + return; } - } - assert(GlobalAdapter->ZeResult != - std::nullopt); // verify that level-zero is initialized - PlatformVec platforms; - - // Absorb the ZE_RESULT_ERROR_UNINITIALIZED and just return 0 Platforms. - if (*GlobalAdapter->ZeResult == ZE_RESULT_ERROR_UNINITIALIZED) { - UR_LOG(ERR, "Level Zero Uninitialized\n"); - result = std::move(platforms); - return; - } - if (*GlobalAdapter->ZeResult != ZE_RESULT_SUCCESS) { - UR_LOG(ERR, "Level Zero initialization failure\n"); - result = ze2urResult(*GlobalAdapter->ZeResult); - return; - } + PlatformVec platforms; - // Check if the user has enabled the default L0 SysMan initialization. - const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { - const char *UrRet = std::getenv("UR_L0_ENABLE_ZESINIT_DEFAULT"); - if (!UrRet) - return 0; - UserForcedSysManInit &= 2; - return std::atoi(UrRet); - }(); +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 + auto [useV2, reason] = shouldUseV2Adapter(); + if (!useV2) { + UR_LOG(INFO, "Skipping L0 V2 adapter: {}", reason); + return; + } +#else + auto [useV1, reason] = shouldUseV1Adapter(); + if (!useV1) { + UR_LOG(INFO, "Skipping L0 V1 adapter: {}", reason); + return; + } +#endif - bool ZesInitNeeded = UrSysmanZesinitEnable && !UrSysManEnvInitEnabled; - // Unless the user has forced the SysMan init, we will check the device - // version to see if the zesInit is needed. - if (UserForcedSysManInit == 0 && - checkDeviceIntelGPUIpVersionOrNewer(0x05004000) == UR_RESULT_SUCCESS) { - if (UrSysManEnvInitEnabled) { - setEnvVar("ZES_ENABLE_SYSMAN", "0"); + // Check if the user has enabled the default L0 SysMan initialization. + const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { + const char *UrRet = std::getenv("UR_L0_ENABLE_ZESINIT_DEFAULT"); + if (!UrRet) + return 0; + UserForcedSysManInit &= 2; + return std::atoi(UrRet); + }(); + + bool ZesInitNeeded = UrSysmanZesinitEnable && !UrSysManEnvInitEnabled; + // Unless the user has forced the SysMan init, we will check the device + // version to see if the zesInit is needed. + if (UserForcedSysManInit == 0 && checkDeviceIntelGPUIpVersionOrNewer( + 0x05004000) == UR_RESULT_SUCCESS) { + if (UrSysManEnvInitEnabled) { + setEnvVar("ZES_ENABLE_SYSMAN", "0"); + } + ZesInitNeeded = true; } - ZesInitNeeded = true; - } - if (ZesInitNeeded) { + if (ZesInitNeeded) { #ifdef UR_STATIC_LEVEL_ZERO - GlobalAdapter->getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp; - GlobalAdapter->getSysManDriversFunctionPtr = zesDriverGet; - GlobalAdapter->sysManInitFunctionPtr = zesInit; + getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp; + getSysManDriversFunctionPtr = zesDriverGet; + sysManInitFunctionPtr = zesInit; #else - GlobalAdapter->getDeviceByUUIdFunctionPtr = - (zes_pfnDriverGetDeviceByUuidExp_t) - ur_loader::LibLoader::getFunctionPtr( - GlobalAdapter->processHandle, "zesDriverGetDeviceByUuidExp"); - GlobalAdapter->getSysManDriversFunctionPtr = - (zes_pfnDriverGet_t)ur_loader::LibLoader::getFunctionPtr( - GlobalAdapter->processHandle, "zesDriverGet"); - GlobalAdapter->sysManInitFunctionPtr = - (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr( - GlobalAdapter->processHandle, "zesInit"); + getDeviceByUUIdFunctionPtr = (zes_pfnDriverGetDeviceByUuidExp_t) + ur_loader::LibLoader::getFunctionPtr(processHandle, + "zesDriverGetDeviceByUuidExp"); + getSysManDriversFunctionPtr = + (zes_pfnDriverGet_t)ur_loader::LibLoader::getFunctionPtr( + processHandle, "zesDriverGet"); + sysManInitFunctionPtr = + (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr(processHandle, + "zesInit"); #endif - } - if (GlobalAdapter->getDeviceByUUIdFunctionPtr && - GlobalAdapter->getSysManDriversFunctionPtr && - GlobalAdapter->sysManInitFunctionPtr) { - ze_init_flags_t L0ZesInitFlags = 0; - UR_LOG(DEBUG, "\nzesInit with flags value of {}\n", - static_cast(L0ZesInitFlags)); - GlobalAdapter->ZesResult = ZE_CALL_NOCHECK( - GlobalAdapter->sysManInitFunctionPtr, (L0ZesInitFlags)); - } else { - GlobalAdapter->ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; - } + } + if (getDeviceByUUIdFunctionPtr && getSysManDriversFunctionPtr && + sysManInitFunctionPtr) { + ze_init_flags_t L0ZesInitFlags = 0; + UR_LOG(DEBUG, "\nzesInit with flags value of {}\n", + static_cast(L0ZesInitFlags)); + ZesResult = ZE_CALL_NOCHECK(sysManInitFunctionPtr, (L0ZesInitFlags)); + } else { + ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; + } - ur_result_t err = initPlatforms(platforms, GlobalAdapter->ZesResult); - if (err == UR_RESULT_SUCCESS) { - result = std::move(platforms); - } else { - result = err; - } - }; + ur_result_t err = initPlatforms(this, platforms, ZesResult); + if (err == UR_RESULT_SUCCESS) { + Platforms = std::move(platforms); + } else { + throw err; + } } void globalAdapterOnDemandCleanup() { @@ -562,15 +599,25 @@ ur_result_t urAdapterGet( /// ::urAdapterGet shall only retrieve that number of platforms. ur_adapter_handle_t *Adapters, /// [out][optional] returns the total number of adapters available. - uint32_t *NumAdapters) { + uint32_t *NumAdapters) try { static std::mutex AdapterConstructionMutex{}; - if (NumEntries > 0 && Adapters) { - std::lock_guard Lock{AdapterConstructionMutex}; + // We need to initialize the adapter even if user only queries + // the number of adapters to decided whether to use V1 or V2. + std::lock_guard Lock{AdapterConstructionMutex}; - if (!GlobalAdapter) { - GlobalAdapter = new ur_adapter_handle_t_(); + if (!GlobalAdapter) { + GlobalAdapter = new ur_adapter_handle_t_(); + } + + if (GlobalAdapter->Platforms.size() == 0) { + if (NumAdapters) { + *NumAdapters = 0; } + return UR_RESULT_ERROR_UNSUPPORTED_VERSION; + } + + if (NumEntries && Adapters) { *Adapters = GlobalAdapter; if (GlobalAdapter->RefCount.retain() == 0) { @@ -583,6 +630,10 @@ ur_result_t urAdapterGet( } return UR_RESULT_SUCCESS; +} catch (ur_result_t result) { + return result; +} catch (...) { + return UR_RESULT_ERROR_UNKNOWN; } ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { diff --git a/source/adapters/level_zero/adapter.hpp b/source/adapters/level_zero/adapter.hpp index 890e39d296..0c0ce9a9f1 100644 --- a/source/adapters/level_zero/adapter.hpp +++ b/source/adapters/level_zero/adapter.hpp @@ -37,11 +37,7 @@ struct ur_adapter_handle_t_ : ur::handle_base { uint32_t ZeInitDriversCount = 0; bool InitDriversSupported = false; - ze_result_t ZeInitDriversResult; - ze_result_t ZeInitResult; - ze_result_t ZesResult; - std::optional ZeResult; - ZeCache> PlatformCache; + PlatformVec Platforms; logger::Logger &logger; HMODULE processHandle = nullptr; diff --git a/source/adapters/level_zero/device.cpp b/source/adapters/level_zero/device.cpp index 897dc5e659..167c7cb856 100644 --- a/source/adapters/level_zero/device.cpp +++ b/source/adapters/level_zero/device.cpp @@ -1583,12 +1583,8 @@ ur_result_t urDeviceCreateWithNativeHandle( // a valid Level Zero device. ur_device_handle_t Dev = nullptr; - if (const auto *platforms = GlobalAdapter->PlatformCache->get_value()) { - for (const auto &p : *platforms) { - Dev = p->getDeviceFromNativeHandle(ZeDevice); - } - } else { - return GlobalAdapter->PlatformCache->get_error(); + for (const auto &p : GlobalAdapter->Platforms) { + Dev = p->getDeviceFromNativeHandle(ZeDevice); } if (Dev == nullptr) diff --git a/source/adapters/level_zero/platform.cpp b/source/adapters/level_zero/platform.cpp index 50a2ce1080..47f2e519bb 100644 --- a/source/adapters/level_zero/platform.cpp +++ b/source/adapters/level_zero/platform.cpp @@ -28,20 +28,15 @@ ur_result_t urPlatformGet( uint32_t *NumPlatforms) { // Platform handles are cached for reuse. This is to ensure consistent // handle pointers across invocations and to improve retrieval performance. - if (const auto *cached_platforms = GlobalAdapter->PlatformCache->get_value(); - cached_platforms) { - uint32_t nplatforms = (uint32_t)cached_platforms->size(); - if (NumPlatforms) { - *NumPlatforms = nplatforms; - } + uint32_t nplatforms = (uint32_t)GlobalAdapter->Platforms.size(); + if (NumPlatforms) { + *NumPlatforms = nplatforms; + } if (Platforms) { for (uint32_t i = 0; i < std::min(nplatforms, NumEntries); ++i) { - Platforms[i] = cached_platforms->at(i).get(); + Platforms[i] = GlobalAdapter->Platforms.at(i).get(); } } - } else { - return GlobalAdapter->PlatformCache->get_error(); - } return UR_RESULT_SUCCESS; } diff --git a/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index 85e6c7e250..1e4beaf363 100644 --- a/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -90,9 +90,13 @@ ur_queue_immediate_in_order_t::queueGetInfo(ur_queue_info_t propName, } ur_result_t ur_queue_immediate_in_order_t::queueGetNativeHandle( - ur_queue_native_desc_t * /*pDesc*/, ur_native_handle_t *phNativeQueue) { + ur_queue_native_desc_t *pDesc, ur_native_handle_t *phNativeQueue) { *phNativeQueue = reinterpret_cast( commandListManager.get_no_lock()->getZeCommandList()); + if (pDesc && pDesc->pNativeData) { + // pNativeData == isImmediateQueue + *(reinterpret_cast(pDesc->pNativeData)) = 1; + } return UR_RESULT_SUCCESS; } diff --git a/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index f0344eba26..2fbe7fa5a1 100644 --- a/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -94,10 +94,14 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo( } ur_result_t ur_queue_immediate_out_of_order_t::queueGetNativeHandle( - ur_queue_native_desc_t * /*pDesc*/, ur_native_handle_t *phNativeQueue) { + ur_queue_native_desc_t *pDesc, ur_native_handle_t *phNativeQueue) { *phNativeQueue = reinterpret_cast( (*commandListManagers.get_no_lock())[getNextCommandListId()] .getZeCommandList()); + if (pDesc && pDesc->pNativeData) { + // pNativeData == isImmediateQueue + *(reinterpret_cast(pDesc->pNativeData)) = 1; + } return UR_RESULT_SUCCESS; } diff --git a/source/adapters/offload/kernel.cpp b/source/adapters/offload/kernel.cpp index b9e9152d43..58c4f6cf7f 100644 --- a/source/adapters/offload/kernel.cpp +++ b/source/adapters/offload/kernel.cpp @@ -21,8 +21,8 @@ urKernelCreate(ur_program_handle_t hProgram, const char *pKernelName, ur_kernel_handle_t *phKernel) { ur_kernel_handle_t Kernel = new ur_kernel_handle_t_; - auto Res = olGetKernel(hProgram->OffloadProgram, pKernelName, - &Kernel->OffloadKernel); + auto Res = olGetSymbol(hProgram->OffloadProgram, pKernelName, + OL_SYMBOL_KIND_KERNEL, &Kernel->OffloadKernel); if (Res != OL_SUCCESS) { delete Kernel; diff --git a/source/adapters/offload/kernel.hpp b/source/adapters/offload/kernel.hpp index e8ff732d70..83866b5974 100644 --- a/source/adapters/offload/kernel.hpp +++ b/source/adapters/offload/kernel.hpp @@ -77,6 +77,6 @@ struct ur_kernel_handle_t_ : RefCounted { } }; - ol_kernel_handle_t OffloadKernel; + ol_symbol_handle_t OffloadKernel; OffloadKernelArguments Args{}; }; diff --git a/source/adapters/offload/program.cpp b/source/adapters/offload/program.cpp index dde21a20b2..cf497c571f 100644 --- a/source/adapters/offload/program.cpp +++ b/source/adapters/offload/program.cpp @@ -108,7 +108,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramCreateWithBinary( phProgram); } - ur_program_handle_t Program = new ur_program_handle_t_(); + ur_program_handle_t Program = new ur_program_handle_t_{}; + Program->URContext = hContext; + Program->Binary = RealBinary; + Program->BinarySizeInBytes = RealLength; auto Res = olCreateProgram(hContext->Device->OffloadDevice, RealBinary, RealLength, &Program->OffloadProgram); @@ -137,6 +140,19 @@ UR_APIEXPORT ur_result_t UR_APICALL urProgramBuildExp(ur_program_handle_t, return UR_RESULT_SUCCESS; } +UR_APIEXPORT ur_result_t UR_APICALL urProgramCompile(ur_context_handle_t, + ur_program_handle_t, + const char *) { + // Do nothing, program is built upon creation + return UR_RESULT_SUCCESS; +} + +UR_APIEXPORT ur_result_t UR_APICALL +urProgramCreateWithIL(ur_context_handle_t, const void *, size_t, + const ur_program_properties_t *, ur_program_handle_t *) { + return UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE; +} + UR_APIEXPORT ur_result_t UR_APICALL urProgramGetInfo(ur_program_handle_t hProgram, ur_program_info_t propName, size_t propSize, void *pPropValue, size_t *pPropSizeRet) { @@ -145,8 +161,42 @@ urProgramGetInfo(ur_program_handle_t hProgram, ur_program_info_t propName, switch (propName) { case UR_PROGRAM_INFO_REFERENCE_COUNT: return ReturnValue(hProgram->RefCount.load()); - default: + case UR_PROGRAM_INFO_CONTEXT: + return ReturnValue(hProgram->URContext); + case UR_PROGRAM_INFO_NUM_DEVICES: + return ReturnValue(1); + case UR_PROGRAM_INFO_DEVICES: + return ReturnValue(&hProgram->URContext->Device, 1); + case UR_PROGRAM_INFO_IL: + return ReturnValue(reinterpret_cast(0), 0); + case UR_PROGRAM_INFO_BINARY_SIZES: + return ReturnValue(&hProgram->BinarySizeInBytes, 1); + case UR_PROGRAM_INFO_BINARIES: { + if (!pPropValue && !pPropSizeRet) { + return UR_RESULT_ERROR_INVALID_NULL_POINTER; + } + + if (pPropValue != nullptr) { + if (propSize < sizeof(void *)) { + return UR_RESULT_ERROR_INVALID_SIZE; + } + + std::memcpy(*reinterpret_cast(pPropValue), hProgram->Binary, + hProgram->BinarySizeInBytes); + } + + if (pPropSizeRet != nullptr) { + *pPropSizeRet = sizeof(void *); + } + break; + } + case UR_PROGRAM_INFO_NUM_KERNELS: + case UR_PROGRAM_INFO_KERNEL_NAMES: + // Program introspection is not available for liboffload (or generally, + // amdgpu/cuda) return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; + default: + return UR_RESULT_ERROR_INVALID_ENUMERATION; } return UR_RESULT_SUCCESS; diff --git a/source/adapters/offload/program.hpp b/source/adapters/offload/program.hpp index 1d0263aad2..abd0f41c18 100644 --- a/source/adapters/offload/program.hpp +++ b/source/adapters/offload/program.hpp @@ -17,4 +17,7 @@ struct ur_program_handle_t_ : RefCounted { ol_program_handle_t OffloadProgram; + ur_context_handle_t URContext; + const uint8_t *Binary; + size_t BinarySizeInBytes; }; diff --git a/source/adapters/offload/ur_interface_loader.cpp b/source/adapters/offload/ur_interface_loader.cpp index ef3902e0b0..3c6a9a86d4 100644 --- a/source/adapters/offload/ur_interface_loader.cpp +++ b/source/adapters/offload/ur_interface_loader.cpp @@ -86,9 +86,9 @@ UR_DLLEXPORT ur_result_t UR_APICALL urGetProgramProcAddrTable( return result; } pDdiTable->pfnBuild = urProgramBuild; - pDdiTable->pfnCompile = nullptr; + pDdiTable->pfnCompile = urProgramCompile; pDdiTable->pfnCreateWithBinary = urProgramCreateWithBinary; - pDdiTable->pfnCreateWithIL = nullptr; + pDdiTable->pfnCreateWithIL = urProgramCreateWithIL; pDdiTable->pfnCreateWithNativeHandle = urProgramCreateWithNativeHandle; pDdiTable->pfnGetBuildInfo = nullptr; pDdiTable->pfnGetFunctionPointer = nullptr; diff --git a/source/loader/ur_adapter_registry.hpp b/source/loader/ur_adapter_registry.hpp index 05c3c3e9ce..cabed246a6 100644 --- a/source/loader/ur_adapter_registry.hpp +++ b/source/loader/ur_adapter_registry.hpp @@ -355,22 +355,6 @@ class AdapterRegistry { } for (const auto &adapterName : adapterNames) { - // Skip legacy L0 adapter if the v2 adapter is requested, and vice versa. - if (std::string(adapterName).find("level_zero") != std::string::npos) { - auto v2Requested = getenv_tobool("UR_LOADER_USE_LEVEL_ZERO_V2", false); - v2Requested |= getenv_tobool("SYCL_UR_USE_LEVEL_ZERO_V2", false); - auto v2Adapter = - std::string(adapterName).find("v2") != std::string::npos; - - if (v2Requested != v2Adapter) { - UR_LOG(INFO, "The adapter '{}' is skipped because {} {}.", - adapterName, - "UR_LOADER_USE_LEVEL_ZERO_V2 or SYCL_UR_USE_LEVEL_ZERO_V2", - v2Requested ? "is set" : "is not set"); - continue; - } - } - std::vector loadPaths; // Adapter search order: diff --git a/source/loader/ur_ldrddi.cpp b/source/loader/ur_ldrddi.cpp index 2ddf9f21c9..d8dfaf8e8a 100644 --- a/source/loader/ur_ldrddi.cpp +++ b/source/loader/ur_ldrddi.cpp @@ -34,21 +34,21 @@ __urdlllocal ur_result_t UR_APICALL urAdapterGet( auto context = getContext(); - size_t adapterIndex = 0; - if (nullptr != phAdapters && NumEntries != 0) { - for (auto &platform : context->platforms) { - if (platform.initStatus != UR_RESULT_SUCCESS) - continue; - platform.dditable.Adapter.pfnGet(1, &phAdapters[adapterIndex], nullptr); - adapterIndex++; - if (adapterIndex == NumEntries) { - break; - } - } + uint32_t numAdapters = 0; + for (auto &platform : context->platforms) { + if (platform.initStatus != UR_RESULT_SUCCESS) + continue; + + uint32_t adapter; + ur_adapter_handle_t *adapterHandle = + numAdapters < NumEntries ? &phAdapters[numAdapters] : nullptr; + platform.dditable.Adapter.pfnGet(1, adapterHandle, &adapter); + + numAdapters += adapter; } if (pNumAdapters != nullptr) { - *pNumAdapters = static_cast(context->platforms.size()); + *pNumAdapters = numAdapters; } return UR_RESULT_SUCCESS; diff --git a/test/conformance/testing/include/uur/checks.h b/test/conformance/testing/include/uur/checks.h index 9bcbc6a8af..d5ad05af97 100644 --- a/test/conformance/testing/include/uur/checks.h +++ b/test/conformance/testing/include/uur/checks.h @@ -41,7 +41,8 @@ inline std::ostream &operator<<(std::ostream &out, const Result &result) { do { \ auto status = ret; \ if (status == UR_RESULT_ERROR_UNSUPPORTED_FEATURE || \ - status == UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION) { \ + status == UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION || \ + status == UR_RESULT_ERROR_COMPILER_NOT_AVAILABLE) { \ GTEST_SKIP(); \ } else { \ ASSERT_EQ(status, UR_RESULT_SUCCESS); \