From 364c2da9fbbe34548c7b0cd55b89ca10fc3b0ff2 Mon Sep 17 00:00:00 2001 From: "Spruit, Neil R" Date: Fri, 7 Apr 2023 02:57:37 +0000 Subject: [PATCH] feature: Add Support for zeMemPutIpcHandle & zeMemGet IPC Handle converters Related-To: LOCI-4172, LOCI-4305, LOCI-4306 - Create a new IPC Memory handle upon call to getIpcMemHandle if the previous handle has been freed. - Release the Ipc Memory Handle when zeMemPutIpcHandle is called. - Create a new IPC Handle for tracking thru zeMemGetAllocProperties when ze_external_memory_export_fd_t is used. - Convert FD to opaque IPC handle and IPC Handle to FD. Signed-off-by: Spruit, Neil R --- level_zero/api/core/ze_core_loader.cpp | 18 ++ .../api/core/ze_memory_api_entrypoints.h | 44 ++- level_zero/core/source/context/context.h | 3 + .../core/source/context/context_imp.cpp | 100 +++++- level_zero/core/source/context/context_imp.h | 17 + .../unit_tests/fixtures/memory_ipc_fixture.h | 4 +- .../memory/test_memory_drm_or_wddm.cpp | 290 ++++++++++++++++++ .../sources/memory/test_memory_wddm.cpp | 127 ++++++++ .../memory_manager/graphics_allocation.h | 2 + shared/source/memory_manager/memory_manager.h | 1 + .../os_interface/linux/drm_allocation.cpp | 8 + .../os_interface/linux/drm_allocation.h | 3 + .../os_interface/linux/drm_memory_manager.cpp | 7 + .../os_interface/linux/drm_memory_manager.h | 1 + .../os_interface/windows/CMakeLists.txt | 1 + .../os_interface/windows/wddm_allocation.h | 4 + .../windows/wddm_allocation_common.cpp | 29 ++ .../windows/wddm_memory_manager.cpp | 11 + .../windows/wddm_memory_manager.h | 2 + .../graphics_allocation_tests.cpp | 8 + .../memory_manager/memory_manager_tests.cpp | 7 + .../linux/drm_memory_manager_tests.cpp | 87 ++++++ ..._device_address_space_drm_or_wddm_test.cpp | 60 ++++ .../windows/wddm_memory_manager_tests.cpp | 60 ++++ 24 files changed, 881 insertions(+), 13 deletions(-) create mode 100644 shared/source/os_interface/windows/wddm_allocation_common.cpp diff --git a/level_zero/api/core/ze_core_loader.cpp b/level_zero/api/core/ze_core_loader.cpp index 3e0dd2784deda..88694cff9f90e 100644 --- a/level_zero/api/core/ze_core_loader.cpp +++ b/level_zero/api/core/ze_core_loader.cpp @@ -79,6 +79,7 @@ zeGetMemProcAddrTable( pDdiTable->pfnGetIpcHandle = L0::zeMemGetIpcHandle; pDdiTable->pfnOpenIpcHandle = L0::zeMemOpenIpcHandle; pDdiTable->pfnCloseIpcHandle = L0::zeMemCloseIpcHandle; + pDdiTable->pfnPutIpcHandle = L0::zeMemPutIpcHandle; driver_ddiTable.core_ddiTable.Mem = *pDdiTable; if (driver_ddiTable.enableTracing) { pDdiTable->pfnAllocShared = zeMemAllocSharedTracing; @@ -638,6 +639,23 @@ zeGetKernelExpProcAddrTable( return result; } +ZE_DLLEXPORT ze_result_t ZE_APICALL +zeGetMemExpProcAddrTable( + ze_api_version_t version, + ze_mem_exp_dditable_t *pDdiTable) { + if (nullptr == pDdiTable) + return ZE_RESULT_ERROR_INVALID_ARGUMENT; + if (ZE_MAJOR_VERSION(driver_ddiTable.version) != ZE_MAJOR_VERSION(version) || + ZE_MINOR_VERSION(driver_ddiTable.version) > ZE_MINOR_VERSION(version)) + return ZE_RESULT_ERROR_UNSUPPORTED_VERSION; + + ze_result_t result = ZE_RESULT_SUCCESS; + pDdiTable->pfnGetIpcHandleFromFileDescriptorExp = L0::zeMemGetIpcHandleFromFileDescriptorExp; + pDdiTable->pfnGetFileDescriptorFromIpcHandleExp = L0::zeMemGetFileDescriptorFromIpcHandleExp; + driver_ddiTable.core_ddiTable.MemExp = *pDdiTable; + return result; +} + ZE_APIEXPORT ze_result_t ZE_APICALL zeGetImageExpProcAddrTable( ze_api_version_t version, diff --git a/level_zero/api/core/ze_memory_api_entrypoints.h b/level_zero/api/core/ze_memory_api_entrypoints.h index a38a1e5d711d6..5bb8037218011 100644 --- a/level_zero/api/core/ze_memory_api_entrypoints.h +++ b/level_zero/api/core/ze_memory_api_entrypoints.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020-2022 Intel Corporation + * Copyright (C) 2020-2023 Intel Corporation * * SPDX-License-Identifier: MIT * @@ -77,6 +77,12 @@ ze_result_t zeMemGetIpcHandle( return L0::Context::fromHandle(hContext)->getIpcMemHandle(ptr, pIpcHandle); } +ze_result_t zeMemPutIpcHandle( + ze_context_handle_t hContext, + ze_ipc_mem_handle_t ipcHandle) { + return L0::Context::fromHandle(hContext)->putIpcMemHandle(ipcHandle); +} + ze_result_t zeMemOpenIpcHandle( ze_context_handle_t hContext, ze_device_handle_t hDevice, @@ -92,6 +98,14 @@ ze_result_t zeMemCloseIpcHandle( return L0::Context::fromHandle(hContext)->closeIpcMemHandle(ptr); } +ze_result_t zeMemGetIpcHandleFromFileDescriptorExp(ze_context_handle_t hContext, uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) { + return L0::Context::fromHandle(hContext)->getIpcHandleFromFd(handle, pIpcHandle); +} + +ze_result_t zeMemGetFileDescriptorFromIpcHandleExp(ze_context_handle_t hContext, ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) { + return L0::Context::fromHandle(hContext)->getFdFromIpcHandle(ipcHandle, pHandle); +} + } // namespace L0 extern "C" { @@ -200,6 +214,14 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeMemOpenIpcHandle( pptr); } +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemPutIpcHandle( + ze_context_handle_t hContext, + ze_ipc_mem_handle_t ipcHandle) { + return L0::zeMemPutIpcHandle( + hContext, + ipcHandle); +} + ZE_APIEXPORT ze_result_t ZE_APICALL zeMemCloseIpcHandle( ze_context_handle_t hContext, const void *ptr) { @@ -207,4 +229,24 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeMemCloseIpcHandle( hContext, ptr); } + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemGetIpcHandleFromFileDescriptorExp( + ze_context_handle_t hContext, + uint64_t handle, + ze_ipc_mem_handle_t *pIpcHandle) { + return L0::zeMemGetIpcHandleFromFileDescriptorExp( + hContext, + handle, + pIpcHandle); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemGetFileDescriptorFromIpcHandleExp( + ze_context_handle_t hContext, + ze_ipc_mem_handle_t ipcHandle, + uint64_t *pHandle) { + return L0::zeMemGetFileDescriptorFromIpcHandleExp( + hContext, + ipcHandle, + pHandle); +} } diff --git a/level_zero/core/source/context/context.h b/level_zero/core/source/context/context.h index 7fce7eea16440..43633270a4755 100644 --- a/level_zero/core/source/context/context.h +++ b/level_zero/core/source/context/context.h @@ -75,8 +75,11 @@ struct Context : _ze_context_handle_t { void **pBase, size_t *pSize) = 0; virtual ze_result_t closeIpcMemHandle(const void *ptr) = 0; + virtual ze_result_t putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) = 0; virtual ze_result_t getIpcMemHandle(const void *ptr, ze_ipc_mem_handle_t *pIpcHandle) = 0; + virtual ze_result_t getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) = 0; + virtual ze_result_t getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) = 0; virtual ze_result_t getIpcMemHandles( const void *ptr, diff --git a/level_zero/core/source/context/context_imp.cpp b/level_zero/core/source/context/context_imp.cpp index 00287b59b2d8e..edaaf0ced59c5 100644 --- a/level_zero/core/source/context/context_imp.cpp +++ b/level_zero/core/source/context/context_imp.cpp @@ -397,8 +397,19 @@ ze_result_t ContextImp::freeMem(const void *ptr, bool blocking) { for (auto pairDevice : this->devices) { this->freePeerAllocations(ptr, blocking, Device::fromHandle(pairDevice.second)); } - this->driverHandle->svmAllocsManager->freeSVMAlloc(const_cast(ptr), blocking); + + std::map::iterator ipcHandleIterator; + auto lockIPC = this->lockIPCHandleMap(); + ipcHandleIterator = this->getIPCHandleMap().begin(); + while (ipcHandleIterator != this->getIPCHandleMap().end()) { + if (ipcHandleIterator->second->ptr == reinterpret_cast(ptr)) { + delete ipcHandleIterator->second; + this->getIPCHandleMap().erase(ipcHandleIterator->first); + break; + } + ipcHandleIterator++; + } return ZE_RESULT_SUCCESS; } @@ -517,6 +528,43 @@ ze_result_t ContextImp::closeIpcMemHandle(const void *ptr) { return this->freeMem(ptr); } +ze_result_t ContextImp::putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) { + IpcMemoryData &ipcData = *reinterpret_cast(ipcHandle.data); + std::map::iterator ipcHandleIterator; + auto lock = this->lockIPCHandleMap(); + ipcHandleIterator = this->getIPCHandleMap().find(ipcData.handle); + if (ipcHandleIterator != this->getIPCHandleMap().end()) { + ipcHandleIterator->second->refcnt -= 1; + if (ipcHandleIterator->second->refcnt == 0) { + auto *memoryManager = driverHandle->getMemoryManager(); + memoryManager->closeInternalHandle(ipcData.handle, ipcHandleIterator->second->handleId, ipcHandleIterator->second->alloc); + delete ipcHandleIterator->second; + this->getIPCHandleMap().erase(ipcData.handle); + } + } + return ZE_RESULT_SUCCESS; +} + +void ContextImp::setIPCHandleData(NEO::GraphicsAllocation *graphicsAllocation, uint64_t handle, IpcMemoryData &ipcData, uint64_t ptrAddress) { + std::map::iterator ipcHandleIterator; + + ipcData = {}; + ipcData.handle = handle; + + auto lock = this->lockIPCHandleMap(); + ipcHandleIterator = this->getIPCHandleMap().find(handle); + if (ipcHandleIterator != this->getIPCHandleMap().end()) { + ipcHandleIterator->second->refcnt += 1; + } else { + IpcHandleTracking *handleTracking = new IpcHandleTracking; + handleTracking->alloc = graphicsAllocation; + handleTracking->refcnt = 1; + handleTracking->ptr = ptrAddress; + handleTracking->ipcData = ipcData; + this->getIPCHandleMap().insert(std::pair(handle, handleTracking)); + } +} + ze_result_t ContextImp::getIpcMemHandle(const void *ptr, ze_ipc_mem_handle_t *pIpcHandle) { NEO::SvmAllocationData *allocData = this->driverHandle->svmAllocsManager->getSVMAlloc(ptr); @@ -525,7 +573,7 @@ ze_result_t ContextImp::getIpcMemHandle(const void *ptr, auto *graphicsAllocation = allocData->gpuAllocations.getDefaultGraphicsAllocation(); uint64_t handle = 0; - int ret = graphicsAllocation->peekInternalHandle(memoryManager, handle); + int ret = graphicsAllocation->createInternalHandle(memoryManager, 0u, handle); if (ret < 0) { return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } @@ -533,8 +581,7 @@ ze_result_t ContextImp::getIpcMemHandle(const void *ptr, memoryManager->registerIpcExportedAllocation(graphicsAllocation); IpcMemoryData &ipcData = *reinterpret_cast(pIpcHandle->data); - ipcData = {}; - ipcData.handle = handle; + setIPCHandleData(graphicsAllocation, handle, ipcData, reinterpret_cast(ptr)); auto type = allocData->memoryType; if (type == HOST_UNIFIED_MEMORY) { ipcData.type = static_cast(InternalIpcMemoryType::IPC_HOST_UNIFIED_MEMORY); @@ -545,6 +592,32 @@ ze_result_t ContextImp::getIpcMemHandle(const void *ptr, return ZE_RESULT_ERROR_INVALID_ARGUMENT; } +ze_result_t ContextImp::getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) { + std::map::iterator ipcHandleIterator; + auto lock = this->lockIPCHandleMap(); + ipcHandleIterator = this->getIPCHandleMap().find(handle); + if (ipcHandleIterator != this->getIPCHandleMap().end()) { + IpcMemoryData &ipcData = *reinterpret_cast(pIpcHandle->data); + ipcData = ipcHandleIterator->second->ipcData; + } else { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + return ZE_RESULT_SUCCESS; +} + +ze_result_t ContextImp::getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) { + IpcMemoryData &ipcData = *reinterpret_cast(ipcHandle.data); + std::map::iterator ipcHandleIterator; + auto lock = this->lockIPCHandleMap(); + ipcHandleIterator = this->getIPCHandleMap().find(ipcData.handle); + if (ipcHandleIterator != this->getIPCHandleMap().end()) { + *pHandle = ipcHandleIterator->first; + } else { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + return ZE_RESULT_SUCCESS; +} + ze_result_t ContextImp::getIpcMemHandles(const void *ptr, uint32_t *numIpcHandles, ze_ipc_mem_handle_t *pIpcHandles) { @@ -570,14 +643,13 @@ ze_result_t ContextImp::getIpcMemHandles(const void *ptr, for (uint32_t i = 0; i < *numIpcHandles; i++) { uint64_t handle = 0; - int ret = allocData->gpuAllocations.getDefaultGraphicsAllocation()->peekInternalHandle(this->driverHandle->getMemoryManager(), i, handle); + int ret = allocData->gpuAllocations.getDefaultGraphicsAllocation()->createInternalHandle(this->driverHandle->getMemoryManager(), i, handle); if (ret < 0) { return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; } IpcMemoryData &ipcData = *reinterpret_cast(pIpcHandles[i].data); - ipcData = {}; - ipcData.handle = handle; + setIPCHandleData(alloc, handle, ipcData, reinterpret_cast(ptr)); ipcData.type = static_cast(ipcType); } @@ -661,10 +733,18 @@ ze_result_t ContextImp::handleAllocationExtensions(NEO::GraphicsAllocation *allo if (type == ZE_MEMORY_TYPE_SHARED) { return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE; } + ze_ipc_mem_handle_t ipcHandle; uint64_t handle = 0; - int ret = alloc->peekInternalHandle(driverHandle->getMemoryManager(), handle); - if (ret < 0) { - return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + auto result = getIpcMemHandle(reinterpret_cast(alloc->getGpuAddress()), &ipcHandle); + if (result != ZE_RESULT_SUCCESS) { + // If this memory is not an SVM Allocation like Images, then retrieve only the handle untracked. + auto ret = alloc->peekInternalHandle(driverHandle->getMemoryManager(), handle); + if (ret < 0) { + return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + } else { + IpcMemoryData &ipcData = *reinterpret_cast(ipcHandle.data); + handle = ipcData.handle; } extendedMemoryExportProperties->fd = static_cast(handle); } else if (extendedProperties->stype == ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_WIN32) { diff --git a/level_zero/core/source/context/context_imp.h b/level_zero/core/source/context/context_imp.h index 0b562efe2c2cc..72893306e8764 100644 --- a/level_zero/core/source/context/context_imp.h +++ b/level_zero/core/source/context/context_imp.h @@ -27,6 +27,15 @@ struct IpcMemoryData { #pragma pack() static_assert(sizeof(IpcMemoryData) <= ZE_MAX_IPC_HANDLE_SIZE, "IpcMemoryData is bigger than ZE_MAX_IPC_HANDLE_SIZE"); +struct IpcHandleTracking { + uint64_t refcnt = 0; + NEO::GraphicsAllocation *alloc = nullptr; + uint32_t handleId = 0; + uint64_t handle = 0; + uint64_t ptr = 0; + struct IpcMemoryData ipcData = {}; +}; + struct ContextImp : Context { ContextImp(DriverHandle *driverHandle); ~ContextImp() override = default; @@ -63,12 +72,15 @@ struct ContextImp : Context { void **pBase, size_t *pSize) override; ze_result_t closeIpcMemHandle(const void *ptr) override; + ze_result_t putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) override; ze_result_t getIpcMemHandle(const void *ptr, ze_ipc_mem_handle_t *pIpcHandle) override; ze_result_t openIpcMemHandle(ze_device_handle_t hDevice, const ze_ipc_mem_handle_t &handle, ze_ipc_memory_flags_t flags, void **ptr) override; + ze_result_t getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) override; + ze_result_t getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) override; ze_result_t getIpcMemHandles( @@ -171,12 +183,17 @@ struct ContextImp : Context { this->numDevices = static_cast(this->deviceHandles.size()); } NEO::VirtualMemoryReservation *findSupportedVirtualReservation(const void *ptr, size_t size); + std::map &getIPCHandleMap() { return this->ipcHandles; }; + [[nodiscard]] std::unique_lock lockIPCHandleMap() { return std::unique_lock(this->ipcHandleMapMutex); }; protected: + void setIPCHandleData(NEO::GraphicsAllocation *graphicsAllocation, uint64_t handle, IpcMemoryData &ipcData, uint64_t ptrAddress); bool isAllocationSuitableForCompression(const StructuresLookupTable &structuresLookupTable, Device &device, size_t allocSize); size_t getPageSizeRequired(size_t size); std::map devices; + std::map ipcHandles; + std::mutex ipcHandleMapMutex; std::vector deviceHandles; DriverHandleImp *driverHandle = nullptr; uint32_t numDevices = 0; diff --git a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h index 57605a479ffa9..d887fa61667f1 100644 --- a/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h +++ b/level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h @@ -71,7 +71,7 @@ struct ContextFdMock : public L0::ContextImp { ze_memory_allocation_properties_t *pMemAllocProperties, ze_device_handle_t *phDevice) override { ze_result_t res = ContextImp::getMemAllocProperties(ptr, pMemAllocProperties, phDevice); - if (ZE_RESULT_SUCCESS == res && pMemAllocProperties->pNext) { + if (ZE_RESULT_SUCCESS == res && pMemAllocProperties->pNext && !memPropTest) { ze_base_properties_t *baseProperties = reinterpret_cast(pMemAllocProperties->pNext); if (baseProperties->stype == ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD) { @@ -104,7 +104,7 @@ struct ContextFdMock : public L0::ContextImp { ze_result_t closeIpcMemHandle(const void *ptr) override { return ZE_RESULT_SUCCESS; } - + bool memPropTest = false; DriverHandleGetFdMock *driverHandle = nullptr; }; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_drm_or_wddm.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_drm_or_wddm.cpp index 19b4fe1b083b3..9e8d7f4da2ef0 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_drm_or_wddm.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_drm_or_wddm.cpp @@ -46,6 +46,296 @@ TEST_F(MemoryIPCTests, EXPECT_EQ(ZE_RESULT_SUCCESS, result); } +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleWithDeviceAllocationAndCallToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleWithDeviceAllocationAndCallToGetFdFromIpcHandleThenIPCHandleReturned) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_external_memory_export_desc_t export_desc = {}; + export_desc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + export_desc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + + ze_device_mem_alloc_desc_t device_alloc_desc = {}; + device_alloc_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + device_alloc_desc.pNext = &export_desc; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &device_alloc_desc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_external_memory_export_fd_t export_fd = {}; + export_fd.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD; + export_fd.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + ze_memory_allocation_properties_t alloc_props = {}; + alloc_props.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + alloc_props.pNext = &export_fd; + ze_device_handle_t deviceHandle; + context->memPropTest = true; + EXPECT_EQ(ZE_RESULT_SUCCESS, context->getMemAllocProperties(ptr, &alloc_props, &deviceHandle)); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + uint64_t read_ipc_handle = 0; + result = context->getFdFromIpcHandle(ipcHandle, &read_ipc_handle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(static_cast(read_ipc_handle), export_fd.fd); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleFromFdThenValidIpcHandleReturned) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_external_memory_export_desc_t export_desc = {}; + export_desc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + export_desc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + + ze_device_mem_alloc_desc_t device_alloc_desc = {}; + device_alloc_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + device_alloc_desc.pNext = &export_desc; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &device_alloc_desc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_external_memory_export_fd_t export_fd = {}; + export_fd.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD; + export_fd.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + ze_memory_allocation_properties_t alloc_props = {}; + alloc_props.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + alloc_props.pNext = &export_fd; + ze_device_handle_t deviceHandle; + context->memPropTest = true; + EXPECT_EQ(ZE_RESULT_SUCCESS, context->getMemAllocProperties(ptr, &alloc_props, &deviceHandle)); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcHandleFromFd(export_fd.fd, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + IpcMemoryData &ipcData = *reinterpret_cast(ipcHandle.data); + EXPECT_EQ(static_cast(ipcData.handle), export_fd.fd); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleFromFdWithInvalidFdThenErrorReturned) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_external_memory_export_desc_t export_desc = {}; + export_desc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + export_desc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + + ze_device_mem_alloc_desc_t device_alloc_desc = {}; + device_alloc_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + device_alloc_desc.pNext = &export_desc; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &device_alloc_desc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcHandleFromFd(0u, &ipcHandle); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, result); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleWithDeviceAllocationAndCallToGetFdFromIpcHandleWithInvalidHandleThenErrorReturned) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_external_memory_export_desc_t export_desc = {}; + export_desc.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_DESC; + export_desc.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + + ze_device_mem_alloc_desc_t device_alloc_desc = {}; + device_alloc_desc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + device_alloc_desc.pNext = &export_desc; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &device_alloc_desc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_external_memory_export_fd_t export_fd = {}; + export_fd.stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD; + export_fd.flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF; + ze_memory_allocation_properties_t alloc_props = {}; + alloc_props.stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES; + alloc_props.pNext = &export_fd; + ze_device_handle_t deviceHandle; + context->memPropTest = true; + EXPECT_EQ(ZE_RESULT_SUCCESS, context->getMemAllocProperties(ptr, &alloc_props, &deviceHandle)); + + ze_ipc_mem_handle_t ipcHandle; + IpcMemoryData &ipcData = *reinterpret_cast(ipcHandle.data); + ipcData.handle = 256; + uint64_t read_ipc_handle = 256; + result = context->getFdFromIpcHandle(ipcHandle, &read_ipc_handle); + EXPECT_EQ(ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY, result); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenMultipleCallsToGetIpcHandleWithDeviceAllocationAndCallsToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenMultipleCallsToGetIpcHandleWithDeviceAllocationAndOneCallToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_NE(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); +} + +TEST_F(MemoryIPCTests, + givenCallsToGetIpcHandleWithDeviceAllocationWithDifferentContextsThenIpcHandleClosed) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + std::unique_ptr context_invalid; + context_invalid = std::make_unique(driverHandle.get()); + EXPECT_NE(context_invalid, nullptr); + context_invalid->getDevices().insert(std::make_pair(device->getRootDeviceIndex(), device->toHandle())); + context_invalid->rootDeviceIndices.push_back(neoDevice->getRootDeviceIndex()); + context_invalid->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context_invalid->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + TEST_F(MemoryIPCTests, whenCallingGetIpcHandleWithHostAllocationThenSuccessIsReturned) { size_t size = 10; diff --git a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp index 7b7cf96a21b6a..3240d5b0d8f5f 100644 --- a/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp +++ b/level_zero/core/test/unit_tests/sources/memory/test_memory_wddm.cpp @@ -46,6 +46,133 @@ TEST_F(MemoryIPCTests, EXPECT_EQ(ZE_RESULT_SUCCESS, result); } +TEST_F(MemoryIPCTests, + givenCallToGetIpcHandleWithDeviceAllocationAndCallToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenMultipleCallsToGetIpcHandleWithDeviceAllocationAndCallsToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + +TEST_F(MemoryIPCTests, + givenMultipleCallsToGetIpcHandleWithDeviceAllocationAndOneCallToPutIpcHandleThenIpcHandleIsReturnedAndReleased) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_NE(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); +} + +TEST_F(MemoryIPCTests, + givenCallsToGetIpcHandleWithDeviceAllocationWithDifferentContextsThenIpcHandleClosed) { + size_t size = 10; + size_t alignment = 1u; + void *ptr = nullptr; + + ze_device_mem_alloc_desc_t deviceDesc = {}; + ze_result_t result = context->allocDeviceMem(device->toHandle(), + &deviceDesc, + size, alignment, &ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + EXPECT_NE(nullptr, ptr); + + std::unique_ptr context_invalid; + context_invalid = std::make_unique(driverHandle.get()); + EXPECT_NE(context_invalid, nullptr); + context_invalid->getDevices().insert(std::make_pair(device->getRootDeviceIndex(), device->toHandle())); + context_invalid->rootDeviceIndices.push_back(neoDevice->getRootDeviceIndex()); + context_invalid->deviceBitfields.insert({neoDevice->getRootDeviceIndex(), neoDevice->getDeviceBitfield()}); + + ze_ipc_mem_handle_t ipcHandle; + result = context->getIpcMemHandle(ptr, &ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + result = context->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context_invalid->putIpcMemHandle(ipcHandle); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); + + EXPECT_EQ(0u, context->getIPCHandleMap().size()); + + result = context->freeMem(ptr); + EXPECT_EQ(ZE_RESULT_SUCCESS, result); +} + TEST_F(MemoryIPCTests, whenCallingOpenIpcHandleWithIpcHandleThenDeviceAllocationIsReturned) { size_t size = 10; diff --git a/shared/source/memory_manager/graphics_allocation.h b/shared/source/memory_manager/graphics_allocation.h index 91a25b42b88ab..299b7890550f3 100644 --- a/shared/source/memory_manager/graphics_allocation.h +++ b/shared/source/memory_manager/graphics_allocation.h @@ -169,7 +169,9 @@ class GraphicsAllocation : public IDNode { bool isResidencyTaskCountBelow(TaskCountType taskCount, uint32_t contextId) const { return !isResident(contextId) || getResidencyTaskCount(contextId) < taskCount; } virtual std::string getAllocationInfoString() const; + virtual int createInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) { return 0; } virtual int peekInternalHandle(MemoryManager *memoryManager, uint64_t &handle) { return 0; } + virtual void clearInternalHandle(uint32_t handleId) { return; } virtual int peekInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) { return 0; diff --git a/shared/source/memory_manager/memory_manager.h b/shared/source/memory_manager/memory_manager.h index f6ff3303bc704..a8093bf92ed99 100644 --- a/shared/source/memory_manager/memory_manager.h +++ b/shared/source/memory_manager/memory_manager.h @@ -121,6 +121,7 @@ class MemoryManager { virtual GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(const std::vector &handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation, bool reuseSharedAllocation) = 0; virtual GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation, bool reuseSharedAllocation) = 0; virtual void closeSharedHandle(GraphicsAllocation *graphicsAllocation){}; + virtual void closeInternalHandle(uint64_t &handle, uint32_t handleId, GraphicsAllocation *graphicsAllocation){}; virtual GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) = 0; virtual bool mapAuxGpuVA(GraphicsAllocation *graphicsAllocation); diff --git a/shared/source/os_interface/linux/drm_allocation.cpp b/shared/source/os_interface/linux/drm_allocation.cpp index fcb382ac0c986..2224aaef22347 100644 --- a/shared/source/os_interface/linux/drm_allocation.cpp +++ b/shared/source/os_interface/linux/drm_allocation.cpp @@ -40,6 +40,14 @@ std::string DrmAllocation::getAllocationInfoString() const { return ss.str(); } +void DrmAllocation::clearInternalHandle(uint32_t handleId) { + handles[handleId] = std::numeric_limits::max(); +} + +int DrmAllocation::createInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) { + return peekInternalHandle(memoryManager, handleId, handle); +} + int DrmAllocation::peekInternalHandle(MemoryManager *memoryManager, uint64_t &handle) { return peekInternalHandle(memoryManager, 0u, handle); } diff --git a/shared/source/os_interface/linux/drm_allocation.h b/shared/source/os_interface/linux/drm_allocation.h index 76b9e62cd8edd..1fc7a7b959e61 100644 --- a/shared/source/os_interface/linux/drm_allocation.h +++ b/shared/source/os_interface/linux/drm_allocation.h @@ -94,6 +94,9 @@ class DrmAllocation : public GraphicsAllocation { uint64_t getHandleAddressBase(uint32_t handleIndex) override; size_t getHandleSize(uint32_t handleIndex) override; + int createInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) override; + + void clearInternalHandle(uint32_t handleId) override; int peekInternalHandle(MemoryManager *memoryManager, uint64_t &handle) override; diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp index 9ab70c9ea9bf9..32265dab0ce2e 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.cpp +++ b/shared/source/os_interface/linux/drm_memory_manager.cpp @@ -1055,6 +1055,13 @@ GraphicsAllocation *DrmMemoryManager::createGraphicsAllocationFromSharedHandle(o return drmAllocation; } +void DrmMemoryManager::closeInternalHandle(uint64_t &handle, uint32_t handleId, GraphicsAllocation *graphicsAllocation) { + DrmAllocation *drmAllocation = static_cast(graphicsAllocation); + drmAllocation->clearInternalHandle(handleId); + [[maybe_unused]] auto status = this->closeFunction(static_cast(handle)); + DEBUG_BREAK_IF(status != 0); +} + void DrmMemoryManager::closeSharedHandle(GraphicsAllocation *gfxAllocation) { DrmAllocation *drmAllocation = static_cast(gfxAllocation); if (drmAllocation->peekSharedHandle() != Sharing::nonSharedResource) { diff --git a/shared/source/os_interface/linux/drm_memory_manager.h b/shared/source/os_interface/linux/drm_memory_manager.h index 01a0bce914916..f37a336471359 100644 --- a/shared/source/os_interface/linux/drm_memory_manager.h +++ b/shared/source/os_interface/linux/drm_memory_manager.h @@ -40,6 +40,7 @@ class DrmMemoryManager : public MemoryManager { GraphicsAllocation *createGraphicsAllocationFromMultipleSharedHandles(const std::vector &handles, AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation, bool reuseSharedAllocation) override; GraphicsAllocation *createGraphicsAllocationFromSharedHandle(osHandle handle, const AllocationProperties &properties, bool requireSpecificBitness, bool isHostIpcAllocation, bool reuseSharedAllocation) override; void closeSharedHandle(GraphicsAllocation *gfxAllocation) override; + void closeInternalHandle(uint64_t &handle, uint32_t handleId, GraphicsAllocation *graphicsAllocation) override; GraphicsAllocation *createGraphicsAllocationFromNTHandle(void *handle, uint32_t rootDeviceIndex, AllocationType allocType) override { return nullptr; } diff --git a/shared/source/os_interface/windows/CMakeLists.txt b/shared/source/os_interface/windows/CMakeLists.txt index 6090d27aa6ed7..af00f09b283ee 100644 --- a/shared/source/os_interface/windows/CMakeLists.txt +++ b/shared/source/os_interface/windows/CMakeLists.txt @@ -51,6 +51,7 @@ set(NEO_CORE_OS_INTERFACE_WDDM ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}wddm_additional_context_flags.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}wddm_allocation.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/wddm_allocation_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}${BRANCH_DIR_SUFFIX}wddm_apply_additional_map_gpu_va_fields.cpp ${CMAKE_CURRENT_SOURCE_DIR}/wddm_engine_mapper.cpp ${CMAKE_CURRENT_SOURCE_DIR}/d3dkmthk_wrapper.h diff --git a/shared/source/os_interface/windows/wddm_allocation.h b/shared/source/os_interface/windows/wddm_allocation.h index 3f6c82e442cd5..8c0c520aa9225 100644 --- a/shared/source/os_interface/windows/wddm_allocation.h +++ b/shared/source/os_interface/windows/wddm_allocation.h @@ -72,6 +72,10 @@ class WddmAllocation : public GraphicsAllocation { handles[handleIndex] = handle; } + void clearInternalHandle(uint32_t handleId) override; + + int createInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) override; + int peekInternalHandle(MemoryManager *memoryManager, uint64_t &handle) override { handle = ntSecureHandle; return handle == 0; diff --git a/shared/source/os_interface/windows/wddm_allocation_common.cpp b/shared/source/os_interface/windows/wddm_allocation_common.cpp new file mode 100644 index 0000000000000..9d21a3fdbb35c --- /dev/null +++ b/shared/source/os_interface/windows/wddm_allocation_common.cpp @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + */ + +#include "shared/source/os_interface/windows/wddm_allocation.h" +#include "shared/source/os_interface/windows/wddm_memory_manager.h" + +namespace NEO { +int WddmAllocation::createInternalHandle(MemoryManager *memoryManager, uint32_t handleId, uint64_t &handle) { + handle = ntSecureHandle; + if (handle == 0) { + HANDLE ntSharedHandle = NULL; + WddmMemoryManager *wddmMemoryManager = reinterpret_cast(memoryManager); + auto status = wddmMemoryManager->createInternalNTHandle(&resourceHandle, &ntSharedHandle, this->getRootDeviceIndex()); + if (status != STATUS_SUCCESS) { + return handle == 0; + } + ntSecureHandle = castToUint64(ntSharedHandle); + handle = ntSecureHandle; + } + return handle == 0; +} +void WddmAllocation::clearInternalHandle(uint32_t handleId) { + ntSecureHandle = 0u; +} +} // namespace NEO \ No newline at end of file diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp index 4b5dccc5769c1..57dc6ad74de1b 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.cpp +++ b/shared/source/os_interface/windows/wddm_memory_manager.cpp @@ -212,6 +212,10 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryUsingKmdAndMapItToC return wddmAllocation.release(); } +NTSTATUS WddmMemoryManager::createInternalNTHandle(D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle, uint32_t rootDeviceIndex) { + return getWddm(rootDeviceIndex).createNTHandle(resourceHandle, ntHandle); +} + GraphicsAllocation *WddmMemoryManager::allocateHugeGraphicsMemory(const AllocationData &allocationData, bool sharedVirtualAddress) { void *hostPtr = nullptr, *alignedPtr = nullptr; size_t alignedSize = 0; @@ -588,6 +592,13 @@ void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation return freeGraphicsMemoryImpl(gfxAllocation); } +void WddmMemoryManager::closeInternalHandle(uint64_t &handle, uint32_t handleId, GraphicsAllocation *graphicsAllocation) { + WddmAllocation *wddmAllocation = static_cast(graphicsAllocation); + wddmAllocation->clearInternalHandle(handleId); + [[maybe_unused]] auto status = SysCalls::closeHandle(reinterpret_cast(reinterpret_cast(handle))); + DEBUG_BREAK_IF(!status); +} + void WddmMemoryManager::freeGraphicsMemoryImpl(GraphicsAllocation *gfxAllocation) { WddmAllocation *input = static_cast(gfxAllocation); DEBUG_BREAK_IF(!validateAllocation(input)); diff --git a/shared/source/os_interface/windows/wddm_memory_manager.h b/shared/source/os_interface/windows/wddm_memory_manager.h index 23f5676699412..0f4055ba504ef 100644 --- a/shared/source/os_interface/windows/wddm_memory_manager.h +++ b/shared/source/os_interface/windows/wddm_memory_manager.h @@ -76,6 +76,8 @@ class WddmMemoryManager : public MemoryManager { void releaseDeviceSpecificMemResources(uint32_t rootDeviceIndex) override{}; void createDeviceSpecificMemResources(uint32_t rootDeviceIndex) override{}; void registerAllocationInOs(GraphicsAllocation *allocation) override; + void closeInternalHandle(uint64_t &handle, uint32_t handleId, GraphicsAllocation *graphicsAllocation) override; + MOCKABLE_VIRTUAL NTSTATUS createInternalNTHandle(D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle, uint32_t rootDeviceIndex); protected: GraphicsAllocation *createGraphicsAllocation(OsHandleStorage &handleStorage, const AllocationData &allocationData) override; diff --git a/shared/test/unit_test/memory_manager/graphics_allocation_tests.cpp b/shared/test/unit_test/memory_manager/graphics_allocation_tests.cpp index bcf87ae86d435..929dbc6dded5f 100644 --- a/shared/test/unit_test/memory_manager/graphics_allocation_tests.cpp +++ b/shared/test/unit_test/memory_manager/graphics_allocation_tests.cpp @@ -239,6 +239,14 @@ TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenInternalHandleIsB EXPECT_EQ(0ull, handle); } +TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenInternalHandleIsBeingObtainedOrCreatedThenZeroIsReturned) { + MockGraphicsAllocation graphicsAllocation; + uint64_t handle = 0; + graphicsAllocation.createInternalHandle(nullptr, 0u, handle); + EXPECT_EQ(0ull, handle); + graphicsAllocation.clearInternalHandle(0u); +} + TEST(GraphicsAllocationTest, givenDefaultGraphicsAllocationWhenGettingNumHandlesThenZeroIsReturned) { MockGraphicsAllocation graphicsAllocation; EXPECT_EQ(0u, graphicsAllocation.getNumHandles()); diff --git a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp index ac93ab42f1a72..bd131d65f35ab 100644 --- a/shared/test/unit_test/memory_manager/memory_manager_tests.cpp +++ b/shared/test/unit_test/memory_manager/memory_manager_tests.cpp @@ -28,6 +28,13 @@ TEST(MemoryManagerTest, WhenCallingHasPageFaultsEnabledThenReturnFalse) { EXPECT_FALSE(memoryManager.hasPageFaultsEnabled(device)); } +TEST(MemoryManagerTest, WhenCallingCloseInternalHandleWithOsAgnosticThenNoChanges) { + MockExecutionEnvironment executionEnvironment(defaultHwInfo.get()); + OsAgnosticMemoryManager memoryManager(executionEnvironment); + uint64_t handle = 0u; + memoryManager.closeInternalHandle(handle, 0u, nullptr); +} + TEST(MemoryManagerTest, WhenCallingIsAllocationTypeToCaptureThenScratchAndPrivateTypesReturnTrue) { MockMemoryManager mockMemoryManager; diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp index bbff49485351d..8af3fd9e5ed0a 100644 --- a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp @@ -537,6 +537,41 @@ TEST_F(DrmMemoryManagerTest, whenPeekInternalHandleIsCalledThenBoIsReturned) { memoryManager->freeGraphicsMemory(allocation); } +TEST_F(DrmMemoryManagerTest, whenCreateInternalHandleIsCalledThenBoIsReturned) { + mock->ioctlExpected.gemUserptr = 1; + mock->ioctlExpected.gemWait = 1; + mock->ioctlExpected.gemClose = 1; + mock->ioctlExpected.handleToPrimeFd = 1; + mock->outputFd = 1337; + auto allocation = static_cast(this->memoryManager->allocateGraphicsMemoryWithProperties(createAllocationProperties(rootDeviceIndex, 10 * MemoryConstants::pageSize, true))); + ASSERT_NE(allocation->getBO(), nullptr); + uint64_t handle = 0; + int ret = allocation->createInternalHandle(this->memoryManager, 0u, handle); + ASSERT_EQ(ret, 0); + ASSERT_EQ(handle, static_cast(1337)); + + memoryManager->freeGraphicsMemory(allocation); +} + +TEST_F(DrmMemoryManagerTest, whenCreateInternalHandleIsCalledThenClearInternalHandleThenSameHandleisReturned) { + mock->ioctlExpected.gemUserptr = 1; + mock->ioctlExpected.gemWait = 1; + mock->ioctlExpected.gemClose = 1; + mock->ioctlExpected.handleToPrimeFd = 2; + mock->outputFd = 1337; + auto allocation = static_cast(this->memoryManager->allocateGraphicsMemoryWithProperties(createAllocationProperties(rootDeviceIndex, 10 * MemoryConstants::pageSize, true))); + ASSERT_NE(allocation->getBO(), nullptr); + uint64_t handle = 0; + int ret = allocation->createInternalHandle(this->memoryManager, 0u, handle); + ASSERT_EQ(ret, 0); + allocation->clearInternalHandle(0u); + ret = allocation->createInternalHandle(this->memoryManager, 0u, handle); + ASSERT_EQ(ret, 0); + ASSERT_EQ(handle, static_cast(1337)); + + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(DrmMemoryManagerTest, whenPeekInternalHandleIsCalledAndObtainFdFromHandleFailsThenErrorIsReturned) { mock->ioctlExpected.gemUserptr = 1; mock->ioctlExpected.gemWait = 1; @@ -584,6 +619,38 @@ TEST_F(DrmMemoryManagerTest, whenCallingPeekInternalHandleSeveralTimesThenSameHa memoryManager->freeGraphicsMemory(allocation); } +TEST_F(DrmMemoryManagerTest, whenCallingCreateInternalHandleSeveralTimesThenSameHandleIsReturned) { + mock->ioctlExpected.gemUserptr = 1; + mock->ioctlExpected.gemWait = 1; + mock->ioctlExpected.gemClose = 1; + mock->ioctlExpected.handleToPrimeFd = 1; + uint64_t expectedFd = 1337; + mock->outputFd = static_cast(expectedFd); + mock->incrementOutputFdAfterCall = true; + auto allocation = static_cast(this->memoryManager->allocateGraphicsMemoryWithProperties(createAllocationProperties(rootDeviceIndex, 10 * MemoryConstants::pageSize, true))); + ASSERT_NE(allocation->getBO(), nullptr); + + EXPECT_EQ(mock->outputFd, static_cast(expectedFd)); + uint64_t handle0 = 0; + int ret = allocation->createInternalHandle(this->memoryManager, 0u, handle0); + ASSERT_EQ(ret, 0); + EXPECT_NE(mock->outputFd, static_cast(expectedFd)); + + uint64_t handle1 = 0; + uint64_t handle2 = 0; + + ret = allocation->createInternalHandle(this->memoryManager, 0u, handle1); + ASSERT_EQ(ret, 0); + ret = allocation->createInternalHandle(this->memoryManager, 0u, handle2); + ASSERT_EQ(ret, 0); + + ASSERT_EQ(handle0, expectedFd); + ASSERT_EQ(handle1, expectedFd); + ASSERT_EQ(handle2, expectedFd); + + memoryManager->freeGraphicsMemory(allocation); +} + TEST_F(DrmMemoryManagerTest, whenPeekInternalHandleWithHandleIdIsCalledThenBoIsReturned) { mock->ioctlExpected.gemUserptr = 1; mock->ioctlExpected.gemWait = 1; @@ -857,6 +924,26 @@ TEST_F(DrmMemoryManagerTest, GivenAllocationWhenClosingSharedHandleThenSucceeds) memoryManager->freeGraphicsMemory(graphicsAllocation); } +TEST_F(DrmMemoryManagerTest, GivenAllocationWhenClosingInternalHandleThenSucceeds) { + mock->ioctlExpected.primeFdToHandle = 1; + mock->ioctlExpected.gemWait = 1; + mock->ioctlExpected.gemClose = 1; + mock->ioctlExpected.handleToPrimeFd = 1; + + osHandle handle = 1u; + uint64_t handleVal = 1u; + this->mock->outputHandle = 2u; + size_t size = 4096u; + AllocationProperties properties(rootDeviceIndex, false, size, AllocationType::SHARED_BUFFER, false, {}); + + auto graphicsAllocation = memoryManager->createGraphicsAllocationFromSharedHandle(handle, properties, false, false, true); + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(this->memoryManager, 0u, handleVal)); + + memoryManager->closeInternalHandle(handleVal, 0u, graphicsAllocation); + + memoryManager->freeGraphicsMemory(graphicsAllocation); +} + TEST_F(DrmMemoryManagerTest, GivenNullptrDrmAllocationWhenTryingToRegisterItThenRegisterSharedBoHandleAllocationDoesNothing) { ASSERT_TRUE(memoryManager->sharedBoHandles.empty()); diff --git a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp index e5a64dd826b63..0a24c8da69853 100644 --- a/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp +++ b/shared/test/unit_test/os_interface/wddm_linux/configure_device_address_space_drm_or_wddm_test.cpp @@ -85,6 +85,14 @@ struct MockWddmLinuxMemoryManager : NEO::WddmMemoryManager { using WddmMemoryManager::mapPhysicalToVirtualMemory; using WddmMemoryManager::unMapPhysicalToVirtualMemory; using WddmMemoryManager::WddmMemoryManager; + NTSTATUS createInternalNTHandle(D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle, uint32_t rootDeviceIndex) override { + if (failCreateInternalNTHandle) { + return 1; + } else { + return WddmMemoryManager::createInternalNTHandle(resourceHandle, ntHandle, rootDeviceIndex); + } + } + bool failCreateInternalNTHandle = false; }; struct WddmLinuxMockHwDeviceIdWddm : public NEO::HwDeviceIdWddm { @@ -676,6 +684,58 @@ TEST_F(WddmLinuxTest, givenAllocatePhysicalDeviceMemoryThenAllocationReturned) { memoryManager.freeGraphicsMemoryImpl(alloc); } +TEST_F(WddmLinuxTest, givenAllocatedMemoryAndCloseInternalHandleThenSharedHandleClosed) { + osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock; + osEnvironment->gdi->createAllocation2 = createAllocation2Mock; + osEnvironment->gdi->mapGpuVirtualAddress = mapGpuVirtualAddressMock; + osEnvironment->gdi->lock2 = lock2Mock; + osEnvironment->gdi->destroyAllocation2 = destroyAllocations2Mock; + + MockWddmLinuxMemoryManager memoryManager{mockExecEnv}; + + NEO::AllocationData allocData = {}; + NEO::MemoryManager::AllocationStatus status = NEO::MemoryManager::AllocationStatus::Error; + allocData.size = 3U; + + auto alloc = memoryManager.allocatePhysicalDeviceMemory(allocData, status); + ASSERT_NE(nullptr, alloc); + uint64_t handle = 0; + EXPECT_EQ(0, alloc->createInternalHandle(&memoryManager, 0u, handle)); + + memoryManager.closeInternalHandle(handle, 0u, alloc); + + EXPECT_EQ(0, alloc->createInternalHandle(&memoryManager, 0u, handle)); + + memoryManager.freeGraphicsMemoryImpl(alloc); +} + +TEST_F(WddmLinuxTest, givenAllocatedMemoryAndCreateInternalHandleFailedThenEmpyHandleReturned) { + osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock; + osEnvironment->gdi->createAllocation2 = createAllocation2Mock; + osEnvironment->gdi->mapGpuVirtualAddress = mapGpuVirtualAddressMock; + osEnvironment->gdi->lock2 = lock2Mock; + osEnvironment->gdi->destroyAllocation2 = destroyAllocations2Mock; + + MockWddmLinuxMemoryManager memoryManager{mockExecEnv}; + + NEO::AllocationData allocData = {}; + NEO::MemoryManager::AllocationStatus status = NEO::MemoryManager::AllocationStatus::Error; + allocData.size = 3U; + + auto alloc = memoryManager.allocatePhysicalDeviceMemory(allocData, status); + ASSERT_NE(nullptr, alloc); + + uint64_t handle = 0; + EXPECT_EQ(0, alloc->createInternalHandle(&memoryManager, 0u, handle)); + + memoryManager.closeInternalHandle(handle, 0u, alloc); + + memoryManager.failCreateInternalNTHandle = true; + EXPECT_EQ(1, alloc->createInternalHandle(&memoryManager, 0u, handle)); + + memoryManager.freeGraphicsMemoryImpl(alloc); +} + TEST_F(WddmLinuxTest, givenAllocatePhysicalLocalDeviceMemoryThenErrorReturned) { osEnvironment->gdi->reserveGpuVirtualAddress = reserveDeviceAddressSpaceMock; osEnvironment->gdi->createAllocation2 = createAllocation2Mock; diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp index dcd6ff8ede6b4..b18447c146927 100644 --- a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp +++ b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_tests.cpp @@ -107,6 +107,14 @@ class MockAllocateGraphicsMemoryUsingKmdAndMapItToCpuVAWddm : public MemoryManag return true; } + NTSTATUS createInternalNTHandle(D3DKMT_HANDLE *resourceHandle, HANDLE *ntHandle, uint32_t rootDeviceIndex) override { + if (failCreateInternalNTHandle) { + return 1; + } else { + return WddmMemoryManager::createInternalNTHandle(resourceHandle, ntHandle, rootDeviceIndex); + } + } + bool failCreateInternalNTHandle = false; }; class WddmMemoryManagerAllocPathTests : public ::testing::Test { @@ -160,4 +168,56 @@ TEST_F(WddmMemoryManagerAllocPathTests, givenAllocateGraphicsMemoryUsingKmdAndMa memoryManager->freeGraphicsMemory(graphicsAllocation); } +} + +TEST_F(WddmMemoryManagerAllocPathTests, GivenValidAllocationThenCreateInternalHandleSucceeds) { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::SVM_CPU; + allocData.forceKMDAllocation = true; + allocData.makeGPUVaDifferentThanCPUPtr = true; + auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false); + + uint64_t handle = 0; + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + memoryManager->closeInternalHandle(handle, 0u, graphicsAllocation); + + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + memoryManager->freeGraphicsMemory(graphicsAllocation); +} + +TEST_F(WddmMemoryManagerAllocPathTests, GivenValidAllocationThenCreateInternalHandleSucceedsAfterMultipleCallsToCreate) { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::SVM_CPU; + allocData.forceKMDAllocation = true; + allocData.makeGPUVaDifferentThanCPUPtr = true; + auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false); + + uint64_t handle = 0; + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + memoryManager->closeInternalHandle(handle, 0u, graphicsAllocation); + + memoryManager->freeGraphicsMemory(graphicsAllocation); +} + +TEST_F(WddmMemoryManagerAllocPathTests, GivenValidAllocationWithFailingCreateInternalHandleThenErrorReturned) { + NEO::AllocationData allocData = {}; + allocData.type = NEO::AllocationType::SVM_CPU; + allocData.forceKMDAllocation = true; + allocData.makeGPUVaDifferentThanCPUPtr = true; + auto graphicsAllocation = memoryManager->allocateGraphicsMemoryUsingKmdAndMapItToCpuVA(allocData, false); + + uint64_t handle = 0; + EXPECT_EQ(0, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + memoryManager->closeInternalHandle(handle, 0u, graphicsAllocation); + + memoryManager->failCreateInternalNTHandle = true; + EXPECT_EQ(1, graphicsAllocation->createInternalHandle(memoryManager, 0u, handle)); + + memoryManager->freeGraphicsMemory(graphicsAllocation); } \ No newline at end of file