Skip to content

Commit

Permalink
feature: Add Support for zeMemPutIpcHandle & zeMemGet IPC Handle conv…
Browse files Browse the repository at this point in the history 
…erters

Related-To: LOCI-4172, LOCI-4305, LOCI-4306

- Create a new IPC Memory handle upon call to getIpcMemHandle if the
previous handle has been freed.
- Release the Ipc Memory Handle when zeMemPutIpcHandle is called.
- Create a new IPC Handle for tracking thru zeMemGetAllocProperties
when ze_external_memory_export_fd_t is used.
- Convert FD to opaque IPC handle and IPC Handle to FD.

Signed-off-by: Spruit, Neil R <neil.r.spruit@intel.com>
  • Loading branch information
@nrspruit @Compute-Runtime-Automation
nrspruit authored and Compute-Runtime-Automation committed Apr 27, 2023
1 parent e2bbed2 commit 364c2da
Show file tree
Hide file tree
Showing 24 changed files with 881 additions and 13 deletions.
18 changes: 18 additions & 0 deletions level_zero/api/core/ze_core_loader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ zeGetMemProcAddrTable(
pDdiTable->pfnGetIpcHandle = L0::zeMemGetIpcHandle;
pDdiTable->pfnOpenIpcHandle = L0::zeMemOpenIpcHandle;
pDdiTable->pfnCloseIpcHandle = L0::zeMemCloseIpcHandle;
pDdiTable->pfnPutIpcHandle = L0::zeMemPutIpcHandle;
driver_ddiTable.core_ddiTable.Mem = *pDdiTable;
if (driver_ddiTable.enableTracing) {
pDdiTable->pfnAllocShared = zeMemAllocSharedTracing;
Expand Down Expand Up @@ -638,6 +639,23 @@ zeGetKernelExpProcAddrTable(
return result;
}

ZE_DLLEXPORT ze_result_t ZE_APICALL
zeGetMemExpProcAddrTable(
ze_api_version_t version,
ze_mem_exp_dditable_t *pDdiTable) {
if (nullptr == pDdiTable)
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
if (ZE_MAJOR_VERSION(driver_ddiTable.version) != ZE_MAJOR_VERSION(version) ||
ZE_MINOR_VERSION(driver_ddiTable.version) > ZE_MINOR_VERSION(version))
return ZE_RESULT_ERROR_UNSUPPORTED_VERSION;

ze_result_t result = ZE_RESULT_SUCCESS;
pDdiTable->pfnGetIpcHandleFromFileDescriptorExp = L0::zeMemGetIpcHandleFromFileDescriptorExp;
pDdiTable->pfnGetFileDescriptorFromIpcHandleExp = L0::zeMemGetFileDescriptorFromIpcHandleExp;
driver_ddiTable.core_ddiTable.MemExp = *pDdiTable;
return result;
}

ZE_APIEXPORT ze_result_t ZE_APICALL
zeGetImageExpProcAddrTable(
ze_api_version_t version,
Expand Down
44 changes: 43 additions & 1 deletion level_zero/api/core/ze_memory_api_entrypoints.h
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2020-2022 Intel Corporation
* Copyright (C) 2020-2023 Intel Corporation
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -77,6 +77,12 @@ ze_result_t zeMemGetIpcHandle(
return L0::Context::fromHandle(hContext)->getIpcMemHandle(ptr, pIpcHandle);
}

ze_result_t zeMemPutIpcHandle(
ze_context_handle_t hContext,
ze_ipc_mem_handle_t ipcHandle) {
return L0::Context::fromHandle(hContext)->putIpcMemHandle(ipcHandle);
}

ze_result_t zeMemOpenIpcHandle(
ze_context_handle_t hContext,
ze_device_handle_t hDevice,
Expand All @@ -92,6 +98,14 @@ ze_result_t zeMemCloseIpcHandle(
return L0::Context::fromHandle(hContext)->closeIpcMemHandle(ptr);
}

ze_result_t zeMemGetIpcHandleFromFileDescriptorExp(ze_context_handle_t hContext, uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) {
return L0::Context::fromHandle(hContext)->getIpcHandleFromFd(handle, pIpcHandle);
}

ze_result_t zeMemGetFileDescriptorFromIpcHandleExp(ze_context_handle_t hContext, ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) {
return L0::Context::fromHandle(hContext)->getFdFromIpcHandle(ipcHandle, pHandle);
}

} // namespace L0

extern "C" {
Expand Down Expand Up @@ -200,11 +214,39 @@ ZE_APIEXPORT ze_result_t ZE_APICALL zeMemOpenIpcHandle(
pptr);
}

ZE_APIEXPORT ze_result_t ZE_APICALL zeMemPutIpcHandle(
ze_context_handle_t hContext,
ze_ipc_mem_handle_t ipcHandle) {
return L0::zeMemPutIpcHandle(
hContext,
ipcHandle);
}

ZE_APIEXPORT ze_result_t ZE_APICALL zeMemCloseIpcHandle(
ze_context_handle_t hContext,
const void *ptr) {
return L0::zeMemCloseIpcHandle(
hContext,
ptr);
}

ZE_APIEXPORT ze_result_t ZE_APICALL zeMemGetIpcHandleFromFileDescriptorExp(
ze_context_handle_t hContext,
uint64_t handle,
ze_ipc_mem_handle_t *pIpcHandle) {
return L0::zeMemGetIpcHandleFromFileDescriptorExp(
hContext,
handle,
pIpcHandle);
}

ZE_APIEXPORT ze_result_t ZE_APICALL zeMemGetFileDescriptorFromIpcHandleExp(
ze_context_handle_t hContext,
ze_ipc_mem_handle_t ipcHandle,
uint64_t *pHandle) {
return L0::zeMemGetFileDescriptorFromIpcHandleExp(
hContext,
ipcHandle,
pHandle);
}
}
3 changes: 3 additions & 0 deletions level_zero/core/source/context/context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,11 @@ struct Context : _ze_context_handle_t {
void **pBase,
size_t *pSize) = 0;
virtual ze_result_t closeIpcMemHandle(const void *ptr) = 0;
virtual ze_result_t putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) = 0;
virtual ze_result_t getIpcMemHandle(const void *ptr,
ze_ipc_mem_handle_t *pIpcHandle) = 0;
virtual ze_result_t getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) = 0;
virtual ze_result_t getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) = 0;
virtual ze_result_t
getIpcMemHandles(
const void *ptr,
Expand Down
100 changes: 90 additions & 10 deletions level_zero/core/source/context/context_imp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -397,8 +397,19 @@ ze_result_t ContextImp::freeMem(const void *ptr, bool blocking) {
for (auto pairDevice : this->devices) {
this->freePeerAllocations(ptr, blocking, Device::fromHandle(pairDevice.second));
}

this->driverHandle->svmAllocsManager->freeSVMAlloc(const_cast<void *>(ptr), blocking);

std::map<uint64_t, IpcHandleTracking *>::iterator ipcHandleIterator;
auto lockIPC = this->lockIPCHandleMap();
ipcHandleIterator = this->getIPCHandleMap().begin();
while (ipcHandleIterator != this->getIPCHandleMap().end()) {
if (ipcHandleIterator->second->ptr == reinterpret_cast<uint64_t>(ptr)) {
delete ipcHandleIterator->second;
this->getIPCHandleMap().erase(ipcHandleIterator->first);
break;
}
ipcHandleIterator++;
}
return ZE_RESULT_SUCCESS;
}

Expand Down Expand Up @@ -517,6 +528,43 @@ ze_result_t ContextImp::closeIpcMemHandle(const void *ptr) {
return this->freeMem(ptr);
}

ze_result_t ContextImp::putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) {
IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(ipcHandle.data);
std::map<uint64_t, IpcHandleTracking *>::iterator ipcHandleIterator;
auto lock = this->lockIPCHandleMap();
ipcHandleIterator = this->getIPCHandleMap().find(ipcData.handle);
if (ipcHandleIterator != this->getIPCHandleMap().end()) {
ipcHandleIterator->second->refcnt -= 1;
if (ipcHandleIterator->second->refcnt == 0) {
auto *memoryManager = driverHandle->getMemoryManager();
memoryManager->closeInternalHandle(ipcData.handle, ipcHandleIterator->second->handleId, ipcHandleIterator->second->alloc);
delete ipcHandleIterator->second;
this->getIPCHandleMap().erase(ipcData.handle);
}
}
return ZE_RESULT_SUCCESS;
}

void ContextImp::setIPCHandleData(NEO::GraphicsAllocation *graphicsAllocation, uint64_t handle, IpcMemoryData &ipcData, uint64_t ptrAddress) {
std::map<uint64_t, IpcHandleTracking *>::iterator ipcHandleIterator;

ipcData = {};
ipcData.handle = handle;

auto lock = this->lockIPCHandleMap();
ipcHandleIterator = this->getIPCHandleMap().find(handle);
if (ipcHandleIterator != this->getIPCHandleMap().end()) {
ipcHandleIterator->second->refcnt += 1;
} else {
IpcHandleTracking *handleTracking = new IpcHandleTracking;
handleTracking->alloc = graphicsAllocation;
handleTracking->refcnt = 1;
handleTracking->ptr = ptrAddress;
handleTracking->ipcData = ipcData;
this->getIPCHandleMap().insert(std::pair<uint64_t, IpcHandleTracking *>(handle, handleTracking));
}
}

ze_result_t ContextImp::getIpcMemHandle(const void *ptr,
ze_ipc_mem_handle_t *pIpcHandle) {
NEO::SvmAllocationData *allocData = this->driverHandle->svmAllocsManager->getSVMAlloc(ptr);
Expand All @@ -525,16 +573,15 @@ ze_result_t ContextImp::getIpcMemHandle(const void *ptr,
auto *graphicsAllocation = allocData->gpuAllocations.getDefaultGraphicsAllocation();

uint64_t handle = 0;
int ret = graphicsAllocation->peekInternalHandle(memoryManager, handle);
int ret = graphicsAllocation->createInternalHandle(memoryManager, 0u, handle);
if (ret < 0) {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
}

memoryManager->registerIpcExportedAllocation(graphicsAllocation);

IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(pIpcHandle->data);
ipcData = {};
ipcData.handle = handle;
setIPCHandleData(graphicsAllocation, handle, ipcData, reinterpret_cast<uint64_t>(ptr));
auto type = allocData->memoryType;
if (type == HOST_UNIFIED_MEMORY) {
ipcData.type = static_cast<uint8_t>(InternalIpcMemoryType::IPC_HOST_UNIFIED_MEMORY);
Expand All @@ -545,6 +592,32 @@ ze_result_t ContextImp::getIpcMemHandle(const void *ptr,
return ZE_RESULT_ERROR_INVALID_ARGUMENT;
}

ze_result_t ContextImp::getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) {
std::map<uint64_t, IpcHandleTracking *>::iterator ipcHandleIterator;
auto lock = this->lockIPCHandleMap();
ipcHandleIterator = this->getIPCHandleMap().find(handle);
if (ipcHandleIterator != this->getIPCHandleMap().end()) {
IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(pIpcHandle->data);
ipcData = ipcHandleIterator->second->ipcData;
} else {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
}
return ZE_RESULT_SUCCESS;
}

ze_result_t ContextImp::getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) {
IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(ipcHandle.data);
std::map<uint64_t, IpcHandleTracking *>::iterator ipcHandleIterator;
auto lock = this->lockIPCHandleMap();
ipcHandleIterator = this->getIPCHandleMap().find(ipcData.handle);
if (ipcHandleIterator != this->getIPCHandleMap().end()) {
*pHandle = ipcHandleIterator->first;
} else {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
}
return ZE_RESULT_SUCCESS;
}

ze_result_t ContextImp::getIpcMemHandles(const void *ptr,
uint32_t *numIpcHandles,
ze_ipc_mem_handle_t *pIpcHandles) {
Expand All @@ -570,14 +643,13 @@ ze_result_t ContextImp::getIpcMemHandles(const void *ptr,

for (uint32_t i = 0; i < *numIpcHandles; i++) {
uint64_t handle = 0;
int ret = allocData->gpuAllocations.getDefaultGraphicsAllocation()->peekInternalHandle(this->driverHandle->getMemoryManager(), i, handle);
int ret = allocData->gpuAllocations.getDefaultGraphicsAllocation()->createInternalHandle(this->driverHandle->getMemoryManager(), i, handle);
if (ret < 0) {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
}

IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(pIpcHandles[i].data);
ipcData = {};
ipcData.handle = handle;
setIPCHandleData(alloc, handle, ipcData, reinterpret_cast<uint64_t>(ptr));
ipcData.type = static_cast<uint8_t>(ipcType);
}

Expand Down Expand Up @@ -661,10 +733,18 @@ ze_result_t ContextImp::handleAllocationExtensions(NEO::GraphicsAllocation *allo
if (type == ZE_MEMORY_TYPE_SHARED) {
return ZE_RESULT_ERROR_UNSUPPORTED_FEATURE;
}
ze_ipc_mem_handle_t ipcHandle;
uint64_t handle = 0;
int ret = alloc->peekInternalHandle(driverHandle->getMemoryManager(), handle);
if (ret < 0) {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
auto result = getIpcMemHandle(reinterpret_cast<void *>(alloc->getGpuAddress()), &ipcHandle);
if (result != ZE_RESULT_SUCCESS) {
// If this memory is not an SVM Allocation like Images, then retrieve only the handle untracked.
auto ret = alloc->peekInternalHandle(driverHandle->getMemoryManager(), handle);
if (ret < 0) {
return ZE_RESULT_ERROR_OUT_OF_HOST_MEMORY;
}
} else {
IpcMemoryData &ipcData = *reinterpret_cast<IpcMemoryData *>(ipcHandle.data);
handle = ipcData.handle;
}
extendedMemoryExportProperties->fd = static_cast<int>(handle);
} else if (extendedProperties->stype == ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_WIN32) {
Expand Down
17 changes: 17 additions & 0 deletions level_zero/core/source/context/context_imp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ struct IpcMemoryData {
#pragma pack()
static_assert(sizeof(IpcMemoryData) <= ZE_MAX_IPC_HANDLE_SIZE, "IpcMemoryData is bigger than ZE_MAX_IPC_HANDLE_SIZE");

struct IpcHandleTracking {
uint64_t refcnt = 0;
NEO::GraphicsAllocation *alloc = nullptr;
uint32_t handleId = 0;
uint64_t handle = 0;
uint64_t ptr = 0;
struct IpcMemoryData ipcData = {};
};

struct ContextImp : Context {
ContextImp(DriverHandle *driverHandle);
~ContextImp() override = default;
Expand Down Expand Up @@ -63,12 +72,15 @@ struct ContextImp : Context {
void **pBase,
size_t *pSize) override;
ze_result_t closeIpcMemHandle(const void *ptr) override;
ze_result_t putIpcMemHandle(ze_ipc_mem_handle_t ipcHandle) override;
ze_result_t getIpcMemHandle(const void *ptr,
ze_ipc_mem_handle_t *pIpcHandle) override;
ze_result_t openIpcMemHandle(ze_device_handle_t hDevice,
const ze_ipc_mem_handle_t &handle,
ze_ipc_memory_flags_t flags,
void **ptr) override;
ze_result_t getIpcHandleFromFd(uint64_t handle, ze_ipc_mem_handle_t *pIpcHandle) override;
ze_result_t getFdFromIpcHandle(ze_ipc_mem_handle_t ipcHandle, uint64_t *pHandle) override;

ze_result_t
getIpcMemHandles(
Expand Down Expand Up @@ -171,12 +183,17 @@ struct ContextImp : Context {
this->numDevices = static_cast<uint32_t>(this->deviceHandles.size());
}
NEO::VirtualMemoryReservation *findSupportedVirtualReservation(const void *ptr, size_t size);
std::map<uint64_t, IpcHandleTracking *> &getIPCHandleMap() { return this->ipcHandles; };
[[nodiscard]] std::unique_lock<std::mutex> lockIPCHandleMap() { return std::unique_lock<std::mutex>(this->ipcHandleMapMutex); };

protected:
void setIPCHandleData(NEO::GraphicsAllocation *graphicsAllocation, uint64_t handle, IpcMemoryData &ipcData, uint64_t ptrAddress);
bool isAllocationSuitableForCompression(const StructuresLookupTable &structuresLookupTable, Device &device, size_t allocSize);
size_t getPageSizeRequired(size_t size);

std::map<uint32_t, ze_device_handle_t> devices;
std::map<uint64_t, IpcHandleTracking *> ipcHandles;
std::mutex ipcHandleMapMutex;
std::vector<ze_device_handle_t> deviceHandles;
DriverHandleImp *driverHandle = nullptr;
uint32_t numDevices = 0;
Expand Down
4 changes: 2 additions & 2 deletions level_zero/core/test/unit_tests/fixtures/memory_ipc_fixture.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ struct ContextFdMock : public L0::ContextImp {
ze_memory_allocation_properties_t *pMemAllocProperties,
ze_device_handle_t *phDevice) override {
ze_result_t res = ContextImp::getMemAllocProperties(ptr, pMemAllocProperties, phDevice);
if (ZE_RESULT_SUCCESS == res && pMemAllocProperties->pNext) {
if (ZE_RESULT_SUCCESS == res && pMemAllocProperties->pNext && !memPropTest) {
ze_base_properties_t *baseProperties =
reinterpret_cast<ze_base_properties_t *>(pMemAllocProperties->pNext);
if (baseProperties->stype == ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD) {
Expand Down Expand Up @@ -104,7 +104,7 @@ struct ContextFdMock : public L0::ContextImp {
ze_result_t closeIpcMemHandle(const void *ptr) override {
return ZE_RESULT_SUCCESS;
}

bool memPropTest = false;
DriverHandleGetFdMock *driverHandle = nullptr;
};

Expand Down
Loading

0 comments on commit 364c2da

Please sign in to comment.