diff --git a/source/adapters/level_zero/usm.cpp b/source/adapters/level_zero/usm.cpp index 616b4c4d57..a25c57e21b 100644 --- a/source/adapters/level_zero/usm.cpp +++ b/source/adapters/level_zero/usm.cpp @@ -1,6 +1,6 @@ //===--------- usm.cpp - Level Zero Adapter -------------------------------===// // -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2024 Intel Corporation // // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM // Exceptions. See LICENSE.TXT @@ -18,6 +18,7 @@ #include "logger/ur_logger.hpp" #include "ur_level_zero.hpp" +#include "ur_util.hpp" #include @@ -766,6 +767,97 @@ umf_result_t L0MemoryProvider::get_min_page_size(void *Ptr, size_t *PageSize) { return UMF_RESULT_SUCCESS; } +typedef struct ze_ipc_data_t { + int pid; + ze_ipc_mem_handle_t zeHandle; +} ze_ipc_data_t; + +umf_result_t L0MemoryProvider::get_ipc_handle_size(size_t *Size) { + UR_ASSERT(Size, UMF_RESULT_ERROR_INVALID_ARGUMENT); + *Size = sizeof(ze_ipc_data_t); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t L0MemoryProvider::get_ipc_handle(const void *Ptr, size_t Size, + void *IpcData) { + std::ignore = Size; + + UR_ASSERT(Ptr && IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData; + auto Ret = ZE_CALL_NOCHECK(zeMemGetIpcHandle, + (Context->ZeContext, Ptr, &zeIpcData->zeHandle)); + if (Ret != ZE_RESULT_SUCCESS) { + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + zeIpcData->pid = ur_getpid(); + + return UMF_RESULT_SUCCESS; +} + +umf_result_t L0MemoryProvider::put_ipc_handle(void *IpcData) { + UR_ASSERT(IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData; + std::ignore = zeIpcData; + + // zeMemPutIpcHandle was introduced in Level Zero 1.6. Before Level Zero 1.6, + // IPC handle was released automatically when corresponding memory buffer + // was freed. +#if (ZE_API_VERSION_CURRENT >= ZE_MAKE_VERSION(1, 6)) + auto Ret = ZE_CALL_NOCHECK(zeMemPutIpcHandle, + (Context->ZeContext, zeIpcData->zeHandle)); + if (Ret != ZE_RESULT_SUCCESS) { + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } +#endif + + return UMF_RESULT_SUCCESS; +} + +umf_result_t L0MemoryProvider::open_ipc_handle(void *IpcData, void **Ptr) { + UR_ASSERT(IpcData && Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT); + ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData; + + int fdLocal = -1; + if (zeIpcData->pid != ur_getpid()) { + int fdRemote = -1; + memcpy(&fdRemote, &zeIpcData->zeHandle, sizeof(fdRemote)); + fdLocal = ur_duplicate_fd(zeIpcData->pid, fdRemote); + if (fdLocal == -1) { + logger::error("duplicating file descriptor from IPC handle failed"); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + memcpy(&zeIpcData->zeHandle, &fdLocal, sizeof(fdLocal)); + } + + auto Ret = + ZE_CALL_NOCHECK(zeMemOpenIpcHandle, (Context->ZeContext, Device->ZeDevice, + zeIpcData->zeHandle, 0, Ptr)); + if (fdLocal != -1) { + ur_close_fd(fdLocal); + } + + if (Ret != ZE_RESULT_SUCCESS) { + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + return UMF_RESULT_SUCCESS; +} + +umf_result_t L0MemoryProvider::close_ipc_handle(void *Ptr, size_t Size) { + std::ignore = Size; + + UR_ASSERT(Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT); + auto Ret = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, (Context->ZeContext, Ptr)); + if (Ret != ZE_RESULT_SUCCESS) { + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + return UMF_RESULT_SUCCESS; +} + ur_result_t L0SharedMemoryProvider::allocateImpl(void **ResultPtr, size_t Size, uint32_t Alignment) { return USMSharedAllocImpl(ResultPtr, Context, Device, /*host flags*/ 0, diff --git a/source/adapters/level_zero/usm.hpp b/source/adapters/level_zero/usm.hpp index 1f230a5f55..2fe74a5ecf 100644 --- a/source/adapters/level_zero/usm.hpp +++ b/source/adapters/level_zero/usm.hpp @@ -85,11 +85,26 @@ class USMMemoryProviderBase { virtual umf_result_t purge_force(void *, size_t) { return UMF_RESULT_ERROR_NOT_SUPPORTED; }; - umf_result_t allocation_merge(void *, void *, size_t) { - return UMF_RESULT_ERROR_UNKNOWN; + virtual umf_result_t allocation_merge(void *, void *, size_t) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + virtual umf_result_t allocation_split(void *, size_t, size_t) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + virtual umf_result_t get_ipc_handle_size(size_t *) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + virtual umf_result_t get_ipc_handle(const void *, size_t, void *) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + virtual umf_result_t put_ipc_handle(void *) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; } - umf_result_t allocation_split(void *, size_t, size_t) { - return UMF_RESULT_ERROR_UNKNOWN; + virtual umf_result_t open_ipc_handle(void *, void **) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + virtual umf_result_t close_ipc_handle(void *, size_t) { + return UMF_RESULT_ERROR_NOT_SUPPORTED; } virtual const char *get_name() { return ""; }; virtual ~USMMemoryProviderBase() = default; @@ -111,6 +126,11 @@ class L0MemoryProvider : public USMMemoryProviderBase { umf_result_t get_min_page_size(void *, size_t *) override; // TODO: Different name for each provider (Host/Shared/SharedRO/Device) const char *get_name() override { return "L0"; }; + umf_result_t get_ipc_handle_size(size_t *) override; + umf_result_t get_ipc_handle(const void *, size_t, void *) override; + umf_result_t put_ipc_handle(void *) override; + umf_result_t open_ipc_handle(void *, void **) override; + umf_result_t close_ipc_handle(void *, size_t) override; }; // Allocation routines for shared memory type diff --git a/source/common/umf_helpers.hpp b/source/common/umf_helpers.hpp index c381d05af7..e2e5b9a467 100644 --- a/source/common/umf_helpers.hpp +++ b/source/common/umf_helpers.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. * See LICENSE.TXT @@ -33,6 +33,35 @@ using provider_unique_handle_t = std::unique_ptr>; +#define DEFINE_CHECK_OP(op) \ + template class HAS_OP_##op { \ + typedef char check_success; \ + typedef long check_fail; \ + template static check_success test(decltype(&U::op)); \ + template static check_fail test(...); \ + \ + public: \ + static constexpr bool value = \ + sizeof(test(0)) == sizeof(check_success); \ + }; \ + \ + template \ + static inline \ + typename std::enable_if::value, umf_result_t>::type \ + CALL_OP_##op(T *t, Args &&...args) { \ + return t->op(std::forward(args)...); \ + }; \ + \ + static inline umf_result_t CALL_OP_##op(...) { \ + return UMF_RESULT_ERROR_NOT_SUPPORTED; \ + } + +DEFINE_CHECK_OP(get_ipc_handle_size); +DEFINE_CHECK_OP(get_ipc_handle); +DEFINE_CHECK_OP(put_ipc_handle); +DEFINE_CHECK_OP(open_ipc_handle); +DEFINE_CHECK_OP(close_ipc_handle); + #define UMF_ASSIGN_OP(ops, type, func, default_return) \ ops.func = [](void *obj, auto... args) { \ try { \ @@ -50,6 +79,15 @@ using provider_unique_handle_t = } \ } +#define UMF_ASSIGN_OP_OPT(ops, type, func, default_return) \ + ops.func = [](void *obj, auto... args) { \ + try { \ + return CALL_OP_##func(reinterpret_cast(obj), args...); \ + } catch (...) { \ + return default_return; \ + } \ + } + namespace detail { template umf_result_t initialize(T *obj, ArgsTuple &&args) { @@ -133,6 +171,12 @@ auto memoryProviderMakeUnique(Args &&...args) { UMF_ASSIGN_OP(ops.ext, T, purge_force, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops.ext, T, allocation_merge, UMF_RESULT_ERROR_UNKNOWN); UMF_ASSIGN_OP(ops.ext, T, allocation_split, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_OPT(ops.ipc, T, get_ipc_handle_size, + UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_OPT(ops.ipc, T, get_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_OPT(ops.ipc, T, put_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_OPT(ops.ipc, T, open_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); + UMF_ASSIGN_OP_OPT(ops.ipc, T, close_ipc_handle, UMF_RESULT_ERROR_UNKNOWN); umf_memory_provider_handle_t hProvider = nullptr; auto ret = umfMemoryProviderCreate(&ops, &argsTuple, &hProvider); diff --git a/source/common/ur_util.cpp b/source/common/ur_util.cpp index e486ff6e1a..d36c73ab91 100644 --- a/source/common/ur_util.cpp +++ b/source/common/ur_util.cpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. * See LICENSE.TXT @@ -9,15 +9,63 @@ */ #include "ur_util.hpp" +#include "logger/ur_logger.hpp" #ifdef _WIN32 #include int ur_getpid(void) { return static_cast(GetCurrentProcessId()); } + +int ur_close_fd(int fd) { return -1; } + +int ur_duplicate_fd(int pid, int fd_in) { + // TODO: find another way to obtain a duplicate of another process's file descriptor + (void)pid; // unused + (void)fd_in; // unused + return -1; +} + #else +#include #include int ur_getpid(void) { return static_cast(getpid()); } -#endif + +int ur_close_fd(int fd) { return close(fd); } + +int ur_duplicate_fd(int pid, int fd_in) { +// pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor. +// Permission to duplicate another process's file descriptor +// is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2)) +// that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface. +// pidfd_getfd(2) is supported since Linux 5.6 +// pidfd_open(2) is supported since Linux 5.3 +#if defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd) + errno = 0; + int pid_fd = syscall(SYS_pidfd_open, pid, 0); + if (pid_fd == -1) { + logger::error("SYS_pidfd_open"); + return -1; + } + + int fd_dup = syscall(SYS_pidfd_getfd, pid_fd, fd_in, 0); + close(pid_fd); + if (fd_dup == -1) { + logger::error("SYS_pidfd_getfd"); + return -1; + } + + return fd_dup; +#else + // TODO: find another way to obtain a duplicate of another process's file descriptor + (void)pid; // unused + (void)fd_in; // unused + errno = ENOTSUP; // unsupported + logger::error("__NR_pidfd_open or __NR_pidfd_getfd not available"); + return -1; +#endif /* defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd) */ +} + +#endif /* _WIN32 */ std::optional ur_getenv(const char *name) { #if defined(_WIN32) diff --git a/source/common/ur_util.hpp b/source/common/ur_util.hpp index e27d7103be..e24c1153c5 100644 --- a/source/common/ur_util.hpp +++ b/source/common/ur_util.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2022-2023 Intel Corporation + * Copyright (C) 2022-2024 Intel Corporation * * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. * See LICENSE.TXT @@ -27,6 +27,8 @@ #include int ur_getpid(void); +int ur_close_fd(int fd); +int ur_duplicate_fd(int pid, int fd_in); /* for compatibility with non-clang compilers */ #if defined(__has_feature) diff --git a/test/adapters/level_zero/CMakeLists.txt b/test/adapters/level_zero/CMakeLists.txt index 237a2bbe9b..a4dbb02c9b 100644 --- a/test/adapters/level_zero/CMakeLists.txt +++ b/test/adapters/level_zero/CMakeLists.txt @@ -69,4 +69,16 @@ if(NOT WIN32) target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE zeCallMap) endif() +add_adapter_test(level_zero_ipc + FIXTURE DEVICES + SOURCES + ipc.cpp + ENVIRONMENT + "UR_ADAPTERS_FORCE_LOAD=\"$\"" +) + +target_link_libraries(test-adapter-level_zero_ipc PRIVATE + ur_umf +) + add_subdirectory(v2) diff --git a/test/adapters/level_zero/ipc.cpp b/test/adapters/level_zero/ipc.cpp new file mode 100644 index 0000000000..58ea56ad7c --- /dev/null +++ b/test/adapters/level_zero/ipc.cpp @@ -0,0 +1,51 @@ +// Copyright (C) 2024 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + +#ifndef ASSERT_UMF_SUCCESS +#define ASSERT_UMF_SUCCESS(ACTUAL) ASSERT_EQ(ACTUAL, UMF_RESULT_SUCCESS) +#endif + +using urL0IpcTest = uur::urContextTest; +UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urL0IpcTest); + +TEST_P(urL0IpcTest, SuccessHostL0Ipc) { + ur_device_usm_access_capability_flags_t hostUSMSupport = 0; + ASSERT_SUCCESS(uur::GetDeviceUSMHostSupport(device, hostUSMSupport)); + if (!hostUSMSupport) { + GTEST_SKIP() << "Host USM is not supported."; + } + + void *ptr = nullptr; + size_t allocSize = sizeof(int); + ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, allocSize, &ptr)); + ASSERT_NE(ptr, nullptr); + + umf_memory_pool_handle_t umfPool = umfPoolByPtr(ptr); + ASSERT_NE(umfPool, nullptr); + + umf_memory_provider_handle_t umfProvider = nullptr; + ASSERT_UMF_SUCCESS(umfPoolGetMemoryProvider(umfPool, &umfProvider)); + + size_t ipcHandleSize = 0; + ASSERT_UMF_SUCCESS( + umfMemoryProviderGetIPCHandleSize(umfProvider, &ipcHandleSize)); + + void *ipcHandle = nullptr; + ASSERT_UMF_SUCCESS( + umfMemoryProviderAlloc(umfProvider, ipcHandleSize, 0, &ipcHandle)); + ASSERT_UMF_SUCCESS( + umfMemoryProviderGetIPCHandle(umfProvider, ptr, allocSize, ipcHandle)); + + ASSERT_UMF_SUCCESS(umfMemoryProviderPutIPCHandle(umfProvider, ipcHandle)); + + ASSERT_UMF_SUCCESS( + umfMemoryProviderFree(umfProvider, ipcHandle, ipcHandleSize)); + + ASSERT_SUCCESS(urUSMFree(context, ptr)); +}