[L0] Implement IPC ops for L0 adapter

source/adapters/level_zero/usm.cpp

@@ -1,6 +1,6 @@
 //===--------- usm.cpp - Level Zero Adapter -------------------------------===//
 //
-// Copyright (C) 2023 Intel Corporation
+// Copyright (C) 2023-2024 Intel Corporation
 //
 // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM
 // Exceptions. See LICENSE.TXT
@@ -18,6 +18,7 @@
 
 #include "logger/ur_logger.hpp"
 #include "ur_level_zero.hpp"
+#include "ur_util.hpp"
 
 #include <umf_helpers.hpp>
 
@@ -766,6 +767,97 @@ umf_result_t L0MemoryProvider::get_min_page_size(void *Ptr, size_t *PageSize) {
   return UMF_RESULT_SUCCESS;
 }
 
+typedef struct ze_ipc_data_t {
+  int pid;
+  ze_ipc_mem_handle_t zeHandle;
+} ze_ipc_data_t;
+
+umf_result_t L0MemoryProvider::get_ipc_handle_size(size_t *Size) {
+  UR_ASSERT(Size, UMF_RESULT_ERROR_INVALID_ARGUMENT);
+  *Size = sizeof(ze_ipc_data_t);
+
+  return UMF_RESULT_SUCCESS;
+}
+
+umf_result_t L0MemoryProvider::get_ipc_handle(const void *Ptr, size_t Size,
+                                              void *IpcData) {
+  std::ignore = Size;
+
+  UR_ASSERT(Ptr && IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT);
+  ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData;
+  auto Ret = ZE_CALL_NOCHECK(zeMemGetIpcHandle,
+                             (Context->ZeContext, Ptr, &zeIpcData->zeHandle));
+  if (Ret != ZE_RESULT_SUCCESS) {
+    return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
+  }
+
+  zeIpcData->pid = ur_getpid();
+
+  return UMF_RESULT_SUCCESS;
+}
+
+umf_result_t L0MemoryProvider::put_ipc_handle(void *IpcData) {
+  UR_ASSERT(IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT);
+  ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData;
+  std::ignore = zeIpcData;
+
+  // zeMemPutIpcHandle was introduced in Level Zero 1.6. Before Level Zero 1.6,
+  // IPC handle was released automatically when corresponding memory buffer
+  // was freed.
+#if (ZE_API_VERSION_CURRENT >= ZE_MAKE_VERSION(1, 6))
+  auto Ret = ZE_CALL_NOCHECK(zeMemPutIpcHandle,
+                             (Context->ZeContext, zeIpcData->zeHandle));
+  if (Ret != ZE_RESULT_SUCCESS) {
+    return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
+  }
+#endif
+
+  return UMF_RESULT_SUCCESS;
+}
+
+umf_result_t L0MemoryProvider::open_ipc_handle(void *IpcData, void **Ptr) {
+  UR_ASSERT(IpcData && Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT);
+  ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData;
+
+  int fdLocal = -1;
+  if (zeIpcData->pid != ur_getpid()) {
+    int fdRemote = -1;
+    memcpy(&fdRemote, &zeIpcData->zeHandle, sizeof(fdRemote));
+    fdLocal = ur_duplicate_fd(zeIpcData->pid, fdRemote);
+    if (fdLocal == -1) {
+      logger::error("duplicating file descriptor from IPC handle failed");
+      return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
+    }
+
+    memcpy(&zeIpcData->zeHandle, &fdLocal, sizeof(fdLocal));
+  }
+
+  auto Ret =
+      ZE_CALL_NOCHECK(zeMemOpenIpcHandle, (Context->ZeContext, Device->ZeDevice,
+                                           zeIpcData->zeHandle, 0, Ptr));
+  if (fdLocal != -1) {
+    ur_close_fd(fdLocal);
+  }
+
+  if (Ret != ZE_RESULT_SUCCESS) {
+    return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
+  }
+
+  return UMF_RESULT_SUCCESS;
+}
+
+umf_result_t L0MemoryProvider::close_ipc_handle(void *Ptr, size_t Size) {
+  std::ignore = Size;
+
+  UR_ASSERT(Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT);
+  auto Ret = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, (Context->ZeContext, Ptr));
+  if (Ret != ZE_RESULT_SUCCESS) {
+    return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC;
+  }
+
+  return UMF_RESULT_SUCCESS;
+}
+
 ur_result_t L0SharedMemoryProvider::allocateImpl(void **ResultPtr, size_t Size,
                                                  uint32_t Alignment) {
   return USMSharedAllocImpl(ResultPtr, Context, Device, /*host flags*/ 0,

source/adapters/level_zero/usm.hpp

@@ -85,11 +85,26 @@ class USMMemoryProviderBase {
   virtual umf_result_t purge_force(void *, size_t) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   };
-  umf_result_t allocation_merge(void *, void *, size_t) {
-    return UMF_RESULT_ERROR_UNKNOWN;
+  virtual umf_result_t allocation_merge(void *, void *, size_t) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+  }
+  virtual umf_result_t allocation_split(void *, size_t, size_t) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+  }
+  virtual umf_result_t get_ipc_handle_size(size_t *) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+  }
+  virtual umf_result_t get_ipc_handle(const void *, size_t, void *) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+  }
+  virtual umf_result_t put_ipc_handle(void *) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  umf_result_t allocation_split(void *, size_t, size_t) {
-    return UMF_RESULT_ERROR_UNKNOWN;
+  virtual umf_result_t open_ipc_handle(void *, void **) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
+  }
+  virtual umf_result_t close_ipc_handle(void *, size_t) {
+    return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
   virtual const char *get_name() { return ""; };
   virtual ~USMMemoryProviderBase() = default;
@@ -111,6 +126,11 @@ class L0MemoryProvider : public USMMemoryProviderBase {
   umf_result_t get_min_page_size(void *, size_t *) override;
   // TODO: Different name for each provider (Host/Shared/SharedRO/Device)
   const char *get_name() override { return "L0"; };
+  umf_result_t get_ipc_handle_size(size_t *) override;
+  umf_result_t get_ipc_handle(const void *, size_t, void *) override;
+  umf_result_t put_ipc_handle(void *) override;
+  umf_result_t open_ipc_handle(void *, void **) override;
+  umf_result_t close_ipc_handle(void *, size_t) override;
 };
 
 // Allocation routines for shared memory type

source/common/umf_helpers.hpp

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright (C) 2023 Intel Corporation
+ * Copyright (C) 2023-2024 Intel Corporation
  *
  * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
  * See LICENSE.TXT
@@ -33,6 +33,35 @@ using provider_unique_handle_t =
     std::unique_ptr<umf_memory_provider_t,
                     std::function<void(umf_memory_provider_handle_t)>>;
 
+#define DEFINE_CHECK_OP(op)                                                    \
+    template <typename T> class HAS_OP_##op {                                  \
+        typedef char check_success;                                            \
+        typedef long check_fail;                                               \
+        template <typename U> static check_success test(decltype(&U::op));     \
+        template <typename U> static check_fail test(...);                     \
+                                                                               \
+      public:                                                                  \
+        static constexpr bool value =                                          \
+            sizeof(test<T>(0)) == sizeof(check_success);                       \
+    };                                                                         \
+                                                                               \
+    template <typename T, typename... Args>                                    \
+    static inline                                                              \
+        typename std::enable_if<HAS_OP_##op<T>::value, umf_result_t>::type     \
+            CALL_OP_##op(T *t, Args &&...args) {                               \
+        return t->op(std::forward<Args>(args)...);                             \
+    };                                                                         \
+                                                                               \
+    static inline umf_result_t CALL_OP_##op(...) {                             \
+        return UMF_RESULT_ERROR_NOT_SUPPORTED;                                 \
+    }
+
+DEFINE_CHECK_OP(get_ipc_handle_size);
+DEFINE_CHECK_OP(get_ipc_handle);
+DEFINE_CHECK_OP(put_ipc_handle);
+DEFINE_CHECK_OP(open_ipc_handle);
+DEFINE_CHECK_OP(close_ipc_handle);
+
 #define UMF_ASSIGN_OP(ops, type, func, default_return)                         \
     ops.func = [](void *obj, auto... args) {                                   \
         try {                                                                  \
@@ -50,6 +79,15 @@ using provider_unique_handle_t =
         }                                                                      \
     }
 
+#define UMF_ASSIGN_OP_OPT(ops, type, func, default_return)                     \
+    ops.func = [](void *obj, auto... args) {                                   \
+        try {                                                                  \
+            return CALL_OP_##func(reinterpret_cast<type *>(obj), args...);     \
+        } catch (...) {                                                        \
+            return default_return;                                             \
+        }                                                                      \
+    }
+
 namespace detail {
 template <typename T, typename ArgsTuple>
 umf_result_t initialize(T *obj, ArgsTuple &&args) {
@@ -133,6 +171,12 @@ auto memoryProviderMakeUnique(Args &&...args) {
     UMF_ASSIGN_OP(ops.ext, T, purge_force, UMF_RESULT_ERROR_UNKNOWN);
     UMF_ASSIGN_OP(ops.ext, T, allocation_merge, UMF_RESULT_ERROR_UNKNOWN);
     UMF_ASSIGN_OP(ops.ext, T, allocation_split, UMF_RESULT_ERROR_UNKNOWN);
+    UMF_ASSIGN_OP_OPT(ops.ipc, T, get_ipc_handle_size,
+                      UMF_RESULT_ERROR_UNKNOWN);
+    UMF_ASSIGN_OP_OPT(ops.ipc, T, get_ipc_handle, UMF_RESULT_ERROR_UNKNOWN);
+    UMF_ASSIGN_OP_OPT(ops.ipc, T, put_ipc_handle, UMF_RESULT_ERROR_UNKNOWN);
+    UMF_ASSIGN_OP_OPT(ops.ipc, T, open_ipc_handle, UMF_RESULT_ERROR_UNKNOWN);
+    UMF_ASSIGN_OP_OPT(ops.ipc, T, close_ipc_handle, UMF_RESULT_ERROR_UNKNOWN);
 
     umf_memory_provider_handle_t hProvider = nullptr;
     auto ret = umfMemoryProviderCreate(&ops, &argsTuple, &hProvider);

source/common/ur_util.cpp

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2024 Intel Corporation
  *
  * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
  * See LICENSE.TXT
@@ -9,15 +9,63 @@
  */
 
 #include "ur_util.hpp"
+#include "logger/ur_logger.hpp"
 
 #ifdef _WIN32
 #include <windows.h>
 int ur_getpid(void) { return static_cast<int>(GetCurrentProcessId()); }
+
+int ur_close_fd(int fd) { return -1; }
+
+int ur_duplicate_fd(int pid, int fd_in) {
+    // TODO: find another way to obtain a duplicate of another process's file descriptor
+    (void)pid;   // unused
+    (void)fd_in; // unused
+    return -1;
+}
+
 #else
 
+#include <sys/syscall.h>
 #include <unistd.h>
 int ur_getpid(void) { return static_cast<int>(getpid()); }
-#endif
+
+int ur_close_fd(int fd) { return close(fd); }
+
+int ur_duplicate_fd(int pid, int fd_in) {
+// pidfd_getfd(2) is used to obtain a duplicate of another process's file descriptor.
+// Permission to duplicate another process's file descriptor
+// is governed by a ptrace access mode PTRACE_MODE_ATTACH_REALCREDS check (see ptrace(2))
+// that can be changed using the /proc/sys/kernel/yama/ptrace_scope interface.
+// pidfd_getfd(2) is supported since Linux 5.6
+// pidfd_open(2) is supported since Linux 5.3
+#if defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd)
+    errno = 0;
+    int pid_fd = syscall(SYS_pidfd_open, pid, 0);
+    if (pid_fd == -1) {
+        logger::error("SYS_pidfd_open");
+        return -1;
+    }
+
+    int fd_dup = syscall(SYS_pidfd_getfd, pid_fd, fd_in, 0);
+    close(pid_fd);
+    if (fd_dup == -1) {
+        logger::error("SYS_pidfd_getfd");
+        return -1;
+    }
+
+    return fd_dup;
+#else
+    // TODO: find another way to obtain a duplicate of another process's file descriptor
+    (void)pid;       // unused
+    (void)fd_in;     // unused
+    errno = ENOTSUP; // unsupported
+    logger::error("__NR_pidfd_open or __NR_pidfd_getfd not available");
+    return -1;
+#endif /* defined(__NR_pidfd_open) && defined(__NR_pidfd_getfd) */
+}
+
+#endif /* _WIN32 */
 
 std::optional<std::string> ur_getenv(const char *name) {
 #if defined(_WIN32)

source/common/ur_util.hpp

@@ -1,6 +1,6 @@
 /*
  *
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2024 Intel Corporation
  *
  * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
  * See LICENSE.TXT
@@ -27,6 +27,8 @@
 #include <vector>
 
 int ur_getpid(void);
+int ur_close_fd(int fd);
+int ur_duplicate_fd(int pid, int fd_in);
 
 /* for compatibility with non-clang compilers */
 #if defined(__has_feature)

test/adapters/level_zero/CMakeLists.txt

@@ -69,4 +69,16 @@ if(NOT WIN32)
     target_link_libraries(test-adapter-level_zero_multi_queue PRIVATE zeCallMap)
 endif()
 
+add_adapter_test(level_zero_ipc
+    FIXTURE DEVICES
+    SOURCES
+        ipc.cpp
+    ENVIRONMENT
+        "UR_ADAPTERS_FORCE_LOAD=\"$<TARGET_FILE:ur_adapter_level_zero>\""
+)
+
+target_link_libraries(test-adapter-level_zero_ipc PRIVATE
+    ur_umf
+)
+
 add_subdirectory(v2)

test/adapters/level_zero/ipc.cpp

@@ -0,0 +1,51 @@
+// Copyright (C) 2024 Intel Corporation
+// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+// See LICENSE.TXT
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <umf/memory_pool.h>
+#include <umf/memory_provider.h>
+#include <uur/fixtures.h>
+
+#ifndef ASSERT_UMF_SUCCESS
+#define ASSERT_UMF_SUCCESS(ACTUAL) ASSERT_EQ(ACTUAL, UMF_RESULT_SUCCESS)
+#endif
+
+using urL0IpcTest = uur::urContextTest;
+UUR_INSTANTIATE_DEVICE_TEST_SUITE_P(urL0IpcTest);
+
+TEST_P(urL0IpcTest, SuccessHostL0Ipc) {
+    ur_device_usm_access_capability_flags_t hostUSMSupport = 0;
+    ASSERT_SUCCESS(uur::GetDeviceUSMHostSupport(device, hostUSMSupport));
+    if (!hostUSMSupport) {
+        GTEST_SKIP() << "Host USM is not supported.";
+    }
+
+    void *ptr = nullptr;
+    size_t allocSize = sizeof(int);
+    ASSERT_SUCCESS(urUSMHostAlloc(context, nullptr, nullptr, allocSize, &ptr));
+    ASSERT_NE(ptr, nullptr);
+
+    umf_memory_pool_handle_t umfPool = umfPoolByPtr(ptr);
+    ASSERT_NE(umfPool, nullptr);
+
+    umf_memory_provider_handle_t umfProvider = nullptr;
+    ASSERT_UMF_SUCCESS(umfPoolGetMemoryProvider(umfPool, &umfProvider));
+
+    size_t ipcHandleSize = 0;
+    ASSERT_UMF_SUCCESS(
+        umfMemoryProviderGetIPCHandleSize(umfProvider, &ipcHandleSize));
+
+    void *ipcHandle = nullptr;
+    ASSERT_UMF_SUCCESS(
+        umfMemoryProviderAlloc(umfProvider, ipcHandleSize, 0, &ipcHandle));
+    ASSERT_UMF_SUCCESS(
+        umfMemoryProviderGetIPCHandle(umfProvider, ptr, allocSize, ipcHandle));
+
+    ASSERT_UMF_SUCCESS(umfMemoryProviderPutIPCHandle(umfProvider, ipcHandle));
+
+    ASSERT_UMF_SUCCESS(
+        umfMemoryProviderFree(umfProvider, ipcHandle, ipcHandleSize));
+
+    ASSERT_SUCCESS(urUSMFree(context, ptr));
+}