Mirror intel/llvm commits

.github/intel-llvm-mirror-base-commit

@@ -1 +1 @@
-c270247bc15562ace836c4c0878046a86bbe87fa
+bf62e0bfd96cf67214624f3ea6ebe7575ed1fd2e

source/adapters/cuda/usm.cpp

@@ -94,10 +94,15 @@ urUSMSharedAlloc(ur_context_handle_t, ur_device_handle_t hDevice,
 
 /// USM: Frees the given USM pointer associated with the context.
 ///
-UR_APIEXPORT ur_result_t UR_APICALL urUSMFree(ur_context_handle_t hContext,
-                                              void *pMem) {
-  (void)hContext; // unused
-  return umf::umf2urResult(umfFree(pMem));
+UR_APIEXPORT ur_result_t UR_APICALL urUSMFree(ur_context_handle_t, void *pMem) {
+  umf_memory_pool_handle_t hPool = NULL;
+  umf_result_t ret = umfPoolByPtr(pMem, &hPool);
+  if (ret == UMF_RESULT_SUCCESS) {
+    return umf::umf2urResult(umfPoolFree(hPool, pMem));
+  } else if (ret == UMF_RESULT_ERROR_INVALID_ARGUMENT) {
+    return mapErrorUR(cuMemFree((CUdeviceptr)pMem));
+  }
+  return umf::umf2urResult(ret);
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL
@@ -178,8 +183,9 @@ urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem,
       return ReturnValue(Device);
     }
     case UR_USM_ALLOC_INFO_POOL: {
-      auto UMFPool = umfPoolByPtr(pMem);
-      if (!UMFPool) {
+      umf_memory_pool_handle_t UMFPool = nullptr;
+      auto UMFResult = umfPoolByPtr(pMem, &UMFPool);
+      if (UMFResult != UMF_RESULT_SUCCESS || !UMFPool) {
         return UR_RESULT_ERROR_INVALID_VALUE;
       }
       ur_usm_pool_handle_t Pool = hContext->getOwningURPool(UMFPool);

source/adapters/hip/usm.cpp

@@ -126,7 +126,8 @@ USMFreeImpl([[maybe_unused]] ur_context_handle_t hContext, void *pMem) {
 /// USM: Frees the given USM pointer associated with the context.
 UR_APIEXPORT ur_result_t UR_APICALL urUSMFree(ur_context_handle_t hContext,
                                               void *pMem) {
-  if (auto Pool = umfPoolByPtr(pMem)) {
+  umf_memory_pool_handle_t Pool = nullptr;
+  if (umfPoolByPtr(pMem, &Pool) == UMF_RESULT_SUCCESS && Pool) {
     return umf::umf2urResult(umfPoolFree(Pool, pMem));
   } else {
     return USMFreeImpl(hContext, pMem);
@@ -230,8 +231,9 @@ urUSMGetMemAllocInfo(ur_context_handle_t hContext, const void *pMem,
       return ReturnValue(Device);
     }
     case UR_USM_ALLOC_INFO_POOL: {
-      auto UMFPool = umfPoolByPtr(pMem);
-      if (!UMFPool) {
+      umf_memory_pool_handle_t UMFPool = nullptr;
+      auto UMFResult = umfPoolByPtr(pMem, &UMFPool);
+      if (UMFResult != UMF_RESULT_SUCCESS || !UMFPool) {
         return UR_RESULT_ERROR_INVALID_VALUE;
       }
       ur_usm_pool_handle_t Pool = hContext->getOwningURPool(UMFPool);
@@ -316,10 +318,11 @@ enum umf_result_t USMMemoryProvider::free(void *Ptr, size_t Size) {
   return UMF_RESULT_SUCCESS;
 }
 
-void USMMemoryProvider::get_last_native_error(const char **ErrMsg,
-                                              int32_t *ErrCode) {
+enum umf_result_t USMMemoryProvider::get_last_native_error(const char **ErrMsg,
+                                                           int32_t *ErrCode) {
   (void)ErrMsg;
   *ErrCode = static_cast<int32_t>(getLastStatusRef());
+  return UMF_RESULT_SUCCESS;
 }
 
 umf_result_t USMMemoryProvider::get_min_page_size(const void *Ptr,

source/adapters/hip/usm.hpp

@@ -61,24 +61,19 @@ class USMMemoryProvider {
   umf_result_t initialize(ur_context_handle_t Ctx, ur_device_handle_t Dev);
   umf_result_t alloc(size_t Size, size_t Align, void **Ptr);
   umf_result_t free(void *Ptr, size_t Size);
-  void get_last_native_error(const char **ErrMsg, int32_t *ErrCode);
+  umf_result_t get_last_native_error(const char **ErrMsg, int32_t *ErrCode);
   umf_result_t get_min_page_size(const void *, size_t *);
   umf_result_t get_recommended_page_size(size_t, size_t *) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   };
-  umf_result_t purge_lazy(void *, size_t) {
-    return UMF_RESULT_ERROR_NOT_SUPPORTED;
-  };
-  umf_result_t purge_force(void *, size_t) {
-    return UMF_RESULT_ERROR_NOT_SUPPORTED;
-  };
-  umf_result_t allocation_merge(void *, void *, size_t) {
-    return UMF_RESULT_ERROR_UNKNOWN;
-  }
-  umf_result_t allocation_split(void *, size_t, size_t) {
-    return UMF_RESULT_ERROR_UNKNOWN;
+  umf_result_t get_name(const char **Name) {
+    if (!Name) {
+      return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    *Name = "HIP";
+    return UMF_RESULT_SUCCESS;
   }
-  const char *get_name() { return "HIP"; }
 
   virtual ~USMMemoryProvider() = default;
 };

source/adapters/level_zero/async_alloc.cpp

@@ -235,18 +235,24 @@ ur_result_t urEnqueueUSMFreeExp(
                (ZeCommandList, WaitList.Length, WaitList.ZeEventList));
   }
 
-  auto hPool = umfPoolByPtr(Mem);
-  if (!hPool) {
+  umf_memory_pool_handle_t hPool = nullptr;
+  auto umfRet = umfPoolByPtr(Mem, &hPool);
+  if (umfRet != UMF_RESULT_SUCCESS || !hPool) {
     return USMFreeHelper(Queue->Context, Mem);
   }
 
   UsmPool *usmPool = nullptr;
-  auto ret = umfPoolGetTag(hPool, (void **)&usmPool);
-  if (ret != UMF_RESULT_SUCCESS || usmPool == nullptr) {
+  umfRet = umfPoolGetTag(hPool, (void **)&usmPool);
+  if (umfRet != UMF_RESULT_SUCCESS || usmPool == nullptr) {
+    return USMFreeHelper(Queue->Context, Mem);
+  }
+
+  size_t size = 0;
+  umfRet = umfPoolMallocUsableSize(hPool, Mem, &size);
+  if (umfRet != UMF_RESULT_SUCCESS) {
     return USMFreeHelper(Queue->Context, Mem);
   }
 
-  size_t size = umfPoolMallocUsableSize(hPool, Mem);
   (*Event)->RefCount.retain();
   usmPool->AsyncPool.insert(Mem, size, *Event, Queue);
 

source/adapters/level_zero/enqueued_pool.cpp

@@ -57,11 +57,14 @@ void EnqueuedPool::insert(void *Ptr, size_t Size, ur_event_handle_t Event,
 bool EnqueuedPool::cleanup() {
   auto Lock = std::lock_guard(Mutex);
   auto FreedAllocations = !Freelist.empty();
+
+  auto umfRet [[maybe_unused]] = UMF_RESULT_SUCCESS;
   for (auto It : Freelist) {
-    auto hPool = umfPoolByPtr(It.Ptr);
+    umf_memory_pool_handle_t hPool = nullptr;
+    umfRet = umfPoolByPtr(It.Ptr, &hPool);
     assert(hPool != nullptr);
 
-    auto umfRet [[maybe_unused]] = umfPoolFree(hPool, It.Ptr);
+    umfRet = umfPoolFree(hPool, It.Ptr);
     assert(umfRet == UMF_RESULT_SUCCESS);
 
     if (It.Event)
@@ -81,11 +84,13 @@ bool EnqueuedPool::cleanupForQueue(void *Queue) {
 
   bool FreedAllocations = false;
 
+  auto umfRet [[maybe_unused]] = UMF_RESULT_SUCCESS;
   while (It != Freelist.end() && It->Queue == Queue) {
-    auto hPool = umfPoolByPtr(It->Ptr);
+    umf_memory_pool_handle_t hPool = nullptr;
+    umfRet = umfPoolByPtr(It->Ptr, &hPool);
     assert(hPool != nullptr);
 
-    auto umfRet [[maybe_unused]] = umfPoolFree(hPool, It->Ptr);
+    umfRet = umfPoolFree(hPool, It->Ptr);
     assert(umfRet == UMF_RESULT_SUCCESS);
 
     if (It->Event)

source/adapters/level_zero/usm.cpp

@@ -44,34 +44,6 @@ ur_result_t getProviderNativeError(const char *providerName,
 usm::DisjointPoolAllConfigs DisjointPoolConfigInstance =
     InitializeDisjointPoolConfig();
 
-ur_result_t umf2urResult(umf_result_t umfResult) {
-  if (umfResult == UMF_RESULT_SUCCESS)
-    return UR_RESULT_SUCCESS;
-
-  switch (umfResult) {
-  case UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY:
-    return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY;
-  case UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC: {
-    auto hProvider = umfGetLastFailedMemoryProvider();
-    if (hProvider == nullptr)
-      return UR_RESULT_ERROR_UNKNOWN;
-
-    ur_result_t Err = UR_RESULT_ERROR_UNKNOWN;
-    umfMemoryProviderGetLastNativeError(hProvider, nullptr,
-                                        reinterpret_cast<int32_t *>(&Err));
-    return Err;
-  }
-  case UMF_RESULT_ERROR_INVALID_ARGUMENT:
-    return UR_RESULT_ERROR_INVALID_ARGUMENT;
-  case UMF_RESULT_ERROR_INVALID_ALIGNMENT:
-    return UR_RESULT_ERROR_UNSUPPORTED_ALIGNMENT;
-  case UMF_RESULT_ERROR_NOT_SUPPORTED:
-    return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
-  default:
-    return UR_RESULT_ERROR_UNKNOWN;
-  };
-}
-
 usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig() {
   const char *PoolUrTraceVal = std::getenv("UR_L0_USM_ALLOCATOR_TRACE");
   const char *PoolPiTraceVal =
@@ -472,8 +444,9 @@ ur_result_t urUSMGetMemAllocInfo(
     return ReturnValue(Size);
   }
   case UR_USM_ALLOC_INFO_POOL: {
-    auto UMFPool = umfPoolByPtr(Ptr);
-    if (!UMFPool) {
+    umf_memory_pool_handle_t UMFPool = nullptr;
+    auto umfRet = umfPoolByPtr(Ptr, &UMFPool);
+    if (umfRet != UMF_RESULT_SUCCESS || !UMFPool) {
       return UR_RESULT_ERROR_INVALID_VALUE;
     }
 
@@ -513,7 +486,7 @@ ur_result_t urUSMPoolCreate(
   } catch (const UsmAllocationException &Ex) {
     return Ex.getError();
   } catch (umf_result_t e) {
-    return umf2urResult(e);
+    return umf::umf2urResult(e);
   } catch (...) {
     return UR_RESULT_ERROR_UNKNOWN;
   }
@@ -808,15 +781,16 @@ typedef struct ze_ipc_data_t {
   ze_ipc_mem_handle_t zeHandle;
 } ze_ipc_data_t;
 
-umf_result_t L0MemoryProvider::get_ipc_handle_size(size_t *Size) {
+umf_result_t L0MemoryProvider::ext_get_ipc_handle_size(size_t *Size) {
   UR_ASSERT(Size, UMF_RESULT_ERROR_INVALID_ARGUMENT);
   *Size = sizeof(ze_ipc_data_t);
 
   return UMF_RESULT_SUCCESS;
 }
 
-umf_result_t L0MemoryProvider::get_ipc_handle(const void *Ptr, size_t /*Size*/,
-                                              void *IpcData) {
+umf_result_t L0MemoryProvider::ext_get_ipc_handle(const void *Ptr,
+                                                  size_t /*Size*/,
+                                                  void *IpcData) {
 
   UR_ASSERT(Ptr && IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT);
   ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData;
@@ -831,7 +805,7 @@ umf_result_t L0MemoryProvider::get_ipc_handle(const void *Ptr, size_t /*Size*/,
   return UMF_RESULT_SUCCESS;
 }
 
-umf_result_t L0MemoryProvider::put_ipc_handle(void *IpcData) {
+umf_result_t L0MemoryProvider::ext_put_ipc_handle(void *IpcData) {
   UR_ASSERT(IpcData, UMF_RESULT_ERROR_INVALID_ARGUMENT);
 
   // zeMemPutIpcHandle was introduced in Level Zero 1.6. Before Level Zero 1.6,
@@ -849,7 +823,7 @@ umf_result_t L0MemoryProvider::put_ipc_handle(void *IpcData) {
   return UMF_RESULT_SUCCESS;
 }
 
-umf_result_t L0MemoryProvider::open_ipc_handle(void *IpcData, void **Ptr) {
+umf_result_t L0MemoryProvider::ext_open_ipc_handle(void *IpcData, void **Ptr) {
   UR_ASSERT(IpcData && Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT);
   ze_ipc_data_t *zeIpcData = (ze_ipc_data_t *)IpcData;
 
@@ -880,7 +854,8 @@ umf_result_t L0MemoryProvider::open_ipc_handle(void *IpcData, void **Ptr) {
   return UMF_RESULT_SUCCESS;
 }
 
-umf_result_t L0MemoryProvider::close_ipc_handle(void *Ptr, size_t /*Size*/) {
+umf_result_t L0MemoryProvider::ext_close_ipc_handle(void *Ptr,
+                                                    size_t /*Size*/) {
 
   UR_ASSERT(Ptr, UMF_RESULT_ERROR_INVALID_ARGUMENT);
   auto Ret = ZE_CALL_NOCHECK(zeMemCloseIpcHandle, (Context->ZeContext, Ptr));
@@ -1307,15 +1282,16 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr,
     Context->MemAllocs.erase(It);
   }
 
-  auto hPool = umfPoolByPtr(Ptr);
-  if (!hPool) {
+  umf_memory_pool_handle_t hPool = nullptr;
+  auto umfRet = umfPoolByPtr(Ptr, &hPool);
+  if (umfRet != UMF_RESULT_SUCCESS || !hPool) {
     if (IndirectAccessTrackingEnabled)
       UR_CALL(ContextReleaseHelper(Context));
     return UR_RESULT_ERROR_INVALID_MEM_OBJECT;
   }
 
-  auto umfRet = umfPoolFree(hPool, Ptr);
+  umfRet = umfPoolFree(hPool, Ptr);
   if (IndirectAccessTrackingEnabled)
     UR_CALL(ContextReleaseHelper(Context));
-  return umf2urResult(umfRet);
+  return umf::umf2urResult(umfRet);
 }

source/adapters/level_zero/usm.hpp

@@ -86,9 +86,10 @@ class USMMemoryProviderBase {
   virtual ur_result_t allocateImpl(void **, size_t, uint32_t) = 0;
 
 public:
-  virtual void get_last_native_error(const char ** /*ErrMsg*/,
-                                     int32_t *ErrCode) {
+  virtual umf_result_t get_last_native_error(const char ** /*ErrMsg*/,
+                                             int32_t *ErrCode) {
     *ErrCode = static_cast<int32_t>(getLastStatusRef());
+    return UMF_RESULT_SUCCESS;
   };
   virtual umf_result_t initialize(ur_context_handle_t, ur_device_handle_t) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
@@ -105,34 +106,28 @@ class USMMemoryProviderBase {
   virtual umf_result_t get_recommended_page_size(size_t, size_t *) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   };
-  virtual umf_result_t purge_lazy(void *, size_t) {
-    return UMF_RESULT_ERROR_NOT_SUPPORTED;
-  };
-  virtual umf_result_t purge_force(void *, size_t) {
-    return UMF_RESULT_ERROR_NOT_SUPPORTED;
-  };
-  virtual umf_result_t allocation_merge(void *, void *, size_t) {
+  virtual umf_result_t ext_get_ipc_handle_size(size_t *) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t allocation_split(void *, size_t, size_t) {
+  virtual umf_result_t ext_get_ipc_handle(const void *, size_t, void *) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t get_ipc_handle_size(size_t *) {
+  virtual umf_result_t ext_put_ipc_handle(void *) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t get_ipc_handle(const void *, size_t, void *) {
+  virtual umf_result_t ext_open_ipc_handle(void *, void **) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t put_ipc_handle(void *) {
+  virtual umf_result_t ext_close_ipc_handle(void *, size_t) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t open_ipc_handle(void *, void **) {
+  virtual umf_result_t ext_ctl(int, const char *, void *, size_t,
+                               umf_ctl_query_type_t) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
   }
-  virtual umf_result_t close_ipc_handle(void *, size_t) {
+  virtual umf_result_t get_name(const char **) {
     return UMF_RESULT_ERROR_NOT_SUPPORTED;
-  }
-  virtual const char *get_name() { return ""; };
+  };
   virtual ~USMMemoryProviderBase() = default;
 };
 
@@ -151,12 +146,19 @@ class L0MemoryProvider : public USMMemoryProviderBase {
   umf_result_t free(void *Ptr, size_t Size) override;
   umf_result_t get_min_page_size(const void *, size_t *) override;
   // TODO: Different name for each provider (Host/Shared/SharedRO/Device)
-  const char *get_name() override { return "Level Zero"; };
-  umf_result_t get_ipc_handle_size(size_t *) override;
-  umf_result_t get_ipc_handle(const void *, size_t, void *) override;
-  umf_result_t put_ipc_handle(void *) override;
-  umf_result_t open_ipc_handle(void *, void **) override;
-  umf_result_t close_ipc_handle(void *, size_t) override;
+  umf_result_t get_name(const char **name) override {
+    if (!name) {
+      return UMF_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+
+    *name = "Level Zero";
+    return UMF_RESULT_SUCCESS;
+  };
+  umf_result_t ext_get_ipc_handle_size(size_t *) override;
+  umf_result_t ext_get_ipc_handle(const void *, size_t, void *) override;
+  umf_result_t ext_put_ipc_handle(void *) override;
+  umf_result_t ext_open_ipc_handle(void *, void **) override;
+  umf_result_t ext_close_ipc_handle(void *, size_t) override;
 };
 
 // Allocation routines for shared memory type