Port [SYCL] [L0] Remove unneeded backwards compatibility of 2023.2 make_queue and get_native

Jaime Arteaga · Jaime Arteaga · commit 2922ae7f37de · 2023-05-26T07:32:39.000-07:00
intel#8871 Signed-off-by: Jaime Arteaga <jaime.a.arteaga.molina@intel.com>
diff --git a/sycl/plugins/level_zero/pi_level_zero.cpp b/sycl/plugins/level_zero/pi_level_zero.cpp
@@ -179,28 +179,6 @@ pi_result piextQueueCreate(pi_context Context, pi_device Device,
   return pi2ur::piextQueueCreate(Context, Device, Properties, Queue);
 }
 
-pi_result piextQueueCreate2(pi_context Context, pi_device Device,
-                            pi_queue_properties *Properties, pi_queue *Queue) {
-  return pi2ur::piextQueueCreate(Context, Device, Properties, Queue);
-}
-
-pi_result piextQueueGetNativeHandle2(pi_queue Queue,
-                                     pi_native_handle *NativeHandle,
-                                     int32_t *NativeHandleDesc) {
-  std::ignore = NativeHandleDesc;
-  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle);
-}
-
-pi_result piextQueueCreateWithNativeHandle2(
-    pi_native_handle NativeHandle, int32_t NativeHandleDesc, pi_context Context,
-    pi_device Device, bool OwnNativeHandle, pi_queue_properties *Properties,
-    pi_queue *Queue) {
-  std::ignore = NativeHandleDesc;
-  std::ignore = Properties;
-  return pi2ur::piextQueueCreateWithNativeHandle(NativeHandle, Context, Device,
-                                                 OwnNativeHandle, Queue);
-}
-
 pi_result piQueueGetInfo(pi_queue Queue, pi_queue_info ParamName,
                          size_t ParamValueSize, void *ParamValue,
                          size_t *ParamValueSizeRet) {
@@ -220,18 +198,23 @@ pi_result piQueueFinish(pi_queue Queue) { return pi2ur::piQueueFinish(Queue); }
 pi_result piQueueFlush(pi_queue Queue) { return pi2ur::piQueueFlush(Queue); }
 
 pi_result piextQueueGetNativeHandle(pi_queue Queue,
-                                    pi_native_handle *NativeHandle) {
+                                    pi_native_handle *NativeHandle,
+                                    int32_t *NativeHandleDesc) {
 
-  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle);
+  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle,
+                                          NativeHandleDesc);
 }
 
 pi_result piextQueueCreateWithNativeHandle(pi_native_handle NativeHandle,
+                                           int32_t NativeHandleDesc,
                                            pi_context Context, pi_device Device,
                                            bool OwnNativeHandle,
+                                           pi_queue_properties *Properties,
                                            pi_queue *Queue) {
 
-  return pi2ur::piextQueueCreateWithNativeHandle(NativeHandle, Context, Device,
-                                                 OwnNativeHandle, Queue);
+  return pi2ur::piextQueueCreateWithNativeHandle(
+      NativeHandle, NativeHandleDesc, Context, Device, OwnNativeHandle,
+      Properties, Queue);
 }
 
 pi_result piMemBufferCreate(pi_context Context, pi_mem_flags Flags, size_t Size,
diff --git a/sycl/plugins/unified_runtime/pi2ur.hpp b/sycl/plugins/unified_runtime/pi2ur.hpp
@@ -1326,17 +1326,10 @@ inline pi_result piQueueCreate(pi_context Context, pi_device Device,
   return pi2ur::piextQueueCreate(Context, Device, Properties, Queue);
 }
 
-inline pi_result piextQueueCreate2(pi_context context, pi_device device,
-                                   pi_queue_properties *properties,
-                                   pi_queue *queue) {
-  return pi2ur::piextQueueCreate(context, device, properties, queue);
-}
-
-inline pi_result piextQueueCreateWithNativeHandle(pi_native_handle NativeHandle,
-                                                  pi_context Context,
-                                                  pi_device Device,
-                                                  bool OwnNativeHandle,
-                                                  pi_queue *Queue) {
+inline pi_result piextQueueCreateWithNativeHandle(
+    pi_native_handle NativeHandle, int32_t NativeHandleDesc, pi_context Context,
+    pi_device Device, bool OwnNativeHandle, pi_queue_properties *Properties,
+    pi_queue *Queue) {
   PI_ASSERT(Context, PI_ERROR_INVALID_CONTEXT);
   PI_ASSERT(NativeHandle, PI_ERROR_INVALID_VALUE);
   PI_ASSERT(Queue, PI_ERROR_INVALID_QUEUE);
@@ -1348,29 +1341,45 @@ inline pi_result piextQueueCreateWithNativeHandle(pi_native_handle NativeHandle,
   ur_native_handle_t UrNativeHandle =
       reinterpret_cast<ur_native_handle_t>(NativeHandle);
   ur_queue_handle_t *UrQueue = reinterpret_cast<ur_queue_handle_t *>(Queue);
-  ur_queue_native_properties_t Properties{};
-  Properties.isNativeHandleOwned = OwnNativeHandle;
-  HANDLE_ERRORS(urQueueCreateWithNativeHandle(UrNativeHandle, UrContext,
-                                              UrDevice, &Properties, UrQueue));
-  return PI_SUCCESS;
-}
+  ur_queue_native_properties_t UrNativeProperties{};
+  UrNativeProperties.isNativeHandleOwned = OwnNativeHandle;
 
-inline pi_result piextQueueCreateWithNativeHandle2(
-    pi_native_handle nativeHandle, int32_t nativeHandleDesc, pi_context context,
-    pi_device device, bool pluginOwnsNativeHandle,
-    pi_queue_properties *Properties, pi_queue *queue) {
-  (void)nativeHandleDesc;
-  (void)Properties;
-  return pi2ur::piextQueueCreateWithNativeHandle(nativeHandle, context, device,
-                                                 pluginOwnsNativeHandle, queue);
+  ur_queue_properties_t UrProperties{};
+  UrProperties.stype = UR_STRUCTURE_TYPE_QUEUE_PROPERTIES;
+  if (Properties[1] & PI_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE)
+    UrProperties.flags |= UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE;
+  if (Properties[1] & PI_QUEUE_FLAG_PROFILING_ENABLE)
+    UrProperties.flags |= UR_QUEUE_FLAG_PROFILING_ENABLE;
+  if (Properties[1] & PI_QUEUE_FLAG_ON_DEVICE)
+    UrProperties.flags |= UR_QUEUE_FLAG_ON_DEVICE;
+  if (Properties[1] & PI_QUEUE_FLAG_ON_DEVICE_DEFAULT)
+    UrProperties.flags |= UR_QUEUE_FLAG_ON_DEVICE_DEFAULT;
+  if (Properties[1] & PI_EXT_ONEAPI_QUEUE_FLAG_DISCARD_EVENTS)
+    UrProperties.flags |= UR_QUEUE_FLAG_DISCARD_EVENTS;
+  if (Properties[1] & PI_EXT_ONEAPI_QUEUE_FLAG_PRIORITY_LOW)
+    UrProperties.flags |= UR_QUEUE_FLAG_PRIORITY_LOW;
+  if (Properties[1] & PI_EXT_ONEAPI_QUEUE_FLAG_PRIORITY_HIGH)
+    UrProperties.flags |= UR_QUEUE_FLAG_PRIORITY_HIGH;
+
+  UrNativeProperties.pNext = &UrProperties;
+
+  // TODO: How to pass this up in the urQueueCreateWithNativeHandle interface?
+  std::ignore = NativeHandleDesc;
+  HANDLE_ERRORS(urQueueCreateWithNativeHandle(
+      UrNativeHandle, UrContext, UrDevice, &UrNativeProperties, UrQueue));
+  return PI_SUCCESS;
 }
 
 inline pi_result piextQueueGetNativeHandle(pi_queue Queue,
-                                           pi_native_handle *NativeHandle) {
+                                           pi_native_handle *NativeHandle,
+                                           int32_t *NativeHandleDesc) {
 
   PI_ASSERT(Queue, PI_ERROR_INVALID_QUEUE);
   PI_ASSERT(NativeHandle, PI_ERROR_INVALID_VALUE);
 
+  // TODO: How to pass this up in the urQueueGetNativeHandle interface?
+  std::ignore = NativeHandleDesc;
+
   ur_queue_handle_t UrQueue = reinterpret_cast<ur_queue_handle_t>(Queue);
 
   ur_native_handle_t UrNativeQueue{};
@@ -1381,14 +1390,6 @@ inline pi_result piextQueueGetNativeHandle(pi_queue Queue,
   return PI_SUCCESS;
 }
 
-inline pi_result piextQueueGetNativeHandle2(pi_queue Queue,
-                                            pi_native_handle *NativeHandle,
-                                            int32_t *NativeHandleDesc) {
-
-  (void)NativeHandleDesc;
-  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle);
-}
-
 inline pi_result piQueueRelease(pi_queue Queue) {
   PI_ASSERT(Queue, PI_ERROR_INVALID_QUEUE);
 
diff --git a/sycl/plugins/unified_runtime/pi_unified_runtime.cpp b/sycl/plugins/unified_runtime/pi_unified_runtime.cpp
@@ -109,28 +109,6 @@ __SYCL_EXPORT pi_result piextQueueCreate(pi_context Context, pi_device Device,
   return pi2ur::piextQueueCreate(Context, Device, Properties, Queue);
 }
 
-__SYCL_EXPORT pi_result piextQueueCreate2(pi_context Context, pi_device Device,
-                                          pi_queue_properties *Properties,
-                                          pi_queue *Queue) {
-  return pi2ur::piextQueueCreate(Context, Device, Properties, Queue);
-}
-
-__SYCL_EXPORT pi_result piextQueueGetNativeHandle2(
-    pi_queue Queue, pi_native_handle *NativeHandle, int32_t *NativeHandleDesc) {
-  std::ignore = NativeHandleDesc;
-  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle);
-}
-
-__SYCL_EXPORT pi_result piextQueueCreateWithNativeHandle2(
-    pi_native_handle NativeHandle, int32_t NativeHandleDesc, pi_context Context,
-    pi_device Device, bool OwnNativeHandle, pi_queue_properties *Properties,
-    pi_queue *Queue) {
-  std::ignore = NativeHandleDesc;
-  std::ignore = Properties;
-  return pi2ur::piextQueueCreateWithNativeHandle(NativeHandle, Context, Device,
-                                                 OwnNativeHandle, Queue);
-}
-
 __SYCL_EXPORT pi_result piQueueRelease(pi_queue Queue) {
   return pi2ur::piQueueRelease(Queue);
 }
@@ -724,16 +702,19 @@ __SYCL_EXPORT pi_result piextContextCreateWithNativeHandle(
       NativeHandle, NumDevices, Devices, OwnNativeHandle, RetContext);
 }
 
-__SYCL_EXPORT pi_result
-piextQueueGetNativeHandle(pi_queue Queue, pi_native_handle *NativeHandle) {
-  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle);
+__SYCL_EXPORT pi_result piextQueueGetNativeHandle(
+    pi_queue Queue, pi_native_handle *NativeHandle, int32_t *NativeHandleDesc) {
+  return pi2ur::piextQueueGetNativeHandle(Queue, NativeHandle,
+                                          NativeHandleDesc);
 }
 
 __SYCL_EXPORT pi_result piextQueueCreateWithNativeHandle(
-    pi_native_handle NativeHandle, pi_context Context, pi_device Device,
-    bool OwnNativeHandle, pi_queue *Queue) {
-  return pi2ur::piextQueueCreateWithNativeHandle(NativeHandle, Context, Device,
-                                                 OwnNativeHandle, Queue);
+    pi_native_handle NativeHandle, int32_t NativeHandleDesc, pi_context Context,
+    pi_device Device, bool OwnNativeHandle, pi_queue_properties *Properties,
+    pi_queue *Queue) {
+  return pi2ur::piextQueueCreateWithNativeHandle(
+      NativeHandle, NativeHandleDesc, Context, Device, OwnNativeHandle,
+      Properties, Queue);
 }
 
 __SYCL_EXPORT pi_result piMemRelease(pi_mem Mem) {
@@ -1068,9 +1049,6 @@ __SYCL_EXPORT pi_result piPluginInit(pi_plugin *PluginInit) {
   _PI_API(piQueueFlush)
   _PI_API(piextQueueGetNativeHandle)
   _PI_API(piextQueueCreateWithNativeHandle)
-  _PI_API(piextQueueCreate2)
-  _PI_API(piextQueueGetNativeHandle2)
-  _PI_API(piextQueueCreateWithNativeHandle2)
 
   _PI_API(piProgramCreate)
   _PI_API(piProgramBuild)
diff --git a/sycl/plugins/unified_runtime/ur/adapters/level_zero/ur_level_zero_queue.cpp b/sycl/plugins/unified_runtime/ur/adapters/level_zero/ur_level_zero_queue.cpp
@@ -463,32 +463,66 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueGetNativeHandle(
   // Lock automatically releases when this goes out of scope.
   std::shared_lock<ur_shared_mutex> lock(Queue->Mutex);
 
-  auto ZeQueue = ur_cast<ze_command_queue_handle_t *>(NativeQueue);
-
-  // Extract a Level Zero compute queue handle from the given PI queue
+  // Get handle to this thread's queue group.
   auto &QueueGroup = Queue->getQueueGroup(false /*compute*/);
-  uint32_t QueueGroupOrdinalUnused;
-  *ZeQueue = QueueGroup.getZeQueue(&QueueGroupOrdinalUnused);
+
+  if (Queue->UsingImmCmdLists) {
+    auto ZeCmdList = ur_cast<ze_command_list_handle_t *>(NativeQueue);
+    // Extract the Level Zero command list handle from the given PI queue
+    *ZeCmdList = QueueGroup.getImmCmdList()->first;
+    // TODO: How to pass this up in the urQueueGetNativeHandle interface?
+    // *NativeHandleDesc = true;
+  } else {
+    auto ZeQueue = ur_cast<ze_command_queue_handle_t *>(NativeQueue);
+
+    // Extract a Level Zero compute queue handle from the given PI queue
+    auto &QueueGroup = Queue->getQueueGroup(false /*compute*/);
+    uint32_t QueueGroupOrdinalUnused;
+    *ZeQueue = QueueGroup.getZeQueue(&QueueGroupOrdinalUnused);
+    // TODO: How to pass this up in the urQueueGetNativeHandle interface?
+    // *NativeHandleDesc = false;
+  }
 
   return UR_RESULT_SUCCESS;
 }
 
+void ur_queue_handle_t_::pi_queue_group_t::setImmCmdList(
+    ze_command_list_handle_t ZeCommandList) {
+  ImmCmdLists = std::vector<ur_command_list_ptr_t>(
+      1,
+      Queue->CommandListMap
+          .insert(std::pair<ze_command_list_handle_t, pi_command_list_info_t>{
+              ZeCommandList, {nullptr, true, false, nullptr, 0}})
+          .first);
+}
+
 UR_APIEXPORT ur_result_t UR_APICALL urQueueCreateWithNativeHandle(
     ur_native_handle_t NativeQueue, ///< [in] the native handle of the queue.
     ur_context_handle_t Context,    ///< [in] handle of the context object
     ur_device_handle_t Device,      ///
-    const ur_queue_native_properties_t *Properties, ///
+    const ur_queue_native_properties_t *NativeProperties, ///
     ur_queue_handle_t
         *RetQueue ///< [out] pointer to the handle of the queue object created.
 ) {
-  auto ZeQueue = ur_cast<ze_command_queue_handle_t>(NativeQueue);
-  // Assume this is the "0" index queue in the compute command-group.
-  std::vector<ze_command_queue_handle_t> ZeQueues{ZeQueue};
+  bool OwnNativeHandle = false;
+  ur_queue_flags_t Flags{};
 
-  // TODO: see what we can do to correctly initialize PI queue for
-  // compute vs. copy Level-Zero queue. Currently we will send
-  // all commands to the "ZeQueue".
-  std::vector<ze_command_queue_handle_t> ZeroCopyQueues;
+  if (NativeProperties) {
+    OwnNativeHandle = NativeProperties->isNativeHandleOwned;
+    if (NativeProperties->pNext) {
+      const ur_base_properties_t *extendedProperties =
+          reinterpret_cast<const ur_base_properties_t *>(
+              NativeProperties->pNext);
+      if (extendedProperties->stype == UR_STRUCTURE_TYPE_QUEUE_PROPERTIES) {
+        const ur_queue_properties_t *UrProperties =
+            reinterpret_cast<const ur_queue_properties_t *>(extendedProperties);
+        Flags = UrProperties->flags;
+      }
+    }
+  }
+
+  // TODO: How to pass this up in the urQueueCreateWithNativeHandle interface?
+  int32_t NativeHandleDesc = 0;
 
   // Get the device handle from first device in the platform
   // Maybe this is not completely correct.
@@ -502,15 +536,42 @@ UR_APIEXPORT ur_result_t UR_APICALL urQueueCreateWithNativeHandle(
                         nullptr));
   }
 
-  try {
-    ur_queue_handle_t_ *Queue =
-        new ur_queue_handle_t_(ZeQueues, ZeroCopyQueues, Context, UrDevice,
-                               Properties->isNativeHandleOwned);
-    *RetQueue = reinterpret_cast<ur_queue_handle_t>(Queue);
-  } catch (const std::bad_alloc &) {
-    return UR_RESULT_ERROR_OUT_OF_RESOURCES;
-  } catch (...) {
-    return UR_RESULT_ERROR_UNKNOWN;
+  // The NativeHandleDesc has value if if the native handle is an immediate
+  // command list.
+  if (NativeHandleDesc == 1) {
+    std::vector<ze_command_queue_handle_t> ComputeQueues{nullptr};
+    std::vector<ze_command_queue_handle_t> CopyQueues;
+
+    try {
+      ur_queue_handle_t_ *Queue = new ur_queue_handle_t_(
+          ComputeQueues, CopyQueues, Context, UrDevice, OwnNativeHandle, Flags);
+      *RetQueue = reinterpret_cast<ur_queue_handle_t>(Queue);
+    } catch (const std::bad_alloc &) {
+      return UR_RESULT_ERROR_OUT_OF_RESOURCES;
+    } catch (...) {
+      return UR_RESULT_ERROR_UNKNOWN;
+    }
+    auto &InitialGroup = (*RetQueue)->ComputeQueueGroupsByTID.begin()->second;
+    InitialGroup.setImmCmdList(ur_cast<ze_command_list_handle_t>(NativeQueue));
+  } else {
+    auto ZeQueue = ur_cast<ze_command_queue_handle_t>(NativeQueue);
+    // Assume this is the "0" index queue in the compute command-group.
+    std::vector<ze_command_queue_handle_t> ZeQueues{ZeQueue};
+
+    // TODO: see what we can do to correctly initialize PI queue for
+    // compute vs. copy Level-Zero queue. Currently we will send
+    // all commands to the "ZeQueue".
+    std::vector<ze_command_queue_handle_t> ZeroCopyQueues;
+
+    try {
+      ur_queue_handle_t_ *Queue = new ur_queue_handle_t_(
+          ZeQueues, ZeroCopyQueues, Context, UrDevice, OwnNativeHandle, Flags);
+      *RetQueue = reinterpret_cast<ur_queue_handle_t>(Queue);
+    } catch (const std::bad_alloc &) {
+      return UR_RESULT_ERROR_OUT_OF_RESOURCES;
+    } catch (...) {
+      return UR_RESULT_ERROR_UNKNOWN;
+    }
   }
 
   return UR_RESULT_SUCCESS;
@@ -757,6 +818,8 @@ ur_queue_handle_t_::ur_queue_handle_t_(
     bool OwnZeCommandQueue, ur_queue_flags_t Properties, int ForceComputeIndex)
     : Context{Context}, Device{Device}, OwnZeCommandQueue{OwnZeCommandQueue},
       Properties(Properties) {
+  // Set the type of commandlists the queue will use.
+  UsingImmCmdLists = Device->useImmediateCommandLists();
   // Compute group initialization.
   // First, see if the queue's device allows for round-robin or it is
   // fixed to one particular compute CCS (it is so for sub-sub-devices).
@@ -766,7 +829,7 @@ ur_queue_handle_t_::ur_queue_handle_t_(
   ComputeQueueGroup.ZeQueues = ComputeQueues;
   // Create space to hold immediate commandlists corresponding to the
   // ZeQueues
-  if (Device->ImmCommandListUsed) {
+  if (UsingImmCmdLists) {
     ComputeQueueGroup.ImmCmdLists = std::vector<ur_command_list_ptr_t>(
         ComputeQueueGroup.ZeQueues.size(), CommandListMap.end());
   }
@@ -798,7 +861,7 @@ ur_queue_handle_t_::ur_queue_handle_t_(
       die("No compute queue available/allowed.");
     }
   }
-  if (Device->ImmCommandListUsed) {
+  if (UsingImmCmdLists) {
     // Create space to hold immediate commandlists corresponding to the
     // ZeQueues
     ComputeQueueGroup.ImmCmdLists = std::vector<ur_command_list_ptr_t>(
diff --git a/sycl/plugins/unified_runtime/ur/adapters/level_zero/ur_level_zero_queue.hpp b/sycl/plugins/unified_runtime/ur/adapters/level_zero/ur_level_zero_queue.hpp
