Fixes cl_gpu_metrics sample to pass test (#19)

jfedorov · web-flow · commit 4f8368040794 · 2023-11-27T14:37:03.000+01:00
* Fixes cl_gpu_metrics sample to pass test

Signed-off-by: gta &lt;julia.fedorova@intel.com&gt;

* Removes unnecessary empty lines

Signed-off-by: gta &lt;julia.fedorova@intel.com&gt;

---------

Signed-off-by: gta &lt;julia.fedorova@intel.com&gt;
diff --git a/samples/cl_gpu_metrics/cl_metric_collector.h b/samples/cl_gpu_metrics/cl_metric_collector.h
@@ -35,8 +35,12 @@ class ClMetricCollector {
     uint32_t sub_device_id =
       sub_device_string.empty() ? 0 : std::stoul(sub_device_string);
 
+    std::string order_string = utils::GetEnv("PTI_DEVICE_PCI_ORDER");
+    bool respect_device_pci_order =
+      order_string.empty() ? false : true;
+
     MetricDevice* metric_device =
-      MetricDevice::Create(device_id, sub_device_id);
+      MetricDevice::Create(device_id, sub_device_id, respect_device_pci_order);
     if (metric_device == nullptr) {
       std::cerr << "[WARNING] Unable to find MD library" << std::endl;
       return nullptr;
@@ -150,6 +154,8 @@ class ClMetricCollector {
     return calculated_reports;
   }
 
+  const utils::DeviceUUID* GetDeviceUUID () { return device_uuid_; };
+
   ClMetricCollector(const ClMetricCollector& copy) = delete;
   ClMetricCollector& operator=(const ClMetricCollector& copy) = delete;
 
@@ -161,6 +167,7 @@ class ClMetricCollector {
     PTI_ASSERT(device_ != nullptr);
     PTI_ASSERT(group_ != nullptr);
     PTI_ASSERT(set_ != nullptr);
+    device_uuid_ = const_cast<utils::DeviceUUID*>(device->GetDeviceUUID());
     EnableMetrics();
   }
 
@@ -256,6 +263,7 @@ class ClMetricCollector {
   std::thread* collector_thread_ = nullptr;
 
   std::vector<uint8_t> metric_storage_;
+  utils::DeviceUUID* device_uuid_ = nullptr;
 };
 
-#endif // PTI_SAMPLES_CL_GPU_METRICS_CL_METRIC_COLLECTOR_H_
+#endif // PTI_SAMPLES_CL_GPU_METRICS_CL_METRIC_COLLECTOR_H_
diff --git a/samples/cl_gpu_metrics/tool.cc b/samples/cl_gpu_metrics/tool.cc
@@ -49,13 +49,31 @@ static std::chrono::steady_clock::time_point start;
 extern "C" PTI_EXPORT
 void Usage() {
   std::cout <<
-    "Usage: ./cl_gpu_metrics[.exe] <application> <args>" <<
+    "Usage: ./cl_gpu_metrics[.exe] [options] <application> <args>" <<
     std::endl;
+  std::cout << "Options:" << std::endl;
+  std::cout <<
+    "--pci-order            " <<
+    "Enumerate devices in the order of their PCI addresses" <<
+    std::endl;
+  std::cout <<
+    "Set PTI_DEVICE_ID and PTI_SUB_DEVICE_ID to collect for specific device/sub-device" <<
+    std::endl;
+
 }
 
 extern "C" PTI_EXPORT
 int ParseArgs(int argc, char* argv[]) {
-  return 1;
+  int app_index = 1;
+  for (int i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "--pci-order") == 0 ){
+      utils::SetEnv("PTI_DEVICE_PCI_ORDER", "1");
+      ++app_index;
+    } else {
+      break;
+    }
+  }
+  return app_index;
 }
 
 extern "C" PTI_EXPORT
@@ -83,9 +101,9 @@ static KernelMap GetKernelMap() {
 
   int gpu_timestamp_id = metric_collector->GetMetricId("QueryBeginTime");
   PTI_ASSERT(gpu_timestamp_id >= 0);
-  int eu_active_id = metric_collector->GetMetricId("EuActive");
+  int eu_active_id = metric_collector->GetMetricId("XVE_ACTIVE");
   PTI_ASSERT(eu_active_id >= 0);
-  int eu_stall_id = metric_collector->GetMetricId("EuStall");
+  int eu_stall_id = metric_collector->GetMetricId("XVE_STALL");
   PTI_ASSERT(eu_stall_id >= 0);
 
   uint32_t report_size = metric_collector->GetReportSize();
@@ -171,7 +189,23 @@ static void PrintResults() {
   }
 
   std::cerr << std::endl;
-  std::cerr << "=== Device Metrics: ===" << std::endl;
+  const utils::DeviceUUID* device_uuid = metric_collector->GetDeviceUUID();
+  if (device_uuid == nullptr) {
+    std::cerr << "=== Device Metrics: ===" << std::endl;
+  } else {
+    std::cerr << "=== Device ( bdf:" << std::hex
+              << (uint32_t)device_uuid->pciBus
+              << ":" << (uint32_t)device_uuid->pciDevice
+              << "." << (uint32_t)device_uuid->pciFunction
+              << std::dec;
+    if (device_uuid->subDeviceId == 0) {
+      std::cerr << " root device";
+    } else{
+      std::cerr << " sub-device: " << (uint32_t)(device_uuid->subDeviceId - 1);
+    }
+    std::cerr << ") Metrics: ===" << std::dec << std::endl;
+  }
+
   std::cerr << std::endl;
   std::cerr << "Total Execution Time (ns): " << time.count() << std::endl;
   std::cerr << "Total Kernel Time (ns): " << total_duration << std::endl;
@@ -251,4 +285,4 @@ void DisableProfiling() {
     delete kernel_collector;
     delete metric_collector;
   }
-}
+}
diff --git a/tests/samples/cl_gpu_metrics.py b/tests/samples/cl_gpu_metrics.py
@@ -59,7 +59,7 @@ def run(path):
   app_folder = utils.get_sample_executable_path("cl_gemm")
   app_file = os.path.join(app_folder, "cl_gemm" + file_extention)
   command = [file_name_prefix + "cl_gpu_metrics" + file_extention,\
-    app_file, "gpu", "1024", "1"]
+    "--pci-order", app_file, "gpu", "1024", "1"]
   stdout, stderr = utils.run_process(command, path)
   if not stdout:
     return "stdout is empty"
diff --git a/utils/metric_device.h b/utils/metric_device.h
@@ -8,6 +8,7 @@
 #define PTI_UTILS_METRIC_DEVICE_H_
 
 #include <string.h>
+#include <map>
 
 #include "metric_utils.h"
 #include "pti_assert.h"
@@ -30,7 +31,7 @@ class MetricDevice {
 
     md::IAdapterGroupLatest* adapter_group = nullptr;
     md::TCompletionCode status = OpenAdapterGroup(&adapter_group);
-    PTI_ASSERT(status == md::CC_OK);
+    PTI_ASSERT(status == md::CC_OK || status == md::CC_ALREADY_INITIALIZED);
     PTI_ASSERT(adapter_group != nullptr);
 
     uint32_t device_count = adapter_group->GetParams()->AdapterCount;
@@ -64,7 +65,9 @@ class MetricDevice {
     return sub_device_count;
   }
 
-  static MetricDevice* Create(uint32_t device_id, uint32_t sub_device_id) {
+  static MetricDevice* Create(uint32_t device_id,
+                              uint32_t sub_device_id,
+                              bool respect_device_pci_order = false ) {
     SharedLibrary* lib = OpenMetricsLibrary();
     if (lib == nullptr) {
       return nullptr;
@@ -87,8 +90,32 @@ class MetricDevice {
       return nullptr;
     }
 
-    PTI_ASSERT(device_id < adapter_group->GetParams()->AdapterCount);
-    md::IAdapterLatest* adapter = adapter_group->GetAdapter(device_id);
+    uint32_t adapter_count = adapter_group->GetParams()->AdapterCount;
+    PTI_ASSERT(device_id < adapter_count);
+    md::IAdapterLatest* adapter = nullptr;
+    if (respect_device_pci_order) {
+      // needed to order adapters by PCI address
+      std::map<md::SAdapterParams_1_9,
+               md::IAdapter_1_11*,
+               utils::ComparatorPciAddress<md::SAdapterParams_1_9> > adapters_map;
+      for (uint32_t i = 0; i < adapter_count; i++){
+        md::IAdapter_1_11* ad = adapter_group->GetAdapter(i);
+        PTI_ASSERT(ad != nullptr);
+        const md::SAdapterParams_1_9* params = ad->GetParams();
+        adapters_map.insert(std::pair<md::SAdapterParams_1_9,md::IAdapter_1_11*>{*params, ad});
+      }
+      uint32_t id = 0;
+      for (const auto& entry : adapters_map ) {
+        if (id == device_id) {
+          adapter = entry.second;
+          break;
+        }
+        id++;
+      }
+
+    } else {
+      adapter = adapter_group->GetAdapter(device_id);
+    }
     PTI_ASSERT(adapter != nullptr);
 
     uint32_t sub_device_count = adapter->GetParams()->SubDevicesCount;
@@ -100,7 +127,8 @@ class MetricDevice {
     }
     PTI_ASSERT(status == md::CC_OK || status == md::CC_ALREADY_INITIALIZED);
 
-    return new MetricDevice(adapter_group, adapter, device, lib);
+    return new MetricDevice(adapter_group, adapter, device, lib,
+                           (sub_device_count == 0) ? 0 : sub_device_id + 1);
   }
 
   ~MetricDevice() {
@@ -170,6 +198,9 @@ class MetricDevice {
     return nullptr;
   }
 
+  const utils::DeviceUUID* GetDeviceUUID() {
+    return &device_uuid_;
+  }
 private:
   static SharedLibrary* OpenMetricsLibrary() {
     SharedLibrary* lib = nullptr;
@@ -184,13 +215,21 @@ class MetricDevice {
 
   MetricDevice(
       md::IAdapterGroupLatest* adapter_group, md::IAdapterLatest* adapter,
-      md::IMetricsDeviceLatest* device, SharedLibrary* lib)
+      md::IMetricsDeviceLatest* device, SharedLibrary* lib, uint32_t sub_device_index = 0)
       : adapter_group_(adapter_group), adapter_(adapter),
-        device_(device), lib_(lib) {}
+        device_(device), lib_(lib) {
+          device_uuid_.vendorID = static_cast<uint16_t>(adapter_->GetParams()->VendorId);
+          device_uuid_.deviceID = static_cast<uint16_t>(adapter_->GetParams()->DeviceId);
+          device_uuid_.pciBus = static_cast<uint8_t>(adapter_->GetParams()->BusNumber);
+          device_uuid_.pciDevice = static_cast<uint8_t>(adapter_->GetParams()->DeviceNumber);
+          device_uuid_.pciFunction = static_cast<uint8_t>(adapter_->GetParams()->FunctionNumber);
+          device_uuid_.subDeviceId = static_cast<uint8_t>(sub_device_index);
+        }
 
   md::IAdapterGroupLatest* adapter_group_ = nullptr;
   md::IAdapterLatest* adapter_ = nullptr;
   md::IMetricsDeviceLatest* device_ = nullptr;
+  utils::DeviceUUID  device_uuid_ ;
   SharedLibrary* lib_ = nullptr;
 };
 
diff --git a/utils/utils.h b/utils/utils.h
@@ -42,6 +42,18 @@
 
 namespace utils {
 
+struct DeviceUUID {
+  uint16_t vendorID;
+  uint16_t deviceID;
+  uint16_t revisionID;
+  uint16_t pciDomain;
+  uint8_t pciBus;
+  uint8_t pciDevice;
+  uint8_t pciFunction;
+  uint8_t reserved[4];
+  uint8_t subDeviceId;
+};
+
 struct Comparator {
   template<typename T>
   bool operator()(const T& left, const T& right) const {
@@ -52,6 +64,19 @@ struct Comparator {
   }
 };
 
+template<typename T>
+struct ComparatorPciAddress {
+  bool operator()(const T& left, const T& right) const {
+    if (left.BusNumber != right.BusNumber) {
+        return (left.BusNumber < right.BusNumber);
+    }
+    if (left.DeviceNumber != right.DeviceNumber) {
+        return (left.DeviceNumber < right.DeviceNumber);
+    }
+    return left.FunctionNumber < right.FunctionNumber;
+  }
+};
+
 #if defined(__gnu_linux__)
 
 inline uint64_t GetTime(clockid_t id) {
