Support workload type for dynamic shaped models (#690)

Author: preetha-intel

onnxruntime/core/providers/openvino/backend_manager.cc

@@ -28,9 +28,10 @@ SessionContext& BackendManager::GetSessionContext() {
   return session_context_;
 }
 
-ov::CompiledModel& BackendManager::GetOVCompiledModel() {
-  ov::CompiledModel& ov_ptr = concrete_backend_->GetOVCompiledModel();
-  return (ov_ptr);
+ov::CompiledModel BackendManager::GetOVCompiledModel() {
+  if (concrete_backend_)
+    return concrete_backend_->GetOVCompiledModel();
+  return ov::CompiledModel();
 }
 
 BackendManager::BackendManager(SessionContext& session_context,

onnxruntime/core/providers/openvino/backend_manager.h

@@ -29,7 +29,7 @@ class BackendManager {
   void ShutdownBackendManager();
   SessionContext& GetSessionContext();
   Status ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& subgraph);
-  ov::CompiledModel& GetOVCompiledModel();
+  ov::CompiledModel GetOVCompiledModel();
 
  private:
   std::unique_ptr<ONNX_NAMESPACE::ModelProto> GetModelProtoFromFusedNode(

onnxruntime/core/providers/openvino/backends/basic_backend.h

@@ -82,7 +82,7 @@ class BasicBackend : public IBackend {
 
   void Infer(OrtKernelContext* context) override;
   ~BasicBackend() override = default;
-  ov::CompiledModel& GetOVCompiledModel() override {
+  ov::CompiledModel GetOVCompiledModel() override {
     return exe_network_.Get();
   }
 

onnxruntime/core/providers/openvino/ibackend.h

@@ -15,7 +15,7 @@ namespace openvino_ep {
 class IBackend {
  public:
   virtual void Infer(OrtKernelContext* context) = 0;
-  virtual ov::CompiledModel& GetOVCompiledModel() = 0;
+  virtual ov::CompiledModel GetOVCompiledModel() = 0;
   virtual ~IBackend() = default;
 };
 using ptr_stream_t = std::unique_ptr<std::istream>;

onnxruntime/core/providers/openvino/openvino_execution_provider.cc

@@ -238,10 +238,20 @@ common::Status OpenVINOExecutionProvider::SetEpDynamicOptions(gsl::span<const ch
         LOGS_DEFAULT(WARNING) << "Supported types are 'Efficient' and 'Default' \n";
       }
       if (workload_type != "") {
-        LOGS_DEFAULT(INFO) << "SetEpDynamicOptions - modifying: " << key << "/" << value;
+        LOGS_DEFAULT(VERBOSE) << "SetEpDynamicOptions - modifying: " << key << "/" << value;
         for (auto& backend : backend_managers_) {
-          ov::CompiledModel& ov_compiled_model = backend.GetOVCompiledModel();
-          ov_compiled_model.set_property(ov::workload_type(workload_type));
+            ov::CompiledModel ov_compiled_model = backend.GetOVCompiledModel();
+            if(ov_compiled_model) {
+              ov_compiled_model.set_property(ov::workload_type(workload_type));
+          } else {
+            LOGS_DEFAULT(VERBOSE) << "Model is not compiled in OV as its dynamic";
+            ov::AnyMap map;
+            map["WORKLOAD_TYPE"] = workload_type;
+            if (session_context_.device_type == "NPU")
+              session_context_.load_config["NPU"] = std::move(map);
+            else
+              ORT_THROW(" WORKLOAD_TYPE property is supported only for NPU");
+          }
         }
       }
     } else {