ci: Upgrade OpenVINO model version to 2024.05.0

build.py

@@ -75,8 +75,8 @@
     "triton_container_version": "24.12",
     "upstream_container_version": "24.12",
     "ort_version": "1.20.1",
-    "ort_openvino_version": "2024.5.0",
-    "standalone_openvino_version": "2024.5.0",
+    "ort_openvino_version": "2024.4.0",
+    "standalone_openvino_version": "2024.4.0",
     "dcgm_version": "3.3.6",
     "vllm_version": "0.5.5",
     "rhel_py_version": "3.12.3",

qa/common/gen_common.py

@@ -24,10 +24,12 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import os
 from typing import List
 
 # Common utilities for model generation scripts
 import numpy as np
+import openvino as ov
 
 np_dtype_string = np.dtype(object)
 
@@ -168,3 +170,18 @@ def np_to_torch_dtype(np_dtype):
     elif np_dtype == np_dtype_string:
         return List[str]
     return None
+
+
+def openvino_save_model(model_version_dir, model):
+    # W/A for error moving to OpenVINO new APIs "Attempt to get a name for a Tensor without names".
+    # For more details, check https://github.com/triton-inference-server/openvino_backend/issues/89
+    if len(model.outputs) == 0:
+        model.outputs[0].get_tensor().set_names({"OUTPUT"})
+    else:
+        for idx, out in enumerate(model.outputs):
+            out.get_tensor().set_names({f"OUTPUT{idx}"})
+
+    os.makedirs(model_version_dir, exist_ok=True)
+    ov.serialize(
+        model, model_version_dir + "/model.xml", model_version_dir + "/model.bin"
+    )

qa/common/gen_qa_dyna_sequence_models.py

@@ -36,6 +36,7 @@
     np_to_tf_dtype,
     np_to_torch_dtype,
     np_to_trt_dtype,
+    openvino_save_model,
 )
 
 FLAGS = None
@@ -1292,28 +1293,19 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape
     )
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
 
-    in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT")
-    start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START")
-    end = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="END")
-    ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY")
-    corrid = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID")
+    in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT")
+    start = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="START")
+    end = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="END")
+    ready = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="READY")
+    corrid = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID")
 
-    tmp1 = ng.add(in0, start)
-    tmp2 = ng.multiply(end, corrid)
-    tmp = ng.add(tmp1, tmp2)
-    op0 = ng.multiply(tmp, ready, name="OUTPUT")
+    tmp1 = ov.opset1.add(in0, start)
+    tmp2 = ov.opset1.multiply(end, corrid)
+    tmp = ov.opset1.add(tmp1, tmp2)
+    op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT")
 
-    function = ng.impl.Function([op0], [in0, start, end, ready, corrid], model_name)
-    ie_network = IENetwork(ng.impl.Function.to_capsule(function))
-
-    try:
-        os.makedirs(model_version_dir)
-    except OSError as ex:
-        pass  # ignore existing dir
-
-    ie_network.serialize(
-        model_version_dir + "/model.xml", model_version_dir + "/model.bin"
-    )
+    model = ov.Model([op0], [in0, start, end, ready, corrid], model_name)
+    openvino_save_model(model_version_dir, model)
 
 
 def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype, shape):
@@ -1560,8 +1552,7 @@ def create_models(models_dir, dtype, shape, no_batch=True):
         import torch
         from torch import nn
     if FLAGS.openvino:
-        from openvino.inference_engine import IENetwork
-        import ngraph as ng
+        import openvino.runtime as ov
 
     import test_util as tu
 

qa/common/gen_qa_identity_models.py

@@ -37,6 +37,7 @@
     np_to_onnx_dtype,
     np_to_tf_dtype,
     np_to_trt_dtype,
+    openvino_save_model,
 )
 
 FLAGS = None
@@ -574,18 +575,14 @@ def create_openvino_modelfile(
         in_name = "INPUT{}".format(io_num)
         out_name = "OUTPUT{}".format(io_num)
         openvino_inputs.append(
-            ng.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name)
+            ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name)
+        )
+        openvino_outputs.append(
+            ov.opset1.result(openvino_inputs[io_num], name=out_name)
         )
-        openvino_outputs.append(ng.result(openvino_inputs[io_num], name=out_name))
-
-    function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name)
-    ie_network = IENetwork(ng.impl.Function.to_capsule(function))
-
-    os.makedirs(model_version_dir, exist_ok=True)
 
-    ie_network.serialize(
-        model_version_dir + "/model.xml", model_version_dir + "/model.bin"
-    )
+    model = ov.Model(openvino_outputs, openvino_inputs, model_name)
+    openvino_save_model(model_version_dir, model)
 
 
 def create_openvino_modelconfig(
@@ -1317,8 +1314,7 @@ def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True):
     ):
         import tensorrt as trt
     if FLAGS.openvino:
-        from openvino.inference_engine import IENetwork
-        import ngraph as ng
+        import openvino.runtime as ov
 
     import test_util as tu