From ade2458bea6c0c391821dcf69f1b236460e0a0bc Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 18 Dec 2024 15:58:06 -0800 Subject: [PATCH 1/5] Revert "Rollback OpenVINO version for model gerneation (#7867)" This reverts commit 47971dd8b59b00e38ecd0d54f39bc142ab16308f. --- qa/common/gen_qa_model_repository | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index b9b8e81d4a..32e2492ad4 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -55,7 +55,7 @@ ONNX_VERSION=1.16.1 ONNX_OPSET=0 # OPENVINO version -OPENVINO_VERSION=2023.3.0 +OPENVINO_VERSION=2024.5.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} From 1b4b1b511da67057f0b3c85c9ba9e70042aeeda0 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 18 Dec 2024 15:58:09 -0800 Subject: [PATCH 2/5] Revert "Update OpenVINO to 2024.5 (#7857)" This reverts commit da9b15354bc4034b2d59c03108856880819ad3d7. --- build.py | 4 ++-- qa/common/gen_qa_dyna_sequence_models.py | 28 ++++++++++------------ qa/common/gen_qa_identity_models.py | 16 ++++++------- qa/common/gen_qa_model_repository | 2 +- qa/common/gen_qa_models.py | 30 +++++++++++++----------- qa/common/gen_qa_reshape_models.py | 14 +++++------ qa/common/gen_qa_sequence_models.py | 20 +++++++--------- 7 files changed, 55 insertions(+), 59 deletions(-) diff --git a/build.py b/build.py index 3c0ba555bc..509b2c4eaa 100755 --- a/build.py +++ b/build.py @@ -75,8 +75,8 @@ "triton_container_version": "24.12", "upstream_container_version": "24.12", "ort_version": "1.20.1", - "ort_openvino_version": "2024.5.0", - "standalone_openvino_version": "2024.5.0", + "ort_openvino_version": "2024.4.0", + "standalone_openvino_version": "2024.4.0", "dcgm_version": "3.3.6", "vllm_version": "0.5.5", "rhel_py_version": "3.12.3", diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index bcb5a3a2b4..94087cb8c6 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -1292,27 +1292,26 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") - start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START") - end = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="END") - ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") - corrid = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID") + in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") + start = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="START") + end = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="END") + ready = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") + corrid = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="CORRID") - tmp1 = ng.add(in0, start) - tmp2 = ng.multiply(end, corrid) - tmp = ng.add(tmp1, tmp2) - op0 = ng.multiply(tmp, ready, name="OUTPUT") + tmp1 = ov.opset1.add(in0, start) + tmp2 = ov.opset1.multiply(end, corrid) + tmp = ov.opset1.add(tmp1, tmp2) + op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT") - function = ng.impl.Function([op0], [in0, start, end, ready, corrid], model_name) - ie_network = IENetwork(ng.impl.Function.to_capsule(function)) + model = ov.Model([op0], [in0, start, end, ready, corrid], model_name) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ie_network.serialize( - model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1560,8 +1559,7 @@ def create_models(models_dir, dtype, shape, no_batch=True): import torch from torch import nn if FLAGS.openvino: - from openvino.inference_engine import IENetwork - import ngraph as ng + import openvino.runtime as ov import test_util as tu diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 7b7066dbb9..abc57ab48a 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -574,17 +574,18 @@ def create_openvino_modelfile( in_name = "INPUT{}".format(io_num) out_name = "OUTPUT{}".format(io_num) openvino_inputs.append( - ng.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name) + ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name=in_name) + ) + openvino_outputs.append( + ov.opset1.result(openvino_inputs[io_num], name=out_name) ) - openvino_outputs.append(ng.result(openvino_inputs[io_num], name=out_name)) - function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name) - ie_network = IENetwork(ng.impl.Function.to_capsule(function)) + model = ov.Model(openvino_outputs, openvino_inputs, model_name) os.makedirs(model_version_dir, exist_ok=True) - ie_network.serialize( - model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1317,8 +1318,7 @@ def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): ): import tensorrt as trt if FLAGS.openvino: - from openvino.inference_engine import IENetwork - import ngraph as ng + import openvino.runtime as ov import test_util as tu diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 32e2492ad4..f4cdc2b6a2 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -55,7 +55,7 @@ ONNX_VERSION=1.16.1 ONNX_OPSET=0 # OPENVINO version -OPENVINO_VERSION=2024.5.0 +OPENVINO_VERSION=2024.4.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index efe3fca1e9..1af7762869 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1809,28 +1809,31 @@ def create_openvino_modelfile( ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT0") - in1 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT1") + in0 = ov.opset1.parameter( + shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT0" + ) + in1 = ov.opset1.parameter( + shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT1" + ) - r0 = ng.add(in0, in1) if not swap else ng.subtract(in0, in1) - r1 = ng.subtract(in0, in1) if not swap else ng.add(in0, in1) + r0 = ov.opset1.add(in0, in1) if not swap else ov.opset1.subtract(in0, in1) + r1 = ov.opset1.subtract(in0, in1) if not swap else ov.opset1.add(in0, in1) - result0 = ng.reshape(r0, batch_dim + output0_shape, special_zero=False) - result1 = ng.reshape(r1, batch_dim + output1_shape, special_zero=False) + result0 = ov.opset1.reshape(r0, batch_dim + output0_shape, special_zero=False) + result1 = ov.opset1.reshape(r1, batch_dim + output1_shape, special_zero=False) - op0 = ng.convert(result0, destination_type=output0_dtype, name="OUTPUT0") - op1 = ng.convert(result1, destination_type=output1_dtype, name="OUTPUT1") + op0 = ov.opset1.convert(result0, destination_type=output0_dtype, name="OUTPUT0") + op1 = ov.opset1.convert(result1, destination_type=output1_dtype, name="OUTPUT1") - function = ng.impl.Function([op0, op1], [in0, in1], model_name) - ie_network = IENetwork(ng.impl.Function.to_capsule(function)) + model = ov.Model([op0, op1], [in0, in1], model_name) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ie_network.serialize( - model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -2486,8 +2489,7 @@ def create_fixed_models( import torch from torch import nn if FLAGS.openvino: - from openvino.inference_engine import IENetwork - import ngraph as ng + import openvino.runtime as ov import test_util as tu diff --git a/qa/common/gen_qa_reshape_models.py b/qa/common/gen_qa_reshape_models.py index 3abb2e978a..a27355574f 100755 --- a/qa/common/gen_qa_reshape_models.py +++ b/qa/common/gen_qa_reshape_models.py @@ -934,13 +934,13 @@ def create_openvino_modelfile( in_name = "INPUT{}".format(io_num) out_name = "OUTPUT{}".format(io_num) openvino_inputs.append( - ng.parameter( + ov.opset1.parameter( shape=batch_dim + input_shapes[io_num], dtype=dtype, name=in_name ) ) openvino_outputs.append( - ng.reshape( + ov.opset1.reshape( openvino_inputs[io_num], batch_dim + output_shapes[io_num], name=out_name, @@ -948,16 +948,15 @@ def create_openvino_modelfile( ) ) - function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name) - ie_network = IENetwork(ng.impl.Function.to_capsule(function)) + model = ov.Model(openvino_outputs, openvino_inputs, model_name) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ie_network.serialize( - model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1463,8 +1462,7 @@ def create_openvino_models( import torch from torch import nn if FLAGS.openvino: - from openvino.inference_engine import IENetwork - import ngraph as ng + import openvino.runtime as ov import test_util as tu diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index 42557c7747..25c35b7603 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -1145,23 +1145,22 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape ) model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") - start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START") - ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") + in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT") + start = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="START") + ready = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="READY") - tmp = ng.add(in0, start) - op0 = ng.multiply(tmp, ready, name="OUTPUT") + tmp = ov.opset1.add(in0, start) + op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT") - function = ng.impl.Function([op0], [in0, start, ready], model_name) - ie_network = IENetwork(ng.impl.Function.to_capsule(function)) + model = ov.Model([op0], [in0, start, ready], model_name) try: os.makedirs(model_version_dir) except OSError as ex: pass # ignore existing dir - ie_network.serialize( - model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" ) @@ -1421,8 +1420,7 @@ def create_models(models_dir, dtype, shape, no_batch=True): import torch from torch import nn if FLAGS.openvino: - from openvino.inference_engine import IENetwork - import ngraph as ng + import openvino.runtime as ov import test_util as tu From 87cc718f35872ec13f6436540f1afad0fee1f99c Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 18 Dec 2024 16:00:36 -0800 Subject: [PATCH 3/5] Upgrade OpenVINO model version to 2024.5.0 --- qa/common/gen_common.py | 17 + qa/common/gen_qa_dyna_sequence_models.py | 11 +- qa/common/gen_qa_identity_models.py | 8 +- qa/common/gen_qa_model_repository | 606 +++++++++++------------ qa/common/gen_qa_models.py | 11 +- qa/common/gen_qa_reshape_models.py | 11 +- qa/common/gen_qa_sequence_models.py | 11 +- 7 files changed, 330 insertions(+), 345 deletions(-) diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index 417ad9477a..2e6cd4497d 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -24,10 +24,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import os from typing import List # Common utilities for model generation scripts import numpy as np +import openvino as ov np_dtype_string = np.dtype(object) @@ -168,3 +170,18 @@ def np_to_torch_dtype(np_dtype): elif np_dtype == np_dtype_string: return List[str] return None + + +def openvino_save_model(model_version_dir, model): + # W/A for error moving to OpenVINO new APIs "Attempt to get a name for a Tensor without names". + # For more details, check https://github.com/triton-inference-server/openvino_backend/issues/89 + if len(model.outputs) == 0: + model.outputs[0].get_tensor().set_names({"OUTPUT"}) + else: + for idx, out in enumerate(model.outputs): + out.get_tensor().set_names({f"OUTPUT{idx}"}) + + os.makedirs(model_version_dir, exist_ok=True) + ov.serialize( + model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" + ) diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 94087cb8c6..7b95e2e66e 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -36,6 +36,7 @@ np_to_tf_dtype, np_to_torch_dtype, np_to_trt_dtype, + openvino_save_model, ) FLAGS = None @@ -1304,15 +1305,7 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape op0 = ov.opset1.multiply(tmp, ready, name="OUTPUT") model = ov.Model([op0], [in0, start, end, ready, corrid], model_name) - - try: - os.makedirs(model_version_dir) - except OSError as ex: - pass # ignore existing dir - - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" - ) + openvino_save_model(model_version_dir, model) def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype, shape): diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index abc57ab48a..7865d24e58 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_tf_dtype, np_to_trt_dtype, + openvino_save_model, ) FLAGS = None @@ -581,12 +582,7 @@ def create_openvino_modelfile( ) model = ov.Model(openvino_outputs, openvino_inputs, model_name) - - os.makedirs(model_version_dir, exist_ok=True) - - ov.serialize( - model, model_version_dir + "/model.xml", model_version_dir + "/model.bin" - ) + openvino_save_model(model_version_dir, model) def create_openvino_modelconfig( diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index f4cdc2b6a2..d1691d566b 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -55,7 +55,7 @@ ONNX_VERSION=1.16.1 ONNX_OPSET=0 # OPENVINO version -OPENVINO_VERSION=2024.4.0 +OPENVINO_VERSION=2024.5.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} @@ -263,305 +263,305 @@ fi fi # [[ "aarch64" != $(uname -m) ]] -# ONNX -cat >$HOST_SRCDIR/$ONNXSCRIPT <$HOST_SRCDIR/$TORCHSCRIPT <$HOST_SRCDIR/$TFSCRIPT <4.24.0" - -python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --models_dir=$DESTDIR -chmod -R 777 $DESTDIR -python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --variable --models_dir=$VARDESTDIR -chmod -R 777 $VARDESTDIR -python3 $SRCDIR/gen_qa_identity_models.py --graphdef --savedmodel --models_dir=$IDENTITYDESTDIR -chmod -R 777 $IDENTITYDESTDIR -python3 $SRCDIR/gen_qa_reshape_models.py --graphdef --savedmodel --variable --models_dir=$RESHAPEDESTDIR -chmod -R 777 $RESHAPEDESTDIR -python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --models_dir=$SEQDESTDIR -chmod -R 777 $SEQDESTDIR -python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --variable --models_dir=$VARSEQDESTDIR -chmod -R 777 $VARSEQDESTDIR -python3 $SRCDIR/gen_qa_dyna_sequence_models.py --graphdef --savedmodel --models_dir=$DYNASEQDESTDIR -chmod -R 777 $DYNASEQDESTDIR -python3 $SRCDIR/gen_qa_noshape_models.py --savedmodel --models_dir=$NOSHAPEDESTDIR -chmod -R 777 $NOSHAPEDESTDIR -python3 $SRCDIR/gen_qa_ragged_models.py --savedmodel --models_dir=$RAGGEDDESTDIR -chmod -R 777 $RAGGEDDESTDIR -python3 $SRCDIR/gen_qa_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_model_repository -python3 $SRCDIR/gen_qa_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_model_repository -python3 $SRCDIR/gen_qa_reshape_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_reshape_model_repository -python3 $SRCDIR/gen_qa_identity_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_identity_model_repository -python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_sequence_model_repository -python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_sequence_model_repository -chmod -R 777 $ENSEMBLEDESTDIR -python3 $SRCDIR/gen_tag_sigdef.py --dir $SIGDEFDESTDIR -chmod -R 777 $SIGDEFDESTDIR -python3 $SRCDIR/gen_qa_tf_parameters.py --models_dir $TFPARAMETERSDIR -chmod -R 777 $TFPARAMETERSDIR -EOF - -chmod a+x $HOST_SRCDIR/$TFSCRIPT -if [ $? -ne 0 ]; then - echo -e "Failed: chmod" - exit 1 -fi - -docker pull $TENSORFLOW_IMAGE -docker run $DOCKER_GPU_ARGS --rm --entrypoint $SRCDIR/$TFSCRIPT \ - --mount type=bind,source=$HOST_SRCDIR,target=$SRCDIR \ - --mount type=bind,source=$HOST_DESTDIR,target=$DESTDIR \ - --mount type=bind,source=$HOST_VARDESTDIR,target=$VARDESTDIR \ - --mount type=bind,source=$HOST_IDENTITYDESTDIR,target=$IDENTITYDESTDIR \ - --mount type=bind,source=$HOST_SIGDEFDESTDIR,target=$SIGDEFDESTDIR \ - --mount type=bind,source=$HOST_RESHAPEDESTDIR,target=$RESHAPEDESTDIR \ - --mount type=bind,source=$HOST_SEQDESTDIR,target=$SEQDESTDIR \ - --mount type=bind,source=$HOST_DYNASEQDESTDIR,target=$DYNASEQDESTDIR \ - --mount type=bind,source=$HOST_VARSEQDESTDIR,target=$VARSEQDESTDIR \ - --mount type=bind,source=$HOST_NOSHAPEDESTDIR,target=$NOSHAPEDESTDIR \ - --mount type=bind,source=$HOST_ENSEMBLEDESTDIR,target=$ENSEMBLEDESTDIR \ - --mount type=bind,source=$HOST_RAGGEDDESTDIR,target=$RAGGEDDESTDIR \ - --mount type=bind,source=$HOST_TFPARAMETERSDESTDIR,target=$TFPARAMETERSDIR \ - $TENSORFLOW_IMAGE -if [ $? -ne 0 ]; then - echo -e "Failed" - exit 1 -fi - -# TensorRT -docker pull ${TENSORRT_IMAGE} -TENSORRT_VERSION="$(docker inspect ${TENSORRT_IMAGE} --format '{{index .Config.Labels "com.nvidia.tensorrt.version"}}' | cut -d . -f -2)" -cat >$HOST_SRCDIR/$TRTSCRIPT <$HOST_SRCDIR/$ONNXSCRIPT <$HOST_SRCDIR/$TORCHSCRIPT <$HOST_SRCDIR/$TFSCRIPT <4.24.0" + +# python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --models_dir=$DESTDIR +# chmod -R 777 $DESTDIR +# python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --variable --models_dir=$VARDESTDIR +# chmod -R 777 $VARDESTDIR +# python3 $SRCDIR/gen_qa_identity_models.py --graphdef --savedmodel --models_dir=$IDENTITYDESTDIR +# chmod -R 777 $IDENTITYDESTDIR +# python3 $SRCDIR/gen_qa_reshape_models.py --graphdef --savedmodel --variable --models_dir=$RESHAPEDESTDIR +# chmod -R 777 $RESHAPEDESTDIR +# python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --models_dir=$SEQDESTDIR +# chmod -R 777 $SEQDESTDIR +# python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --variable --models_dir=$VARSEQDESTDIR +# chmod -R 777 $VARSEQDESTDIR +# python3 $SRCDIR/gen_qa_dyna_sequence_models.py --graphdef --savedmodel --models_dir=$DYNASEQDESTDIR +# chmod -R 777 $DYNASEQDESTDIR +# python3 $SRCDIR/gen_qa_noshape_models.py --savedmodel --models_dir=$NOSHAPEDESTDIR +# chmod -R 777 $NOSHAPEDESTDIR +# python3 $SRCDIR/gen_qa_ragged_models.py --savedmodel --models_dir=$RAGGEDDESTDIR +# chmod -R 777 $RAGGEDDESTDIR +# python3 $SRCDIR/gen_qa_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_model_repository +# python3 $SRCDIR/gen_qa_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_model_repository +# python3 $SRCDIR/gen_qa_reshape_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_reshape_model_repository +# python3 $SRCDIR/gen_qa_identity_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_identity_model_repository +# python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_sequence_model_repository +# python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_sequence_model_repository +# chmod -R 777 $ENSEMBLEDESTDIR +# python3 $SRCDIR/gen_tag_sigdef.py --dir $SIGDEFDESTDIR +# chmod -R 777 $SIGDEFDESTDIR +# python3 $SRCDIR/gen_qa_tf_parameters.py --models_dir $TFPARAMETERSDIR +# chmod -R 777 $TFPARAMETERSDIR +# EOF + +# chmod a+x $HOST_SRCDIR/$TFSCRIPT +# if [ $? -ne 0 ]; then +# echo -e "Failed: chmod" +# exit 1 +# fi + +# docker pull $TENSORFLOW_IMAGE +# docker run $DOCKER_GPU_ARGS --rm --entrypoint $SRCDIR/$TFSCRIPT \ +# --mount type=bind,source=$HOST_SRCDIR,target=$SRCDIR \ +# --mount type=bind,source=$HOST_DESTDIR,target=$DESTDIR \ +# --mount type=bind,source=$HOST_VARDESTDIR,target=$VARDESTDIR \ +# --mount type=bind,source=$HOST_IDENTITYDESTDIR,target=$IDENTITYDESTDIR \ +# --mount type=bind,source=$HOST_SIGDEFDESTDIR,target=$SIGDEFDESTDIR \ +# --mount type=bind,source=$HOST_RESHAPEDESTDIR,target=$RESHAPEDESTDIR \ +# --mount type=bind,source=$HOST_SEQDESTDIR,target=$SEQDESTDIR \ +# --mount type=bind,source=$HOST_DYNASEQDESTDIR,target=$DYNASEQDESTDIR \ +# --mount type=bind,source=$HOST_VARSEQDESTDIR,target=$VARSEQDESTDIR \ +# --mount type=bind,source=$HOST_NOSHAPEDESTDIR,target=$NOSHAPEDESTDIR \ +# --mount type=bind,source=$HOST_ENSEMBLEDESTDIR,target=$ENSEMBLEDESTDIR \ +# --mount type=bind,source=$HOST_RAGGEDDESTDIR,target=$RAGGEDDESTDIR \ +# --mount type=bind,source=$HOST_TFPARAMETERSDESTDIR,target=$TFPARAMETERSDIR \ +# $TENSORFLOW_IMAGE +# if [ $? -ne 0 ]; then +# echo -e "Failed" +# exit 1 +# fi + +# # TensorRT +# docker pull ${TENSORRT_IMAGE} +# TENSORRT_VERSION="$(docker inspect ${TENSORRT_IMAGE} --format '{{index .Config.Labels "com.nvidia.tensorrt.version"}}' | cut -d . -f -2)" +# cat >$HOST_SRCDIR/$TRTSCRIPT < Date: Thu, 19 Dec 2024 13:02:46 -0800 Subject: [PATCH 4/5] Restore openvino versions in build.py --- build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.py b/build.py index 509b2c4eaa..3c0ba555bc 100755 --- a/build.py +++ b/build.py @@ -75,8 +75,8 @@ "triton_container_version": "24.12", "upstream_container_version": "24.12", "ort_version": "1.20.1", - "ort_openvino_version": "2024.4.0", - "standalone_openvino_version": "2024.4.0", + "ort_openvino_version": "2024.5.0", + "standalone_openvino_version": "2024.5.0", "dcgm_version": "3.3.6", "vllm_version": "0.5.5", "rhel_py_version": "3.12.3", From cca039d7c4b1be2a81233c0625a3e24f0eb4d24c Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 19 Dec 2024 17:21:39 -0800 Subject: [PATCH 5/5] Remove commented lines --- qa/common/gen_qa_model_repository | 604 +++++++++++++++--------------- 1 file changed, 302 insertions(+), 302 deletions(-) diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index d1691d566b..32e2492ad4 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -263,305 +263,305 @@ fi fi # [[ "aarch64" != $(uname -m) ]] -# # ONNX -# cat >$HOST_SRCDIR/$ONNXSCRIPT <$HOST_SRCDIR/$TORCHSCRIPT <$HOST_SRCDIR/$TFSCRIPT <4.24.0" - -# python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --models_dir=$DESTDIR -# chmod -R 777 $DESTDIR -# python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --variable --models_dir=$VARDESTDIR -# chmod -R 777 $VARDESTDIR -# python3 $SRCDIR/gen_qa_identity_models.py --graphdef --savedmodel --models_dir=$IDENTITYDESTDIR -# chmod -R 777 $IDENTITYDESTDIR -# python3 $SRCDIR/gen_qa_reshape_models.py --graphdef --savedmodel --variable --models_dir=$RESHAPEDESTDIR -# chmod -R 777 $RESHAPEDESTDIR -# python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --models_dir=$SEQDESTDIR -# chmod -R 777 $SEQDESTDIR -# python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --variable --models_dir=$VARSEQDESTDIR -# chmod -R 777 $VARSEQDESTDIR -# python3 $SRCDIR/gen_qa_dyna_sequence_models.py --graphdef --savedmodel --models_dir=$DYNASEQDESTDIR -# chmod -R 777 $DYNASEQDESTDIR -# python3 $SRCDIR/gen_qa_noshape_models.py --savedmodel --models_dir=$NOSHAPEDESTDIR -# chmod -R 777 $NOSHAPEDESTDIR -# python3 $SRCDIR/gen_qa_ragged_models.py --savedmodel --models_dir=$RAGGEDDESTDIR -# chmod -R 777 $RAGGEDDESTDIR -# python3 $SRCDIR/gen_qa_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_model_repository -# python3 $SRCDIR/gen_qa_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_model_repository -# python3 $SRCDIR/gen_qa_reshape_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_reshape_model_repository -# python3 $SRCDIR/gen_qa_identity_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_identity_model_repository -# python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_sequence_model_repository -# python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_sequence_model_repository -# chmod -R 777 $ENSEMBLEDESTDIR -# python3 $SRCDIR/gen_tag_sigdef.py --dir $SIGDEFDESTDIR -# chmod -R 777 $SIGDEFDESTDIR -# python3 $SRCDIR/gen_qa_tf_parameters.py --models_dir $TFPARAMETERSDIR -# chmod -R 777 $TFPARAMETERSDIR -# EOF - -# chmod a+x $HOST_SRCDIR/$TFSCRIPT -# if [ $? -ne 0 ]; then -# echo -e "Failed: chmod" -# exit 1 -# fi - -# docker pull $TENSORFLOW_IMAGE -# docker run $DOCKER_GPU_ARGS --rm --entrypoint $SRCDIR/$TFSCRIPT \ -# --mount type=bind,source=$HOST_SRCDIR,target=$SRCDIR \ -# --mount type=bind,source=$HOST_DESTDIR,target=$DESTDIR \ -# --mount type=bind,source=$HOST_VARDESTDIR,target=$VARDESTDIR \ -# --mount type=bind,source=$HOST_IDENTITYDESTDIR,target=$IDENTITYDESTDIR \ -# --mount type=bind,source=$HOST_SIGDEFDESTDIR,target=$SIGDEFDESTDIR \ -# --mount type=bind,source=$HOST_RESHAPEDESTDIR,target=$RESHAPEDESTDIR \ -# --mount type=bind,source=$HOST_SEQDESTDIR,target=$SEQDESTDIR \ -# --mount type=bind,source=$HOST_DYNASEQDESTDIR,target=$DYNASEQDESTDIR \ -# --mount type=bind,source=$HOST_VARSEQDESTDIR,target=$VARSEQDESTDIR \ -# --mount type=bind,source=$HOST_NOSHAPEDESTDIR,target=$NOSHAPEDESTDIR \ -# --mount type=bind,source=$HOST_ENSEMBLEDESTDIR,target=$ENSEMBLEDESTDIR \ -# --mount type=bind,source=$HOST_RAGGEDDESTDIR,target=$RAGGEDDESTDIR \ -# --mount type=bind,source=$HOST_TFPARAMETERSDESTDIR,target=$TFPARAMETERSDIR \ -# $TENSORFLOW_IMAGE -# if [ $? -ne 0 ]; then -# echo -e "Failed" -# exit 1 -# fi - -# # TensorRT -# docker pull ${TENSORRT_IMAGE} -# TENSORRT_VERSION="$(docker inspect ${TENSORRT_IMAGE} --format '{{index .Config.Labels "com.nvidia.tensorrt.version"}}' | cut -d . -f -2)" -# cat >$HOST_SRCDIR/$TRTSCRIPT <$HOST_SRCDIR/$ONNXSCRIPT <$HOST_SRCDIR/$TORCHSCRIPT <$HOST_SRCDIR/$TFSCRIPT <4.24.0" + +python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --models_dir=$DESTDIR +chmod -R 777 $DESTDIR +python3 $SRCDIR/gen_qa_models.py --graphdef --savedmodel --variable --models_dir=$VARDESTDIR +chmod -R 777 $VARDESTDIR +python3 $SRCDIR/gen_qa_identity_models.py --graphdef --savedmodel --models_dir=$IDENTITYDESTDIR +chmod -R 777 $IDENTITYDESTDIR +python3 $SRCDIR/gen_qa_reshape_models.py --graphdef --savedmodel --variable --models_dir=$RESHAPEDESTDIR +chmod -R 777 $RESHAPEDESTDIR +python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --models_dir=$SEQDESTDIR +chmod -R 777 $SEQDESTDIR +python3 $SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --variable --models_dir=$VARSEQDESTDIR +chmod -R 777 $VARSEQDESTDIR +python3 $SRCDIR/gen_qa_dyna_sequence_models.py --graphdef --savedmodel --models_dir=$DYNASEQDESTDIR +chmod -R 777 $DYNASEQDESTDIR +python3 $SRCDIR/gen_qa_noshape_models.py --savedmodel --models_dir=$NOSHAPEDESTDIR +chmod -R 777 $NOSHAPEDESTDIR +python3 $SRCDIR/gen_qa_ragged_models.py --savedmodel --models_dir=$RAGGEDDESTDIR +chmod -R 777 $RAGGEDDESTDIR +python3 $SRCDIR/gen_qa_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_model_repository +python3 $SRCDIR/gen_qa_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_model_repository +python3 $SRCDIR/gen_qa_reshape_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_reshape_model_repository +python3 $SRCDIR/gen_qa_identity_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_identity_model_repository +python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --models_dir=$ENSEMBLEDESTDIR/qa_sequence_model_repository +python3 $SRCDIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$ENSEMBLEDESTDIR/qa_variable_sequence_model_repository +chmod -R 777 $ENSEMBLEDESTDIR +python3 $SRCDIR/gen_tag_sigdef.py --dir $SIGDEFDESTDIR +chmod -R 777 $SIGDEFDESTDIR +python3 $SRCDIR/gen_qa_tf_parameters.py --models_dir $TFPARAMETERSDIR +chmod -R 777 $TFPARAMETERSDIR +EOF + +chmod a+x $HOST_SRCDIR/$TFSCRIPT +if [ $? -ne 0 ]; then + echo -e "Failed: chmod" + exit 1 +fi + +docker pull $TENSORFLOW_IMAGE +docker run $DOCKER_GPU_ARGS --rm --entrypoint $SRCDIR/$TFSCRIPT \ + --mount type=bind,source=$HOST_SRCDIR,target=$SRCDIR \ + --mount type=bind,source=$HOST_DESTDIR,target=$DESTDIR \ + --mount type=bind,source=$HOST_VARDESTDIR,target=$VARDESTDIR \ + --mount type=bind,source=$HOST_IDENTITYDESTDIR,target=$IDENTITYDESTDIR \ + --mount type=bind,source=$HOST_SIGDEFDESTDIR,target=$SIGDEFDESTDIR \ + --mount type=bind,source=$HOST_RESHAPEDESTDIR,target=$RESHAPEDESTDIR \ + --mount type=bind,source=$HOST_SEQDESTDIR,target=$SEQDESTDIR \ + --mount type=bind,source=$HOST_DYNASEQDESTDIR,target=$DYNASEQDESTDIR \ + --mount type=bind,source=$HOST_VARSEQDESTDIR,target=$VARSEQDESTDIR \ + --mount type=bind,source=$HOST_NOSHAPEDESTDIR,target=$NOSHAPEDESTDIR \ + --mount type=bind,source=$HOST_ENSEMBLEDESTDIR,target=$ENSEMBLEDESTDIR \ + --mount type=bind,source=$HOST_RAGGEDDESTDIR,target=$RAGGEDDESTDIR \ + --mount type=bind,source=$HOST_TFPARAMETERSDESTDIR,target=$TFPARAMETERSDIR \ + $TENSORFLOW_IMAGE +if [ $? -ne 0 ]; then + echo -e "Failed" + exit 1 +fi + +# TensorRT +docker pull ${TENSORRT_IMAGE} +TENSORRT_VERSION="$(docker inspect ${TENSORRT_IMAGE} --format '{{index .Config.Labels "com.nvidia.tensorrt.version"}}' | cut -d . -f -2)" +cat >$HOST_SRCDIR/$TRTSCRIPT <