From 361acab06ed76bb787de23b0c48013121a4e4954 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 17 May 2019 12:19:07 +0200 Subject: [PATCH] updated openvino mode ensemble to 0.2 version --- .../openvino_imagenet_ensemble/README.md | 54 ++++++++++++ .../resources/model/Makefile | 4 +- .../seldon_ov_predict_ensemble.json | 88 ++++++++++++++----- 3 files changed, 120 insertions(+), 26 deletions(-) create mode 100644 examples/models/openvino_imagenet_ensemble/README.md diff --git a/examples/models/openvino_imagenet_ensemble/README.md b/examples/models/openvino_imagenet_ensemble/README.md new file mode 100644 index 0000000000..6f5f2466d4 --- /dev/null +++ b/examples/models/openvino_imagenet_ensemble/README.md @@ -0,0 +1,54 @@ +# Inference pipeline example with models ensemble and python_openvino image + +## Overview +The pipeline presented here includes the following components: + +* an [input transformer](resources/transformer) that converts jpeg compressed content into NumPy array +* two [model components](resources/model) that executes inference requests using ResNet and DenseNet models +* a [combiner component](resources/combiner) that implements the ensemble of models +* an [output transformer](resources/transformer) that converts an array of classification probabilities to a human-readable top1 class name + +![pipeline graph](pipeline1.png) + +The model component is based on [python_openvino](../../../wrappers/s2i/python_openvino) model wrapper +which includes OpenVINO inference engine python API. It can be used as a base for all kind of models implementation +using optimized inference execution and OpenVINO models. + + +# Deployment and execution +Try executing the pipeline by following the steps in the [jupyter notebook](openvino_imagenet_ensemble.ipynb). + +It is recommended it run it on Kubernetes nodes with 32vCPU nodes. When less resources is available on the nodes, +adjust the env variables in the pipeline OMP_THREADS to match the physical cores you allocated (vCPU/2). + +This pipeline uses public docker images, public pre-trained models and includes sample images. +Only the kubernetes instance is needed to follow it and reproduce in your environment. + +You can use included grpc client to connect to the deployed pipeline and submit the inference requests. +```bash +python seldon_grpc_client.py --help +usage: seldon_grpc_client.py [-h] [--repeats REPEATS] [--debug] + [--test-input TEST_INPUT] + [--ambassador AMBASSADOR] + +optional arguments: + -h, --help show this help message and exit + --repeats REPEATS + --debug + --test-input TEST_INPUT + --ambassador AMBASSADOR + +example: +python seldon_grpc_client.py --ambassador IP:port --test-input input_images.txt +``` + + +## References: + +[intel.ai blog post](https://www.intel.ai/inference-performance-boost-with-seldon-on-intel-xeon-scalable-processors) + +[seldon model wrapper with openvino python API](../../../wrappers/s2i/python_openvino) + +[Inference Engine Dev Guide](https://docs.openvinotoolkit.org/latest/_docs_IE_DG_Deep_Learning_Inference_Engine_DevGuide.html) + +[OpenVINO model zoo](https://github.com/opencv/open_model_zoo) \ No newline at end of file diff --git a/examples/models/openvino_imagenet_ensemble/resources/model/Makefile b/examples/models/openvino_imagenet_ensemble/resources/model/Makefile index 19665ed1b1..f5b3c4eaef 100644 --- a/examples/models/openvino_imagenet_ensemble/resources/model/Makefile +++ b/examples/models/openvino_imagenet_ensemble/resources/model/Makefile @@ -1,8 +1,8 @@ -IMAGE_VERSION=0.1 +IMAGE_VERSION=0.2 IMAGE_NAME=docker.io/seldonio/openvino-demo-prediction build: - s2i build -E environment_grpc . seldonio/seldon-core-s2i-openvino:0.1 $(IMAGE_NAME):$(IMAGE_VERSION) + s2i build -E environment_grpc . seldonio/seldon-core-s2i-openvino:0.2 $(IMAGE_NAME):$(IMAGE_VERSION) push_to_dockerhub: docker push $(IMAGE_NAME):$(IMAGE_VERSION) diff --git a/examples/models/openvino_imagenet_ensemble/seldon_ov_predict_ensemble.json b/examples/models/openvino_imagenet_ensemble/seldon_ov_predict_ensemble.json index d2db23eccb..4dbffebdf0 100644 --- a/examples/models/openvino_imagenet_ensemble/seldon_ov_predict_ensemble.json +++ b/examples/models/openvino_imagenet_ensemble/seldon_ov_predict_ensemble.json @@ -63,7 +63,15 @@ }, { "name": "prediction1", - "image": "seldonio/openvino-demo-prediction:0.1", + "image": "seldonio/openvino-demo-prediction:0.2", + "resources": { + "requests": { + "cpu": "1" + }, + "limits": { + "cpu": "32" + } + }, "env": [ { "name": "XML_PATH", @@ -73,6 +81,18 @@ "name": "BIN_PATH", "value": "gs://intelai_public_models/densenet_169/1/densenet_169_i8.bin" }, + { + "name": "KMP_AFFINITY", + "value": "granularity=fine,verbose,compact,1,0" + }, + { + "name": "KMP_BLOCKTIME", + "value": "1" + }, + { + "name": "OMP_NUM_THREADS", + "value": "16" + }, { "name": "http_proxy", "value": "" @@ -93,33 +113,53 @@ }, { "name": "prediction2", - "image": "seldonio/openvino-demo-prediction:0.1", + "image": "seldonio/openvino-demo-prediction:0.2", + "resources": { + "requests": { + "cpu": "1" + }, + "limits": { + "cpu": "32" + } + }, "env": [ - { - "name": "XML_PATH", - "value": "gs://intelai_public_models/resnet_50_i8/1/resnet_50_i8.xml" - }, - { - "name": "BIN_PATH", - "value": "gs://intelai_public_models/resnet_50_i8/1/resnet_50_i8.bin" - }, - { - "name": "http_proxy", + { + "name": "XML_PATH", + "value": "gs://intelai_public_models/resnet_50_i8/1/resnet_50_i8.xml" + }, + { + "name": "BIN_PATH", + "value": "gs://intelai_public_models/resnet_50_i8/1/resnet_50_i8.bin" + }, + { + "name": "KMP_AFFINITY", + "value": "granularity=fine,verbose,compact,1,0" + }, + { + "name": "KMP_BLOCKTIME", + "value": "1" + }, + { + "name": "OMP_NUM_THREADS", + "value": "16" + }, + { + "name": "http_proxy", + "value": "" + }, + { + "name": "https_proxy", "value": "" - }, - { - "name": "https_proxy", - "value": "" - }, - { + }, + { "name": "TRACING", "value": "1" - }, - { - "name": "JAEGER_AGENT_HOST", - "value": "jaeger-agent" - } - ] + }, + { + "name": "JAEGER_AGENT_HOST", + "value": "jaeger-agent" + } + ] } ], "terminationGracePeriodSeconds": 1