initial version

examples/models/openvino_R4/Makefile

@@ -0,0 +1,12 @@
+
+
+build_image:
+	s2i build -E environment_grpc . seldon_python_openvino seldon-openvino-prediction
+
+
+clean:
+	rm -rf model
+	rm -rf proto/__pycache__
+	rm -f proto/*.py
+	rm -r proto/*.proto
+	rm -rf tensorflow

examples/models/openvino_R4/Prediction.py

@@ -0,0 +1,100 @@
+import numpy as np
+import logging
+import datetime
+import os
+import sys
+from urllib.parse import urlparse
+from google.cloud import storage
+from openvino.inference_engine import IENetwork, IEPlugin
+
+
+def get_logger(name):
+    logger = logging.getLogger(name)
+    log_formatter = logging.Formatter("%(asctime)s - %(name)s - "
+                                      "%(levelname)s - %(message)s")
+    logger.setLevel('DEBUG')
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(log_formatter)
+    logger.addHandler(console_handler)
+
+    return logger
+
+logger = get_logger(__name__)
+
+
+def gs_download_file(path):
+    if path is None:
+        return None
+    parsed_path = urlparse(path)
+    bucket_name = parsed_path.netloc
+    file_path = parsed_path.path[1:]
+    gs_client = storage.Client()
+    bucket = gs_client.get_bucket(bucket_name)
+    blob = bucket.blob(file_path)
+    tmp_path = os.path.join('/tmp', file_path.split(os.sep)[-1])
+    blob.download_to_filename(tmp_path)
+    return tmp_path
+
+
+def s3_download_file(path):
+    if path is None:
+        return None
+    s3_endpoint = os.getenv('S3_ENDPOINT')
+    s3_client = boto3.client('s3', endpoint_url=s3_endpoint)
+    parsed_path = urlparse(path)
+    bucket_name = parsed_path.netloc
+    file_path = parsed_path.path[1:]
+    tmp_path = os.path.join('/tmp', file_path.split(os.sep)[-1])
+    s3_transfer = boto3.s3.transfer.S3Transfer(s3_client)
+    s3_transfer.download_file(bucket_name, file_path, tmp_path)
+    return tmp_path
+
+
+def GetLocalPath(requested_path):
+    parsed_path = urlparse(requested_path)
+    if parsed_path.scheme == '':
+        return requested_path
+    elif parsed_path.scheme == 'gs':
+        return gs_download_file(path=requested_path)
+    elif parsed_path.scheme == 's3':
+        return s3_download_file(path=requested_path)
+
+
+class Prediction(object):
+    def __init__(self):
+        try:
+            xml_path = os.environ["XML_PATH"]
+            bin_path = os.environ["BIN_PATH"]
+
+        except KeyError:
+            print("Please set the environment variables XML_PATH, BIN, PATH, INPUT and OUTPUT")
+            sys.exit(1)
+
+        xml_local_path = GetLocalPath(xml_path)
+        bin_local_path = GetLocalPath(bin_path)
+        print('path object', xml_local_path)
+
+        CPU_EXTENSION = os.getenv('CPU_EXTENSION', '/usr/local/lib/libcpu_extension.so')
+
+        plugin = IEPlugin(device='CPU', plugin_dirs=None)
+        if CPU_EXTENSION:
+            plugin.add_cpu_extension(CPU_EXTENSION)
+        net = IENetwork.from_ir(model=xml_local_path, weights=bin_local_path)
+        self.input_blob = next(iter(net.inputs))
+        self.out_blob = next(iter(net.outputs))
+        self.batch_size = net.inputs[self.input_blob].shape[0]
+        self.inputs = net.inputs
+        self.outputs = net.outputs
+        self.exec_net = plugin.load(network=net, num_requests=self.batch_size)
+
+
+    def predict(self,X,feature_names):
+        start_time = datetime.datetime.now()
+        results = self.exec_net.infer(inputs={self.input_blob: X})
+        predictions = results[self.out_blob]
+        end_time = datetime.datetime.now()
+        duration = (end_time - start_time).total_seconds() * 1000
+        logger.debug("Processing time: {:.2f} ms".format(duration))
+        return predictions.astype(np.float64)
+

examples/models/openvino_R4/README.md

@@ -0,0 +1,8 @@
+# OpenVINO prediction component
+
+Model configuration is implemented using environment variables:
+
+XML_PATH  - s3, gs or local path to xml file in OpenVINO model server
+
+BIN_PATH - s3, gs or local path to bin file in OpenVINO model server
+

examples/models/openvino_R4/client.py

@@ -0,0 +1,103 @@
+import json
+import requests
+import base64
+from proto import prediction_pb2
+from proto import prediction_pb2_grpc
+import grpc
+import numpy as np
+import pickle
+import tensorflow as tf
+import cv2
+import datetime
+import tensorflow as tf
+
+def image_2_vector(input_file):
+    nparr = np.fromfile(input_file, dtype=np.float32)
+    print("nparr",nparr.dtype,nparr.shape)
+    img = cv2.imdecode(nparr, cv2.IMREAD_ANYCOLOR)
+    print("img",img.dtype,img.shape)
+    print("Initial size",img.shape)
+    image = cv2.resize(img, (w, h))
+    print("image",image.dtype)
+    print("Converted size",image.shape)
+
+    vector = image.reshape((w * h * 3))
+    print("vector shape",vector.shape, "vector type", vector.dtype )
+    return vector
+
+def image_2_bytes(input_file):
+    with open(input_file, "rb") as binary_file:
+        # Read the whole file at once
+        data = binary_file.read()
+
+        #data = data.tobytes()
+        #print(data)
+        print("binary data size:", len(data), type(data))
+    return data
+
+def run(function,image_path,iterations=1):
+    w = 224
+    h = 224
+
+    # NOTE(gRPC Python Team): .close() is possible on a channel and should be
+    # used in circumstances in which the with statement does not fit the needs
+    # of the code.
+    with grpc.insecure_channel('localhost:5000') as channel:
+        stub = prediction_pb2_grpc.ModelStub(channel)
+        print("seldon stub", stub)
+        start_time = datetime.datetime.now()
+        processing_times = np.zeros((0),int)
+
+        img = cv2.imread(image_path)
+        print("img type", type(img))
+        print("img",img.shape)
+        print("Initial size",img.shape)
+        image = cv2.resize(img, (w, h))
+        image = image.reshape(1, w, h, 3)
+        image = image.transpose((0,3,1,2))
+        print("image",image.dtype)
+        print("Converted size",image.shape)
+
+        if function == "tensor":
+            datadef = prediction_pb2.DefaultData(
+                names = 'x',
+                tensor = prediction_pb2.Tensor(
+                    shape = image.shape,
+                    values = image.ravel().tolist()
+                )
+            )
+        elif function == "tftensor":
+            print("Create tftensor")
+            datadef = prediction_pb2.DefaultData(
+                names = 'x',
+                tftensor = tf.make_tensor_proto(image)
+            )
+
+        GRPC_request = prediction_pb2.SeldonMessage(
+            data = datadef
+        )
+
+        for I in range(iterations):
+            start_time = datetime.datetime.now()
+            response = stub.Predict(request=GRPC_request)
+            end_time = datetime.datetime.now()
+            duration = (end_time - start_time).total_seconds() * 1000
+            processing_times = np.append(processing_times,np.array([int(duration)]))
+
+        print('processing time for all iterations')
+        for x in processing_times:
+            print(x,"ms")
+        print('processing_statistics')
+        print('average time:',round(np.average(processing_times),1), 'ms; average speed:', round(1000/np.average(processing_times),1),'fps')
+        print('median time:',round(np.median(processing_times),1), 'ms; median speed:',round(1000/np.median(processing_times),1),'fps')
+        print('max time:',round(np.max(processing_times),1), 'ms; max speed:',round(1000/np.max(processing_times),1),'fps')
+        print('min time:',round(np.min(processing_times),1),'ms; min speed:',round(1000/np.min(processing_times),1),'fps')
+        print('time percentile 90:',round(np.percentile(processing_times,90),1),'ms; speed percentile 90:',round(1000/np.percentile(processing_times,90),1),'fps')
+        print('time percentile 50:',round(np.percentile(processing_times,50),1),'ms; speed percentile 50:',round(1000/np.percentile(processing_times,50),1),'fps')
+        print('time standard deviation:',round(np.std(processing_times)))
+        print('time variance:',round(np.var(processing_times)))
+
+
+#run("tensor","./dog.jpeg",iterations=100)
+
+run("tftensor","./dog.jpeg",iterations=100)

examples/models/openvino_R4/environment_grpc

@@ -0,0 +1,8 @@
+MODEL_NAME=Prediction
+API_TYPE=GRPC
+SERVICE_TYPE=MODEL
+XML_PATH=gs://inference-eu/models_zoo/resnet_V1_50/frozen_resnet_V1_50.xml
+BIN_PATH=gs://inference-eu/models_zoo/resnet_V1_50/frozen_resnet_V1_50.bin
+GOOGLE_APPLICATION_CREDENTIALS=/etc/gcp.json
+PERSISTENCE=0
+

examples/models/openvino_R4/requirements.txt

@@ -0,0 +1,2 @@
+google-cloud-storage==1.13.0
+boto3==1.9.34

wrappers/s2i/python/Makefile

@@ -44,6 +44,10 @@ test_local:
 	cat Dockerfile.local.tmpl | sed  -e "s|%PYTHON_VERSION%|$(PYTHON_VERSION)|" > Dockerfile
 	docker build -t $(IMAGE_NAME)-candidate .
 	IMAGE_NAME=$(IMAGE_NAME)-candidate test/run
+.PHONY: build_python_openvino
+build_python_openvino:
+	docker build -f Dockerfile_openvino_base --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -t openvino_base:latest .
+	docker build -f Dockerfile_model_openvino --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -t seldon_model_openvino .	
 
 .PHONY: clean
 clean:

wrappers/s2i/python/README.md

@@ -13,6 +13,8 @@ e.g. from 0.3-SNAPSHOT to release 0.3 and create 0.4-SNAPSHOT
     ``` ./update_python_version.sh X X+1```, e.g ```./update_python_version.sh 0.2 0.3```
 
 
+ * building openvino python base image
+```docker build -f Dockerfile_openvino_base --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy -t openvino_base:latest . ```