From ca9cdd7c659660f0c703a6af45cab48de7115e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Tue, 27 Sep 2022 16:24:12 +0200 Subject: [PATCH 001/130] ovmsclient 2022.2 release changes [develop] (#1440) --- client/python/ovmsclient/lib/Makefile | 2 +- client/python/ovmsclient/lib/README.md | 4 ++-- client/python/ovmsclient/lib/docs/http_client.md | 4 +--- client/python/ovmsclient/lib/docs/pypi_overview.md | 6 +++--- client/python/ovmsclient/lib/setup.py | 6 +++--- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/client/python/ovmsclient/lib/Makefile b/client/python/ovmsclient/lib/Makefile index bf48eb3e6c..c224605d81 100644 --- a/client/python/ovmsclient/lib/Makefile +++ b/client/python/ovmsclient/lib/Makefile @@ -18,7 +18,7 @@ VIRTUALENV_EXE := python3 -m virtualenv -p python3 VIRTUALENV_DIR := .venv ACTIVATE="$(VIRTUALENV_DIR)/bin/activate" -PACKAGE_PATH := dist/ovmsclient-2022.1-py3-none-any.whl +PACKAGE_PATH := dist/ovmsclient-2022.2-py3-none-any.whl .PHONY: build-deps build-package build test clean style diff --git a/client/python/ovmsclient/lib/README.md b/client/python/ovmsclient/lib/README.md index a52ec83c21..41ded7bc8c 100644 --- a/client/python/ovmsclient/lib/README.md +++ b/client/python/ovmsclient/lib/README.md @@ -44,7 +44,7 @@ Assuming you have TFS API built, you can use `make build-package` target to buil **To install the package run:** ```bash -pip3 install --force-reinstall --no-deps dist/ovmsclient-2022.1-py3-none-any.whl +pip3 install --force-reinstall --no-deps dist/ovmsclient-2022.2-py3-none-any.whl ``` *Note*: For development purposes you may want to repeatedly reinstall the package. @@ -61,7 +61,7 @@ make build-package ``` - `make test` - runs tests on `ovmsclient` package. By default the package located in `dist/` directory is used. To specify custom package path pass `PACKAGE_PATH` option like: - `make test PACKAGE_PATH=/opt/packages/ovmsclient-0.2-py3-none-any.whl` + `make test PACKAGE_PATH=/opt/packages/ovmsclient-2022.2-py3-none-any.whl` ```bash make test ``` diff --git a/client/python/ovmsclient/lib/docs/http_client.md b/client/python/ovmsclient/lib/docs/http_client.md index ba7f03530e..731afd3742 100644 --- a/client/python/ovmsclient/lib/docs/http_client.md +++ b/client/python/ovmsclient/lib/docs/http_client.md @@ -162,9 +162,7 @@ Request prediction on provided inputs. | Key | Value type | |---|---| | input_name | string | - | input_data | python scalar, python list, numpy scalar, numpy array, TensorProto | - - If provided **input_data** is not TensorProto, the `make_tensor_proto` function with default parameters will be called internally. + | input_data | python scalar, python list, numpy scalar, numpy array | - `model_name`: name of the requested model. Accepted types: `string`. - `model_version` (optional): version of the requested model. Accepted types: `positive integer`. Value 0 is special and means the latest served version will be chosen (only in OVMS, TFS requires specific version number provided). Default value: 0. diff --git a/client/python/ovmsclient/lib/docs/pypi_overview.md b/client/python/ovmsclient/lib/docs/pypi_overview.md index e1d1e09fda..7d66a086d0 100644 --- a/client/python/ovmsclient/lib/docs/pypi_overview.md +++ b/client/python/ovmsclient/lib/docs/pypi_overview.md @@ -9,7 +9,7 @@ The `ovmsclient` package works both with OpenVINO™ Model Server and Tensor The `ovmsclient` can replace `tensorflow-serving-api` package with reduced footprint and simplified interface. -See [API reference](https://github.com/openvinotoolkit/model_server/blob/releases/2022/1/client/python/ovmsclient/lib/docs/README.md) for usage details. +See [API reference](https://github.com/openvinotoolkit/model_server/blob/releases/2022/2/client/python/ovmsclient/lib/docs/README.md) for usage details. ## Usage example @@ -23,7 +23,7 @@ client = ovmsclient.make_grpc_client("localhost:9000") # Get model metadata to learn about model inputs model_metadata = client.get_model_metadata(model_name="model") -# If model has only one input, get its name like that +# If model has only one input, get its name input_name = next(iter(model_metadata["inputs"])) # Read the image file @@ -38,4 +38,4 @@ results = client.predict(inputs=inputs, model_name="model") ``` -Learn more on `ovmsclient` [documentation site](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/client/python/ovmsclient/lib). \ No newline at end of file +Learn more on `ovmsclient` [documentation site](https://github.com/openvinotoolkit/model_server/tree/releases/2022/2/client/python/ovmsclient/lib). \ No newline at end of file diff --git a/client/python/ovmsclient/lib/setup.py b/client/python/ovmsclient/lib/setup.py index 8f2349500f..e080d585cc 100644 --- a/client/python/ovmsclient/lib/setup.py +++ b/client/python/ovmsclient/lib/setup.py @@ -49,17 +49,17 @@ def run(self): setuptools.setup( name="ovmsclient", - version="2022.1", + version="2022.2", license="Apache License 2.0", author="Intel Corporation", author_email="ovms.engineering@intel.com", description="Python client for OpenVINO Model Server", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/client/python/ovmsclient/lib", + url="https://github.com/openvinotoolkit/model_server/tree/releases/2022/2/client/python/ovmsclient/lib", cmdclass={ "build_apis": BuildApis, }, packages=setuptools.find_namespace_packages(include=["ovmsclient*", "tensorflow*", "tensorflow_serving*"]), - install_requires=["grpcio==1.47.0", "protobuf==3.19.4", "numpy>=1.16.6,<=1.23.1", "requests==2.27.1"], + install_requires=["grpcio==1.47.0", "protobuf>=3.19.4,<4.0.0", "numpy>=1.16.6,<=1.23.1", "requests==2.27.1"], ) From 8b50de61669106bd3de7031626213c02db21dbed Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Thu, 29 Sep 2022 09:33:16 +0200 Subject: [PATCH 002/130] updates for automatic documentation tests (#1444) Co-authored-by: ngrozae --- client/python/kserve-api/samples/README.md | 3 +- .../bert_question_answering/python/README.md | 12 +-- docs/metrics.md | 78 ++++++++++--------- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/client/python/kserve-api/samples/README.md b/client/python/kserve-api/samples/README.md index e807b03b46..7e7f1eff05 100644 --- a/client/python/kserve-api/samples/README.md +++ b/client/python/kserve-api/samples/README.md @@ -544,7 +544,7 @@ python3 ./http_infer_resnet.py --help usage: http_infer_resnet.py [-h] --images_numpy_path IMAGES_NUMPY_PATH [--labels_numpy_path LABELS_NUMPY_PATH] [--http_address HTTP_ADDRESS] [--http_port HTTP_PORT] [--input_name INPUT_NAME] [--output_name OUTPUT_NAME] [--transpose_input {False,True}] [--transpose_method {nchw2nhwc,nhwc2nchw}] [--iterations ITERATIONS] [--batchsize BATCHSIZE] [--model_name MODEL_NAME] - [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] + [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] [--binary_data] Sends requests via KServe REST API using images in numpy format. It displays performance statistics and optionally the model accuracy @@ -576,6 +576,7 @@ optional arguments: Define pipeline name, must be same as is in service --dag-batch-size-auto Add demultiplexer dimension at front + --binary_data Send input data in binary format ``` - Usage Example #1 - Input data placed in JSON object. diff --git a/demos/bert_question_answering/python/README.md b/demos/bert_question_answering/python/README.md index 3f89528112..17b7f46c15 100644 --- a/demos/bert_question_answering/python/README.md +++ b/demos/bert_question_answering/python/README.md @@ -51,10 +51,10 @@ On October 25, 2019, Google Search announced that they had started applying BERT [ INFO ] ---answer: 0.22 Bidirectional Encoder Representations from Transformers [ INFO ] Bidirectional Encoder Representations from Transformers (BERT) is a transformer-based machine learning technique for natural language processing (NLP) pre-training developed by Google. BERT was created and published in 2018 by Jacob Devlin and his colleagues from Google.[1][2] In 2019, Google announced that it had begun leveraging BERT in its search engine, and by late 2020 it was using BERT in almost every English-language query. A 2020 literature survey concluded that "in a little over a year, BERT has become a ubiquitous baseline in NLP experiments", counting over 150 research publications analyzing and improving the model.[3] -The original English-language BERT has two models:[1] (1) the BERTBASE: 12 encoders with 12 bidirectional self-attention heads, and (2) the BERTLARGE: 24 encoders with 16 bidirectional self-attention heads. Both models are pre-trained from unlabeled data extracted from the BooksCorpus[4] with 800M words and English Wikipedia with 2,500M words.[5] -BERT is at its core a transformer language model with a variable number of encoder layers and self-attention heads. The architecture is "almost identical" to the original transformer implementation in Vaswani et al. (2017).[6] -BERT was pretrained on two tasks: language modelling (15% of tokens were masked and BERT was trained to predict them from context) and next sentence prediction (BERT was trained to predict if a chosen next sentence was probable or not given the first sentence). As a result of the training process, BERT learns contextual embeddings for words. After pretraining, which is computationally expensive, BERT can be finetuned with less resources on smaller datasets to optimize its performance on specific tasks.[1][7] +The original English-language BERT has two models:[1] (1) the BERTBASE: 12 encoders with 12 bidirectional self-attention heads, and (2) the BERTLARGE: 24 encoders with 16 bidirectional self-attention heads. Both models are pre-trained from unlabeled data extracted from the BooksCorpus[4] with 800M words and English Wikipedia with 2,500M words. +BERT is at its core a transformer language model with a variable number of encoder layers and self-attention heads. The architecture is "almost identical" to the original transformer implementation in Vaswani et al. (2017).[5] +BERT was pretrained on two tasks: language modeling (15% of tokens were masked and BERT was trained to predict them from context) and next sentence prediction (BERT was trained to predict if a chosen next sentence was probable or not given the first sentence). As a result of the training process, BERT learns contextual embeddings for words. After pretraining, which is computationally expensive, BERT can be finetuned with fewer resources on smaller datasets to optimize its performance on specific tasks.[1][6] -[ INFO ] ---answer: 0.20 BERT won the Best Long Paper Award -[ INFO ] BERT won the Best Long Paper Award at the 2019 Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL).[20] -``` \ No newline at end of file +[ INFO ] ---answer: 0.36 The research paper describing BERT +[ INFO ] The research paper describing BERT won the Best Long Paper Award at the 2019 Annual Conference of the North American Chapter of the Association for Computational Linguistics (NAACL).[20] +``` diff --git a/docs/metrics.md b/docs/metrics.md index e4b5339036..6ee01f4670 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -22,7 +22,7 @@ They are exposed on the `/metrics` endpoint. ## Available metrics families -Metrics from default list are enabled with the `metrics_enabled` flag or json configuration. +Metrics from default list are enabled with the `metrics_enable` flag or json configuration. However, you can enable also additional metrics by listing all the metrics you want to enable in the `metric_list` flag or json configuration. @@ -62,7 +62,7 @@ Metrics endpoint is using the same port as the REST interface for running the mo It is required to enable REST in the model server by setting the parameter --rest_port. -To enable default metrics set you need to specify the `metrics_enabled` flag or json setting: +To enable default metrics set you need to specify the `metrics_enable` flag or json setting: CLI @@ -70,28 +70,19 @@ CLI docker run --rm -d -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ --model_name resnet --model_path gs://ovms-public-eu/resnet50 --port 9000 \ --rest_port 8000 \ - --metrics_enabled - ``` - -CONFIG CMD - - ```bash - docker run --rm -d -v -d -v ${PWD}/workspace:/workspace openvino/model_server --config_path /workspace/config.json -p 8000:8000 -p 9000:9000 openvino/model_server:latest \ - --rest_port 8000 - --port 9000 + --metrics_enable ``` CONFIG JSON ```bash - { + mkdir workspace + echo '{ "model_config_list": [ { "config": { "name": "resnet", - "base_path": "/workspace/resnet-50-tf", - "layout": "NHWC:NCHW", - "shape": "(1,224,224,3)" + "base_path": "gs://ovms-public-eu/resnet50" } } ], @@ -102,7 +93,15 @@ CONFIG JSON "enable" : true } } - } + }' >> workspace/config.json + ``` + +CONFIG CMD + + ```bash + docker run --rm -d -v ${PWD}/workspace:/workspace -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ + --config_path /workspace/config.json \ + --port 9000 --rest_port 8000 ``` ## Change the default list of metrics @@ -117,23 +116,14 @@ CLI docker run --rm -d -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ --model_name resnet --model_path gs://ovms-public-eu/resnet50 --port 9000 \ --rest_port 8000 \ - --metrics_enabled \ + --metrics_enable \ --metrics_list ovms_requests_success,ovms_infer_req_queue_size ``` -CONFIG CMD - - ```bash - docker run --rm -d -v -d -v ${PWD}/workspace:/workspace openvino/model_server \ - --config_path /workspace/config.json -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ - --rest_port 8000 \ - --port 9000 - ``` - CONFIG JSON ```bash - { + echo '{ "model_config_list": [ { "config": { @@ -150,18 +140,26 @@ CONFIG JSON "metrics_list": ["ovms_requests_success", "ovms_infer_req_queue_size"] } } - } - ``` + }' > workspace/config.json + ``` + +CONFIG CMD + + ```bash + docker run --rm -d -v -d -v ${PWD}/workspace:/workspace -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ + --config_path /workspace/config.json \ + --port 9000 --rest_port 8000 + ``` CONFIG JSON WITH ALL METRICS ENABLED -```bash - { + ```bash + echo '{ "model_config_list": [ { "config": { "name": "resnet", - "base_path": "/workspace/resnet-50-tf" + "base_path": "gs://ovms-public-eu/resnet50" } } ], @@ -182,15 +180,23 @@ CONFIG JSON WITH ALL METRICS ENABLED "ovms_infer_req_queue_size"] } } - } - ``` + }' > workspace/config.json + ``` + +CONFIG CMD + + ```bash + docker run --rm -d -v -d -v ${PWD}/workspace:/workspace -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ + --config_path /workspace/config.json \ + --port 9000 --rest_port 8000 + ``` ## Example response from metrics endpoint To use data from metrics endpoint you can use the curl command: -```bash + ```bash curl http://localhost:8000/metrics -``` + ``` [Example metrics output](https://raw.githubusercontent.com/openvinotoolkit/model_server/v2022.2/docs/metrics_output.out) ## Metrics implementation for DAG pipelines From 6ec2ccb8817786f44e042bd9fb4d614b362d1634 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Thu, 29 Sep 2022 13:33:22 +0200 Subject: [PATCH 003/130] CVS-87064_doc_fix (#1447) * AUTO plugin update * THROUGHPUT update Co-authored-by: ngrozae --- docs/accelerators.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/accelerators.md b/docs/accelerators.md index 3ca1f78b81..d7f3f44464 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -181,7 +181,7 @@ Make sure you have passed the devices and access to the devices you want to use Below is an example of the command with AUTO Plugin as target device. It includes extra docker parameters to enable GPU (/dev/dri) , beside CPU. ```bash - docker run --rm -d --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1)\ + docker run --rm -d --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) \ -u $(id -u):$(id -g) -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest \ --model_path /opt/model --model_name resnet --port 9001 \ --target_device AUTO @@ -203,13 +203,13 @@ LATENCY --target_device AUTO ``` -THROUGHTPUT +THROUGHPUT ```bash docker run --rm -d --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest \ --model_path /opt/model --model_name resnet --port 9001 \ - --plugin_config '{"PERFORMANCE_HINT": "THROUGHTPUT"}' \ + --plugin_config '{"PERFORMANCE_HINT": "THROUGHPUT"}' \ --target_device AUTO ``` From bbee2226ceb217bb81a5d72a9b78e4c0ef091fb2 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 4 Oct 2022 14:42:45 +0200 Subject: [PATCH 004/130] Update links to 2022.2 (#1445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miłosz Żeglarski Co-authored-by: bstrzele Co-authored-by: Dariusz Trawinski --- README.md | 46 +++++++++---------- client/python/ovmsclient/lib/README.md | 4 +- client/python/ovmsclient/samples/README.md | 2 +- demos/image_classification/go/README.md | 2 +- demos/image_classification/python/README.md | 2 +- demos/model_ensemble/python/README.md | 2 +- .../person_vehicle_bike_detection/README.md | 2 +- .../python/README.md | 27 ++++++----- docs/accelerators.md | 14 +++--- docs/architecture.md | 4 +- docs/docker_container.md | 4 +- docs/dynamic_shape_dynamic_model.md | 2 +- docs/home.md | 6 +-- docs/host.md | 4 +- docs/model_cache.md | 2 +- docs/models_repository.md | 4 +- docs/ovms_quickstart.md | 6 +-- docs/parameters.md | 8 ++-- docs/performance_tuning.md | 8 ++-- docs/security_considerations.md | 2 +- docs/shape_batch_size_and_layout.md | 2 +- docs/stateful_models.md | 10 ++-- docs/tf_model_binary_input.md | 6 +-- .../image_transformation/README.md | 6 +-- src/example/SampleCpuExtension/README.md | 2 +- tests/functional/model/models_information.py | 17 ++----- 26 files changed, 93 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 84cafc33db..5c118f8670 100644 --- a/README.md +++ b/README.md @@ -12,23 +12,23 @@ Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage. Read [release notes](https://github.com/openvinotoolkit/model_server/releases) to find out what’s new. -Review the [Architecture concept](https://docs.openvino.ai/2022.1/ovms_docs_architecture.html) document for more details. +Review the [Architecture concept](https://docs.openvino.ai/2022.2/ovms_docs_architecture.html) document for more details. Key features: - support for multiple frameworks, such as Caffe, TensorFlow, MXNet, PaddlePaddle and ONNX -- online deployment of new [model versions](https://docs.openvino.ai/2022.1/ovms_docs_model_version_policy.html) -- [configuration updates in runtime](https://docs.openvino.ai/2022.1/ovms_docs_online_config_changes.html) +- online deployment of new [model versions](https://docs.openvino.ai/2022.2/ovms_docs_model_version_policy.html) +- [configuration updates in runtime](https://docs.openvino.ai/2022.2/ovms_docs_online_config_changes.html) - support for AI accelerators, such as -[Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), -[GPU](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_GPU.html), and -[HDDL](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_HDDL.html) -- works with [Bare Metal Hosts](docs/host.md) as well as [Docker containers](https://docs.openvino.ai/2022.1/ovms_docs_docker_container.html) -- [model reshaping](https://docs.openvino.ai/2022.1/ovms_docs_shape_batch_layout.html) in runtime -- [directed Acyclic Graph Scheduler](https://docs.openvino.ai/2022.1/ovms_docs_dag.html) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead -- [custom nodes in DAG pipelines](https://docs.openvino.ai/2022.1/ovms_docs_custom_node_development.html) - allowing model inference and data transformations to be implemented with a custom node C/C++ dynamic library -- [serving stateful models](https://docs.openvino.ai/2022.1/ovms_docs_stateful_models.html) - models that operate on sequences of data and maintain their state between inference requests -- [binary format of the input data](https://docs.openvino.ai/2022.1/ovms_docs_binary_input.html) - data can be sent in JPEG or PNG formats to reduce traffic and offload the client applications -- [model caching](https://docs.openvino.ai/2022.1/ovms_docs_model_cache.html) - cache the models on first load and re-use models from cache on subsequent loads +[Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), +[GPU](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html), and +[HDDL](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_HDDL.html) +- works with [Bare Metal Hosts](docs/host.md) as well as [Docker containers](https://docs.openvino.ai/2022.2/ovms_docs_docker_container.html) +- [model reshaping](https://docs.openvino.ai/2022.2/ovms_docs_shape_batch_layout.html) in runtime +- [directed Acyclic Graph Scheduler](https://docs.openvino.ai/2022.2/ovms_docs_dag.html) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead +- [custom nodes in DAG pipelines](https://docs.openvino.ai/2022.2/ovms_docs_custom_node_development.html) - allowing model inference and data transformations to be implemented with a custom node C/C++ dynamic library +- [serving stateful models](https://docs.openvino.ai/2022.2/ovms_docs_stateful_models.html) - models that operate on sequences of data and maintain their state between inference requests +- [binary format of the input data](https://docs.openvino.ai/2022.2/ovms_docs_binary_input.html) - data can be sent in JPEG or PNG formats to reduce traffic and offload the client applications +- [model caching](https://docs.openvino.ai/2022.2/ovms_docs_model_cache.html) - cache the models on first load and re-use models from cache on subsequent loads - [metrics](https://docs.openvino.ai/2022.2/ovms_docs_metrics.html) - metrics compatible with Prometheus standard @@ -40,28 +40,28 @@ They are stored in: ## Run OpenVINO Model Server -A demonstration on how to use OpenVINO Model Server can be found in [our quick-start guide](https://docs.openvino.ai/2022.1/ovms_docs_quick_start_guide.html). +A demonstration on how to use OpenVINO Model Server can be found in [our quick-start guide](https://docs.openvino.ai/2022.2/ovms_docs_quick_start_guide.html). For more information on using Model Server in various scenarios you can check the following guides: -* [Model repository configuration](https://docs.openvino.ai/2022.1/ovms_docs_models_repository.html) +* [Model repository configuration](https://docs.openvino.ai/2022.2/ovms_docs_models_repository.html) -* [Using a docker container](https://docs.openvino.ai/2022.1/ovms_docs_docker_container.html) +* [Using a docker container](https://docs.openvino.ai/2022.2/ovms_docs_docker_container.html) -* [Landing on bare metal or virtual machine](https://docs.openvino.ai/2022.1/ovms_docs_baremetal.html) +* [Landing on bare metal or virtual machine](https://docs.openvino.ai/2022.2/ovms_docs_baremetal.html) -* [Performance tuning](https://docs.openvino.ai/2022.1/ovms_docs_performance_tuning.html) +* [Performance tuning](https://docs.openvino.ai/2022.2/ovms_docs_performance_tuning.html) -* [Directed Acyclic Graph Scheduler](https://docs.openvino.ai/2022.1/ovms_docs_dag.html) +* [Directed Acyclic Graph Scheduler](https://docs.openvino.ai/2022.2/ovms_docs_dag.html) -* [Custom nodes development](https://docs.openvino.ai/2022.1/ovms_docs_custom_node_development.html) +* [Custom nodes development](https://docs.openvino.ai/2022.2/ovms_docs_custom_node_development.html) -* [Serving stateful models](https://docs.openvino.ai/2022.1/ovms_docs_stateful_models.html) +* [Serving stateful models](https://docs.openvino.ai/2022.2/ovms_docs_stateful_models.html) -* [Deploy using a Kubernetes Helm Chart](https://docs.openvino.ai/2022.1/ovms_deploy_helm_chart.html) +* [Deploy using a Kubernetes Helm Chart](https://docs.openvino.ai/2022.2/ovms_deploy_helm_chart.html) * [Deployment using Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) -* [Using binary input data](https://docs.openvino.ai/2022.1/ovms_docs_binary_input.html) +* [Using binary input data](https://docs.openvino.ai/2022.2/ovms_docs_binary_input.html) diff --git a/client/python/ovmsclient/lib/README.md b/client/python/ovmsclient/lib/README.md index 41ded7bc8c..920ab830cd 100644 --- a/client/python/ovmsclient/lib/README.md +++ b/client/python/ovmsclient/lib/README.md @@ -100,7 +100,7 @@ model_status = client.get_model_status(model_name="model") model_metadata = client.get_model_metadata(model_name="model") # Exemplary model_metadata. Values for model: -# https://docs.openvino.ai/2022.1/omz_models_model_resnet_50_tf.html +# https://docs.openvino.ai/2022.2/omz_models_model_resnet_50_tf.html # #{ # "model_version": 1, @@ -123,7 +123,7 @@ model_metadata = client.get_model_metadata(model_name="model") **Create and send predict request with binary input data:** ```python # Assuming requesting model with inputs and outputs as in: -# https://docs.openvino.ai/2022.1/omz_models_model_resnet_50_tf.html +# https://docs.openvino.ai/2022.2/omz_models_model_resnet_50_tf.html with open(, 'rb') as f: img = f.read() diff --git a/client/python/ovmsclient/samples/README.md b/client/python/ovmsclient/samples/README.md index f08acdf161..6a9252b8f1 100644 --- a/client/python/ovmsclient/samples/README.md +++ b/client/python/ovmsclient/samples/README.md @@ -31,7 +31,7 @@ Install samples dependencies: pip3 install -r requirements.txt ``` -Download [Resnet50-tf Model](https://docs.openvino.ai/2022.1/omz_models_model_resnet_50_tf.html) and convert it into Intermediate Representation format: +Download [Resnet50-tf Model](https://docs.openvino.ai/2022.2/omz_models_model_resnet_50_tf.html) and convert it into Intermediate Representation format: ```bash mkdir models docker run -u $(id -u):$(id -g) -v ${PWD}/models:/models openvino/ubuntu20_dev:latest omz_downloader --name resnet-50-tf --output_dir /models diff --git a/demos/image_classification/go/README.md b/demos/image_classification/go/README.md index c715195296..119b2bf535 100644 --- a/demos/image_classification/go/README.md +++ b/demos/image_classification/go/README.md @@ -11,7 +11,7 @@ cd model_server/demos/image_classification/go ## Get the model -To run end to end flow and get correct results, please download `resnet-50-tf` model and convert it to IR format by following [instructions available on the OpenVINO Model Zoo page](https://docs.openvino.ai/2022.1/omz_models_model_resnet_50_tf.html) +To run end to end flow and get correct results, please download `resnet-50-tf` model and convert it to IR format by following [instructions available on the OpenVINO Model Zoo page](https://docs.openvino.ai/2022.2/omz_models_model_resnet_50_tf.html) Place converted model files (XML and BIN) under the following path: `/resnet-50-tf/1` diff --git a/demos/image_classification/python/README.md b/demos/image_classification/python/README.md index 16021ad68d..15d34a65fa 100644 --- a/demos/image_classification/python/README.md +++ b/demos/image_classification/python/README.md @@ -2,7 +2,7 @@ ## Overview -The script [image_classification.py](https://github.com/openvinotoolkit/model_server/blob/releases/2022/1/demos/image_classification/python/image_classification.py) reads all images and their labels specified in the text file. It then classifies them with [ResNet50](https://docs.openvino.ai/2022.1/omz_models_model_resnet50_binary_0001.html) model and presents accuracy results. +The script [image_classification.py](https://github.com/openvinotoolkit/model_server/blob/releases/2022/1/demos/image_classification/python/image_classification.py) reads all images and their labels specified in the text file. It then classifies them with [ResNet50](https://docs.openvino.ai/2022.2/omz_models_model_resnet50_binary_0001.html) model and presents accuracy results. ## Download ResNet50 model diff --git a/demos/model_ensemble/python/README.md b/demos/model_ensemble/python/README.md index 3b17417eda..b07f1bcdeb 100644 --- a/demos/model_ensemble/python/README.md +++ b/demos/model_ensemble/python/README.md @@ -3,7 +3,7 @@ This guide shows how to implement a model ensemble using the [DAG Scheduler](../../../docs/dag_scheduler.md). - Let's consider you develop an application to perform image classification. There are many different models that can be used for this task. The goal is to combine results from inferences executed on two different models and calculate argmax to pick the most probable classification label. -- For this task, select two models: [googlenet-v2](https://docs.openvino.ai/2022.1/omz_models_public_googlenet_v2_tf_googlenet_v2_tf.html) and [resnet-50](https://docs.openvino.ai/2022.1/omz_models_public_resnet_50_tf_resnet_50_tf.html). Additionally, create own model **argmax** to combine and select top result. The aim is to perform this task on the server side with no intermediate results passed over the network. The server should take care of feeding inputs/outputs in subsequent models. Both - googlenet and resnet predictions should run in parallel. +- For this task, select two models: [googlenet-v2](https://docs.openvino.ai/2022.2/omz_models_public_googlenet_v2_tf_googlenet_v2_tf.html) and [resnet-50](https://docs.openvino.ai/2022.2/omz_models_public_resnet_50_tf_resnet_50_tf.html). Additionally, create own model **argmax** to combine and select top result. The aim is to perform this task on the server side with no intermediate results passed over the network. The server should take care of feeding inputs/outputs in subsequent models. Both - googlenet and resnet predictions should run in parallel. - Diagram for this pipeline would look like this: ![diagram](model_ensemble_diagram.svg) diff --git a/demos/real_time_stream_analysis/python/use_cases/person_vehicle_bike_detection/README.md b/demos/real_time_stream_analysis/python/use_cases/person_vehicle_bike_detection/README.md index e5ebcf9bf7..dd6134c622 100644 --- a/demos/real_time_stream_analysis/python/use_cases/person_vehicle_bike_detection/README.md +++ b/demos/real_time_stream_analysis/python/use_cases/person_vehicle_bike_detection/README.md @@ -2,7 +2,7 @@ ## Download Model -Model used in this example is [person-vehicle-bike-detection-2002](https://docs.openvino.ai/2022.1/omz_models_model_person_vehicle_bike_detection_2002.html). +Model used in this example is [person-vehicle-bike-detection-2002](https://docs.openvino.ai/2022.2/omz_models_model_person_vehicle_bike_detection_2002.html). Create `workspace` and model directory and download model in IR format: ```bash mkdir -p workspace/person-vehcile-bike-detection-2002/1 diff --git a/demos/speech_recognition_with_kaldi_model/python/README.md b/demos/speech_recognition_with_kaldi_model/python/README.md index 517d7a2bb0..6c58cba818 100644 --- a/demos/speech_recognition_with_kaldi_model/python/README.md +++ b/demos/speech_recognition_with_kaldi_model/python/README.md @@ -22,34 +22,33 @@ pip3 install -r requirements.txt ### Getting ready with rm_lstm4f stateful model To run this example you will need to download the rm_lstm4f model with input and score ark files and convert it to IR format. -- Download the model from [rm_lstm4f](https://download.01.org/openvinotoolkit/models_contrib/speech/kaldi/rm_lstm4f/) +- Download the model from [rm_lstm4f](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/) -```bash -mkdir models && cd models -wget -r -np -nH --cut-dirs=5 -R *index.html* https://download.01.org/openvinotoolkit/models_contrib/speech/kaldi/rm_lstm4f/ -``` -This command downloads following files: +Those commands will download nessesary files: -``` -rm_lstm4f.counts rm_lstm4f.nnet rm_lstm4f.mapping rm_lstm4f.md +```bash +mkdir models && cd models +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.counts +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.nnet +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.mapping ``` -rm_lstm4f model files in Kaldi format. +rm_lstm4f model files in Kaldi format: -``` -test_feat_1_10.ark +```bash +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/test_feat_1_10.ark ``` [Kaldi's](http://kaldi-asr.org/doc/io.html) binary archive file with input data for the model -``` -test_score_1_10.ark +```bash +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/test_score_1_10.ark ``` [Kaldi's](http://kaldi-asr.org/doc/io.html) binary archive file with reference model results -- [Convert model to IR](https://docs.openvino.ai/2022.1/openvino_inference_engine_samples_speech_sample_README.html) +- [Convert model to IR](https://docs.openvino.ai/2022.2/openvino_inference_engine_samples_speech_sample_README.html) ```bash docker run -u $(id -u):$(id -g) -v $(pwd):/models:rw openvino/ubuntu20_dev:latest mo --framework kaldi --input_model /models/rm_lstm4f.nnet --counts /models/rm_lstm4f.counts --remove_output_softmax --output_dir /models/rm_lstm4f/1 diff --git a/docs/accelerators.md b/docs/accelerators.md index d7f3f44464..3f0be8657c 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -15,7 +15,7 @@ mv ${PWD}/models/public/resnet-50-tf/FP32 ${PWD}/models/public/resnet-50-tf/1 ## Starting the server with the Intel® Neural Compute Stick 2 [Intel Movidius Neural Compute Stick 2](https://software.intel.com/en-us/neural-compute-stick) can be employed by OVMS OpenVINO Model Server via -[the MYRIAD plugin](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_MYRIAD.html). It must be visible and accessible on the host machine. +[the MYRIAD plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html). It must be visible and accessible on the host machine. NCS devices should be reported by the `lsusb` command, printing out `ID 03e7:2485`. @@ -38,7 +38,7 @@ To start the server with Neural Compute Stick use either of the two options: To run a container that is using the HDDL accelerator, _hddldaemon_ must be running on the host machine. You must set up the environment (the OpenVINO package must be pre-installed) and start _hddldaemon_ on the host before starting a container. -Refer to the steps from [OpenVINO installation guides](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_docker_linux.html#running-the-image-on-intel-vision-accelerator-design-with-intel-movidius-vpus). +Refer to the steps from [OpenVINO installation guides](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_docker_linux.html#running-the-image-on-intel-vision-accelerator-design-with-intel-movidius-vpus). An example of a command starting a server with HDDL: ```bash @@ -58,13 +58,13 @@ Check out our recommendations for [throughput optimization on HDDL](performance_ ## Starting a Docker Container with Intel GPU -The [GPU plugin](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_GPU.html) uses the Intel Compute Library for +The [GPU plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html) uses the Intel Compute Library for Deep Neural Networks ([clDNN](https://01.org/cldnn)) to infer deep neural networks. For inference execution, it employs Intel® Processor Graphics including Intel® HD Graphics, Intel® Iris® Graphics, Intel® Iris® Xe Graphics, and Intel® Iris® Xe MAX graphics. Before using GPU as OpenVINO Model Server target device, you need to: -- install the required drivers - refer to [OpenVINO installation guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html#step-5-optional-configure-inference-on-non-cpu-devices) +- install the required drivers - refer to [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_linux.html#step-5-optional-configure-inference-on-non-cpu-devices) - start the docker container with the additional parameter of `--device /dev/dri` to pass the device context - set the parameter of `--target_device` to `GPU`. - use the `openvino/model_server:latest-gpu` image, which contains GPU dependencies @@ -109,7 +109,7 @@ make docker_build INSTALL_DRIVER_VERSION=22.10.22597 ## Using Multi-Device Plugin If you have multiple inference devices available (e.g. Myriad VPUs and CPU) you can increase inference throughput by enabling the Multi-Device Plugin. -It distributes Inference requests among multiple devices, balancing out the load. For more detailed information read OpenVINO’s [Multi-Device plugin documentation](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_Running_on_multiple_devices.html) documentation. +It distributes Inference requests among multiple devices, balancing out the load. For more detailed information read OpenVINO’s [Multi-Device plugin documentation](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_Running_on_multiple_devices.html) documentation. To use this feature in OpenVINO Model Server, you can choose one of two ways: @@ -150,7 +150,7 @@ The total throughput will be roughly equal to the sum of CPU and Intel Movidius ## Using Heterogeneous Plugin -The [HETERO plugin](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_Hetero_execution.html) makes it possible to distribute inference load of one model +The [HETERO plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_Hetero_execution.html) makes it possible to distribute inference load of one model among several computing devices. That way different parts of the deep learning network can be executed by devices best suited to their type of calculations. OpenVINO automatically divides the network to optimize the process. @@ -172,7 +172,7 @@ echo '{"model_config_list": [ ## Using AUTO Plugin -[Auto Device](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_supported_plugins_AUTO.html) (or AUTO in short) is a new special “virtual” or “proxy” device in the OpenVINO toolkit, it doesn’t bind to a specific type of HW device. +[Auto Device](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_AUTO.html) (or AUTO in short) is a new special “virtual” or “proxy” device in the OpenVINO toolkit, it doesn’t bind to a specific type of HW device. AUTO solves the complexity in application required to code a logic for the HW device selection (through HW devices) and then, on the deducing the best optimization settings on that device. AUTO always chooses the best device, if compiling model fails on this device, AUTO will try to compile it on next best device until one of them succeeds. Make sure you have passed the devices and access to the devices you want to use in for the docker image. For example with: diff --git a/docs/architecture.md b/docs/architecture.md index beb36874b5..0bb5c6761b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,12 +2,12 @@ - OpenVINO™ Model Server provides a C++ implementation of the gRPC and RESTful API interfaces compatible with [Tensorflow Serving](https://www.tensorflow.org/tfx/guide/serving). -- In the backend, OpenVINO™ Model Server uses [OpenVINO™ Runtime](https://docs.openvino.ai/2022.1/index.html) libraries from OpenVINO™ toolkit. This speeds up execution on the CPU and enables it on AI accelerators, like [Neural Compute Stick 2](https://software.intel.com/content/www/us/en/main/hardware/neural-compute-stick.html), iGPU(Integrated Graphics Processing Unit), and [HDDL](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_ivad_vpu.html). +- In the backend, OpenVINO™ Model Server uses [OpenVINO™ Runtime](https://docs.openvino.ai/2022.2/index.html) libraries from OpenVINO™ toolkit. This speeds up execution on the CPU and enables it on AI accelerators, like [Neural Compute Stick 2](https://software.intel.com/content/www/us/en/main/hardware/neural-compute-stick.html), iGPU(Integrated Graphics Processing Unit), and [HDDL](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_ivad_vpu.html). - API requests in gRPC code backbone are created based on [TensorFlow Serving Core Framework](https://www.tensorflow.org/tfx/guide/serving) with tuned implementation of request handling. -- Services are designed via a set of C++ classes managing AI models in the Intermediate Representation format. [OpenVINO™ Runtime](https://docs.openvino.ai/2022.1/index.html) executes the model's operations. +- Services are designed via a set of C++ classes managing AI models in the Intermediate Representation format. [OpenVINO™ Runtime](https://docs.openvino.ai/2022.2/index.html) executes the model's operations. ![serving](serving-c.png) diff --git a/docs/docker_container.md b/docs/docker_container.md index 2af8d4569b..d600ebe48d 100644 --- a/docs/docker_container.md +++ b/docs/docker_container.md @@ -6,7 +6,7 @@ This is a step-by-step guide on how to deploy OpenVINO™ Model Server on Li - [Docker Engine](https://docs.docker.com/engine/) installed ([How to Install Docker Engine](https://docs.docker.com/engine/install/)) - Intel® Core™ processor (6-12th gen.) or Intel® Xeon® processor -- (optional) AI accelerators [supported by OpenVINO](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) +- (optional) AI accelerators [supported by OpenVINO](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) - Linux, macOS or Windows via [WSL](https://docs.microsoft.com/en-us/windows/wsl/) **NOTE:** accelerators are only tested on bare-metal Linux hosts. @@ -115,7 +115,7 @@ make docker_build BASE_OS=ubuntu OVMS_CPP_DOCKER_IMAGE=ovms_dg2 INSTALL_DRIVER_V If you have multiple inference devices available (e.g. Myriad VPUs and CPU) you can increase inference throughput by enabling the Multi-Device Plugin. With Multi-Device Plugin enabled, inference requests will be load balanced between multiple devices. -For more detailed information read [OpenVino's Multi-Device plugin documentation](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_Running_on_multiple_devices.html). +For more detailed information read [OpenVino's Multi-Device plugin documentation](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_Running_on_multiple_devices.html). In order to use this feature in OpenVino™ Model Server, following steps are required: diff --git a/docs/dynamic_shape_dynamic_model.md b/docs/dynamic_shape_dynamic_model.md index a70c783288..aae42a41be 100644 --- a/docs/dynamic_shape_dynamic_model.md +++ b/docs/dynamic_shape_dynamic_model.md @@ -8,7 +8,7 @@ Enable dynamic shape by setting the `shape` parameter to range or undefined: - `--shape "(1,3,200:500,200:500)"` when model is supposed to support height and width values in a range of 200-500. Note that any dimension can support range of values, height and width are only examples here. > Note that some models do not support dynamic dimensions. Learn more about supported model graph layers including all limitations -on [Shape Inference Document](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_ShapeInference.html). +on [Shape Inference Document](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_ShapeInference.html). Another option to use dynamic shape feature is to export the model with dynamic dimension using Model Optimizer. OpenVINO Model Server will inherit the dynamic shape and no additional settings are needed. diff --git a/docs/home.md b/docs/home.md index 9329155504..dc87d69fc8 100644 --- a/docs/home.md +++ b/docs/home.md @@ -47,9 +47,9 @@ Key features: - online deployment of new [model versions](model_version_policy.md) - [configuration updates in runtime](online_config_changes.md) - support for AI accelerators, such as -[Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), -[GPU](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_GPU.html), and -[HDDL](https://docs.openvino.ai/2022.1/openvino_docs_OV_UG_supported_plugins_HDDL.html) +[Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), +[GPU](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html), and +[HDDL](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_HDDL.html) - works with [Bare Metal Hosts](host.md) as well as [Docker containers](docker_container.md) - [model reshaping](shape_batch_size_and_layout.md) in runtime - [directed Acyclic Graph Scheduler](dag_scheduler.md) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead diff --git a/docs/host.md b/docs/host.md index 2eff906174..bf92c83579 100644 --- a/docs/host.md +++ b/docs/host.md @@ -9,7 +9,7 @@ OpenVINO Model Server can be hosted on a bare metal server, virtual machine, or OpenVINO Model Server execution on baremetal is tested on Ubuntu 20.04.x. For other operating systems we recommend using [OVMS docker containers](./docker_container.md). -For supported hardware, refer to [supported configurations](https://docs.openvino.ai/2022.1/_docs_IE_DG_supported_plugins_Supported_Devices.html). +For supported hardware, refer to [supported configurations](https://docs.openvino.ai/2022.2/_docs_IE_DG_supported_plugins_Supported_Devices.html). Always verify if your model is supported by the VPU Plugins and convert it to the OpenVINO format, using [OpenVINO Model Optimizer](https://software.intel.com/en-us/articles/OpenVINO-ModelOptimizer). ## Installing Model Server @@ -47,4 +47,4 @@ Refer to [Running Model Server using Docker Container](./docker_container.md) to > **NOTE**: > When AI accelerators are used for inference execution, additional steps may be required to install their drivers and dependencies. Learn more about it -> Learn more about it on [OpenVINO installation guide](https://docs.openvino.ai/2022.1/openvino_docs_install_guides_installing_openvino_linux.html). +> Learn more about it on [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_linux.html). diff --git a/docs/model_cache.md b/docs/model_cache.md index 8628fc9beb..47d94572bd 100644 --- a/docs/model_cache.md +++ b/docs/model_cache.md @@ -1,7 +1,7 @@ # Model Cache {#ovms_docs_model_cache} ## Overview -The Model Server can leverage a [OpenVINO™ model cache functionality](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_Model_caching_overview.html), to speed up subsequent model loading on a target device. +The Model Server can leverage a [OpenVINO™ model cache functionality](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_Model_caching_overview.html), to speed up subsequent model loading on a target device. The cached files make the Model Server initialization usually faster. The boost depends on a model and a target device. The most noticable improvement will be observed with GPU devices. On other devices, like CPU, it is possible to observe no speed up effect or even slower loading process depending on used model. Test the setup before final deployment. diff --git a/docs/models_repository.md b/docs/models_repository.md index 382e502f33..c8d8aca01c 100644 --- a/docs/models_repository.md +++ b/docs/models_repository.md @@ -1,12 +1,12 @@ # Model Repository {#ovms_docs_models_repository} The AI models served by OpenVINO™ Model Server must be in either of the three formats: -- [OpenVINO IR](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets), where the graph is represented in .bin and .xml files +- [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets), where the graph is represented in .bin and .xml files - [ONNX](https://onnx.ai/), using the .onnx file - [PaddlePaddle](https://www.paddlepaddle.org.cn/en), using .pdiparams and .pdmodel files To use models trained in other formats you need to convert them first. To do so, use -OpenVINO’s [Model Optimizer](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) for IR, or different +OpenVINO’s [Model Optimizer](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) for IR, or different [converters](https://onnx.ai/supported-tools.html) for ONNX. The models need to be placed and mounted in a particular directory structure and according to the following rules: diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index 4691dc0dab..8f4c4e8969 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -1,10 +1,10 @@ # Quickstart Guide {#ovms_docs_quick_start_guide} -OpenVINO Model Server can perform inference using pre-trained models in either [OpenVINO IR](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets) +OpenVINO Model Server can perform inference using pre-trained models in either [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets) or [ONNX](https://onnx.ai/) format. You can get them by: -- downloading proper models from [Open Model Zoo](https://download.01.org/opencv/2021/openvinotoolkit/2021.1/open_model_zoo/models_bin/) -- converting other formats using [Model Optimizer](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) +- downloading proper models from [Open Model Zoo](https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/) +- converting other formats using [Model Optimizer](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) To quickly start using OpenVINO™ Model Server follow these steps: 1. Prepare Docker diff --git a/docs/parameters.md b/docs/parameters.md index 7d7ab751e3..93f24f8d45 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -7,17 +7,17 @@ |---|---|---| | `"model_name"/"name"` | `string` | Model name exposed over gRPC and REST API.(use `model_name` in command line, `name` in json config) | | `"model_path"/"base_path"` | `string` | If using a Google Cloud Storage, Azure Storage or S3 path, see [cloud storage guide](./using_cloud_storage.md). The path may look as follows:
`"/opt/ml/models/model"`
`"gs://bucket/models/model"`
`"s3://bucket/models/model"`
`"azure://bucket/models/model"`
(use `model_path` in command line, `base_path` in json config) | -| `"shape"` | `tuple/json/"auto"` | `shape` is optional and takes precedence over `batch_size`. The `shape` argument changes the model that is enabled in the model server to fit the parameters. `shape` accepts three forms of the values: * `auto` - The model server reloads the model with the shape that matches the input data matrix. * a tuple, such as `(1,3,224,224)` - The tuple defines the shape to use for all incoming requests for models with a single input. * A dictionary of shapes, such as `{"input1":"(1,3,224,224)","input2":"(1,3,50,50)", "input3":"auto"}` - This option defines the shape of every included input in the model.Some models don't support the reshape operation.If the model can't be reshaped, it remains in the original parameters and all requests with incompatible input format result in an error. See the logs for more information about specific errors.Learn more about supported model graph layers including all limitations at [Shape Inference Document](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_ShapeInference.html). | +| `"shape"` | `tuple/json/"auto"` | `shape` is optional and takes precedence over `batch_size`. The `shape` argument changes the model that is enabled in the model server to fit the parameters. `shape` accepts three forms of the values: * `auto` - The model server reloads the model with the shape that matches the input data matrix. * a tuple, such as `(1,3,224,224)` - The tuple defines the shape to use for all incoming requests for models with a single input. * A dictionary of shapes, such as `{"input1":"(1,3,224,224)","input2":"(1,3,50,50)", "input3":"auto"}` - This option defines the shape of every included input in the model.Some models don't support the reshape operation.If the model can't be reshaped, it remains in the original parameters and all requests with incompatible input format result in an error. See the logs for more information about specific errors.Learn more about supported model graph layers including all limitations at [Shape Inference Document](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_ShapeInference.html). | | `"batch_size"` | `integer/"auto"` | Optional. By default, the batch size is derived from the model, defined through the OpenVINO Model Optimizer. `batch_size` is useful for sequential inference requests of the same batch size.Some models, such as object detection, don't work correctly with the `batch_size` parameter. With these models, the output's first dimension doesn't represent the batch size. You can set the batch size for these models by using network reshaping and setting the `shape` parameter appropriately.The default option of using the Model Optimizer to determine the batch size uses the size of the first dimension in the first input for the size. For example, if the input shape is `(1, 3, 225, 225)`, the batch size is set to `1`. If you set `batch_size` to a numerical value, the model batch size is changed when the service starts.`batch_size` also accepts a value of `auto`. If you use `auto`, then the served model batch size is set according to the incoming data at run time. The model is reloaded each time the input data changes the batch size. You might see a delayed response upon the first request. | | `"layout" `| `json/string` | `layout` is optional argument which allows to define or change the layout of model input and output tensors. To change the layout (add the transposition step), specify `:`. Example: `NHWC:NCHW` means that user will send input data in `NHWC` layout while the model is in `NCHW` layout.

When specified without colon separator, it doesn't add a transposition but can determine the batch dimension. E.g. `--layout CN` makes prediction service treat second dimension as batch size.

When the model has multiple inputs or the output layout has to be changed, use a json format. Set the mapping, such as: `{"input1":"NHWC:NCHW","input2":"HWN:NHW","output1":"CN:NC"}`.

If not specified, layout is inherited from model.

[Read more](shape_batch_size_and_layout.md#changing-model-inputoutput-layout) | | `"model_version_policy"` | `json/string` | Optional.The model version policy lets you decide which versions of a model that the OpenVINO Model Server is to serve. By default, the server serves the latest version. One reason to use this argument is to control the server memory consumption.The accepted format is in json or string. Examples:
`{"latest": { "num_versions":2 }`
`{"specific": { "versions":[1, 3] } }`
`{"all": {} }` | -| `"plugin_config"` | `json/string` | List of device plugin parameters. For full list refer to [OpenVINO documentation](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) and [performance tuning guide](./performance_tuning.md). Example:
`{"CPU_THROUGHPUT_STREAMS": "CPU_THROUGHPUT_AUTO"}` | +| `"plugin_config"` | `json/string` | List of device plugin parameters. For full list refer to [OpenVINO documentation](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) and [performance tuning guide](./performance_tuning.md). Example:
`{"CPU_THROUGHPUT_STREAMS": "CPU_THROUGHPUT_AUTO"}` | | `"nireq"` | `integer` | The size of internal request queue. When set to 0 or no value is set value is calculated automatically based on available resources.| | `"target_device"` | `string` | Device name to be used to execute inference operations. Accepted values are: `"CPU"/"HDDL"/"GPU"/"MYRIAD"/"MULTI"/"HETERO"` | | `"stateful"` | `bool` | If set to true, model is loaded as stateful. | | `"idle_sequence_cleanup"` | `bool` | If set to true, model will be subject to periodic sequence cleaner scans. See [idle sequence cleanup](stateful_models.md). | | `"max_sequence_number"` | `uint32` | Determines how many sequences can be handled concurrently by a model instance. | -| `"low_latency_transformation"` | `bool` | If set to true, model server will apply [low latency transformation](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) on model load. | +| `"low_latency_transformation"` | `bool` | If set to true, model server will apply [low latency transformation](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) on model load. | ## Server configuration options @@ -34,7 +34,7 @@ Configuration options for the server are defined only via command-line options a | `file_system_poll_wait_seconds` | `integer` | Time interval between config and model versions changes detection in seconds. Default value is 1. Zero value disables changes monitoring. | | `sequence_cleaner_poll_wait_minutes` | `integer` | Time interval (in minutes) between next sequence cleaner scans. Sequences of the models that are subjects to idle sequence cleanup that have been inactive since the last scan are removed. Zero value disables sequence cleaner. See [idle sequence cleanup](stateful_models.md). | | `custom_node_resources_cleaner_interval` | `integer` | Time interval (in seconds) between two consecutive resources cleanup scans. Default is 1. Must be greater than 0. See [custom node development](custom_node_development.md). | -| `cpu_extension` | `string` | Optional path to a library with [custom layers implementation](https://docs.openvino.ai/2022.1/openvino_docs_Extensibility_UG_Intro.html). | +| `cpu_extension` | `string` | Optional path to a library with [custom layers implementation](https://docs.openvino.ai/2022.2/openvino_docs_Extensibility_UG_Intro.html). | | `log_level` | `"DEBUG"/"INFO"/"ERROR"` | Serving logging level | | `log_path` | `string` | Optional path to the log file. | | `cache_dir` | `string` | Path to the model cache storage. Caching will be enabled if this parameter is defined or the default path /opt/cache exists | diff --git a/docs/performance_tuning.md b/docs/performance_tuning.md index 02cee1d900..1bc3682296 100644 --- a/docs/performance_tuning.md +++ b/docs/performance_tuning.md @@ -44,7 +44,7 @@ GPU #### LATENCY This mode prioritizes low latency, providing short response time for each inference job. It performs best for tasks where inference is required for a single input image, like a medical analysis of an ultrasound scan image. It also fits the tasks of real-time or nearly real-time applications, such as an industrial robot's response to actions in its environment or obstacle avoidance for autonomous vehicles. -Note that currently the `PERFORMANCE_HINT` property is supported by CPU and GPU devices only. [More information](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_supported_plugins_AUTO.html#performance-hints). +Note that currently the `PERFORMANCE_HINT` property is supported by CPU and GPU devices only. [More information](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_AUTO.html#performance-hints). To enable Performance Hints for your application, use the following command: @@ -118,7 +118,7 @@ In case of using CPU plugin to run the inference, it might be also beneficial to | CPU_THROUGHPUT_STREAMS | Specifies number of CPU "execution" streams for the throughput mode | -> **NOTE:** For additional information about all parameters read [OpenVINO supported plugins](https://docs.openvino.ai/2022.1/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). +> **NOTE:** For additional information about all parameters read [OpenVINO supported plugins](https://docs.openvino.ai/2022.2/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). - Example: 1. While passing the plugin configuration, omit the `KEY_` phase. @@ -132,7 +132,7 @@ docker run --rm -d --cpuset-cpus 0,1,2,3 -v ${PWD}/models/public/resnet-50-tf:/o ``` ## CPU Power Management Settings -To save power, the OS can decrease the CPU frequency and increase a volatility of the latency values. Similarly the Intel® Turbo Boost Technology may also affect the stability of results. For best reproducibility, consider locking the frequency to the processor base frequency (refer to the https://ark.intel.com/ for your specific CPU). For example, in Linux setting the relevant values for the /sys/devices/system/cpu/cpu* entries does the trick. [Read more](https://docs.openvino.ai/2022.1/openvino_docs_optimization_guide_dldt_optimization_guide.html). High-level commands like cpupower also exists: +To save power, the OS can decrease the CPU frequency and increase a volatility of the latency values. Similarly the Intel® Turbo Boost Technology may also affect the stability of results. For best reproducibility, consider locking the frequency to the processor base frequency (refer to the https://ark.intel.com/ for your specific CPU). For example, in Linux setting the relevant values for the /sys/devices/system/cpu/cpu* entries does the trick. [Read more](https://docs.openvino.ai/2022.2/openvino_docs_optimization_guide_dldt_optimization_guide.html). High-level commands like cpupower also exists: ``` $ cpupower frequency-set --min 3.1GHz ``` @@ -157,7 +157,7 @@ The default value is 1 second which ensures prompt response to creating new mode Depending on the device employed to run the inference operation, you can tune the execution behavior with a set of parameters. Each device is handled by its OpenVINO plugin. -> **NOTE**: For additional information, read [supported configuration parameters for all plugins](https://docs.openvino.ai/2022.1/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). +> **NOTE**: For additional information, read [supported configuration parameters for all plugins](https://docs.openvino.ai/2022.2/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). Model's plugin configuration is a dictionary of param:value pairs passed to OpenVINO Plugin on network load. It can be set with `plugin_config` parameter. diff --git a/docs/security_considerations.md b/docs/security_considerations.md index cb1b792238..72b8d2dd5e 100644 --- a/docs/security_considerations.md +++ b/docs/security_considerations.md @@ -18,5 +18,5 @@ OpenVINO Model Server currently does not provide access restrictions and traffic See also: - [Securing OVMS with NGINX](../extras/nginx-mtls-auth/README.md) -- [Securing models with OVSA](https://docs.openvino.ai/2022.1/ovsa_get_started.html) +- [Securing models with OVSA](https://docs.openvino.ai/2022.2/ovsa_get_started.html) diff --git a/docs/shape_batch_size_and_layout.md b/docs/shape_batch_size_and_layout.md index 67ef43606a..614a267156 100644 --- a/docs/shape_batch_size_and_layout.md +++ b/docs/shape_batch_size_and_layout.md @@ -28,7 +28,7 @@ it ignores the batch_size value. - JSON object e.g. `{"input1":"(1,3,224,224)","input2":"(1,3,50,50)"}` - it defines a shape of every included input in the model *Note:* Some models do not support the reshape operation. Learn more about supported model graph layers including all limitations -on [Shape Inference Document](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_ShapeInference.html). +on [Shape Inference Document](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_ShapeInference.html). In case the model can't be reshaped, it will remain in the original parameters and all requests with incompatible input format will get an error. The model server will also report such problems in the logs. diff --git a/docs/stateful_models.md b/docs/stateful_models.md index 154235f6e3..b8cb2c7538 100644 --- a/docs/stateful_models.md +++ b/docs/stateful_models.md @@ -22,11 +22,13 @@ Some models might take the whole sequence of data as an input and iterate over t Serving stateful model in OpenVINO Model Server is very similar to serving stateless models. The only difference is that for stateful models you need to set `stateful` flag in the model configuration. -* Download and prepare example model from [rm_lstm4f](https://download.01.org/openvinotoolkit/models_contrib/speech/kaldi/rm_lstm4f/) +* Download and prepare example model from [rm_lstm4f](https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/) ```bash mkdir models && cd models -wget -r -np -nH --cut-dirs=5 -R *index.html* https://download.01.org/openvinotoolkit/models_contrib/speech/kaldi/rm_lstm4f/ +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.counts +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.nnet +wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/rm_lstm4f/rm_lstm4f.mapping docker run -u $(id -u):$(id -g) -v $(pwd):/models:rw openvino/ubuntu20_dev:latest mo --framework kaldi --input_model /models/rm_lstm4f.nnet --counts /models/rm_lstm4f.counts --remove_output_softmax --output_dir /models/rm_lstm4f/1 ``` @@ -69,7 +71,7 @@ docker run -d -u $(id -u):$(id -g) -v $(pwd)/rm_lstm4f:/models/stateful_model -v | `stateful` | `bool` | If set to true, model is loaded as stateful. | false | | `idle_sequence_cleanup` | `bool` | If set to true, model will be subject to periodic sequence cleaner scans.
See [idle sequence cleanup](#stateful_cleanup). | true | | `max_sequence_number` | `uint32` | Determines how many sequences can be handled concurrently by a model instance. | 500 | -| `low_latency_transformation` | `bool` | If set to true, model server will apply [low latency transformation](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_network_state_intro.html#lowlatency_transformation) on model load. | false | +| `low_latency_transformation` | `bool` | If set to true, model server will apply [low latency transformation](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_network_state_intro.html#lowlatency_transformation) on model load. | false | **Note:** Setting `idle_sequence_cleanup`, `max_sequence_number` and `low_latency_transformation` require setting `stateful` to true. @@ -307,7 +309,7 @@ If set to `true` sequence cleaner will check that model. Otherwise, sequence cle There are limitations for using stateful models with OVMS: - Support inference execution only using CPU as the target device. - - Support Kaldi models with memory layers and non-Kaldi models with Tensor Iterator. See this [docs about stateful networks](https://docs.openvino.ai/2022.1/openvino_docs_IE_DG_network_state_intro.html) to learn about stateful networks representation in OpenVINO. + - Support Kaldi models with memory layers and non-Kaldi models with Tensor Iterator. See this [docs about stateful networks](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_network_state_intro.html) to learn about stateful networks representation in OpenVINO. - [Auto batch size and shape](shape_batch_size_and_layout.md) are **not** available in stateful models. - Stateful model instances **cannot** be used in [DAGs](dag_scheduler.md). - Requests ordering is guaranteed only when a single client sends subsequent requests in a synchronous manner. Concurrent interaction with the same sequence might negatively affect the accuracy of the results. diff --git a/docs/tf_model_binary_input.md b/docs/tf_model_binary_input.md index df823765f2..47d3f88905 100644 --- a/docs/tf_model_binary_input.md +++ b/docs/tf_model_binary_input.md @@ -4,7 +4,7 @@ This guide shows how to convert TensorFlow models and deploy them with the OpenV - In this example TensorFlow model [ResNet](https://github.com/tensorflow/models/tree/v2.2.0/official/r1/resnet) will be used. -- TensorFlow model can be converted into Intermediate Representation format using model_optimizer tool. There are several formats for storing TensorFlow model. In this guide, we present conversion from SavedModel format. More information about conversion process can be found on the [model optimizer documentation](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html#savedmodel_format). +- TensorFlow model can be converted into Intermediate Representation format using model_optimizer tool. There are several formats for storing TensorFlow model. In this guide, we present conversion from SavedModel format. More information about conversion process can be found on the [model optimizer documentation](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html#savedmodel_format). - Binary input format has several requirements for the model and ovms configuration. More information can be found in [binary inputs documentation](binary_input.md). ## Steps @@ -29,10 +29,10 @@ docker run -u $(id -u):$(id -g) -v ${PWD}/resnet_v2/:/resnet openvino/ubuntu20_d *Note:* Some models might require other parameters such as `--scale` parameter. - `--reverse_input_channels` - required for models that are trained with images in RGB order. -- `--mean_values` , `--scale` - should be provided if input pre-processing operations are not a part of topology- and the pre-processing relies on the application providing input data. They can be determined in several ways described in [conversion parameters guide](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html). In this example [model pre-processing script](https://github.com/tensorflow/models/blob/v2.2.0/official/r1/resnet/imagenet_preprocessing.py) was used to determine them. +- `--mean_values` , `--scale` - should be provided if input pre-processing operations are not a part of topology- and the pre-processing relies on the application providing input data. They can be determined in several ways described in [conversion parameters guide](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html). In this example [model pre-processing script](https://github.com/tensorflow/models/blob/v2.2.0/official/r1/resnet/imagenet_preprocessing.py) was used to determine them. -*Note:* You can find out more about [TensorFlow Model conversion into Intermediate Representation](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html) if your model is stored in other formats. +*Note:* You can find out more about [TensorFlow Model conversion into Intermediate Representation](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Convert_Model_From_TensorFlow.html) if your model is stored in other formats. This operation will create model files in `${PWD}/resnet_v2/models/resnet/1/` folder. ```bash diff --git a/src/custom_nodes/image_transformation/README.md b/src/custom_nodes/image_transformation/README.md index 4a865f0be3..834122e984 100644 --- a/src/custom_nodes/image_transformation/README.md +++ b/src/custom_nodes/image_transformation/README.md @@ -48,9 +48,9 @@ make BASE_OS=redhat NODES=image_transformation | target_image_color_order | Output image color order. If specified and differs from original_image_color_order, color order conversion will be performed | `BGR` | | | original_image_layout | Input image layout. This is required to determine image shape from input shape | | ✓ | | target_image_layout | Output image layout. If specified and differs from original_image_layout, layout conversion will be performed | | | -| scale | All values will be divided by this value. When `scale_values` is specified, this value is ignored. [read more](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | -| scale_values | Scale values to be used for the input image per channel. Input data will be divided by those values. Values should be provided in the same order as output image color order. [read more](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | -| mean_values | Mean values to be used for the input image per channel. Values will be substracted from each input image data value. Values should be provided in the same order as output image color order. [read more](https://docs.openvino.ai/2022.1/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | +| scale | All values will be divided by this value. When `scale_values` is specified, this value is ignored. [read more](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | +| scale_values | Scale values to be used for the input image per channel. Input data will be divided by those values. Values should be provided in the same order as output image color order. [read more](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | +| mean_values | Mean values to be used for the input image per channel. Values will be substracted from each input image data value. Values should be provided in the same order as output image color order. [read more](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model_General.html) | | | | debug | Defines if debug messages should be displayed | false | | > **_NOTE:_** Substracting mean values is performed before division by scale values. diff --git a/src/example/SampleCpuExtension/README.md b/src/example/SampleCpuExtension/README.md index 9ea74a33f7..781797f716 100644 --- a/src/example/SampleCpuExtension/README.md +++ b/src/example/SampleCpuExtension/README.md @@ -32,4 +32,4 @@ $ docker run -it --rm -p 9000:9000 -v `pwd`/lib/ubuntu:/extension:ro -v `pwd`/re --port 9000 --model_name resnet --model_path /resnet --cpu_extension /extension/libcustom_relu_cpu_extension.so ``` -> **NOTE**: Learn more about [OpenVINO extensibility](https://docs.openvino.ai/2022.1/openvino_docs_Extensibility_UG_Intro.html) +> **NOTE**: Learn more about [OpenVINO extensibility](https://docs.openvino.ai/2022.2/openvino_docs_Extensibility_UG_Intro.html) diff --git a/tests/functional/model/models_information.py b/tests/functional/model/models_information.py index cecdc75b0e..294942ba73 100644 --- a/tests/functional/model/models_information.py +++ b/tests/functional/model/models_information.py @@ -20,11 +20,7 @@ import config -MODEL_REPOSITORY_SERVER = "https://download.01.org" -OPENCV_OPENVINO_TOOLKIT = "opencv/2020/openvinotoolkit" -OPENCV_PUBLIC = "opencv/public_models" -OPENVINO_VERSION = "2020.2" -OPEN_MODEL_ZOO_BIN = "open_model_zoo/models_bin" +MODEL_REPOSITORY_SERVER = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin" BUILD_DIR = "1" BUILD_012020 = "012020" PRECISION = "FP32" @@ -33,13 +29,9 @@ PERSON_VEHICLE_BIKE_DETECTION_MODEL = "person-vehicle-bike-detection-crossroad-0078" RESNET_50 = "resnet-50-tf" RESNET_V1_50 = "resnet_v1-50" -OPEN_MODEL_ZOO_MODELS_LOCATION = "{repo}/{opencv}/{version}/{bin}/{build}".format(repo=MODEL_REPOSITORY_SERVER, - opencv=OPENCV_OPENVINO_TOOLKIT, - version=OPENVINO_VERSION, - bin=OPEN_MODEL_ZOO_BIN, - build=BUILD_DIR) +OPEN_MODEL_ZOO_MODELS_LOCATION = "{repo}/{build}".format(repo=MODEL_REPOSITORY_SERVER, + build=BUILD_DIR) URL_OPEN_MODEL_ZOO_FORMAT = "{model_location}/{model}/{precision}/{model}" -URL_PUBLIC_MODEL_FORMAT = "{repo}/{opencv}/{build}/{model}/{model_version}" class AgeGender: @@ -109,8 +101,7 @@ class Resnet: output_shape = (1, 1001) rest_request_format = 'column_name' model_path = os.path.join(config.models_path, name) - url = URL_PUBLIC_MODEL_FORMAT.format(repo=MODEL_REPOSITORY_SERVER, opencv=OPENCV_PUBLIC, build=BUILD_012020, - model=RESNET_50, model_version=RESNET_V1_50) + url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/resnet-50-tf/" + RESNET_V1_50 local_conversion_dir = "tensorflow_format" download_extensions = [".pb"] version = 1 From 0cadfac57ac3c0ee4320e63824dd5deeaf84150a Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Wed, 5 Oct 2022 10:25:09 +0200 Subject: [PATCH 005/130] CVS-87066_doc_fix (#1446) * docs/docker_container.md updated * model path update for gpu docker command * added git clone command Co-authored-by: ngrozae --- docs/docker_container.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/docker_container.md b/docs/docker_container.md index d600ebe48d..185e7defbb 100644 --- a/docs/docker_container.md +++ b/docs/docker_container.md @@ -87,15 +87,15 @@ To do so, use either of these commands: Running the inference operation on GPU requires the ovms process security context account to have correct permissions. It has to belong to the render group identified by the command: -``` +```bash stat -c "group_name=%G group_id=%g" /dev/dri/render* ``` The default account in the docker image is already preconfigured. In case you change the security context, use the following command to start the ovms container: -``` +```bash docker run --rm -it --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ --v /opt/model:/opt/model -p 9001:9001 openvino/model_server:latest-gpu \ ---model_path /opt/model --model_name my_model --port 9001 --target_device GPU +-p 9001:9001 openvino/model_server:latest-gpu \ +--model_name resnet --model_path gs://ovms-public-eu/resnet50-binary --port 9001 --target_device GPU ``` *Note:* The public docker image includes the OpenCL drivers for GPU in version 21.38.21026. @@ -123,7 +123,15 @@ Set target_device for the model in configuration json file to MULTI:DEVICE_1,DEV Below is exemplary config.json setting up Multi-Device Plugin for resnet model, using Intel® Movidius™ Neural Compute Stick and CPU devices: ``` -make docker_build BASE_OS=ubuntu +{ + "model_config_list": [ + {"config": { + "name": "resnet", + "base_path": "/opt/model", + "batch_size": "1", + "target_device": "MULTI:MYRIAD,CPU"} + }] +} ``` Additionally, you can use the `INSTALL_DRIVER_VERSION` argument command to choose which GPU driver version is used by the produced image. @@ -135,6 +143,8 @@ Currently, the following versions are available: Example: ```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server make docker_build INSTALL_DRIVER_VERSION=21.38.21026 ``` If not provided, version 21.38.21026 is used for Redhat and 21.48.21782 is used for Ubuntu. From b64bc673b55c8bb31d49ffdf6cb2633203662a1f Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Wed, 5 Oct 2022 13:03:48 +0200 Subject: [PATCH 006/130] removed redundant dockerfile (#1402) --- .../python/asr_demo/Dockerfile | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 demos/speech_recognition_with_kaldi_model/python/asr_demo/Dockerfile diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/Dockerfile b/demos/speech_recognition_with_kaldi_model/python/asr_demo/Dockerfile deleted file mode 100644 index cebb73fe00..0000000000 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright (c) 2021-2022 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM debian:10 -LABEL maintainer="rick@scriptix.io" - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - g++ \ - make \ - automake \ - autoconf \ - bzip2 \ - unzip \ - wget \ - sox \ - libtool \ - git \ - subversion \ - python2.7 \ - python3 \ - zlib1g-dev \ - ca-certificates \ - gfortran \ - patch \ - ffmpeg \ - vim && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -s /usr/bin/python3 /usr/bin/python - -RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi #EOL -RUN cd /opt/kaldi/tools && \ - ./extras/install_mkl.sh && \ - make -j $(nproc) && \ - cd /opt/kaldi/src && \ - ./configure --shared && \ - make depend -j $(nproc) && \ - make -j $(nproc) && \ - find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \ - find /opt/intel -type f -name "*.a" -exec rm {} \; && \ - find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \ - rm -rf /opt/kaldi/.git - -RUN cd /opt/kaldi/egs/aspire/s5 && \ - wget https://kaldi-asr.org/models/1/0001_aspire_chain_model_with_hclg.tar.bz2 && \ - tar -xvf 0001_aspire_chain_model_with_hclg.tar.bz2 && \ - rm -f 0001_aspire_chain_model_with_hclg.tar.bz2 - -RUN apt-get install -y virtualenv - -RUN git clone -b stateful_client_extension https://github.com/openvinotoolkit/model_server.git /opt/model_server && \ - cd /opt/model_server && \ - virtualenv -p python3 .venv && \ - . .venv/bin/activate && \ - pip install tensorflow-serving-api==2.* kaldi-python-io==1.2.1 && \ - echo "source /opt/model_server/.venv/bin/activate" | tee -a /root/.bashrc && \ - mkdir /opt/workspace - -WORKDIR /opt/workspace/ - From 5ed252c276da83d56a3db3a9d5a95106aa84f8b4 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Wed, 5 Oct 2022 13:04:22 +0200 Subject: [PATCH 007/130] removed kaldi live demo (#1449) --- .../python/asr_demo/README.md | 71 ------ .../python/asr_demo/live-demo.py | 233 ------------------ .../python/asr_demo/run_auto.sh | 47 ---- 3 files changed, 351 deletions(-) delete mode 100644 demos/speech_recognition_with_kaldi_model/python/asr_demo/live-demo.py delete mode 100755 demos/speech_recognition_with_kaldi_model/python/asr_demo/run_auto.sh diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md b/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md index a760de66bc..abf1de1b43 100644 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md +++ b/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md @@ -137,74 +137,3 @@ cat /opt/workspace/sample.wav.txt ```bash /opt/workspace/sample.wav today we have a very nice weather ``` - -### 7. Live speech recognition -You can also run the live-demo.py client on windows machine to record wav files with your microphone and send them to a ssh enabled server with mentioned kaldi and ovms containers setup. - -On server side run the instructions steps from 1 to 5 but instead of commands in step 6, run the following commands: - -Create the data directory as wav files input directory. -```bash -export DATA_DIR=$HOME/asr_demo/data -mkdir -p $DATA_DIR -``` - -Start kaldi container built in the step 2 in interactive mode with $DATA_DIR mounted as /opt/data: -```bash -docker run --rm -it --network="host" -v $DATA_DIR:/opt/data kaldi:latest bash -``` - -The run_auto.sh script works as the run.sh script from step 6. However instead of taking the wav file from the command line argument -it detects wav files in $DATA_DIR and then runs the speech recognition on them. -Run speech recognition loop on the server: -```bash -/opt/model_server/demos/speech_recognition_with_kaldi_model/python/asr_demo/run_auto.sh localhost 9000 -``` - -Install the required packages on client side. -PyAudio will be used to record audio from microphone and paramiko is used as scp client to copy recorded files to $DATA_DIR on the server.: -```bash -#CLIENT SIDE: -python -m pip install PyAudio -python -m pip install paramiko -``` - -Checkout the repository with demo script: -```bash -#CLIENT SIDE: -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server/demos/speech_recognition_with_kaldi_model/python/asr_demo -``` - -The live-demo.py script is a modified version of the script from https://github.com/kaldi-asr/kaldi.git repository from the \kaldi\egs\vystadial_cz\online_demo\live-demo.py path. -Run the live-demo.py script to record and send audio files to the server: - - IP of the unix server with the running ovms and kaldi containers from steps 1 to 5. - - is the path of the $HOME directory from steps 1 to 5. - - is the owner of the $HOME path and a user of the server used to run steps 1 to 5. -```bash -python live-demo.py localhost $HOME/asr_demo/data $USER -``` - -The script will ask you to provide password for the user to connect to the server with scp. -Once connected you can start to record audio by pressing 'r' key and stop it with the same 'r' key to see the detection results. -Below is the example console output: -```bash -Password: -Connection success. -Press r key to toggle recording -Press c key to exit -Recording started... -Recording stopped -Sending file 1619781288.2140305-utt.wav -Sending from \1619781288.2140305-utt.wav -Sending to /asr_demo/data/1619781288.2140305-utt.wav -File sent in 0.33 seconds -Waiting for /asr_demo/data/1619781288.2140305-utt.wav.txt -Got model response in 9.46 seconds -DETECTED TEXT: it's a beautiful day -Recording started... -Recording stopped -... -``` - -The script will also save recorded wav files and detected text in the current working directory. diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/live-demo.py b/demos/speech_recognition_with_kaldi_model/python/asr_demo/live-demo.py deleted file mode 100644 index f02c5a0768..0000000000 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/live-demo.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -# Copyright (c) 2013, Ondrej Platek, Ufal MFF UK -# Modifications copyright (c) 2021 Intel Corporation -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED -# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -# MERCHANTABLITY OR NON-INFRINGEMENT. -# See the Apache 2 License for the specific language governing permissions and -# limitations under the License. # - -from __future__ import unicode_literals -from __future__ import print_function - -import pyaudio -import sys -import time -import wave -import threading -import paramiko -import os -import getpass - - -CHANNELS, RATE, FORMAT = 1, 8000, pyaudio.paInt16 -BATCH_SIZE = 4560 -PORT = 22 - -class LiveDemo(object): - - def __init__(self, host, host_path, user, password, dec_args): - self.batch_size = BATCH_SIZE - self.args = dec_args - self.pin, self.stream = None, None - self.frames = [] - self.utt_frames, self.new_frames = 0, 0 - self.utt_end, self.dialog_end, self.recording = False, False, False - self.host_path = host_path - - try: - self.transport = paramiko.Transport((host, PORT)) - start = time.time() - self.transport.connect(username = user, password = password) - self.sftp = paramiko.SFTPClient.from_transport(self.transport) - print("Connection success.") - except Exception as e: - print("Connection issue: ", e) - - - def setup(self): - self.pin = pyaudio.PyAudio() - self.stream = self.pin.open(format=FORMAT, channels=CHANNELS, - rate=RATE, input=True, frames_per_buffer=self.batch_size, - stream_callback=self.get_audio_callback()) - self.utt_frames, self.new_frames = 0, 0 - self.utt_end, self.dialog_end, self.recording = False, False, False - self.frames = [] - - def tear_down(self): - if self.stream is not None: - self.stream.stop_stream() - self.stream.close() - if self.pin is not None: - self.pin.terminate() - p, stream = None, None - self.frames = [] - - def get_audio_callback(self): - def frame_in(in_data, frame_count, time_info, status): - self.frames.append(in_data) - return in_data, pyaudio.paContinue - return frame_in - - def _user_control_windows(self): - import msvcrt - '''Simply stupid sollution how to control state of recogniser.''' - self.utt_end, self.dialog_end, self.recording = False, False, False - print('Press r key to toggle recording') - print('Press c key to exit') - try: - while True: - if msvcrt.kbhit(): - c = msvcrt.getch().decode("utf-8").lower() - if c == 'r': - self.recording = not self.recording - if self.recording: - print('Recording started...') - self.frames = [] - else: - print('Recording stopped') - self.utt_end = True - elif c == 'c': - self.dialog_end = True - print('\nMarked end of dialogue\n') - break - finally: - print("""Exit""") - - def _user_control_unix(self): - import select - import tty - import termios - '''Simply stupid sollution how to control state of recogniser.''' - self.utt_end, self.dialog_end, self.recording = False, False, False - print('Press r key to toggle recording') - print('Press c key to exit') - - old_settings = termios.tcgetattr(sys.stdin) - try: - tty.setcbreak(sys.stdin.fileno()) - # if is data on input - while (select.select([sys.stdin], [], [], 1) == ([sys.stdin], [], [])): - c = sys.stdin.read(1) - if c == 'r': - self.recording = not self.recording - if self.recording: - print('Recording started...') - self.frames = [] - else: - print('Recording stopped') - self.utt_end = True - elif c == 'c': - self.dialog_end = True - print('\nMarked end of dialogue\n') - break - finally: - termios.tcsetattr(sys.stdin, termios.TCSADRAIN, old_settings) - print("""Exit""") - - def run(self): - if 'nt' in os.name: - x = threading.Thread(target=self._user_control_windows, args=()) - x.start() - else: - x = threading.Thread(target=self._user_control_unix, args=()) - x.start() - - while True: - time.sleep(0.1) - if self.utt_end: - start = time.time() - self.save_utt_wav(str(start)) - self.utt_end = False - self.utt_frames = 0 - # zero out frames - self.frames = [] - if self.dialog_end: - x.join() - break - - self.sftp.close() - self.transport.close() - - def save_utt_wav(self, timestamp): - filename = timestamp+'-utt.wav' - wf = wave.open(filename, 'wb') - wf.setnchannels(CHANNELS) - wf.setframerate(RATE) - wf.setsampwidth(self.pin.get_sample_size(FORMAT)) - wf.writeframes(b''.join(self.frames)) - wf.close() - self.send_file(filename) - - def send_file(self,local_file): - print("Sending file " + local_file) - start = time.time() - remote_path = self.host_path + local_file - this_dir_path = os.path.dirname(os.path.realpath(__file__)) - local_path = os.path.join(this_dir_path, local_file) - print("Sending from " + local_path) - print("Sending to " + remote_path) - try: - self.sftp.put(local_path, remote_path) - except Exception as ex: - print("Wrong server path:" + self.host_path, ex) - return - - print("File sent in {:.2f} seconds".format(time.time() - start) ) - start = time.time() - print("Waiting for " + remote_path + ".txt") - while True and not self.dialog_end: - time.sleep(0.1) - try: - response_file = self.sftp.open(remote_path + ".txt") - print("Got model response in {:.2f} seconds".format(time.time() - start) ) - for line in response_file.readlines(): - if line == "" or len(line.split(".wav")) < 2: - print("Nothing detected") - else: - detection = line.split(".wav")[1] - print("DETECTED TEXT: " + detection) - f=open(local_path+".txt", "a") - f.write(detection +"\n") - f.close() - break - except: - print("Response not ready yet...") - -def print_help(): - print("The live-demo.py script is a modified version of the script from https://github.com/kaldi-asr/kaldi.git repository from the \kaldi\egs\vystadial_cz\online_demo\live-demo.py path.") - print("Run the live-demo.py script to record and send audio files to the server.") - print("Prerequisites is running run_auto.sh script on server, kaldi container and ovms container as described in the instructions from:") - print("https://github.com/openvinotoolkit/model_server/blob/releases/2022/1/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md") - print("\nUsage: python live-demo.py /asr_demo/data \n") - print(" - IP of the unix server with the running ovms and kaldi containers.") - print(" - is the path of the $HOME directory.") - print(" - is the owner of the $HOME path and a user of the server.") - - -if __name__ == '__main__': - print('Python args: %s' % str(sys.argv), file=sys.stderr) - if len(sys.argv) < 4: - print_help() - else: - host = str(sys.argv[1]) - host_path = str(sys.argv[2]) - user = str(sys.argv[3]) - - try: - password = getpass.getpass() - except Exception as error: - print('ERROR', error) - - argv = sys.argv[4:] - demo = LiveDemo(host, host_path, user, password, argv) - demo.setup() - demo.run() diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/run_auto.sh b/demos/speech_recognition_with_kaldi_model/python/asr_demo/run_auto.sh deleted file mode 100755 index 9aee5a83c0..0000000000 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/run_auto.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Variables setup -export OVMS_PATH=/opt/model_server -export KALDI_PATH=/opt/kaldi -export ASPIRE_PATH=$KALDI_PATH/egs/aspire/s5 -export DATA_PATH=/opt/data - -# Extract features -source $OVMS_PATH/.venv/bin/activate -cd $DATA_PATH -while true -do -for i in *.wav; do - [ -f "$i" ] || break - I=`wc -c < echo $i` - J=`wc -c < echo $i` - if [ $I -ne $J ]; then - sleep 0.1 - fi - - rm -rf $ASPIRE_PATH/data/conversion* || true - cd $OVMS_PATH/demos/speech_recognition_with_kaldi_model/python - ./asr_demo/prepare_model_inputs.sh $i - python grpc_stateful_client.py --input_path $DATA_PATH/feats.ark,$DATA_PATH/ivectors.ark --output_path $DATA_PATH/scores.ark --grpc_address $1 --grpc_port $2 --input_name input,ivector --output_name Final_affine --model_name aspire --cw_l 17 --cw_r 12 - ./asr_demo/read_model_output.sh $i - cd $DATA_PATH - rm scores.ark ivectors.ark feats.* sample.wav out.txt || true - rm $i || true - rm -rf $ASPIRE_PATH/data/conversion* || true -done -done From 975c3cf2ff9e8f5516fa5d234db490d745639f36 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Thu, 6 Oct 2022 11:26:46 +0200 Subject: [PATCH 008/130] protobuf version from 3.19.3 to 3.19.5 (#1450) --- client/python/kserve-api/samples/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/python/kserve-api/samples/requirements.txt b/client/python/kserve-api/samples/requirements.txt index 3674c75940..42e7cf9c8f 100644 --- a/client/python/kserve-api/samples/requirements.txt +++ b/client/python/kserve-api/samples/requirements.txt @@ -1,4 +1,4 @@ -protobuf==3.19.3 +protobuf==3.19.5 tritonclient[all]==2.22.3 requests==2.27.1 grpcio From b5b9d664e4fd4d796b4b3722bb5762c48916b6eb Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 6 Oct 2022 17:09:16 +0200 Subject: [PATCH 009/130] Check coverage (#1438) * Check coverage --- Dockerfile.redhat | 14 +++++++++++++- Dockerfile.ubuntu | 10 ++++++++++ Makefile | 3 ++- check_coverage.bat | 26 ++++++++++++++++++++++++++ lib_search.py | 4 ++-- 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100755 check_coverage.bat diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 53951fb8b2..baea6abef4 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -27,6 +27,8 @@ ARG TEMP_DIR=/tmp/openvino_installer ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools ARG DL_DIR=/tmp ARG JOBS +ARG CHECK_COVERAGE=1 + # build_type=[ opt, dbg ] ARG build_type=dbg ARG debug_bazel_flags=--strip=never\ --copt="-g"\ -c\ dbg @@ -41,11 +43,13 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n boost169-thread \ boost169-system \ boost169-date-time \ + bc \ cmake \ - gcc-c++ \ curl \ + gcc-c++ \ gdb \ git \ + java-11-openjdk-devel \ libgusb.x86_64 \ libusbx \ libcurl-devel \ @@ -172,6 +176,14 @@ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; sed -i -e "s ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ +COPY check_coverage.bat /ovms/ +# Test Coverage +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; wget https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16-1.noarch.rpm &&\ + yum install -y lcov-1.16-1.noarch.rpm &&\ + bazel coverage --combined_report=lcov //src:ovms_test &&\ + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ + ./check_coverage.bat && exit $? + RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 2217b0cc6a..02dbdca7df 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -26,6 +26,7 @@ ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools ARG DL_DIR=/tmp ARG JOBS ARG APT_OV_PACKAGE=openvino-2022.1.0 +ARG CHECK_COVERAGE=1 # build_type=[ opt, dbg ] ARG build_type=dbg @@ -43,13 +44,16 @@ RUN apt update && apt install -y \ libboost-thread1.71.0 \ libboost-system1.71.0 \ libboost-date-time1.71.0 \ + bc \ build-essential \ cmake \ automake \ autoconf \ curl \ + default-jdk \ gdb \ git \ + lcov \ libusb-dev \ libusb-1.0-0-dev \ libcurl4-openssl-dev \ @@ -215,6 +219,12 @@ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; sed -i -e "s RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; sed -i -e "s#REPLACE_OPENVINO_NAME#`git --git-dir /openvino/.git log -n 1 | head -n 1 | cut -d' ' -f2 | head -c 12`#g" /ovms/src/version.hpp ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ +COPY check_coverage.bat /ovms/ +# Test Coverage +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; bazel coverage --combined_report=lcov //src:ovms_test &&\ + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ + ./check_coverage.bat && exit $? + RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Makefile b/Makefile index 73b4338db1..59d2c0365b 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ BASE_OS_TAG_UBUNTU ?= 20.04 BASE_OS_TAG_REDHAT ?= 8.6 INSTALL_RPMS_FROM_URL ?= +CHECK_COVERAGE ?=1 # NOTE: when changing any value below, you'll need to adjust WORKSPACE file by hand: # - uncomment source build section, comment binary section @@ -180,7 +181,7 @@ endif --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy=$(HTTPS_PROXY) --build-arg no_proxy=$(NO_PROXY) \ --build-arg ovms_metadata_file=.workspace/metadata.json --build-arg ov_source_branch="$(OV_SOURCE_BRANCH)" \ --build-arg ov_use_binary=$(OV_USE_BINARY) --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL) \ - --build-arg APT_OV_PACKAGE=$(APT_OV_PACKAGE) \ + --build-arg APT_OV_PACKAGE=$(APT_OV_PACKAGE) --build-arg CHECK_COVERAGE=$(CHECK_COVERAGE) \ --build-arg build_type=$(BAZEL_BUILD_TYPE) --build-arg debug_bazel_flags=$(BAZEL_DEBUG_FLAGS) \ --build-arg minitrace_flags=$(MINITRACE_FLAGS) \ --build-arg PROJECT_NAME=${PROJECT_NAME} \ diff --git a/check_coverage.bat b/check_coverage.bat new file mode 100755 index 0000000000..b5b8e8e204 --- /dev/null +++ b/check_coverage.bat @@ -0,0 +1,26 @@ +#!/bin/bash + +#Ubuntu +#MIN_LINES_COV=74.9 +#MIN_FUNCTION_COV=86.5 + +#Rhel +MIN_LINES_COV=73.3 +MIN_FUNCTION_COV=73.6 + + +LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` +FUNC_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | tail -n 1` + +if (( $(echo "$MIN_LINES_COV > $LINES_COV" | bc -l) )); then + echo "Error: $LINES_COV % Lines coverage is lower than minimal $MIN_LINES_COV %" + exit 1 +fi + +if (( $(echo "$MIN_FUNCTION_COV > $FUNC_COV" | bc -l) )); then + echo "Error: $FUNCTION_COV % Functions coverage is lower than minimal $MIN_FUNCTION_COV %" + exit 1 +fi + +echo "Coverage check success" +exit 0 diff --git a/lib_search.py b/lib_search.py index 5f1f4c223c..d1fb9d3f21 100644 --- a/lib_search.py +++ b/lib_search.py @@ -64,7 +64,7 @@ def check_dir(start_dir): 'add.xml', 'tftext.patch', 'net_http.patch', 'clang-format', 'missing_headers.txt', 'listen.patch', 'Doxyfile', 'increment_1x3x4x5.xml', 'rest_sdk_v2.10.16.patch', 'azure_sdk.patch', 'model.xml', 'ovms-c/dist', 'client_requirements.txt', 'REST_age_gender.ipynb', 'libevent/BUILD', 'forbidden_functions.txt', - 'resnet_images.txt', 'vehicle_images.txt', 'opencv_cmake_flags.txt', 'metrics_output.out'] + 'resnet_images.txt', 'vehicle_images.txt', 'opencv_cmake_flags.txt', 'metrics_output.out', 'check_coverage.bat'] exclude_directories = ['/dist/', 'extras/ovms-operator', 'extras/openvino-operator-openshift', 'release_files/thirdparty-licenses'] @@ -99,7 +99,7 @@ def check_func(start_dir): 'openvino.LICENSE.txt', 'c-ares.LICENSE.txt', 'zlib.LICENSE.txt', 'boost.LICENSE.txt', 'libuuid.LICENSE.txt', 'input_images.txt', 'REST_age_gender.ipynb', 'dummy.xml', 'listen.patch', 'add.xml', 'requirements.txt', 'missing_headers.txt', 'libevent/BUILD', 'azure_sdk.patch', 'rest_sdk_v2.10.16.patch', - 'forbidden_functions.txt', 'missing_headers.txt', 'metrics_output.out', 'summator.xml'] + 'forbidden_functions.txt', 'missing_headers.txt', 'metrics_output.out', 'summator.xml', 'check_coverage.bat'] exclude_directories = ['/dist/', 'extras/ovms-operator'] From f3a70752210c13e16b2fe8ec14be9e0ade24402a Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Fri, 7 Oct 2022 10:09:54 +0200 Subject: [PATCH 010/130] Remove layout arg for resnet binary inference (#1437) * removed layout arg * removed layout arg for http binary --- client/python/ovmsclient/samples/README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/client/python/ovmsclient/samples/README.md b/client/python/ovmsclient/samples/README.md index 6a9252b8f1..72c7b0a434 100644 --- a/client/python/ovmsclient/samples/README.md +++ b/client/python/ovmsclient/samples/README.md @@ -160,12 +160,6 @@ Image #8 has been classified as snail Image #9 has been classified as zebra ``` -To serve Resnet with support for binary input data, the model needs to be configured with NHWC layout. That can be acheived by starting the OVMS container with `--layout NHWC:NCHW` parameter. -new OVMS instance with `--layout NHWC:NCHW` parameter. -```bash -docker run -d --rm -v ${PWD}/models/public/resnet-50-tf:/models/public/resnet-50-tf -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/public/resnet-50-tf --port 9000 --rest_port 8000 --layout NHWC:NCHW -``` - ### Predict binary format #### **Make prediction using images in binary format:** @@ -382,12 +376,6 @@ Image #8 has been classified as snail Image #9 has been classified as zebra ``` -To serve Resnet with support for binary input data, the model needs to be configured with NHWC layout. That can be acheived by starting the OVMS container with `--layout NHWC:NCHW` parameter. -new OVMS instance with `--layout NHWC:NCHW` parameter. -```bash -docker run -d --rm -v ${PWD}/models/public/resnet-50-tf:/models/public/resnet-50-tf -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path /models/public/resnet-50-tf --port 9000 --rest_port 8000 --layout NHWC:NCHW -``` - ### Predict binary format #### **Make prediction using images in binary format:** From 6449603b04832b16225383a7a564892a60e1588a Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Fri, 7 Oct 2022 10:12:02 +0200 Subject: [PATCH 011/130] fixed input and output names (#1420) --- .../speech_recognition_with_kaldi_model/python/asr_demo/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/run.sh b/demos/speech_recognition_with_kaldi_model/python/asr_demo/run.sh index 68e0ef58c4..474dd7b5a5 100755 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/run.sh +++ b/demos/speech_recognition_with_kaldi_model/python/asr_demo/run.sh @@ -25,6 +25,6 @@ export DATA_PATH=/tmp source $OVMS_PATH/.venv/bin/activate cd $OVMS_PATH/demos/speech_recognition_with_kaldi_model/python ./asr_demo/prepare_model_inputs.sh $1 -python grpc_stateful_client.py --input_path $DATA_PATH/feats.ark,$DATA_PATH/ivectors.ark --output_path $DATA_PATH/scores.ark --grpc_address $2 --grpc_port $3 --input_name input,ivector --output_name Final_affine --model_name aspire --cw_l 17 --cw_r 12 +python grpc_stateful_client.py --input_path $DATA_PATH/feats.ark,$DATA_PATH/ivectors.ark --output_path $DATA_PATH/scores.ark --grpc_address $2 --grpc_port $3 --input_name input:0,ivector:0 --output_name Final_affine:0 --model_name aspire --cw_l 17 --cw_r 12 ./asr_demo/read_model_output.sh $1 rm $DATA_PATH/* From 4431faa7a8ecf900eb325d250b4f4154ab9b4523 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Fri, 7 Oct 2022 12:31:54 +0200 Subject: [PATCH 012/130] CVS-93694_CVS-93695_CVS-93696_CVS-93698_CVS-93699_doc_test_fixes (#1463) * bash marks added * east_ocr + face_blur + image_transformation + model_zoo_intel_object_detection Co-authored-by: ngrozae --- src/custom_nodes/east_ocr/README.md | 4 ++-- src/custom_nodes/face_blur/README.md | 4 ++-- src/custom_nodes/horizontal_ocr/README.md | 4 ++-- src/custom_nodes/image_transformation/README.md | 4 ++-- src/custom_nodes/model_zoo_intel_object_detection/README.md | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/custom_nodes/east_ocr/README.md b/src/custom_nodes/east_ocr/README.md index 45a81dff35..3fe58f24e1 100644 --- a/src/custom_nodes/east_ocr/README.md +++ b/src/custom_nodes/east_ocr/README.md @@ -12,14 +12,14 @@ and confidence levels for the filtered list of detections. # Building custom node library You can build the shared library of the custom node simply by running command in the context of custom node examples directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server && cd model_server/src/custom_nodes make NODES=east_ocr ``` It will compile the library inside a docker container and save the results in `lib//` folder. You can also select base OS between RH 8.5 (redhat) and Ubuntu 20.04 (ubuntu) by setting `BASE_OS` environment variable. -``` +```bash make BASE_OS=redhat NODES=east_ocr ``` diff --git a/src/custom_nodes/face_blur/README.md b/src/custom_nodes/face_blur/README.md index 9cd34a6259..1263493c37 100644 --- a/src/custom_nodes/face_blur/README.md +++ b/src/custom_nodes/face_blur/README.md @@ -23,14 +23,14 @@ All [OpenVINO Model Zoo](https://github.com/openvinotoolkit/open_model_zoo/tree/ # Building custom node library You can build the shared library of the custom node simply by running command in the context of custom node examples directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server && cd model_server/src/custom_nodes make NODES=face_blur ``` It will compile the library inside a docker container and save the results in `lib//` folder. You can also select base OS between RH 8.5 (redhat) and Ubuntu 20.04 (ubuntu) by setting `BASE_OS` environment variable. -``` +```bash make BASE_OS=redhat NODES=face_blur ``` diff --git a/src/custom_nodes/horizontal_ocr/README.md b/src/custom_nodes/horizontal_ocr/README.md index 2bcfdb9e9b..b128f7eba5 100644 --- a/src/custom_nodes/horizontal_ocr/README.md +++ b/src/custom_nodes/horizontal_ocr/README.md @@ -13,14 +13,14 @@ This custom node can be used to process video frames via [camera example](../../ # Building custom node library You can build the shared library of the custom node simply by running command in the context of custom node examples directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server && cd model_server/src/custom_nodes make NODES=horizontal_ocr ``` It will compile the library inside a docker container and save the results in `lib//` folder. You can also select base OS between RH 8.5 (redhat) and Ubuntu 20.04 (ubuntu) by setting `BASE_OS` environment variable. -``` +```bash make BASE_OS=redhat NODES=horizontal_ocr ``` diff --git a/src/custom_nodes/image_transformation/README.md b/src/custom_nodes/image_transformation/README.md index 834122e984..75a209496e 100644 --- a/src/custom_nodes/image_transformation/README.md +++ b/src/custom_nodes/image_transformation/README.md @@ -14,14 +14,14 @@ In other cases conversion applies which reduces performance of this node. # Building custom node library You can build the shared library of the custom node simply by running command in the context of custom node examples directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server && cd model_server/src/custom_nodes make NODES=image_transformation ``` It will compile the library inside a docker container and save the results in `lib//` folder. You can also select base OS between RH 8.5 (redhat) and Ubuntu 20.04 (ubuntu) by setting `BASE_OS` environment variable. -``` +```bash make BASE_OS=redhat NODES=image_transformation ``` diff --git a/src/custom_nodes/model_zoo_intel_object_detection/README.md b/src/custom_nodes/model_zoo_intel_object_detection/README.md index ea092f896a..23a486e065 100644 --- a/src/custom_nodes/model_zoo_intel_object_detection/README.md +++ b/src/custom_nodes/model_zoo_intel_object_detection/README.md @@ -30,14 +30,14 @@ Public [OpenVINO Model Zoo](https://github.com/openvinotoolkit/open_model_zoo/tr # Building custom node library You can build the shared library of the custom node simply by running command in the context of custom node examples directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server && cd model_server/src/custom_nodes make NODES=model_zoo_intel_object_detection ``` It will compile the library inside a docker container and save the results in `lib//` folder. You can also select base OS between RH 8.5 (redhat) and Ubuntu 20.04 (ubuntu) by setting `BASE_OS` environment variable. -``` +```bash make BASE_OS=redhat NODES=model_zoo_intel_object_detection ``` From 2130b12a4c4a9a57ee1e480fc8ec9696d49513ed Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 7 Oct 2022 15:01:45 +0200 Subject: [PATCH 013/130] Get coverage target (#1464) * Get coverage target --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 59d2c0365b..f4f5f4ef2f 100644 --- a/Makefile +++ b/Makefile @@ -221,6 +221,13 @@ endif http_proxy=$(HTTP_PROXY) https_proxy=$(HTTPS_PROXY) no_proxy=$(NO_PROXY) ./build.sh "$(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)" "$(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG)" "$(BASE_OS)" && \ docker tag $(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG) $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)-nginx-mtls +# Ci build expects index.html in genhtml directory +get_coverage: + @echo "Copying coverage report from build image to genhtml..." + @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) bash + @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . + @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) + test_checksec: @echo "Running checksec on ovms binary..." @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) bash From 0779cb720830b61992d90ae8ec7ef5cd65525a95 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Mon, 10 Oct 2022 14:21:22 +0200 Subject: [PATCH 014/130] CVS-88936_doc_test_fixes (#1467) * expected output updates * demos/optical_character_recognition/python/README.md updated due to required changes in test Co-authored-by: ngrozae --- client/python/ovmsclient/samples/README.md | 25 ++++++++++++------- .../python/README.md | 3 ++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/client/python/ovmsclient/samples/README.md b/client/python/ovmsclient/samples/README.md index 72c7b0a434..e55cf5880e 100644 --- a/client/python/ovmsclient/samples/README.md +++ b/client/python/ovmsclient/samples/README.md @@ -93,6 +93,7 @@ usage: grpc_get_model_metadata.py [-h] [--service_url SERVICE_URL] [--model_version MODEL_VERSION] [--timeout TIMEOUT] +Get information about the status of served models over gRPC interace optional arguments: -h, --help show this help message and exit @@ -127,6 +128,7 @@ usage: grpc_predict_resnet.py [-h] --images_numpy IMAGES_NUMPY [--model_version MODEL_VERSION] [--iterations ITERATIONS] [--timeout TIMEOUT] +Make prediction using images in numerical format optional arguments: -h, --help show this help message and exit @@ -168,12 +170,13 @@ Image #9 has been classified as zebra ```bash python grpc_predict_binary_resnet.py --help -usage: grpc_predict_binary_resnet.py [-h] [--images_dir IMAGES_DIR] +usage: grpc_predict_binary_resnet.py [-h] --images_dir IMAGES_DIR [--service_url SERVICE_URL] [--model_name MODEL_NAME] [--model_version MODEL_VERSION] [--timeout TIMEOUT] +Make prediction using images in binary format optional arguments: -h, --help show this help message and exit @@ -227,13 +230,14 @@ docker run -d --rm -v ${PWD}/models/vehicle-detection:/models/vehicle-detection ```bash python grpc_predict_binary_vehicle_detection.py --help -usage: grpc_predict_binary_vehicle_detection.py [-h] [--images_dir IMAGES_DIR] +usage: grpc_predict_binary_vehicle_detection.py [-h] --images_dir IMAGES_DIR [--service_url SERVICE_URL] [--model_name MODEL_NAME] [--model_version MODEL_VERSION] - [--output_dir OUTPUT_DIR] + --output_dir OUTPUT_DIR [--timeout TIMEOUT] +Make vehicle detection prediction using images in binary format optional arguments: -h, --help show this help message and exit @@ -278,6 +282,7 @@ usage: http_get_model_status.py [-h] [--service_url SERVICE_URL] [--model_version MODEL_VERSION] [--timeout TIMEOUT] +Get information about the status of served models over HTTP interace optional arguments: -h, --help show this help message and exit @@ -311,6 +316,7 @@ usage: http_get_model_metadata.py [-h] [--service_url SERVICE_URL] [--model_version MODEL_VERSION] [--timeout TIMEOUT] +Get information about the status of served models over HTTP interace optional arguments: -h, --help show this help message and exit @@ -345,8 +351,10 @@ usage: http_predict_resnet.py [-h] --images_numpy IMAGES_NUMPY [--model_version MODEL_VERSION] [--iterations ITERATIONS] [--timeout TIMEOUT] +Make prediction using images in numerical format + optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit --images_numpy IMAGES_NUMPY Path to a .npy file with data to infer --service_url SERVICE_URL @@ -390,10 +398,10 @@ usage: http_predict_binary_resnet.py [-h] --images_dir IMAGES_DIR [--model_version MODEL_VERSION] [--timeout TIMEOUT] - +Make prediction using images in binary format optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit --images_dir IMAGES_DIR Path to a directory with images in JPG or PNG format --service_url SERVICE_URL @@ -446,7 +454,7 @@ usage: http_predict_binary_vehicle_detection.py [-h] --images_dir IMAGES_DIR --output_dir OUTPUT_DIR [--timeout TIMEOUT] - +Make vehicle detection prediction using images in binary format optional arguments: -h, --help show this help message and exit @@ -467,6 +475,5 @@ optional arguments: ```bash python http_predict_binary_vehicle_detection.py --images_dir ../../../../demos/common/static/images/cars/ --output_dir ./output --service_url localhost:8000 -Making directory for output: ./output Detection results in file: ./output/road1.jpg -``` \ No newline at end of file +``` diff --git a/demos/optical_character_recognition/python/README.md b/demos/optical_character_recognition/python/README.md index b3c4d9a350..5bc2109900 100644 --- a/demos/optical_character_recognition/python/README.md +++ b/demos/optical_character_recognition/python/README.md @@ -113,7 +113,8 @@ The custom node east_ocr can be built inside a docker container via the followin ```bash git clone https://github.com/openvinotoolkit/model_server.git cd model_server/src/custom_nodes -export BASE_OS=ubuntu # replace to 'redhat` if using UBI base image +# replace to 'redhat` if using UBI base image +export BASE_OS=ubuntu make NODES=east_ocr BASE_OS=${BASE_OS} cd ../../../ ``` From 15741d5af00537ea4ef945ce64aa393690556129 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 10 Oct 2022 14:39:23 +0200 Subject: [PATCH 015/130] Remove coverage libraries (#1466) --- DockerfileMakePackage | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 3793c9db59..a9a3ab49ce 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -49,6 +49,14 @@ RUN rm -f /ovms_release/lib/libsampleloader* RUN rm -f /ovms_release/lib/lib_node* RUN rm -f /ovms_release/lib/libcustom_node* +# Remove coverage libaries +RUN if [ -f /ovms_release/lib/libjava.so ] ; then true ; else exit 0 ; fi ;cd /ovms_release/lib/ &&\ +rm -rf libatk-wrapper.so libattach.so libawt_headless.so libawt.so libawt_xawt.so libdt_socket.so \ +libextnet.so libfontmanager.so libinstrument.so libj2gss.so libj2pcsc.so libj2pkcs11.so libjaas.so \ +libjavajpeg.so libjava.so libjawt.so libjdwp.so libjimage.so libjli.so libjsig.so libjsound.so libjvm.so \ +liblcms.so libmanagement_agent.so libmanagement_ext.so libmanagement.so libmlib_image.so libnet.so libnio.so \ +libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libunpack.so libverify.so libzip.so + RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/3rdparty/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; From f55b01dc76808bae182047891a9ff9377ecae9ba Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Tue, 11 Oct 2022 09:45:22 +0200 Subject: [PATCH 016/130] changed openvino installation in cpu extension example (#1465) --- src/example/SampleCpuExtension/Dockerfile.ubuntu | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/example/SampleCpuExtension/Dockerfile.ubuntu b/src/example/SampleCpuExtension/Dockerfile.ubuntu index 1c6265a704..3c91b933ac 100644 --- a/src/example/SampleCpuExtension/Dockerfile.ubuntu +++ b/src/example/SampleCpuExtension/Dockerfile.ubuntu @@ -20,19 +20,19 @@ FROM $BASE_IMAGE as base_build ARG TEMP_DIR=/tmp/openvino_installer ARG DLDT_PACKAGE_URL ARG APT_OV_PACKAGE +ARG DLDT_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz RUN apt update && apt install -y build-essential wget make python3 WORKDIR / -# OV toolkit package -RUN echo "installing apt package" && \ - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \ - apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \ - echo "deb https://apt.repos.intel.com/openvino/2022 focal main" | tee /etc/apt/sources.list.d/intel-openvino-2022.list && \ - apt update && \ - apt install -y $APT_OV_PACKAGE +RUN mkdir -p $TEMP_DIR && cd $TEMP_DIR/ && \ + wget $DLDT_PACKAGE_URL && \ + mkdir /opt/intel && \ + tar -zxf l_openvino_toolkit*.tgz -C /opt/intel && \ + ln -s /opt/intel/l_openvino_toolkit* /opt/intel/openvino && \ + ln -s /opt/intel/l_openvino_toolkit* /opt/intel/openvino_2022 WORKDIR /workspace COPY Makefile ov_extension.cpp CustomReluOp.cpp CustomReluOp.hpp ./ From 099544c9a9c1ae11873efa2f1158c4a33cf593a2 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Tue, 11 Oct 2022 12:22:50 +0200 Subject: [PATCH 017/130] git clone + make added (#1468) Co-authored-by: ngrozae --- src/example/SampleCpuExtension/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/example/SampleCpuExtension/README.md b/src/example/SampleCpuExtension/README.md index 781797f716..679af818d2 100644 --- a/src/example/SampleCpuExtension/README.md +++ b/src/example/SampleCpuExtension/README.md @@ -12,6 +12,14 @@ Compile the library by running `make cpu_extension BASE_OS=ubuntu` in root direc Shared library will be generated in the `lib` folder. Such library can be used to run Model Server, using `--cpu_extension` argument. +```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server +# replace to 'redhat` if using UBI base image +export BASE_OS=ubuntu +make cpu_extension BASE_OS=${BASE_OS} +``` + ## Preparing resnet50 model In order to demonstrate the usage of cpu_extension library some small modifications in resnet model are needed. @@ -28,7 +36,7 @@ sed -i '0,/ReLU/s//CustomReLU/' resnet50-binary-0001/1/resnet50-binary-0001.xml ## Deploying OVMS ```bash -$ docker run -it --rm -p 9000:9000 -v `pwd`/lib/ubuntu:/extension:ro -v `pwd`/resnet50-binary-0001:/resnet openvino/model_server \ +$ docker run -it --rm -p 9000:9000 -v `pwd`/lib/${BASE_OS}:/extension:ro -v `pwd`/resnet50-binary-0001:/resnet openvino/model_server \ --port 9000 --model_name resnet --model_path /resnet --cpu_extension /extension/libcustom_relu_cpu_extension.so ``` From da3a6ad6fc4df994414abf4141311b12150f2653 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 12 Oct 2022 09:30:55 +0200 Subject: [PATCH 018/130] CVS-92055 Use macro for metric names (#1452) - use extern in string constants not to duplicate global declarations --- src/entry_node.cpp | 2 + src/entry_node.hpp | 2 +- src/exit_node.cpp | 3 + src/exit_node.hpp | 2 +- src/metric_config.cpp | 13 + src/metric_config.hpp | 32 ++- src/model_metric_reporter.cpp | 18 +- src/modelconfig.cpp | 3 + src/modelconfig.hpp | 4 +- src/modelmanager.cpp | 1 + src/modelmanager.hpp | 2 +- src/test/ensemble_config_change_stress.cpp | 31 +-- src/test/metric_config_test.cpp | 92 ++++--- src/test/metrics_flow_test.cpp | 276 +++++++++++---------- 14 files changed, 272 insertions(+), 209 deletions(-) diff --git a/src/entry_node.cpp b/src/entry_node.cpp index 048242ee0d..8d64ac7d78 100644 --- a/src/entry_node.cpp +++ b/src/entry_node.cpp @@ -36,6 +36,8 @@ namespace ovms { +const std::string ENTRY_NODE_NAME = "request"; + template Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) { OVMS_PROFILE_FUNCTION(); diff --git a/src/entry_node.hpp b/src/entry_node.hpp index 57c22f92a2..1ec91159cf 100644 --- a/src/entry_node.hpp +++ b/src/entry_node.hpp @@ -26,7 +26,7 @@ namespace ovms { -const std::string ENTRY_NODE_NAME = "request"; +extern const std::string ENTRY_NODE_NAME; template class EntryNode : public Node { diff --git a/src/exit_node.cpp b/src/exit_node.cpp index 4e1e4cab9e..fd88638196 100644 --- a/src/exit_node.cpp +++ b/src/exit_node.cpp @@ -30,6 +30,9 @@ #include "exitnodesession.hpp" namespace ovms { + +const std::string EXIT_NODE_NAME = "response"; + template Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { OVMS_PROFILE_FUNCTION(); diff --git a/src/exit_node.hpp b/src/exit_node.hpp index d3500736f3..0818163824 100644 --- a/src/exit_node.hpp +++ b/src/exit_node.hpp @@ -28,7 +28,7 @@ namespace ovms { -const std::string EXIT_NODE_NAME = "response"; +extern const std::string EXIT_NODE_NAME; template class ExitNode : public Node { diff --git a/src/metric_config.cpp b/src/metric_config.cpp index f365c4986f..76dbc20042 100644 --- a/src/metric_config.cpp +++ b/src/metric_config.cpp @@ -30,6 +30,19 @@ namespace ovms { +const std::string METRIC_NAME_REQUESTS_SUCCESS = "ovms_requests_success"; +const std::string METRIC_NAME_REQUESTS_FAIL = "ovms_requests_fail"; + +const std::string METRIC_NAME_STREAMS = "ovms_streams"; +const std::string METRIC_NAME_INFER_REQ_QUEUE_SIZE = "ovms_infer_req_queue_size"; + +const std::string METRIC_NAME_INFER_REQ_ACTIVE = "ovms_infer_req_active"; + +const std::string METRIC_NAME_INFERENCE_TIME = "ovms_inference_time_us"; +const std::string METRIC_NAME_CURRENT_REQUESTS = "ovms_current_requests"; +const std::string METRIC_NAME_REQUEST_TIME = "ovms_request_time_us"; +const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME = "ovms_wait_for_infer_req_time_us"; + bool MetricConfig::validateEndpointPath(const std::string& endpoint) { std::regex valid_endpoint_regex("^/[a-zA-Z0-9]*$"); return std::regex_match(endpoint, valid_endpoint_regex); diff --git a/src/metric_config.hpp b/src/metric_config.hpp index bf62bbed55..39e87c768b 100644 --- a/src/metric_config.hpp +++ b/src/metric_config.hpp @@ -24,6 +24,20 @@ #include "status.hpp" namespace ovms { + +extern const std::string METRIC_NAME_REQUESTS_SUCCESS; +extern const std::string METRIC_NAME_REQUESTS_FAIL; + +extern const std::string METRIC_NAME_STREAMS; +extern const std::string METRIC_NAME_INFER_REQ_QUEUE_SIZE; + +extern const std::string METRIC_NAME_INFER_REQ_ACTIVE; + +extern const std::string METRIC_NAME_INFERENCE_TIME; +extern const std::string METRIC_NAME_CURRENT_REQUESTS; +extern const std::string METRIC_NAME_REQUEST_TIME; +extern const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME; + /** * @brief This class represents metrics configuration */ @@ -60,16 +74,16 @@ class MetricConfig { bool validateEndpointPath(const std::string& endpoint); std::unordered_set additionalMetricFamilies = { - {"ovms_infer_req_queue_size"}, - {"ovms_infer_req_active"}}; + {METRIC_NAME_INFER_REQ_QUEUE_SIZE}, + {METRIC_NAME_INFER_REQ_ACTIVE}}; std::unordered_set defaultMetricFamilies = { - {"ovms_current_requests"}, - {"ovms_requests_success"}, - {"ovms_requests_fail"}, - {"ovms_request_time_us"}, - {"ovms_streams"}, - {"ovms_inference_time_us"}, - {"ovms_wait_for_infer_req_time_us"}}; + {METRIC_NAME_CURRENT_REQUESTS}, + {METRIC_NAME_REQUESTS_SUCCESS}, + {METRIC_NAME_REQUESTS_FAIL}, + {METRIC_NAME_REQUEST_TIME}, + {METRIC_NAME_STREAMS}, + {METRIC_NAME_INFERENCE_TIME}, + {METRIC_NAME_WAIT_FOR_INFER_REQ_TIME}}; }; } // namespace ovms diff --git a/src/model_metric_reporter.cpp b/src/model_metric_reporter.cpp index acad4c7fca..0b3ac95596 100644 --- a/src/model_metric_reporter.cpp +++ b/src/model_metric_reporter.cpp @@ -50,7 +50,7 @@ ServableMetricReporter::ServableMetricReporter(const MetricConfig* metricConfig, this->buckets.emplace_back(floor(BUCKET_MULTIPLIER * pow(BUCKET_POWER_BASE, i))); } - std::string familyName = "ovms_requests_success"; + std::string familyName = METRIC_NAME_REQUESTS_SUCCESS; auto family = registry->createFamily(familyName, "Number of successful requests to a model or a DAG."); THROW_IF_NULL(family, "cannot create family"); @@ -139,7 +139,7 @@ ServableMetricReporter::ServableMetricReporter(const MetricConfig* metricConfig, THROW_IF_NULL(this->requestSuccessRestModelReady, "cannot create metric"); } - familyName = "ovms_requests_fail"; + familyName = METRIC_NAME_REQUESTS_FAIL; family = registry->createFamily(familyName, "Number of failed requests to a model or a DAG."); THROW_IF_NULL(family, "cannot create family"); @@ -228,7 +228,7 @@ ServableMetricReporter::ServableMetricReporter(const MetricConfig* metricConfig, THROW_IF_NULL(this->requestFailRestModelReady, "cannot create metric"); } - familyName = "ovms_request_time_us"; + familyName = METRIC_NAME_REQUEST_TIME; auto requestTimeFamily = registry->createFamily(familyName, "Processing time of requests to a model or a DAG."); THROW_IF_NULL(requestTimeFamily, "cannot create family"); @@ -258,7 +258,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri return; } - std::string familyName = "ovms_inference_time_us"; + std::string familyName = METRIC_NAME_INFERENCE_TIME; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Inference execution time in the OpenVINO backend."); @@ -269,7 +269,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri THROW_IF_NULL(this->inferenceTime, "cannot create metric"); } - familyName = "ovms_wait_for_infer_req_time_us"; + familyName = METRIC_NAME_WAIT_FOR_INFER_REQ_TIME; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Request waiting time in the scheduling queue."); @@ -280,7 +280,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri THROW_IF_NULL(this->waitForInferReqTime, "cannot create metric"); } - familyName = "ovms_streams"; + familyName = METRIC_NAME_STREAMS; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Number of OpenVINO execution streams."); @@ -290,7 +290,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri THROW_IF_NULL(this->streams, "cannot create metric"); } - familyName = "ovms_infer_req_queue_size"; + familyName = METRIC_NAME_INFER_REQ_QUEUE_SIZE; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Inference request queue size (nireq)."); @@ -300,7 +300,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri THROW_IF_NULL(this->inferReqQueueSize, "cannot create metric"); } - familyName = "ovms_infer_req_active"; + familyName = METRIC_NAME_INFER_REQ_ACTIVE; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Number of currently consumed inference request from the processing queue."); @@ -310,7 +310,7 @@ ModelMetricReporter::ModelMetricReporter(const MetricConfig* metricConfig, Metri THROW_IF_NULL(this->inferReqActive, "cannot create metric"); } - familyName = "ovms_current_requests"; + familyName = METRIC_NAME_CURRENT_REQUESTS; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, "Number of inference requests currently in process."); diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index b3c95ff4d3..8d70511c22 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -32,6 +32,9 @@ namespace ovms { +const std::string ANONYMOUS_INPUT_NAME = "ANONYMOUS_INPUT_NAME"; +const std::string MAPPING_CONFIG_JSON = "mapping_config.json"; + bool ModelConfig::isDeviceUsed(const std::string& device) const { if (this->isSingleDeviceUsed(device)) return true; diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index edfd32d343..addd1a2d49 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -38,8 +38,8 @@ using mapping_config_t = std::unordered_map; using plugin_config_t = std::map; using custom_loader_options_config_t = std::map; -const std::string ANONYMOUS_INPUT_NAME = "ANONYMOUS_INPUT_NAME"; -const std::string MAPPING_CONFIG_JSON = "mapping_config.json"; +extern const std::string ANONYMOUS_INPUT_NAME; +extern const std::string MAPPING_CONFIG_JSON; const uint32_t DEFAULT_MAX_SEQUENCE_NUMBER = 500; /** diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index 9dfba449c9..e3f93594f9 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -60,6 +60,7 @@ namespace ovms { static uint16_t MAX_CONFIG_JSON_READ_RETRY_COUNT = 2; +const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; ModelManager::ModelManager(const std::string& modelCacheDirectory, MetricRegistry* registry) : ieCore(std::make_unique()), diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index ecdc8905f5..6cc7e35e16 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -42,7 +42,7 @@ namespace ovms { const uint32_t DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS = 10000; -const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; +extern const std::string DEFAULT_MODEL_CACHE_DIRECTORY; class Config; class IVersionReader; diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index 7c6b39d2ee..7ad490cc7c 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -42,16 +42,18 @@ using testing::Return; static const std::string PIPELINE_1_DUMMY_NAME = "pipeline1Dummy"; -static const char* stressTestPipelineOneDummyConfig = R"( +std::string createStressTestPipelineOneDummyConfig() { + return R"( { "monitoring": { "metrics": { "enable": true, "metrics_list": [ - "ovms_current_requests", - "ovms_infer_req_active", - "ovms_requests_success", - "ovms_infer_req_queue_size"] + ")" + + METRIC_NAME_CURRENT_REQUESTS + + R"(",")" + METRIC_NAME_INFER_REQ_ACTIVE + + R"(",")" + METRIC_NAME_REQUESTS_SUCCESS + + R"(",")" + METRIC_NAME_INFER_REQ_QUEUE_SIZE + R"("] } }, "model_config_list": [ @@ -93,6 +95,7 @@ static const char* stressTestPipelineOneDummyConfig = R"( } ] })"; +} static const char* stressTestPipelineOneDummyRemovedConfig = R"( { "model_config_list": [ @@ -1030,7 +1033,7 @@ class StressPipelineConfigChanges : public TestWithTempDir { int arg_count = 5; ovms::Config::instance().parse(arg_count, n_argv); modelPath = directoryPath + "/dummy/"; - SetUpConfig(stressTestPipelineOneDummyConfig); + SetUpConfig(createStressTestPipelineOneDummyConfig()); std::filesystem::copy("/ovms/src/test/dummy", modelPath, std::filesystem::copy_options::recursive); } void defaultVersionRemove() { @@ -1126,17 +1129,17 @@ class StressPipelineConfigChanges : public TestWithTempDir { ASSERT_THAT(metricOutput, ::testing::HasSubstr(metricName + std::string{"{name=\"dummy\",version=\"1\"} "})) << "cannot find dummys " << metricName << " metric\n" << metricOutput; std::regex findActualMetricRgx(std::string{".*"} + metricName + std::string{"\\{name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); - std::regex findRequestsSuccessMetricRgx(std::string{".*ovms_requests_success\\{api=\"TensorFlowServing\",interface=\"gRPC\",method=\"Predict\",name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); + std::regex findRequestsSuccessMetricRgx(std::string{".*"} + METRIC_NAME_REQUESTS_SUCCESS + std::string{"\\{api=\"TensorFlowServing\",interface=\"gRPC\",method=\"Predict\",name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); std::smatch match; ASSERT_TRUE(std::regex_search(metricOutput, match, findActualMetricRgx)) << "cannot find dummys " << metricName << " metric\n" << metricOutput; auto actualVal = ovms::stoi64(match[1]); - ASSERT_TRUE(std::regex_search(metricOutput, match, findRequestsSuccessMetricRgx)) << "cannot find dummys ovms_requests_success metric\n" + ASSERT_TRUE(std::regex_search(metricOutput, match, findRequestsSuccessMetricRgx)) << "cannot find dummys " << METRIC_NAME_REQUESTS_SUCCESS << " metric\n" << metricOutput; auto requestsSuccessCounter = ovms::stoi64(match[1]); - ASSERT_TRUE(requestsSuccessCounter.has_value()) << "cannot parse ovms_requests_success\n" + ASSERT_TRUE(requestsSuccessCounter.has_value()) << "cannot parse " << METRIC_NAME_REQUESTS_SUCCESS << "\n" << metricOutput; - SPDLOG_DEBUG("ovms_requests_success value: {}", requestsSuccessCounter.value()); + SPDLOG_DEBUG("{} value: {}", METRIC_NAME_REQUESTS_SUCCESS, requestsSuccessCounter.value()); ASSERT_TRUE(actualVal.has_value()) << "cannot parse " << metricName << " metric to number\n" << metricOutput; // In case of sporadic error here consider checking ovms_requests_success value (if 0, it could mean the load did not start yet (could happen on slower machines)) @@ -1145,8 +1148,8 @@ class StressPipelineConfigChanges : public TestWithTempDir { } void checkActiveNireqSmallerThanTotal() { std::string metricOutput = manager.getMetricRegistry()->collect(); - std::regex findNireqTotalRgx(std::string{".*ovms_infer_req_queue_size\\{name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); - std::regex findNireqActiveRgx(std::string{".*ovms_infer_req_active\\{name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); + std::regex findNireqTotalRgx(std::string{".*"} + METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"\\{name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); + std::regex findNireqActiveRgx(std::string{".*"} + METRIC_NAME_INFER_REQ_ACTIVE + std::string{"\\{name=\"dummy\",version=\"1\"\\} (.*)\n.*"}); std::smatch match; ASSERT_TRUE(std::regex_search(metricOutput, match, findNireqTotalRgx)) << "cannot find dummys total nireq in metric\n" << metricOutput; @@ -1160,8 +1163,8 @@ class StressPipelineConfigChanges : public TestWithTempDir { } void testCurrentRequestsMetric() { SPDLOG_INFO("{} start", __FUNCTION__); - checkMetricGreaterThan("ovms_current_requests", 0); - checkMetricGreaterThan("ovms_infer_req_active", 0); + checkMetricGreaterThan(METRIC_NAME_CURRENT_REQUESTS, 0); + checkMetricGreaterThan(METRIC_NAME_INFER_REQ_ACTIVE, 0); checkActiveNireqSmallerThanTotal(); SPDLOG_INFO("{} end", __FUNCTION__); } diff --git a/src/test/metric_config_test.cpp b/src/test/metric_config_test.cpp index 860bfe2404..649b2aa57e 100644 --- a/src/test/metric_config_test.cpp +++ b/src/test/metric_config_test.cpp @@ -14,6 +14,8 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include + #include #include @@ -80,7 +82,8 @@ class MetricsConfigNegativeTest : public MetricsConfigTest { } }; -static const char* modelMetricsChangedConfig = R"( +std::string createModelMetricsChangedConfig() { + return R"( { "model_config_list": [ { @@ -99,13 +102,16 @@ static const char* modelMetricsChangedConfig = R"( "metrics": { "enable" : true, - "metrics_list": ["ovms_requests_success", "ovms_infer_req_queue_size"] + "metrics_list": [")" + + METRIC_NAME_REQUESTS_SUCCESS + + std::string{"\", \""} + METRIC_NAME_INFER_REQ_QUEUE_SIZE + R"("] } } })"; +} TEST_F(MetricsConfigNegativeTest, MissingPort) { - SetUpConfig(modelMetricsChangedConfig); + SetUpConfig(createModelMetricsChangedConfig()); std::filesystem::copy("/ovms/src/test/dummy", modelPath, std::filesystem::copy_options::recursive); createConfigFileWithContent(ovmsConfig, configFilePath); @@ -148,7 +154,7 @@ TEST_F(MetricsConfigTest, DefaultValues) { } TEST_F(MetricsConfigTest, ChangedValues) { - SetUpConfig(modelMetricsChangedConfig); + SetUpConfig(createModelMetricsChangedConfig()); std::filesystem::copy("/ovms/src/test/dummy", modelPath, std::filesystem::copy_options::recursive); createConfigFileWithContent(ovmsConfig, configFilePath); @@ -161,12 +167,13 @@ TEST_F(MetricsConfigTest, ChangedValues) { ASSERT_EQ(metricConfig.metricsEnabled, true); // ASSERT_EQ(metricConfig.endpointsPath, "/newmetrics"); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_success")); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size")); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_requests_fail"), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE)); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL), false); } -static const char* modelMetricsBadListConfig = R"( +std::string createModelMetricsBadListConfig() { + return R"( { "model_config_list": [ { @@ -185,13 +192,15 @@ static const char* modelMetricsBadListConfig = R"( "metrics": { "enable" : true, - "metrics_list": ["ovms_request_success", "ovms_infer_req_queue_size"] + "metrics_list": ["bad_name", ")" + + METRIC_NAME_INFER_REQ_QUEUE_SIZE + R"("] } } })"; +} TEST_F(MetricsConfigTest, BadFamilyConfig) { - SetUpConfig(modelMetricsBadListConfig); + SetUpConfig(createModelMetricsBadListConfig()); std::filesystem::copy("/ovms/src/test/dummy", modelPath, std::filesystem::copy_options::recursive); createConfigFileWithContent(ovmsConfig, configFilePath); @@ -202,7 +211,7 @@ TEST_F(MetricsConfigTest, BadFamilyConfig) { } TEST_F(MetricsConfigTest, InitOnce) { - SetUpConfig(modelMetricsChangedConfig); + SetUpConfig(createModelMetricsChangedConfig()); std::filesystem::copy("/ovms/src/test/dummy", modelPath, std::filesystem::copy_options::recursive); createConfigFileWithContent(ovmsConfig, configFilePath); @@ -220,9 +229,9 @@ TEST_F(MetricsConfigTest, InitOnce) { ASSERT_EQ(metricConfig2.metricsEnabled, true); // ASSERT_EQ(metricConfig2.endpointsPath, "/newmetrics"); - ASSERT_TRUE(metricConfig2.isFamilyEnabled("ovms_requests_success")); - ASSERT_TRUE(metricConfig2.isFamilyEnabled("ovms_infer_req_queue_size")); - ASSERT_EQ(metricConfig2.isFamilyEnabled("ovms_requests_fail"), false); + ASSERT_TRUE(metricConfig2.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); + ASSERT_TRUE(metricConfig2.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE)); + ASSERT_EQ(metricConfig2.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL), false); } static const char* modelMetricsAllEnabledConfig = R"( @@ -260,10 +269,10 @@ TEST_F(MetricsConfigTest, MetricsAllEnabledTest) { const auto& metricConfig = manager.getMetricConfig(); ASSERT_EQ(metricConfig.metricsEnabled, true); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_success")); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); // Non default metric - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size"), false); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_fail")); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL)); } static const char* modelMetricsBadEndpoint = R"( @@ -471,30 +480,35 @@ TEST_F(MetricsCli, DefaultCliReading) { MetricConfig metricConfig; ASSERT_EQ(metricConfig.metricsEnabled, false); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_requests_success"), false); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size"), false); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_requests_fail"), false); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS), false); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE), false); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL), false); - auto status = metricConfig.loadFromCLIString(true, "ovms_requests_success, ovms_requests_fail"); + std::stringstream ss; + ss << METRIC_NAME_REQUESTS_SUCCESS << ", " << METRIC_NAME_REQUESTS_FAIL; + auto status = metricConfig.loadFromCLIString(true, ss.str()); ASSERT_TRUE(status.ok()); ASSERT_EQ(metricConfig.metricsEnabled, true); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_success")); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size"), false); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_fail")); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL)); } TEST_F(MetricsCli, WorkingCliReading) { MetricConfig metricConfig; - auto status = metricConfig.loadFromCLIString(true, "ovms_requests_success, ovms_infer_req_queue_size"); + + std::stringstream ss; + ss << METRIC_NAME_REQUESTS_SUCCESS << ", " << METRIC_NAME_INFER_REQ_QUEUE_SIZE; + auto status = metricConfig.loadFromCLIString(true, ss.str()); ASSERT_TRUE(status.ok()); ASSERT_EQ(metricConfig.metricsEnabled, true); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_success")); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size")); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_requests_fail"), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE)); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL), false); } TEST_F(MetricsCli, DefaultEmptyList) { @@ -508,9 +522,9 @@ TEST_F(MetricsCli, DefaultEmptyList) { ASSERT_TRUE(status.ok()); ASSERT_EQ(metricConfig.metricsEnabled, true); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_success")); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size"), false); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_fail")); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS)); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL)); } TEST_F(MetricsCli, BadCliReading) { @@ -528,19 +542,25 @@ TEST_F(MetricsCli, BadCliReading) { TEST_F(MetricsCli, DisabledMetrics) { MetricConfig metricConfig; - auto status = metricConfig.loadFromCLIString(false, "ovms_infer_req_queue_size, ovms_requests_fail"); + + std::stringstream ss; + ss << METRIC_NAME_INFER_REQ_QUEUE_SIZE << ", " << METRIC_NAME_REQUESTS_FAIL; + auto status = metricConfig.loadFromCLIString(false, ss.str()); ASSERT_TRUE(status.ok()); ASSERT_EQ(metricConfig.metricsEnabled, false); ASSERT_EQ(metricConfig.endpointsPath, "/metrics"); - ASSERT_EQ(metricConfig.isFamilyEnabled("ovms_requests_success"), false); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_infer_req_queue_size")); - ASSERT_TRUE(metricConfig.isFamilyEnabled("ovms_requests_fail")); + ASSERT_EQ(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_SUCCESS), false); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_INFER_REQ_QUEUE_SIZE)); + ASSERT_TRUE(metricConfig.isFamilyEnabled(METRIC_NAME_REQUESTS_FAIL)); } TEST_F(MetricsCli, MetricsEnabledCliRestPortDefault) { MetricConfig metricConfig; - auto status = metricConfig.loadFromCLIString(true, "ovms_infer_req_queue_size, ovms_requests_fail"); + + std::stringstream ss; + ss << METRIC_NAME_INFER_REQ_QUEUE_SIZE << ", " << METRIC_NAME_REQUESTS_FAIL; + auto status = metricConfig.loadFromCLIString(true, ss.str()); ASSERT_EQ(status, StatusCode::OK); } diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 7e7b42fec6..4823efea46 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -73,61 +73,6 @@ void checkRequestsCounter(const std::string& collectedMetricData, const std::str } } -const char* pipelineDummyDemux = R"({ - "monitoring": { - "metrics": { - "enable": true, - "metrics_list": [ - "ovms_infer_req_queue_size", - "ovms_infer_req_active", - "ovms_current_requests", - "ovms_requests_success", - "ovms_requests_fail", - "ovms_request_time_us", - "ovms_streams", - "ovms_inference_time_us", - "ovms_wait_for_infer_req_time_us" - ] - } - }, - "model_config_list": [ - {"config": { - "name": "dummy", - "nireq": 2, - "plugin_config": {"CPU_THROUGHPUT_STREAMS": 4}, - "base_path": "/ovms/src/test/dummy"}} - ], - "pipeline_config_list": [ - { - "name": "dummy_demux", - "inputs": [ - "b" - ], - "demultiply_count": 0, - "nodes": [ - { - "name": "dummy-node", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": { - "node_name": "request", - "data_item": "b"}}], - "outputs": [ - {"data_item": "a", - "alias": "a"}] - } - ], - "outputs": [ - {"a": { - "node_name": "dummy-node", - "data_item": "a"}} - ] - } - ] -} -)"; - class ServableManagerModuleWithMockedManager : public ServableManagerModule { ConstructorEnabledModelManager& mockedManager; @@ -178,13 +123,15 @@ class MetricFlowTest : public TestWithTempDir { std::optional modelVersion = std::nullopt; std::optional modelVersionLabel{std::nullopt}; + std::string prepareConfigContent(); + void SetUp() override { TestWithTempDir::SetUp(); char* n_argv[] = {(char*)"ovms", (char*)"--config_path", (char*)"/unused", (char*)"--rest_port", (char*)"8080"}; // Workaround to have rest_port parsed in order to enable metrics int arg_count = 5; ovms::Config::instance().parse(arg_count, n_argv); std::string fileToReload = this->directoryPath + "/config.json"; - createConfigFileWithContent(pipelineDummyDemux, fileToReload); + createConfigFileWithContent(this->prepareConfigContent(), fileToReload); ASSERT_EQ(server.getManager().loadConfig(fileToReload), StatusCode::OK); } }; @@ -235,28 +182,28 @@ TEST_F(MetricFlowTest, GrpcPredict) { } // ovms_requests_success - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", modelName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "gRPC", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_streams{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_streams{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); } TEST_F(MetricFlowTest, GrpcGetModelMetadata) { @@ -280,8 +227,8 @@ TEST_F(MetricFlowTest, GrpcGetModelMetadata) { ASSERT_EQ(impl.GetModelMetadata(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, GrpcGetModelStatus) { @@ -303,8 +250,8 @@ TEST_F(MetricFlowTest, GrpcGetModelStatus) { ASSERT_EQ(impl.GetModelStatus(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, GrpcModelInfer) { @@ -348,28 +295,28 @@ TEST_F(MetricFlowTest, GrpcModelInfer) { ASSERT_EQ(impl.ModelInfer(nullptr, &request, &response).error_code(), grpc::StatusCode::INVALID_ARGUMENT); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "ModelInfer", "KServe", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "ModelInfer", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "ModelInfer", "KServe", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "ModelInfer", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", modelName, 1, "gRPC", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", dagName, 1, "gRPC", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "gRPC", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "gRPC", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_streams{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_streams{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); } TEST_F(MetricFlowTest, GrpcModelMetadata) { @@ -391,8 +338,8 @@ TEST_F(MetricFlowTest, GrpcModelMetadata) { ASSERT_EQ(impl.ModelMetadata(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, GrpcModelReady) { @@ -414,8 +361,8 @@ TEST_F(MetricFlowTest, GrpcModelReady) { ASSERT_EQ(impl.ModelReady(nullptr, &request, &response).error_code(), grpc::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "gRPC", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "gRPC", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "gRPC", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "gRPC", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, RestPredict) { @@ -445,28 +392,28 @@ TEST_F(MetricFlowTest, RestPredict) { ASSERT_EQ(handler.processPredictRequest(dagName, modelVersion, modelVersionLabel, request, &response), ovms::StatusCode::REST_COULD_NOT_PARSE_INSTANCE); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "Predict", "TensorFlowServing", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", modelName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "REST", "Predict", "TensorFlowServing", numberOfFailedRequests); // ran by real request - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_streams{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_streams{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); } TEST_F(MetricFlowTest, RestGetModelMetadata) { @@ -482,8 +429,8 @@ TEST_F(MetricFlowTest, RestGetModelMetadata) { ASSERT_EQ(handler.processModelMetadataRequest(dagName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "GetModelMetadata", "TensorFlowServing", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, RestGetModelStatus) { @@ -499,8 +446,8 @@ TEST_F(MetricFlowTest, RestGetModelStatus) { ASSERT_EQ(handler.processModelStatusRequest(dagName, modelVersion, modelVersionLabel, &response), ovms::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "GetModelStatus", "TensorFlowServing", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, RestModelInfer) { @@ -535,28 +482,28 @@ TEST_F(MetricFlowTest, RestModelInfer) { ASSERT_EQ(handler.processInferKFSRequest(components, response, request), ovms::StatusCode::JSON_INVALID); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "ModelInfer", "KServe", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "ModelInfer", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "ModelInfer", "KServe", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "ModelInfer", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", modelName, 1, "REST", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_fail", dagName, 1, "REST", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, modelName, 1, "REST", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_FAIL, dagName, 1, "REST", "ModelInfer", "KServe", numberOfFailedRequests); // ran by real request - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_request_time_us_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"gRPC\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(0))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_REQUEST_TIME + std::string{"_count{interface=\"REST\",name=\""} + dagName + std::string{"\",version=\"1\"} "} + std::to_string(numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_inference_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFERENCE_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_wait_for_infer_req_time_us_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + std::string{"_count{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_streams{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_streams{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(4))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_STREAMS + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); - EXPECT_THAT(server.collect(), HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); - EXPECT_THAT(server.collect(), Not(HasSubstr(std::string{"ovms_infer_req_queue_size{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); + EXPECT_THAT(server.collect(), HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + modelName + std::string{"\",version=\"1\"} "} + std::to_string(2))); + EXPECT_THAT(server.collect(), Not(HasSubstr(METRIC_NAME_INFER_REQ_QUEUE_SIZE + std::string{"{name=\""} + dagName + std::string{"\",version=\"1\"} "}))); } TEST_F(MetricFlowTest, RestModelMetadata) { @@ -575,8 +522,8 @@ TEST_F(MetricFlowTest, RestModelMetadata) { ASSERT_EQ(handler.processModelMetadataKFSRequest(components, response, request), ovms::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "ModelMetadata", "KServe", numberOfSuccessRequests); // ran by real request } TEST_F(MetricFlowTest, ModelReady) { @@ -595,6 +542,63 @@ TEST_F(MetricFlowTest, ModelReady) { ASSERT_EQ(handler.processModelReadyKFSRequest(components, response, request), ovms::StatusCode::OK); } - checkRequestsCounter(server.collect(), "ovms_requests_success", modelName, 1, "REST", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request - checkRequestsCounter(server.collect(), "ovms_requests_success", dagName, 1, "REST", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request + checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, dagName, 1, "REST", "ModelReady", "KServe", numberOfSuccessRequests); // ran by real request +} + +std::string MetricFlowTest::prepareConfigContent() { + return std::string{R"({ + "monitoring": { + "metrics": { + "enable": true, + "metrics_list": [)"} + + R"(")" + METRIC_NAME_INFER_REQ_QUEUE_SIZE + + R"(",")" + METRIC_NAME_INFER_REQ_ACTIVE + + R"(",")" + METRIC_NAME_CURRENT_REQUESTS + + R"(",")" + METRIC_NAME_REQUESTS_SUCCESS + + R"(",")" + METRIC_NAME_REQUESTS_FAIL + + R"(",")" + METRIC_NAME_REQUEST_TIME + + R"(",")" + METRIC_NAME_STREAMS + + R"(",")" + METRIC_NAME_INFERENCE_TIME + + R"(",")" + METRIC_NAME_WAIT_FOR_INFER_REQ_TIME + + R"("] + } + }, + "model_config_list": [ + {"config": { + "name": "dummy", + "nireq": 2, + "plugin_config": {"CPU_THROUGHPUT_STREAMS": 4}, + "base_path": "/ovms/src/test/dummy"}} + ], + "pipeline_config_list": [ + { + "name": "dummy_demux", + "inputs": [ + "b" + ], + "demultiply_count": 0, + "nodes": [ + { + "name": "dummy-node", + "model_name": "dummy", + "type": "DL model", + "inputs": [ + {"b": { + "node_name": "request", + "data_item": "b"}}], + "outputs": [ + {"data_item": "a", + "alias": "a"}] + } + ], + "outputs": [ + {"a": { + "node_name": "dummy-node", + "data_item": "a"}} + ] + } + ] + } + )"; } From 5cfe7b34b4d25aaed5a06df026222c4ee6cfdaaf Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 12 Oct 2022 11:44:36 +0200 Subject: [PATCH 019/130] Clean response after every test, move sporadically missing MODEL_VERSION_MISSING status to allowed (from required) (#1458) - clean the response after every stress test --- src/test/ensemble_config_change_stress.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index 7ad490cc7c..d3ca2559ac 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -1328,7 +1328,6 @@ class StressPipelineConfigChanges : public TestWithTempDir { const std::set& allowedLoadResults, std::unordered_map>& createPipelineRetCodesCounters) { tensorflow::serving::GetModelMetadataRequest request; - tensorflow::serving::GetModelMetadataResponse response; startSignal.get(); // stressIterationsCounter is additional safety measure auto stressIterationsCounter = stressIterationsLimit; @@ -1339,6 +1338,7 @@ class StressPipelineConfigChanges : public TestWithTempDir { break; } auto status = ovms::GetModelMetadataImpl::createGrpcRequest(pipelineName, 1, &request); + tensorflow::serving::GetModelMetadataResponse response; status = ovms::GetModelMetadataImpl::getModelStatus(&request, &response, manager, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelMetadata)); createPipelineRetCodesCounters[status.getCode()]++; EXPECT_TRUE((requiredLoadResults.find(status.getCode()) != requiredLoadResults.end()) || @@ -1363,7 +1363,6 @@ class StressPipelineConfigChanges : public TestWithTempDir { const std::set& allowedLoadResults, std::unordered_map>& createPipelineRetCodesCounters) { tensorflow::serving::GetModelStatusRequest request; - tensorflow::serving::GetModelStatusResponse response; startSignal.get(); // stressIterationsCounter is additional safety measure // for getModelStatus requests it must be much higher since the response time is much lower @@ -1376,6 +1375,7 @@ class StressPipelineConfigChanges : public TestWithTempDir { break; } auto status = ovms::GetModelStatusImpl::createGrpcRequest(getServableName(), 1, &request); + tensorflow::serving::GetModelStatusResponse response; status = ovms::GetModelStatusImpl::getModelStatus(&request, &response, manager, ovms::ExecutionContext(ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::GetModelStatus)); createPipelineRetCodesCounters[status.getCode()]++; EXPECT_TRUE((requiredLoadResults.find(status.getCode()) != requiredLoadResults.end()) || @@ -1904,10 +1904,11 @@ TEST_F(StressPipelineCustomNodesConfigChanges, ChangeCustomLibraryParamDuringGet TEST_F(StressModelConfigChanges, AddModelDuringGetModelStatusLoad) { bool performWholeConfigReload = true; // we just need to have all model versions rechecked std::set requiredLoadResults = { - StatusCode::MODEL_NAME_MISSING, // until first model is loaded - StatusCode::MODEL_VERSION_MISSING, // this should be hit if test is stressing enough, pottentially to be moved to allowed - StatusCode::OK}; // we expect full continuouity of operation - std::set allowedLoadResults = {}; + StatusCode::MODEL_NAME_MISSING, // until first model is loaded + StatusCode::OK}; // we expect full continuouity of operation + std::set allowedLoadResults = { + StatusCode::MODEL_VERSION_MISSING // this should be hit if test is stressing enough, sporadically does not happen + }; performStressTest( &StressPipelineConfigChanges::triggerGetPipelineStatusInALoop, &StressPipelineConfigChanges::addFirstModel, From 14d58ad7b676cdb8b7022b9bf747a40623adbec4 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 12 Oct 2022 11:47:50 +0200 Subject: [PATCH 020/130] Remove lib (#1469) --- DockerfileMakePackage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DockerfileMakePackage b/DockerfileMakePackage index a9a3ab49ce..145613921d 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -55,7 +55,7 @@ rm -rf libatk-wrapper.so libattach.so libawt_headless.so libawt.so libawt_xawt. libextnet.so libfontmanager.so libinstrument.so libj2gss.so libj2pcsc.so libj2pkcs11.so libjaas.so \ libjavajpeg.so libjava.so libjawt.so libjdwp.so libjimage.so libjli.so libjsig.so libjsound.so libjvm.so \ liblcms.so libmanagement_agent.so libmanagement_ext.so libmanagement.so libmlib_image.so libnet.so libnio.so \ -libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libunpack.so libverify.so libzip.so +libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libsystemconf.so libunpack.so libverify.so libzip.so RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; From 43a3f8058e90961eba3a7e8d66453034c9b5d084 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Thu, 13 Oct 2022 09:24:10 +0200 Subject: [PATCH 021/130] CVS-93700_doc_test_fixes (#1471) * git clone + docker + help expected output * model + batch_size updates * review updates Co-authored-by: ngrozae --- tests/performance/README.md | 38 ++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/performance/README.md b/tests/performance/README.md index 28ea1e8c7a..c3afa785f7 100644 --- a/tests/performance/README.md +++ b/tests/performance/README.md @@ -1,20 +1,36 @@ # Measure performance ## Prerequisites + +Clone OVMS repository. +```bash +$ git clone https://github.com/openvinotoolkit/model_server.git +$ cd model_server/tests/performance +``` + Enable virtualenv in project root directory, install requirements.txt. ```bash $ virtualenv .venv $ . .venv/bin/activate -$ pip3 install -r requirements.txt +$ pip3 install -r ../requirements.txt +``` + +```bash +$ docker run -p 9178:9178 openvino/model_server:latest --model_name resnet \ +--model_path gs://ovms-public-eu/resnet50 --port 9178 --batch_size 2 ``` ## Latency ```bash $ python3 grpc_latency.py --help usage: grpc_latency.py [-h] --images_numpy_path IMAGES_NUMPY_PATH + [--labels_numpy_path LABELS_NUMPY_PATH] [--grpc_address GRPC_ADDRESS] [--grpc_port GRPC_PORT] - [--input_name INPUT_NAME] [--iterations ITERATIONS] - [--batchsize BATCHSIZE] [--model_name MODEL_NAME] - [--report_every REPORT_EVERY] [--id ID] + [--input_name INPUT_NAME] [--output_name OUTPUT_NAME] + [--iterations ITERATIONS] [--batchsize BATCHSIZE] + [--model_name MODEL_NAME] + [--model_version MODEL_VERSION] + [--report_every REPORT_EVERY] [--precision PRECISION] + [--id ID] Sends requests via TFS gRPC API using images in numpy format. It measures performance statistics. @@ -22,13 +38,17 @@ performance statistics. optional arguments: -h, --help show this help message and exit --images_numpy_path IMAGES_NUMPY_PATH - numpy in shape [n,w,h,c] or [n,c,h,w] + image in numpy format + --labels_numpy_path LABELS_NUMPY_PATH + labels in numpy format --grpc_address GRPC_ADDRESS Specify url to grpc service. default:localhost --grpc_port GRPC_PORT - Specify port to grpc service. default: 9000 + Specify port to grpc service. default: 9178 --input_name INPUT_NAME Specify input tensor name. default: input + --output_name OUTPUT_NAME + Specify output tensor name. default: prob --iterations ITERATIONS Number of requests iterations, as default use number of images in numpy memmap. default: 0 (consume all @@ -37,15 +57,19 @@ optional arguments: Number of images in a single request. default: 1 --model_name MODEL_NAME Define model name in payload. default: resnet + --model_version MODEL_VERSION + Model version number. default: 1 --report_every REPORT_EVERY Report performance every X iterations + --precision PRECISION + input precision --id ID Helps identifying client ``` ### Example usage: ```bash -$ python3 grpc_latency.py --grpc_address localhost --grpc_port 9178 --images_numpy_path imgs.npy --iteration 1000 --batchsize 1 --report_every 100 --input_name "data" +$ python3 grpc_latency.py --grpc_address localhost --grpc_port 9178 --images_numpy_path imgs.npy --iteration 1000 --batchsize 2 --report_every 100 --input_name "data" ``` ```bash [--] Starting iterations From 435a9f4e1e427c84cf05a7d16f75192441dce209 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 13 Oct 2022 10:39:16 +0200 Subject: [PATCH 022/130] Introducing cppclean scaning (#1430) * Add script for veryfing count of warnings from cppclean Script has hardcoded value of warnings. This is to be lowered with each incoming fix. * Add additional step in sdl-check to check cppclean as well * Remove unused forward declares, mark local functions as static * Introduce forward declares instead of includes when possible * Sort lib_search.py excludes This MR does not introduce any functional changes. At most it moves code from hpp to cpp files. JIRA:CVS-91117 --- .gitignore | 2 + Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 3 +- Makefile | 2 + check_coverage.bat | 4 +- cppclean.sh | 49 ++++++++ lib_search.py | 117 +++++++++++++++--- src/BUILD | 1 + src/azurefilesystem.cpp | 2 +- src/binaryutils.cpp | 53 ++++---- src/binaryutils.hpp | 2 +- src/cleaner_utils.cpp | 36 ++++++ src/cleaner_utils.hpp | 21 +--- src/config.cpp | 8 ++ src/config.hpp | 10 +- src/custom_node.cpp | 5 +- src/custom_node.hpp | 5 +- src/custom_node_library_manager.hpp | 2 +- src/custom_node_output_allocator.cpp | 1 + src/custom_node_output_allocator.hpp | 2 +- src/customloaders.cpp | 1 + src/customloaders.hpp | 5 +- src/customnodesession.cpp | 2 + src/customnodesession.hpp | 6 +- src/dl_node.cpp | 37 ++++++ src/dl_node.hpp | 36 ++---- src/dlnodesession.cpp | 1 + src/dlnodesession.hpp | 2 +- src/entry_node.cpp | 1 + src/executingstreamidguard.hpp | 1 + src/exitnodesession.cpp | 1 + src/exitnodesession.hpp | 3 +- src/gatherexitnodeinputhandler.cpp | 1 + src/gatherexitnodeinputhandler.hpp | 1 + src/gathernodeinputhandler.cpp | 1 + src/get_model_metadata_impl.cpp | 6 + src/get_model_metadata_impl.hpp | 4 +- src/global_sequences_viewer.cpp | 3 +- src/global_sequences_viewer.hpp | 5 +- src/grpcservermodule.cpp | 9 +- src/grpcservermodule.hpp | 5 +- src/http_rest_api_handler.cpp | 28 +++-- src/http_rest_api_handler.hpp | 6 +- src/httpservermodule.hpp | 3 +- src/kfs_grpc_inference_service.cpp | 9 +- src/kfs_grpc_inference_service.hpp | 4 +- src/layout.cpp | 7 ++ src/layout.hpp | 15 +-- src/layout_configuration.cpp | 3 +- src/layout_configuration.hpp | 4 +- src/logging.cpp | 4 +- src/metric_config.cpp | 1 + src/metric_config.hpp | 3 +- src/metric_module.cpp | 3 +- src/metric_module.hpp | 5 +- src/model.cpp | 6 +- src/model.hpp | 15 ++- src/model_service.cpp | 3 + src/model_service.hpp | 8 +- src/modelconfig.cpp | 33 +++++ src/modelconfig.hpp | 25 +--- src/modelinstance.cpp | 18 +-- src/modelinstance.hpp | 12 +- src/modelmanager.cpp | 18 +-- src/modelmanager.hpp | 13 +- src/modelversionstatus.hpp | 2 +- src/node.cpp | 2 + src/node.hpp | 8 +- src/node_library_utils.cpp | 1 + src/node_library_utils.hpp | 2 +- src/nodeinfo.hpp | 4 +- src/nodeinputhandler.cpp | 2 + src/nodeinputhandler.hpp | 4 +- src/nodesession.cpp | 3 +- src/nodesession.hpp | 2 +- src/nodestreamidguard.hpp | 1 + src/ov_utils.cpp | 3 +- src/ov_utils.hpp | 3 +- src/pipeline.cpp | 5 +- src/pipeline.hpp | 9 +- src/pipeline_factory.cpp | 3 + src/pipeline_factory.hpp | 4 +- src/pipelinedefinition.cpp | 26 +++- src/pipelinedefinition.hpp | 24 ++-- src/pipelinedefinitionstatus.hpp | 3 - src/pipelineeventqueue.hpp | 1 + src/predict_request_validation_utils.cpp | 5 +- src/predict_request_validation_utils.hpp | 2 +- src/prediction_service.cpp | 2 + src/profilermodule.hpp | 2 +- src/rest_parser.cpp | 11 +- src/rest_parser.hpp | 2 +- src/rest_utils.cpp | 15 +-- src/rest_utils.hpp | 2 +- src/sequence.cpp | 3 + src/sequence.hpp | 5 +- src/sequence_manager.cpp | 2 + src/sequence_manager.hpp | 5 +- src/serialization.cpp | 10 +- src/servablemanagermodule.cpp | 1 + src/servablemanagermodule.hpp | 3 +- src/server.cpp | 15 +-- src/server.hpp | 1 - src/shape.cpp | 1 + src/shape.hpp | 3 +- src/statefulmodelinstance.cpp | 2 + src/statefulmodelinstance.hpp | 4 +- src/tensorinfo.cpp | 6 +- src/test/custom_loader_test.cpp | 1 + src/test/demultiplexer_node_test.cpp | 4 + src/test/ensemble_config_change_stress.cpp | 1 + src/test/ensemble_flow_custom_node_tests.cpp | 10 ++ src/test/ensemble_metadata_test.cpp | 2 + src/test/ensemble_tests.cpp | 1 + src/test/gather_node_test.cpp | 2 + .../get_model_metadata_validation_test.cpp | 1 + .../get_pipeline_metadata_response_test.cpp | 1 + src/test/kfs_rest_parser_test.cpp | 1 + src/test/kfs_rest_test.cpp | 1 + src/test/layout_test.cpp | 1 + src/test/model_cache_test.cpp | 1 + src/test/model_service_test.cpp | 1 + src/test/modelinstance_test.cpp | 1 + src/test/modelmanager_test.cpp | 1 + src/test/prediction_service_test.cpp | 1 + src/test/rest_utils_test.cpp | 1 + src/test/sequence_manager_test.cpp | 1 + src/test/server_test.cpp | 1 + src/test/stateful_modelinstance_test.cpp | 1 + src/test/tfs_rest_parser_column_test.cpp | 5 + src/test/tfs_rest_parser_row_test.cpp | 5 + tests/requirements.txt | 1 + 132 files changed, 660 insertions(+), 304 deletions(-) create mode 100755 cppclean.sh create mode 100644 src/cleaner_utils.cpp diff --git a/.gitignore b/.gitignore index 9073a02061..93a01ebb1b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ __pycache__ dist/ lib/ !client/python/ovmsclient/lib +cppclean_src +cppclean_test diff --git a/Dockerfile.redhat b/Dockerfile.redhat index baea6abef4..f0591c1161 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -70,9 +70,9 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n which \ yum-utils \ unzip \ + vim \ xz && \ yum clean all - # Set up Bazel ENV BAZEL_VERSION 3.7.2 WORKDIR /bazel diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 02dbdca7df..97a3f4fde8 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -76,10 +76,10 @@ RUN apt update && apt install -y \ unzip \ wget \ unzip \ + vim \ xz-utils && \ apt clean - # Set up Bazel ENV BAZEL_VERSION 3.7.2 WORKDIR /bazel @@ -239,3 +239,4 @@ RUN ./bazel-bin/src/./ovms --version RUN ./bazel-bin/src/./ovms COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE + diff --git a/Makefile b/Makefile index f4f5f4ef2f..e3b6292d6e 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,8 @@ style: venv clang-format @git diff --exit-code || (echo "clang-format changes not commited. Commit those changes first"; exit 1) @git diff --exit-code --staged || (echo "clang-format changes not commited. Commit those changes first"; exit 1) @. $(ACTIVATE); echo ${PWD}; cpplint ${STYLE_CHECK_OPTS} ${STYLE_CHECK_DIRS} + @echo "Checking cppclean..." + @. $(ACTIVATE); bash -c "./cppclean.sh" sdl-check: venv @echo "Checking SDL requirements..." diff --git a/check_coverage.bat b/check_coverage.bat index b5b8e8e204..5265d590fe 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,8 +5,8 @@ #MIN_FUNCTION_COV=86.5 #Rhel -MIN_LINES_COV=73.3 -MIN_FUNCTION_COV=73.6 +MIN_LINES_COV=72.4 +MIN_FUNCTION_COV=73.2 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` diff --git a/cppclean.sh b/cppclean.sh new file mode 100755 index 0000000000..175a57867c --- /dev/null +++ b/cppclean.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +CPPCLEAN_RESULTS_FILE_SRC="cppclean_src" +CPPCLEAN_RESULTS_FILE_TEST="cppclean_test" +cppclean ./src/ 2>&1 | grep -v test > ${CPPCLEAN_RESULTS_FILE_SRC}; +cppclean ./src/ 2>&1 | grep test > ${CPPCLEAN_RESULTS_FILE_TEST}; +NO_WARNINGS=$(wc -l ${CPPCLEAN_RESULTS_FILE_SRC} | awk '{print $1}') +NO_WARNINGS_TEST=$(wc -l ${CPPCLEAN_RESULTS_FILE_TEST} | awk '{print $1}') +echo "Number of warnings:" ${NO_WARNINGS} +echo "Number of warnings in tests:" ${NO_WARNINGS_TEST} + +trap "cat ${CPPCLEAN_RESULTS_FILE_SRC}" err exit +if [ $(grep "use a forward declaration instead" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 6 ]; then + echo "Failed due to not using forward declarations where possible"; + exit 1; +fi +if [ $(grep "not found in any directly #included header" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 14 ]; then + echo "Failed probably due to not using static keyword with functions definitions"; + exit 1; +fi +if [ $(grep " not used$" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 14 ]; then + echo "Failed probably due to unnecessary forward include"; + exit 1; +fi +if [ ${NO_WARNINGS} -gt 183 ]; then + exit 1 +else + exit 0; +fi +if [ ${NO_WARNINGS_TEST} -gt 131 ]; then + exit 1 +else + exit 0; +fi +exit 0 diff --git a/lib_search.py b/lib_search.py index d1fb9d3f21..388e7839d1 100644 --- a/lib_search.py +++ b/lib_search.py @@ -57,27 +57,70 @@ def check_function(fd): def check_dir(start_dir): no_header = [] - - exclude_files = ['bazel-', '__pycache__', '.venv', '.jpg', '.wav', '.png', '.json', '.git', '.md', '.svg', '.proto', 'gif', - '.jpeg', '.groovy', '.bin', '.npy', '.bandit', '.pdf', '.pytest_cache', '.tar.gz', 'docx', - 'input_images.txt', 'requirements.txt', 'index.html', 'dummy.xml', 'summator.xml', 'tf.patch', 'LICENSE', - 'add.xml', 'tftext.patch', 'net_http.patch', 'clang-format', 'missing_headers.txt', 'listen.patch', 'Doxyfile', - 'increment_1x3x4x5.xml', 'rest_sdk_v2.10.16.patch', 'azure_sdk.patch', 'model.xml', 'ovms-c/dist', - 'client_requirements.txt', 'REST_age_gender.ipynb', 'libevent/BUILD', 'forbidden_functions.txt', - 'resnet_images.txt', 'vehicle_images.txt', 'opencv_cmake_flags.txt', 'metrics_output.out', 'check_coverage.bat'] + exclude_files = [ + '.bandit', + '.bin', + '.git', + '.groovy', + '.jpeg', + '.jpg', + '.json', + '.md', + '.npy', + '.pdf', + '.png', + '.proto', + '.pytest_cache', + '.svg', + '.tar.gz', + '.venv', + '.wav', + 'Doxyfile', + 'LICENSE', + 'REST_age_gender.ipynb', + '__pycache__', + 'add.xml', + 'azure_sdk.patch', + 'bazel-', + 'check_coverage.bat', + 'clang-format', + 'client_requirements.txt', + 'cppclean_src', + 'cppclean_test', + 'docx', + 'dummy.xml', + 'forbidden_functions.txt', + 'gif', + 'increment_1x3x4x5.xml', + 'index.html', + 'input_images.txt', + 'libevent/BUILD', + 'listen.patch', + 'metrics_output.out', + 'missing_headers.txt', + 'model.xml', + 'net_http.patch', + 'opencv_cmake_flags.txt', + 'ovms-c/dist', + 'requirements.txt', + 'resnet_images.txt', + 'rest_sdk_v2.10.16.patch', + 'summator.xml', + 'tf.patch', + 'tftext.patch', + 'vehicle_images.txt', + ] exclude_directories = ['/dist/', 'extras/ovms-operator', 'extras/openvino-operator-openshift', 'release_files/thirdparty-licenses'] for (d_path, _, file_set) in os.walk(start_dir): for f_name in file_set: - skip = False for excluded in exclude_directories: if excluded in d_path: skip = True print('Warning - Skipping directory - ' + d_path + ' for file - ' + f_name) break - if skip: continue @@ -93,13 +136,53 @@ def check_dir(start_dir): def check_func(start_dir): not_ok = [] - exclude_files = ['__pycache__', '.venv', '.pytest_cache', '.vscode', 'ovms-c/dist', '.git', '.tar.gz', 'docx', - '.npy', '.png', '.svg', '.bin', '.jpeg', '.jpg', 'license.txt', 'md', '.groovy', '.json' ,'bazel-', - 'Doxyfile', 'clang-format','net_http.patch', 'tftext.patch', 'tf.patch', 'client_requirements.txt', - 'openvino.LICENSE.txt', 'c-ares.LICENSE.txt', 'zlib.LICENSE.txt', 'boost.LICENSE.txt', - 'libuuid.LICENSE.txt', 'input_images.txt', 'REST_age_gender.ipynb', 'dummy.xml', 'listen.patch', 'add.xml', - 'requirements.txt', 'missing_headers.txt', 'libevent/BUILD', 'azure_sdk.patch', 'rest_sdk_v2.10.16.patch', - 'forbidden_functions.txt', 'missing_headers.txt', 'metrics_output.out', 'summator.xml', 'check_coverage.bat'] + exclude_files = [ + '.bin', + '.git', + '.groovy', + '.jpeg', + '.jpg', + '.json' , + '.npy', + '.png', + '.pytest_cache', + '.svg', + '.tar.gz', + '.venv', + '.vscode', + 'Doxyfile', + 'REST_age_gender.ipynb', + '__pycache__', + 'add.xml', + 'azure_sdk.patch', + 'bazel-', + 'boost.LICENSE.txt', + 'c-ares.LICENSE.txt', + 'check_coverage.bat' + 'clang-format', + 'client_requirements.txt', + 'docx', + 'dummy.xml', + 'forbidden_functions.txt', + 'input_images.txt', + 'libevent/BUILD', + 'libuuid.LICENSE.txt', + 'license.txt', + 'listen.patch', + 'md', + 'metrics_output.out', + 'missing_headers.txt', + 'missing_headers.txt', + 'net_http.patch', + 'openvino.LICENSE.txt', + 'ovms-c/dist', + 'requirements.txt', + 'rest_sdk_v2.10.16.patch', + 'summator.xml', + 'tf.patch', + 'tftext.patch', + 'zlib.LICENSE.txt', + ] exclude_directories = ['/dist/', 'extras/ovms-operator'] diff --git a/src/BUILD b/src/BUILD index 1d9756772c..824ec871bf 100644 --- a/src/BUILD +++ b/src/BUILD @@ -53,6 +53,7 @@ cc_library( "azurestorage.cpp", "azurefilesystem.cpp", "azurefilesystem.hpp", + "cleaner_utils.cpp", "cleaner_utils.hpp", "config.cpp", "config.hpp", diff --git a/src/azurefilesystem.cpp b/src/azurefilesystem.cpp index 6b92031f45..a16e4b4e10 100644 --- a/src/azurefilesystem.cpp +++ b/src/azurefilesystem.cpp @@ -27,7 +27,7 @@ namespace ovms { const std::string AzureFileSystem::AZURE_URL_FILE_PREFIX = "azfs://"; const std::string AzureFileSystem::AZURE_URL_BLOB_PREFIX = "az://"; -as::cloud_storage_account createDefaultOrAnonymousAccount() { +static as::cloud_storage_account createDefaultOrAnonymousAccount() { try { const char* env_cred = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); diff --git a/src/binaryutils.cpp b/src/binaryutils.cpp index f11ae6d8a2..72696df4e5 100644 --- a/src/binaryutils.cpp +++ b/src/binaryutils.cpp @@ -40,7 +40,7 @@ namespace ovms { -int getMatTypeFromTensorPrecision(ovms::Precision tensorPrecision) { +static int getMatTypeFromTensorPrecision(ovms::Precision tensorPrecision) { switch (tensorPrecision) { case ovms::Precision::FP32: return CV_32F; @@ -63,7 +63,7 @@ int getMatTypeFromTensorPrecision(ovms::Precision tensorPrecision) { } } -bool isPrecisionEqual(int matPrecision, ovms::Precision tensorPrecision) { +static bool isPrecisionEqual(int matPrecision, ovms::Precision tensorPrecision) { int convertedTensorPrecision = getMatTypeFromTensorPrecision(tensorPrecision); if (convertedTensorPrecision == matPrecision) { return true; @@ -71,7 +71,7 @@ bool isPrecisionEqual(int matPrecision, ovms::Precision tensorPrecision) { return false; } -cv::Mat convertStringToMat(const std::string& image) { +static cv::Mat convertStringToMat(const std::string& image) { OVMS_PROFILE_FUNCTION(); std::vector data(image.begin(), image.end()); cv::Mat dataMat(data, true); @@ -84,7 +84,7 @@ cv::Mat convertStringToMat(const std::string& image) { } } -Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision requestedPrecision) { +static Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision requestedPrecision) { OVMS_PROFILE_FUNCTION(); int type = getMatTypeFromTensorPrecision(requestedPrecision); if (type == -1) { @@ -96,7 +96,7 @@ Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision return StatusCode::OK; } -Status validateLayout(const std::shared_ptr& tensorInfo) { +static Status validateLayout(const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); static const std::string binarySupportedLayout = "N...HWC"; if (!tensorInfo->getLayout().createIntersection(Layout(binarySupportedLayout), tensorInfo->getShape().size()).has_value()) { @@ -108,20 +108,20 @@ Status validateLayout(const std::shared_ptr& tensorInfo) { return StatusCode::OK; } -bool resizeNeeded(const cv::Mat& image, const dimension_value_t height, const dimension_value_t width) { +static bool resizeNeeded(const cv::Mat& image, const dimension_value_t height, const dimension_value_t width) { if (height != image.rows || width != image.cols) { return true; } return false; } -Status resizeMat(const cv::Mat& src, cv::Mat& dst, const dimension_value_t height, const dimension_value_t width) { +static Status resizeMat(const cv::Mat& src, cv::Mat& dst, const dimension_value_t height, const dimension_value_t width) { OVMS_PROFILE_FUNCTION(); cv::resize(src, dst, cv::Size(width, height)); return StatusCode::OK; } -Status validateNumberOfChannels(const std::shared_ptr& tensorInfo, +static Status validateNumberOfChannels(const std::shared_ptr& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage) { OVMS_PROFILE_FUNCTION(); @@ -152,7 +152,7 @@ Status validateNumberOfChannels(const std::shared_ptr& tensorInfo, return StatusCode::OK; } -Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* firstBatchImage) { +static Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* firstBatchImage) { OVMS_PROFILE_FUNCTION(); if (input.cols == firstBatchImage->cols && input.rows == firstBatchImage->rows) { return StatusCode::OK; @@ -162,7 +162,7 @@ Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* fi return StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH; } -bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, +static bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, const int batchSize) { OVMS_PROFILE_FUNCTION(); if (!tensorInfo->getBatchSize().has_value()) { @@ -171,7 +171,7 @@ bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, return !tensorInfo->getBatchSize().value().match(batchSize); } -Status validateInput(const std::shared_ptr& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage, bool enforceResolutionAlignment) { +static Status validateInput(const std::shared_ptr& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage, bool enforceResolutionAlignment) { // Binary inputs are supported for any endpoint that is compatible with N...HWC layout. // With unknown layout, there is no way to deduce expected endpoint input resolution. // This forces binary utility to create tensors with resolution inherited from first batch of binary input image (request). @@ -187,7 +187,7 @@ Status validateInput(const std::shared_ptr& tensorInfo, const cv::Ma return validateNumberOfChannels(tensorInfo, input, firstBatchImage); } -Status validateTensor(const std::shared_ptr& tensorInfo, +static Status validateTensor(const std::shared_ptr& tensorInfo, const tensorflow::TensorProto& src) { OVMS_PROFILE_FUNCTION(); auto status = validateLayout(tensorInfo); @@ -218,7 +218,7 @@ Status validateTensor(const std::shared_ptr& tensorInfo, return StatusCode::OK; } -Status validateTensor(const std::shared_ptr& tensorInfo, +static Status validateTensor(const std::shared_ptr& tensorInfo, const ::inference::ModelInferRequest::InferInputTensor& src) { OVMS_PROFILE_FUNCTION(); auto status = validateLayout(tensorInfo); @@ -253,7 +253,7 @@ Status validateTensor(const std::shared_ptr& tensorInfo, return StatusCode::OK; } -Dimension getTensorInfoHeightDim(const std::shared_ptr& tensorInfo) { +static Dimension getTensorInfoHeightDim(const std::shared_ptr& tensorInfo) { size_t numberOfShapeDimensions = tensorInfo->getShape().size(); if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { throw std::logic_error("wrong number of shape dimensions"); @@ -262,7 +262,7 @@ Dimension getTensorInfoHeightDim(const std::shared_ptr& tensorInfo) return tensorInfo->getShape()[position]; } -Dimension getTensorInfoWidthDim(const std::shared_ptr& tensorInfo) { +static Dimension getTensorInfoWidthDim(const std::shared_ptr& tensorInfo) { size_t numberOfShapeDimensions = tensorInfo->getShape().size(); if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { throw std::logic_error("wrong number of shape dimensions"); @@ -271,7 +271,7 @@ Dimension getTensorInfoWidthDim(const std::shared_ptr& tensorInfo) { return tensorInfo->getShape()[position]; } -void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& image) { +static void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& image) { if (height.isAny()) { height = image.rows; } else if (height.isDynamic()) { @@ -300,7 +300,7 @@ void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& } } -bool isResizeSupported(const std::shared_ptr& tensorInfo) { +static bool isResizeSupported(const std::shared_ptr& tensorInfo) { for (const auto& dim : tensorInfo->getShape()) { if (dim.isAny()) { return false; @@ -314,24 +314,24 @@ bool isResizeSupported(const std::shared_ptr& tensorInfo) { return true; } -const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i) { +inline static const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i) { return tensor.string_val(i); } -const std::string& getBinaryInput(const ::inference::ModelInferRequest::InferInputTensor& tensor, size_t i) { +inline static const std::string& getBinaryInput(const ::inference::ModelInferRequest::InferInputTensor& tensor, size_t i) { return tensor.contents().bytes_contents(i); } -int getBinaryInputsSize(const tensorflow::TensorProto& tensor) { +inline static int getBinaryInputsSize(const tensorflow::TensorProto& tensor) { return tensor.string_val_size(); } -int getBinaryInputsSize(const ::inference::ModelInferRequest::InferInputTensor& tensor) { +inline static int getBinaryInputsSize(const ::inference::ModelInferRequest::InferInputTensor& tensor) { return tensor.contents().bytes_contents_size(); } template -Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo) { +static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); Dimension targetHeight = getTensorInfoHeightDim(tensorInfo); Dimension targetWidth = getTensorInfoWidthDim(tensorInfo); @@ -387,7 +387,8 @@ Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector< return StatusCode::OK; } -shape_t getShapeFromImages(const std::vector& images, const std::shared_ptr& tensorInfo) { + +static shape_t getShapeFromImages(const std::vector& images, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); shape_t dims; dims.push_back(images.size()); @@ -400,7 +401,7 @@ shape_t getShapeFromImages(const std::vector& images, const std::shared return dims; } -ov::Tensor createTensorFromMats(const std::vector& images, const std::shared_ptr& tensorInfo) { +static ov::Tensor createTensorFromMats(const std::vector& images, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); ov::Shape shape = getShapeFromImages(images, tensorInfo); ov::element::Type precision = tensorInfo->getOvPrecision(); @@ -413,7 +414,7 @@ ov::Tensor createTensorFromMats(const std::vector& images, const std::s return tensor; } -ov::Tensor convertMatsToTensor(std::vector& images, const std::shared_ptr& tensorInfo) { +static ov::Tensor convertMatsToTensor(std::vector& images, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); switch (tensorInfo->getPrecision()) { case ovms::Precision::FP32: @@ -436,7 +437,7 @@ ov::Tensor convertMatsToTensor(std::vector& images, const std::shared_p } template -Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo) { +static Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); auto status = validateTensor(tensorInfo, src); if (status != StatusCode::OK) { diff --git a/src/binaryutils.hpp b/src/binaryutils.hpp index b41806bb61..02b40a6b2e 100644 --- a/src/binaryutils.hpp +++ b/src/binaryutils.hpp @@ -17,10 +17,10 @@ #include -#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { +class Status; template Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); } // namespace ovms diff --git a/src/cleaner_utils.cpp b/src/cleaner_utils.cpp new file mode 100644 index 0000000000..36fb8dbdb0 --- /dev/null +++ b/src/cleaner_utils.cpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "cleaner_utils.hpp" + +#include "global_sequences_viewer.hpp" +#include "modelmanager.hpp" + +namespace ovms { +FunctorSequenceCleaner::FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer) : + globalSequencesViewer(globalSequencesViewer) {} + +void FunctorSequenceCleaner::cleanup() { + globalSequencesViewer.removeIdleSequences(); +} + +FunctorResourcesCleaner::FunctorResourcesCleaner(ModelManager& modelManager) : + modelManager(modelManager) {} + +void FunctorResourcesCleaner::cleanup() { + modelManager.cleanupResources(); +} +} // namespace ovms diff --git a/src/cleaner_utils.hpp b/src/cleaner_utils.hpp index 3ac0cea675..535728ffb6 100644 --- a/src/cleaner_utils.hpp +++ b/src/cleaner_utils.hpp @@ -15,29 +15,20 @@ //***************************************************************************** #pragma once -#include "global_sequences_viewer.hpp" -#include "modelmanager.hpp" - namespace ovms { +class GlobalSequencesViewer; +class ModelManager; struct FunctorSequenceCleaner { GlobalSequencesViewer& globalSequencesViewer; - FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer) : - globalSequencesViewer(globalSequencesViewer) {} - - virtual void cleanup() { - globalSequencesViewer.removeIdleSequences(); - } + FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer); + virtual void cleanup(); }; struct FunctorResourcesCleaner { ModelManager& modelManager; - FunctorResourcesCleaner(ModelManager& modelManager) : - modelManager(modelManager) {} - - virtual void cleanup() { - modelManager.cleanupResources(); - } + FunctorResourcesCleaner(ModelManager& modelManager); + virtual void cleanup(); }; } // namespace ovms diff --git a/src/config.cpp b/src/config.cpp index a46f438204..662c512e02 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -25,6 +25,7 @@ #include #include "logging.hpp" +#include "modelconfig.hpp" #include "version.hpp" namespace ovms { @@ -36,6 +37,13 @@ const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; const std::string DEFAULT_REST_WORKERS_STRING{std::to_string(DEFAULT_REST_WORKERS)}; const uint64_t MAX_REST_WORKERS = 10'000; +uint32_t Config::maxSequenceNumber() const { + if (!result->count("max_sequence_number")) { + return DEFAULT_MAX_SEQUENCE_NUMBER; + } + return result->operator[]("max_sequence_number").as(); +} + Config& Config::parse(int argc, char** argv) { try { options = std::make_unique(argv[0], "OpenVINO Model Server"); diff --git a/src/config.hpp b/src/config.hpp index e73e2dc228..6b507c116e 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -21,8 +21,6 @@ #include -#include "modelconfig.hpp" - namespace ovms { /** * @brief Provides all the configuration options from command line @@ -59,6 +57,7 @@ class Config { * @brief Gets the instance of the config */ static Config& instance() { + // TODO to remove singleton static Config instance; return instance; @@ -323,12 +322,7 @@ class Config { * * @return uint */ - uint32_t maxSequenceNumber() const { - if (!result->count("max_sequence_number")) { - return DEFAULT_MAX_SEQUENCE_NUMBER; - } - return result->operator[]("max_sequence_number").as(); - } + uint32_t maxSequenceNumber() const; /** * @brief Get the log level diff --git a/src/custom_node.cpp b/src/custom_node.cpp index 3feb6c3c6e..d9dfaea0df 100644 --- a/src/custom_node.cpp +++ b/src/custom_node.cpp @@ -17,11 +17,14 @@ #include +#include "custom_node_interface.h" // NOLINT +#include "custom_node_library_internal_manager_wrapper.hpp" #include "custom_node_output_allocator.hpp" #include "customnodesession.hpp" #include "logging.hpp" -#include "node_library.hpp" #include "node_library_utils.hpp" +#include "pipelineeventqueue.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/custom_node.hpp b/src/custom_node.hpp index 358dbc4af1..06f9299045 100644 --- a/src/custom_node.hpp +++ b/src/custom_node.hpp @@ -20,15 +20,16 @@ #include #include -#include "custom_node_interface.h" // NOLINT -#include "custom_node_library_internal_manager_wrapper.hpp" #include "node.hpp" +#include "node_library.hpp" #include "nodeinfo.hpp" #include "pipelineeventqueue.hpp" namespace ovms { class NodeLibrary; +class Status; +class CNLIMWrapper; class CustomNode : public Node { NodeLibrary library; diff --git a/src/custom_node_library_manager.hpp b/src/custom_node_library_manager.hpp index 0a899f0e57..919ea06ae4 100644 --- a/src/custom_node_library_manager.hpp +++ b/src/custom_node_library_manager.hpp @@ -20,9 +20,9 @@ #include #include "node_library.hpp" -#include "status.hpp" namespace ovms { +class Status; class CustomNodeLibraryManager { std::unordered_map libraries; diff --git a/src/custom_node_output_allocator.cpp b/src/custom_node_output_allocator.cpp index a4a4b6e536..8879612689 100644 --- a/src/custom_node_output_allocator.cpp +++ b/src/custom_node_output_allocator.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include "custom_node_output_allocator.hpp" +#include "custom_node_interface.h" // NOLINT #include "logging.hpp" namespace ovms { diff --git a/src/custom_node_output_allocator.hpp b/src/custom_node_output_allocator.hpp index b2b0a84e43..c2b9aa2e46 100644 --- a/src/custom_node_output_allocator.hpp +++ b/src/custom_node_output_allocator.hpp @@ -25,7 +25,7 @@ namespace ovms { bool operator==(const CustomNodeTensor& t1, const CustomNodeTensor& t2); class CustomNodeOutputAllocator : public ov::AllocatorImpl { - struct CustomNodeTensor tensor; + struct ::CustomNodeTensor tensor; NodeLibrary nodeLibrary; void* customNodeLibraryInternalManager; diff --git a/src/customloaders.cpp b/src/customloaders.cpp index 77aa1193ad..1e92eb551a 100644 --- a/src/customloaders.cpp +++ b/src/customloaders.cpp @@ -19,6 +19,7 @@ #include #include "customloaderinterface.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/customloaders.hpp b/src/customloaders.hpp index 7d8b4333e6..39c9f035e8 100644 --- a/src/customloaders.hpp +++ b/src/customloaders.hpp @@ -21,10 +21,9 @@ #include #include -#include "customloaderinterface.hpp" -#include "status.hpp" - namespace ovms { +class CustomLoaderInterface; +class Status; /** * @brief Provides all customloaders */ diff --git a/src/customnodesession.cpp b/src/customnodesession.cpp index 58773377b7..0a6303ee08 100644 --- a/src/customnodesession.cpp +++ b/src/customnodesession.cpp @@ -20,6 +20,7 @@ #include #include +#include "custom_node_interface.h" // NOLINT #include "custom_node_output_allocator.hpp" #include "logging.hpp" #include "node.hpp" @@ -28,6 +29,7 @@ #include "nodeinputhandler.hpp" #include "pipelineeventqueue.hpp" #include "profiler.hpp" +#include "status.hpp" #include "timer.hpp" namespace ovms { diff --git a/src/customnodesession.hpp b/src/customnodesession.hpp index 2c0db19b85..bd62d574de 100644 --- a/src/customnodesession.hpp +++ b/src/customnodesession.hpp @@ -20,16 +20,18 @@ #include -#include "custom_node_interface.h" // NOLINT #include "nodesession.hpp" #include "pipelineeventqueue.hpp" -#include "status.hpp" #include "tensormap.hpp" +class CustomNodeTensor; +class CustomNodeParam; + namespace ovms { class Node; class NodeLibrary; +class Status; class CustomNodeSession : public NodeSession { TensorMap resultTensors; diff --git a/src/dl_node.cpp b/src/dl_node.cpp index 5458183e04..8418a164e7 100644 --- a/src/dl_node.cpp +++ b/src/dl_node.cpp @@ -19,9 +19,13 @@ #include #include "dlnodesession.hpp" +#include "executingstreamidguard.hpp" #include "logging.hpp" #include "metric.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" +#include "nodestreamidguard.hpp" #include "ov_utils.hpp" #include "ovinferrequestsqueue.hpp" #include "prediction_service_utils.hpp" @@ -31,6 +35,39 @@ namespace ovms { const uint WAIT_FOR_STREAM_ID_TIMEOUT_MICROSECONDS = 1; +Status DLNode::getRealInputName(ModelInstance& model, const std::string& alias, std::string* result) const { + auto it = model.getInputsInfo().find(alias); + if (it == model.getInputsInfo().end()) { + return StatusCode::INVALID_MISSING_INPUT; + } + *result = it->second->getName(); + return StatusCode::OK; +} + +Status DLNode::getRealOutputName(ModelInstance& model, const std::string& alias, std::string* result) const { + auto it = nodeOutputNameAlias.find(alias); + const auto& modelOutputName = it != nodeOutputNameAlias.end() ? it->second : alias; + auto jt = model.getOutputsInfo().find(modelOutputName); + if (jt == model.getOutputsInfo().end()) { + return StatusCode::INVALID_MISSING_OUTPUT; + } + *result = jt->second->getName(); + return StatusCode::OK; +} + +DLNode::DLNode(const std::string& nodeName, + const std::string& modelName, + std::optional modelVersion, + ModelManager& modelManager, + std::unordered_map nodeOutputNameAlias, + std::optional demultiplyCount, std::set gatherFromNode) : + Node(nodeName, demultiplyCount, gatherFromNode), + modelName(modelName), + modelVersion(modelVersion), + modelManager(modelManager), + nodeOutputNameAlias(nodeOutputNameAlias) { +} + Status DLNode::execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) { auto& nodeSession = getNodeSession(sessionKey); auto& dlNodeSession = static_cast(nodeSession); diff --git a/src/dl_node.hpp b/src/dl_node.hpp index a0b4f219bf..6a275c5851 100644 --- a/src/dl_node.hpp +++ b/src/dl_node.hpp @@ -24,14 +24,15 @@ #include #include "executingstreamidguard.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" +#include "model_version_policy.hpp" // for model_version_t typename #include "modelversion.hpp" #include "node.hpp" -#include "nodestreamidguard.hpp" namespace ovms { +class ModelInstance; +class ModelInstanceUnloadGuard; +class NodeStreamIdGuard; class ModelManager; class DLNode : public Node { @@ -49,13 +50,7 @@ class DLNode : public Node { DLNode(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, ModelManager& modelManager, std::unordered_map nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : - Node(nodeName, demultiplyCount, gatherFromNode), - modelName(modelName), - modelVersion(modelVersion), - modelManager(modelManager), - nodeOutputNameAlias(nodeOutputNameAlias) { - } + std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}); Status execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) override; @@ -68,25 +63,8 @@ class DLNode : public Node { void release(session_key_t sessionId) override; private: - Status getRealInputName(ModelInstance& model, const std::string& alias, std::string* result) const { - auto it = model.getInputsInfo().find(alias); - if (it == model.getInputsInfo().end()) { - return StatusCode::INVALID_MISSING_INPUT; - } - *result = it->second->getName(); - return StatusCode::OK; - } - - Status getRealOutputName(ModelInstance& model, const std::string& alias, std::string* result) const { - auto it = nodeOutputNameAlias.find(alias); - const auto& modelOutputName = it != nodeOutputNameAlias.end() ? it->second : alias; - auto jt = model.getOutputsInfo().find(modelOutputName); - if (jt == model.getOutputsInfo().end()) { - return StatusCode::INVALID_MISSING_OUTPUT; - } - *result = jt->second->getName(); - return StatusCode::OK; - } + Status getRealInputName(ModelInstance& model, const std::string& alias, std::string* result) const; + Status getRealOutputName(ModelInstance& model, const std::string& alias, std::string* result) const; Status executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest& infer_request); bool tryDisarm(const session_key_t& sessionKey, const uint microseconds = 1) override; diff --git a/src/dlnodesession.cpp b/src/dlnodesession.cpp index 587b5f5fa2..a0240ed923 100644 --- a/src/dlnodesession.cpp +++ b/src/dlnodesession.cpp @@ -29,6 +29,7 @@ #include "ov_utils.hpp" #include "profiler.hpp" #include "shape.hpp" +#include "status.hpp" #include "tensorinfo.hpp" #include "timer.hpp" diff --git a/src/dlnodesession.hpp b/src/dlnodesession.hpp index ca9a798a3e..a078cc5627 100644 --- a/src/dlnodesession.hpp +++ b/src/dlnodesession.hpp @@ -25,7 +25,6 @@ #include "modelversion.hpp" #include "nodesession.hpp" #include "pipelineeventqueue.hpp" -#include "status.hpp" namespace ovms { @@ -34,6 +33,7 @@ class ModelInstance; class Node; class NodeStreamIdGuard; class ModelInstanceUnloadGuard; +class Status; class TensorInfo; class DLNodeSession : public NodeSession { diff --git a/src/entry_node.cpp b/src/entry_node.cpp index 8d64ac7d78..b30afffc94 100644 --- a/src/entry_node.cpp +++ b/src/entry_node.cpp @@ -24,6 +24,7 @@ #include "binaryutils.hpp" #include "deserialization.hpp" #include "logging.hpp" +#include "nodesession.hpp" #include "ov_utils.hpp" #include "predict_request_validation_utils.hpp" #include "profiler.hpp" diff --git a/src/executingstreamidguard.hpp b/src/executingstreamidguard.hpp index e526bf04dc..5bf70222f0 100644 --- a/src/executingstreamidguard.hpp +++ b/src/executingstreamidguard.hpp @@ -20,6 +20,7 @@ class InferRequest; } namespace ovms { +class OVInferRequestsQueue; class ModelMetricReporter; class OVInferRequestsQueue; diff --git a/src/exitnodesession.cpp b/src/exitnodesession.cpp index 715d466608..afd33b977b 100644 --- a/src/exitnodesession.cpp +++ b/src/exitnodesession.cpp @@ -18,6 +18,7 @@ #include #include "gatherexitnodeinputhandler.hpp" +#include "nodesessionmetadata.hpp" namespace ovms { diff --git a/src/exitnodesession.hpp b/src/exitnodesession.hpp index fb7aac0da2..1a77b8fce1 100644 --- a/src/exitnodesession.hpp +++ b/src/exitnodesession.hpp @@ -21,10 +21,11 @@ #include "nodeinputhandler.hpp" #include "nodesession.hpp" -#include "nodesessionmetadata.hpp" #include "tensormap.hpp" namespace ovms { +class CollapseDetails; +class NodeSessionMetadata; template class ExitNodeSession : public NodeSession { diff --git a/src/gatherexitnodeinputhandler.cpp b/src/gatherexitnodeinputhandler.cpp index 08e73a81d1..1de478dacc 100644 --- a/src/gatherexitnodeinputhandler.cpp +++ b/src/gatherexitnodeinputhandler.cpp @@ -18,6 +18,7 @@ #include #include "logging.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/gatherexitnodeinputhandler.hpp b/src/gatherexitnodeinputhandler.hpp index 43756052f2..fd470cec5e 100644 --- a/src/gatherexitnodeinputhandler.hpp +++ b/src/gatherexitnodeinputhandler.hpp @@ -33,6 +33,7 @@ #include "logging.hpp" #include "profiler.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/gathernodeinputhandler.cpp b/src/gathernodeinputhandler.cpp index 08a79fb059..e65eff7c80 100644 --- a/src/gathernodeinputhandler.cpp +++ b/src/gathernodeinputhandler.cpp @@ -22,6 +22,7 @@ #include "nodesessionmetadata.hpp" #include "ov_utils.hpp" #include "profiler.hpp" +#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { diff --git a/src/get_model_metadata_impl.cpp b/src/get_model_metadata_impl.cpp index be23070e65..26b84c7986 100644 --- a/src/get_model_metadata_impl.cpp +++ b/src/get_model_metadata_impl.cpp @@ -17,10 +17,16 @@ #include +#include "execution_context.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" #include "pipelinedefinition.hpp" +#include "pipelinedefinitionstatus.hpp" +#include "pipelinedefinitionunloadguard.hpp" #include "servablemanagermodule.hpp" #include "server.hpp" +#include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" using google::protobuf::util::JsonPrintOptions; diff --git a/src/get_model_metadata_impl.hpp b/src/get_model_metadata_impl.hpp index 2aeda80377..1a200668b8 100644 --- a/src/get_model_metadata_impl.hpp +++ b/src/get_model_metadata_impl.hpp @@ -25,18 +25,18 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "execution_context.hpp" -#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { using proto_signature_map_t = google::protobuf::Map; +class ExecutionContext; class ModelInstance; class ModelManager; class PipelineDefinition; class Server; +class Status; class GetModelMetadataImpl { ModelManager& modelManager; diff --git a/src/global_sequences_viewer.cpp b/src/global_sequences_viewer.cpp index d0d4f43d78..c551ea1a61 100644 --- a/src/global_sequences_viewer.cpp +++ b/src/global_sequences_viewer.cpp @@ -23,12 +23,13 @@ #include "logging.hpp" #include "model.hpp" #include "modelversion.hpp" +#include "sequence_manager.hpp" #include "statefulmodelinstance.hpp" #include "status.hpp" namespace ovms { -static std::string separator = "_"; +static const std::string separator = "_"; Status GlobalSequencesViewer::registerForCleanup(std::string modelName, model_version_t modelVersion, std::shared_ptr sequenceManager) { std::string registration_id = modelName + separator + std::to_string(modelVersion); diff --git a/src/global_sequences_viewer.hpp b/src/global_sequences_viewer.hpp index b15ef71bee..87a12fc259 100644 --- a/src/global_sequences_viewer.hpp +++ b/src/global_sequences_viewer.hpp @@ -24,10 +24,11 @@ #include #include -#include "sequence_manager.hpp" -#include "status.hpp" +#include "modelversion.hpp" namespace ovms { +class SequenceManager; +class Status; const uint32_t DEFAULT_SEQUENCE_CLEANER_INTERVAL = 5; // in minutes class GlobalSequencesViewer { private: diff --git a/src/grpcservermodule.cpp b/src/grpcservermodule.cpp index 032d8b2fe2..55c779cdb6 100644 --- a/src/grpcservermodule.cpp +++ b/src/grpcservermodule.cpp @@ -45,11 +45,9 @@ using grpc::ServerBuilder; namespace ovms { -} // namespace ovms -using namespace ovms; static const int GIGABYTE = 1024 * 1024 * 1024; -bool isPortAvailable(uint64_t port) { +static bool isPortAvailable(uint64_t port) { struct sockaddr_in addr; int s = socket(AF_INET, SOCK_STREAM, 0); if (s == -1) { @@ -75,7 +73,7 @@ struct GrpcChannelArgument { // Parses a comma separated list of gRPC channel arguments into list of // ChannelArgument. -Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std::vector& result) { +static Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std::vector& result) { const std::vector channel_arguments = tokenize(channel_arguments_str, ','); for (const std::string& channel_argument : channel_arguments) { @@ -91,7 +89,7 @@ Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std::vecto return StatusCode::OK; } -uint getGRPCServersCount(const ovms::Config& config) { +static uint getGRPCServersCount(const ovms::Config& config) { const char* environmentVariableBuffer = std::getenv("GRPC_SERVERS"); if (environmentVariableBuffer) { auto result = stou32(environmentVariableBuffer); @@ -185,3 +183,4 @@ const GetModelMetadataImpl& GRPCServerModule::getTFSModelMetadataImpl() const { KFSInferenceServiceImpl& GRPCServerModule::getKFSGrpcImpl() const { return this->kfsGrpcInferenceService; } +} // namespace ovms diff --git a/src/grpcservermodule.hpp b/src/grpcservermodule.hpp index ab640a8106..cc825b7759 100644 --- a/src/grpcservermodule.hpp +++ b/src/grpcservermodule.hpp @@ -20,13 +20,14 @@ #include +#include "kfs_grpc_inference_service.hpp" #include "model_service.hpp" +#include "module.hpp" #include "prediction_service.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" namespace ovms { class Config; +class Server; class GRPCServerModule : public Module { Server& server; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 22664357f7..e873a75138 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include "config.hpp" @@ -35,12 +37,18 @@ #include "metric_registry.hpp" #include "model_metric_reporter.hpp" #include "model_service.hpp" +#include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "pipeline.hpp" #include "pipelinedefinition.hpp" +#include "pipelinedefinitionunloadguard.hpp" #include "prediction_service_utils.hpp" #include "rest_parser.hpp" #include "rest_utils.hpp" +#include "servablemanagermodule.hpp" #include "server.hpp" +#include "status.hpp" #include "stringutils.hpp" #include "timer.hpp" @@ -231,7 +239,7 @@ void HttpRestApiHandler::parseParams(Value& scope, Document& doc) { } std::string HttpRestApiHandler::preprocessInferRequest(std::string request_body) { - static std::unordered_map types = { + static const std::unordered_map types = { {"BOOL", "bool_contents"}, {"INT8", "int_contents"}, {"INT16", "int_contents"}, @@ -251,7 +259,11 @@ std::string HttpRestApiHandler::preprocessInferRequest(std::string request_body) for (SizeType i = 0; i < inputs.Size(); i++) { Value data = inputs[i].GetObject()["data"].GetArray(); Value contents(rapidjson::kObjectType); - Value datatype(types[inputs[i].GetObject()["datatype"].GetString()].c_str(), doc.GetAllocator()); + auto it = types.find(inputs[i].GetObject()["datatype"].GetString()); + if (it == types.end()) + return ""; + // TODO confirm its not an issue + Value datatype(it->second.c_str(), doc.GetAllocator()); contents.AddMember(datatype, data, doc.GetAllocator()); inputs[i].AddMember("contents", contents, doc.GetAllocator()); parseParams(inputs[i], doc); @@ -268,7 +280,7 @@ std::string HttpRestApiHandler::preprocessInferRequest(std::string request_body) return buffer.GetString(); } -Status convertStringToVectorOfSizes(const std::string& comma_separated_numbers, std::vector& sizes) { +static Status convertStringToVectorOfSizes(const std::string& comma_separated_numbers, std::vector& sizes) { std::stringstream streamData(comma_separated_numbers); std::vector sizes_; @@ -286,7 +298,7 @@ Status convertStringToVectorOfSizes(const std::string& comma_separated_numbers, return StatusCode::OK; } -Status parseBinaryInput(::inference::ModelInferRequest_InferInputTensor* input, size_t binary_input_size, const char* buffer) { +static Status parseBinaryInput(::inference::ModelInferRequest_InferInputTensor* input, size_t binary_input_size, const char* buffer) { if (input->datatype() == "FP32") { for (size_t i = 0; i < binary_input_size; i += sizeof(float)) { auto value = input->mutable_contents()->mutable_fp32_contents()->Add(); @@ -348,7 +360,7 @@ Status parseBinaryInput(::inference::ModelInferRequest_InferInputTensor* input, #define CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE " contents is not empty. Content field should be empty when using binary inputs extension." -Status validateContentFieldsEmptiness(::inference::ModelInferRequest_InferInputTensor* input) { +static Status validateContentFieldsEmptiness(::inference::ModelInferRequest_InferInputTensor* input) { if (input->datatype() == "FP32") { if (input->contents().fp32_contents_size() > 0) { SPDLOG_DEBUG("FP32" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); @@ -411,7 +423,7 @@ Status validateContentFieldsEmptiness(::inference::ModelInferRequest_InferInputT return StatusCode::OK; } -Status handleBinaryInputs(::inference::ModelInferRequest& grpc_request, const std::string& request_body, size_t endOfJson) { +static Status handleBinaryInputs(::inference::ModelInferRequest& grpc_request, const std::string& request_body, size_t endOfJson) { const char* binary_inputs = &(request_body[endOfJson]); size_t binary_inputs_size = request_body.length() - endOfJson; @@ -626,7 +638,7 @@ Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestCompo return StatusCode::OK; } -Status parseInferenceHeaderContentLength(HttpRequestComponents& requestComponents, +static Status parseInferenceHeaderContentLength(HttpRequestComponents& requestComponents, const std::vector>& headers) { for (auto header : headers) { if (header.first == "Inference-Header-Content-Length") { @@ -1003,7 +1015,7 @@ Status HttpRestApiHandler::processModelStatusRequest( return StatusCode::OK; } -std::string createErrorJsonWithMessage(std::string message) { +inline static std::string createErrorJsonWithMessage(std::string message) { return "{\n\t\"error\": \"" + message + "\"\n}"; } diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index a5125055a3..379d978f14 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -27,16 +27,16 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "modelmanager.hpp" #include "rest_parser.hpp" -#include "status.hpp" namespace ovms { class ServableMetricReporter; -class ModelMetricReporter; class KFSInferenceServiceImpl; class GetModelMetadataImpl; class Server; +class ModelManager; +class Status; + enum RequestType { Predict, GetModelStatus, GetModelMetadata, diff --git a/src/httpservermodule.hpp b/src/httpservermodule.hpp index a4222ec211..ed3814f965 100644 --- a/src/httpservermodule.hpp +++ b/src/httpservermodule.hpp @@ -19,10 +19,11 @@ #include #include "http_server.hpp" -#include "server.hpp" +#include "module.hpp" namespace ovms { class Config; +class Server; class HTTPServerModule : public Module { std::unique_ptr server; Server& ovmsServer; diff --git a/src/kfs_grpc_inference_service.cpp b/src/kfs_grpc_inference_service.cpp index 576910c24d..0b2261d579 100644 --- a/src/kfs_grpc_inference_service.cpp +++ b/src/kfs_grpc_inference_service.cpp @@ -20,15 +20,22 @@ #include #include "deserialization.hpp" +#include "execution_context.hpp" #include "metric.hpp" #include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" #include "ovinferrequestsqueue.hpp" +#include "pipeline.hpp" #include "pipelinedefinition.hpp" +#include "pipelinedefinitionstatus.hpp" +#include "pipelinedefinitionunloadguard.hpp" #include "prediction_service_utils.hpp" #include "serialization.hpp" #include "servablemanagermodule.hpp" #include "server.hpp" +#include "status.hpp" +#include "stringutils.hpp" #include "tensorinfo.hpp" #include "timer.hpp" #include "version.hpp" @@ -278,7 +285,7 @@ Status KFSInferenceServiceImpl::buildResponse( return StatusCode::OK; } -void addReadyVersions(Model& model, +static void addReadyVersions(Model& model, ::inference::ModelMetadataResponse* response) { auto modelVersions = model.getModelVersionsMapCopy(); for (auto& [modelVersion, modelInstance] : modelVersions) { diff --git a/src/kfs_grpc_inference_service.hpp b/src/kfs_grpc_inference_service.hpp index 04a4009f03..1e87553099 100644 --- a/src/kfs_grpc_inference_service.hpp +++ b/src/kfs_grpc_inference_service.hpp @@ -21,14 +21,14 @@ #include -#include "execution_context.hpp" #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" #include "src/kfserving_api/grpc_predict_v2.pb.h" -#include "status.hpp" namespace ovms { +class Status; using inference::GRPCInferenceService; +class ExecutionContext; class Model; class ModelManager; class ServableMetricReporter; diff --git a/src/layout.cpp b/src/layout.cpp index 92b7ea2903..6a19de4ebe 100644 --- a/src/layout.cpp +++ b/src/layout.cpp @@ -20,11 +20,18 @@ #include #include "logging.hpp" +#include "status.hpp" const char* DEFAULT_LAYOUT = "N..."; const char* UNSPECIFIED_LAYOUT = "..."; namespace ovms { +const std::string Layout::ALLOWED_DIMENSION_LETTERS = "NCHWD"; +const char Layout::ETC_CHAR = '.'; +const char Layout::UNDEFINED_DIMENSION_CHAR = '?'; +const std::string Layout::ALLOWED_DIMENSION_LETTERS_AND_CHARS = ALLOWED_DIMENSION_LETTERS + ETC_CHAR + UNDEFINED_DIMENSION_CHAR; +const std::string Layout::ETC_LAYOUT_DELIMETER = "..."; +const std::string Layout::BATCH_DIMENSION_LETTER = "N"; const Layout& Layout::getDefaultLayout() { static const Layout defaultLayout{DEFAULT_LAYOUT}; return defaultLayout; diff --git a/src/layout.hpp b/src/layout.hpp index 42fb215ba3..f5f633315a 100644 --- a/src/layout.hpp +++ b/src/layout.hpp @@ -22,16 +22,10 @@ #include #include "shape.hpp" -#include "status.hpp" namespace ovms { -static const std::string ALLOWED_DIMENSION_LETTERS = "NCHWD"; -static const char ETC_CHAR = '.'; -static const char UNDEFINED_DIMENSION_CHAR = '?'; -static const std::string ALLOWED_DIMENSION_LETTERS_AND_CHARS = ALLOWED_DIMENSION_LETTERS + ETC_CHAR + UNDEFINED_DIMENSION_CHAR; -static const std::string ETC_LAYOUT_DELIMETER = "..."; -static const std::string BATCH_DIMENSION_LETTER = "N"; +class Status; class Layout : public std::string { std::optional batchIndex = std::nullopt; @@ -42,6 +36,13 @@ class Layout : public std::string { std::string::size_type getNumberOfKnownDimensions() const; public: + static const std::string ALLOWED_DIMENSION_LETTERS; + static const char ETC_CHAR; + static const char UNDEFINED_DIMENSION_CHAR; + static const std::string ALLOWED_DIMENSION_LETTERS_AND_CHARS; + static const std::string ETC_LAYOUT_DELIMETER; + static const std::string BATCH_DIMENSION_LETTER; + Layout() = default; Layout(const std::string& str); diff --git a/src/layout_configuration.cpp b/src/layout_configuration.cpp index 7462e3f76d..6a1a6085d7 100644 --- a/src/layout_configuration.cpp +++ b/src/layout_configuration.cpp @@ -19,6 +19,7 @@ #include #include +#include "status.hpp" #include "stringutils.hpp" namespace ovms { @@ -46,7 +47,7 @@ Status LayoutConfiguration::fromString(const std::string& configurationStr, Layo std::transform(configurationCopy.begin(), configurationCopy.end(), configurationCopy.begin(), ::toupper); - if (configurationCopy.find_first_not_of(ALLOWED_DIMENSION_LETTERS_AND_CHARS + LAYOUT_CONFIGURATION_DELIMETER) != std::string::npos) + if (configurationCopy.find_first_not_of(Layout::ALLOWED_DIMENSION_LETTERS_AND_CHARS + LAYOUT_CONFIGURATION_DELIMETER) != std::string::npos) return StatusCode::LAYOUT_WRONG_FORMAT; size_t delimCount = std::count(configurationCopy.begin(), configurationCopy.end(), LAYOUT_CONFIGURATION_DELIMETER); diff --git a/src/layout_configuration.hpp b/src/layout_configuration.hpp index 11d56cffba..3444bc6c9c 100644 --- a/src/layout_configuration.hpp +++ b/src/layout_configuration.hpp @@ -19,13 +19,13 @@ #include #include "layout.hpp" -#include "status.hpp" namespace ovms { -static const char LAYOUT_CONFIGURATION_DELIMETER = ':'; +class Status; class LayoutConfiguration { + static const char LAYOUT_CONFIGURATION_DELIMETER = ':'; Layout tensor; Layout model; diff --git a/src/logging.cpp b/src/logging.cpp index 49a3d8c763..4d1613ec40 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -28,7 +28,7 @@ std::shared_ptr sequence_manager_logger = std::make_shared logger) { +static void set_log_level(const std::string log_level, std::shared_ptr logger) { logger->set_level(spdlog::level::info); if (!log_level.empty()) { if (log_level == "DEBUG") { @@ -47,7 +47,7 @@ void set_log_level(const std::string log_level, std::shared_ptr } } -void register_loggers(const std::string log_level, std::vector sinks) { +static void register_loggers(const std::string log_level, std::vector sinks) { auto serving_logger = std::make_shared("serving", begin(sinks), end(sinks)); serving_logger->set_pattern(default_pattern); gcs_logger->set_pattern(default_pattern); diff --git a/src/metric_config.cpp b/src/metric_config.cpp index 76dbc20042..841e689202 100644 --- a/src/metric_config.cpp +++ b/src/metric_config.cpp @@ -26,6 +26,7 @@ #include "logging.hpp" #include "rapidjson/document.h" #include "schema.hpp" +#include "status.hpp" #include "stringutils.hpp" namespace ovms { diff --git a/src/metric_config.hpp b/src/metric_config.hpp index 39e87c768b..45a7fa436d 100644 --- a/src/metric_config.hpp +++ b/src/metric_config.hpp @@ -21,8 +21,6 @@ #include -#include "status.hpp" - namespace ovms { extern const std::string METRIC_NAME_REQUESTS_SUCCESS; @@ -38,6 +36,7 @@ extern const std::string METRIC_NAME_CURRENT_REQUESTS; extern const std::string METRIC_NAME_REQUEST_TIME; extern const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME; +class Status; /** * @brief This class represents metrics configuration */ diff --git a/src/metric_module.cpp b/src/metric_module.cpp index 91b50c6605..77584057d1 100644 --- a/src/metric_module.cpp +++ b/src/metric_module.cpp @@ -15,11 +15,12 @@ //***************************************************************************** #include "metric_module.hpp" -#include "config.hpp" #include "metric_registry.hpp" namespace ovms { +MetricModule::~MetricModule() = default; + MetricModule::MetricModule() : registry(std::make_unique()) {} diff --git a/src/metric_module.hpp b/src/metric_module.hpp index 22376fb2e9..8ae41999d4 100644 --- a/src/metric_module.hpp +++ b/src/metric_module.hpp @@ -16,12 +16,12 @@ #pragma once #include -#include "metric_registry.hpp" -#include "server.hpp" +#include "module.hpp" namespace ovms { class Config; +class MetricRegistry; class MetricModule : public Module { protected: @@ -29,6 +29,7 @@ class MetricModule : public Module { public: MetricModule(); + ~MetricModule(); int start(const Config& config) override; void shutdown() override; diff --git a/src/model.cpp b/src/model.cpp index 8730778682..c7a6993eaf 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -21,14 +21,18 @@ #include #include +#include "customloaderinterface.hpp" #include "customloaders.hpp" +#include "filesystem.hpp" #include "localfilesystem.hpp" #include "logging.hpp" +#include "modelinstance.hpp" #include "pipelinedefinition.hpp" +#include "statefulmodelinstance.hpp" namespace ovms { -StatusCode downloadModels(std::shared_ptr& fs, ModelConfig& config, std::shared_ptr versions) { +static StatusCode downloadModels(std::shared_ptr& fs, ModelConfig& config, std::shared_ptr versions) { if (versions->size() == 0) { return StatusCode::OK; } diff --git a/src/model.hpp b/src/model.hpp index c7b3e8cce0..1cbe26597a 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -23,14 +23,23 @@ #include #include -#include "filesystem.hpp" +#include "logging.hpp" #include "modelchangesubscription.hpp" -#include "modelinstance.hpp" -#include "statefulmodelinstance.hpp" +#include "modelversion.hpp" + +namespace ov { +class Core; +} namespace ovms { +class FileSystem; +class GlobalSequencesViewer; +class ModelConfig; +class ModelInstance; class PipelineDefinition; +class MetricConfig; class MetricRegistry; +class Status; /* * @brief This class represent inference models */ class Model { diff --git a/src/model_service.cpp b/src/model_service.cpp index aed8ebd4ef..1aa60b9be2 100644 --- a/src/model_service.cpp +++ b/src/model_service.cpp @@ -30,9 +30,12 @@ #include "tensorflow_serving/apis/model_service.pb.h" #pragma GCC diagnostic pop +#include "execution_context.hpp" +#include "modelinstance.hpp" #include "modelmanager.hpp" #include "pipelinedefinition.hpp" #include "servablemanagermodule.hpp" +#include "server.hpp" #include "status.hpp" using google::protobuf::util::JsonPrintOptions; diff --git a/src/model_service.hpp b/src/model_service.hpp index 5f999ca14a..fafc52e8d7 100644 --- a/src/model_service.hpp +++ b/src/model_service.hpp @@ -28,12 +28,14 @@ #include "tensorflow_serving/apis/model_service.pb.h" #pragma GCC diagnostic pop -#include "execution_context.hpp" -#include "modelmanager.hpp" -#include "status.hpp" +#include "modelversion.hpp" namespace ovms { +class ExecutionContext; +class ModelManager; +class ModelVersionStatus; class Server; +class Status; void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status); diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index 8d70511c22..33eda09755 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -27,10 +27,43 @@ #include #include "logging.hpp" +#include "model_version_policy.hpp" #include "schema.hpp" #include "stringutils.hpp" namespace ovms { +ModelConfig::ModelConfig(const std::string& name, + const std::string& basePath, + const std::string& targetDevice, + const std::string& configBatchSize, + uint64_t nireq, + bool stateful, + bool idleSequenceCleanup, + bool lowLatencyTransformation, + uint32_t maxSequenceNumber, + const std::string& cacheDir, + model_version_t version, + const std::string& localPath) : + name(name), + basePath(basePath), + localPath(localPath), + targetDevice(targetDevice), + modelVersionPolicy(ModelVersionPolicy::getDefaultVersionPolicy()), + nireq(nireq), + stateful(stateful), + idleSequenceCleanup(idleSequenceCleanup), + lowLatencyTransformation(lowLatencyTransformation), + maxSequenceNumber(maxSequenceNumber), + cacheDir(cacheDir), + version(version), + pluginConfig({}), + layout(""), + shapes({}), + layouts({}), + mappingInputs({}), + mappingOutputs({}) { + setBatchingParams(configBatchSize); +} const std::string ANONYMOUS_INPUT_NAME = "ANONYMOUS_INPUT_NAME"; const std::string MAPPING_CONFIG_JSON = "mapping_config.json"; diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index addd1a2d49..d9e4ab5c4b 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -28,11 +28,12 @@ #include "layout_configuration.hpp" #include "metric_config.hpp" -#include "model_version_policy.hpp" +#include "modelversion.hpp" #include "shape.hpp" #include "status.hpp" namespace ovms { +class ModelVersionPolicy; using mapping_config_t = std::unordered_map; using plugin_config_t = std::map; @@ -213,27 +214,7 @@ class ModelConfig { uint32_t maxSequenceNumber = DEFAULT_MAX_SEQUENCE_NUMBER, const std::string& cacheDir = "", model_version_t version = 0, - const std::string& localPath = "") : - name(name), - basePath(basePath), - localPath(localPath), - targetDevice(targetDevice), - modelVersionPolicy(ModelVersionPolicy::getDefaultVersionPolicy()), - nireq(nireq), - stateful(stateful), - idleSequenceCleanup(idleSequenceCleanup), - lowLatencyTransformation(lowLatencyTransformation), - maxSequenceNumber(maxSequenceNumber), - cacheDir(cacheDir), - version(version), - pluginConfig({}), - layout(""), - shapes({}), - layouts({}), - mappingInputs({}), - mappingOutputs({}) { - setBatchingParams(configBatchSize); - } + const std::string& localPath = ""); /** * @brief Compares two ModelConfig instances and decides if models should be reloaded diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 11c2784706..42b6effc69 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -29,6 +29,7 @@ #include #include "config.hpp" +#include "customloaderinterface.hpp" #include "customloaders.hpp" #include "deserialization.hpp" #include "executingstreamidguard.hpp" @@ -37,12 +38,15 @@ #include "layout_configuration.hpp" #include "logging.hpp" #include "model_metric_reporter.hpp" +#include "modelconfig.hpp" +#include "modelinstanceunloadguard.hpp" #include "ov_utils.hpp" #include "predict_request_validation_utils.hpp" #include "prediction_service_utils.hpp" #include "profiler.hpp" #include "serialization.hpp" #include "shape.hpp" +#include "status.hpp" #include "stringutils.hpp" #include "tensorinfo.hpp" #include "timer.hpp" @@ -86,7 +90,7 @@ void ModelInstance::unsubscribe(PipelineDefinition& pd) { subscriptionManager.unsubscribe(pd); } -Status getRequestedShape(const ModelConfig& config, const DynamicModelParameter& parameter, const std::string& name, Shape& shapeOut) { +static Status getRequestedShape(const ModelConfig& config, const DynamicModelParameter& parameter, const std::string& name, Shape& shapeOut) { Shape shape; auto mappedName = config.getMappingInputByKey(name); if (config.getBatchSize().has_value() || parameter.isBatchSizeRequested()) { @@ -107,7 +111,7 @@ Status getRequestedShape(const ModelConfig& config, const DynamicModelParameter& return StatusCode::OK; } -bool hasInputWithName(std::shared_ptr& model, const std::string& name) { +static bool hasInputWithName(std::shared_ptr& model, const std::string& name) { try { model->input(name); return true; @@ -116,7 +120,7 @@ bool hasInputWithName(std::shared_ptr& model, const std::string& name } } -bool hasOutputWithName(std::shared_ptr& model, const std::string& name) { +static bool hasOutputWithName(std::shared_ptr& model, const std::string& name) { try { model->output(name); return true; @@ -125,7 +129,7 @@ bool hasOutputWithName(std::shared_ptr& model, const std::string& nam } } -Status validateConfigurationAgainstNetwork(const ModelConfig& config, std::shared_ptr& model) { +static Status validateConfigurationAgainstNetwork(const ModelConfig& config, std::shared_ptr& model) { if (config.isShapeAnonymousFixed() && model->inputs().size() > 1) { Status status = StatusCode::ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED; SPDLOG_LOGGER_WARN(modelmanager_logger, status.string()); @@ -197,7 +201,7 @@ const Layout ModelInstance::getReportedTensorLayout(const ModelConfig& config, c return layout; } -Status applyLayoutConfiguration(const ModelConfig& config, std::shared_ptr& model, const std::string& modelName, model_version_t modelVersion) { +static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_ptr& model, const std::string& modelName, model_version_t modelVersion) { ov::preprocess::PrePostProcessor preproc(model); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Applying layout configuration: {}", config.layoutConfigurationToString()); @@ -503,7 +507,7 @@ Status ModelInstance::loadOutputTensors(const ModelConfig& config) { } // Temporary methods. To be replaces with proper storage class. -bool dirExists(const std::string& path) { +static bool dirExists(const std::string& path) { if (FileSystem::isPathEscaped(path)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); return false; @@ -517,7 +521,7 @@ bool dirExists(const std::string& path) { return false; } -std::string findFilePathWithExtension(const std::string& path, const std::string& extension) { +static std::string findFilePathWithExtension(const std::string& path, const std::string& extension) { struct dirent* entry; if (FileSystem::isPathEscaped(path)) { SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index 13aecd2670..f6e3144e72 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -31,16 +31,12 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "customloaderconfig.hpp" -#include "customloaderinterface.hpp" #include "model_metric_reporter.hpp" #include "modelchangesubscription.hpp" #include "modelconfig.hpp" #include "modelinstanceunloadguard.hpp" #include "modelversionstatus.hpp" #include "ovinferrequestsqueue.hpp" -#include "sequence_processing_spec.hpp" -#include "status.hpp" #include "tensorinfo.hpp" namespace inference { @@ -49,6 +45,11 @@ class ModelInferResponse; } // namespace inference namespace ovms { +class MetricRegistry; +class ModelConfig; +class ModelInstanceUnloadGuard; +class PipelineDefinition; +class Status; class DynamicModelParameter { public: @@ -73,9 +74,6 @@ class DynamicModelParameter { std::map shapes; }; -class PipelineDefinition; -class MetricRegistry; - /** * @brief This class contains all the information about model */ diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index e3f93594f9..bb8c50a3d4 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -37,7 +37,10 @@ #include "azurefilesystem.hpp" #include "cleaner_utils.hpp" #include "config.hpp" +#include "custom_node_library_internal_manager_wrapper.hpp" #include "custom_node_library_manager.hpp" +#include "customloaderconfig.hpp" +#include "customloaderinterface.hpp" #include "customloaders.hpp" #include "entry_node.hpp" // need for ENTRY_NODE_NAME #include "exit_node.hpp" // need for EXIT_NODE_NAME @@ -47,6 +50,7 @@ #include "logging.hpp" #include "metric_config.hpp" #include "metric_registry.hpp" +#include "modelinstance.hpp" // for logging #include "node_library.hpp" #include "openssl/md5.h" #include "ov_utils.hpp" @@ -59,7 +63,7 @@ namespace ovms { -static uint16_t MAX_CONFIG_JSON_READ_RETRY_COUNT = 2; +static constexpr uint16_t MAX_CONFIG_JSON_READ_RETRY_COUNT = 2; const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; ModelManager::ModelManager(const std::string& modelCacheDirectory, MetricRegistry* registry) : @@ -278,7 +282,7 @@ Status ModelManager::startFromFile(const std::string& jsonFilename) { return StatusCode::OK; } -void processNodeInputs(const std::string nodeName, const rapidjson::Value::ConstMemberIterator& itro, pipeline_connections_t& connections) { +static void processNodeInputs(const std::string nodeName, const rapidjson::Value::ConstMemberIterator& itro, pipeline_connections_t& connections) { for (const auto& nodeInput : itro->value.GetArray()) { for (const auto& objectNameValue : nodeInput.GetObject()) { const std::string inputName = objectNameValue.name.GetString(); @@ -302,7 +306,7 @@ void processNodeInputs(const std::string nodeName, const rapidjson::Value::Const } } -void processPipelineInputs(const rapidjson::Value::ConstMemberIterator& pipelineInputsPtr, const std::string& nodeName, std::unordered_map& nodeOutputNameAlias, const std::string& pipelineName) { +static void processPipelineInputs(const rapidjson::Value::ConstMemberIterator& pipelineInputsPtr, const std::string& nodeName, std::unordered_map& nodeOutputNameAlias, const std::string& pipelineName) { for (const auto& pipelineInput : pipelineInputsPtr->value.GetArray()) { const std::string pipelineInputName = pipelineInput.GetString(); SPDLOG_DEBUG("Mapping node:{} output:{}, under alias:{}", @@ -314,7 +318,7 @@ void processPipelineInputs(const rapidjson::Value::ConstMemberIterator& pipeline } } -void processNodeOutputs(const rapidjson::Value::ConstMemberIterator& nodeOutputsItr, const std::string& nodeName, const std::string& modelName, std::unordered_map& nodeOutputNameAlias) { +static void processNodeOutputs(const rapidjson::Value::ConstMemberIterator& nodeOutputsItr, const std::string& nodeName, const std::string& modelName, std::unordered_map& nodeOutputNameAlias) { for (const auto& nodeOutput : nodeOutputsItr->value.GetArray()) { const std::string modelOutputName = nodeOutput.GetObject()["data_item"].GetString(); const std::string nodeOutputName = nodeOutput.GetObject()["alias"].GetString(); @@ -324,7 +328,7 @@ void processNodeOutputs(const rapidjson::Value::ConstMemberIterator& nodeOutputs } } -void processDLNodeConfig(const rapidjson::Value& nodeConfig, DLNodeInfo& info) { +static void processDLNodeConfig(const rapidjson::Value& nodeConfig, DLNodeInfo& info) { info.modelName = nodeConfig["model_name"].GetString(); if (nodeConfig.HasMember("version")) { info.modelVersion = nodeConfig["version"].GetUint64(); @@ -336,7 +340,7 @@ void processDLNodeConfig(const rapidjson::Value& nodeConfig, DLNodeInfo& info) { firstErrorStatus = status; \ } -Status processCustomNodeConfig(const rapidjson::Value& nodeConfig, CustomNodeInfo& info, const std::string& pipelineName, ModelManager& manager) { +static Status processCustomNodeConfig(const rapidjson::Value& nodeConfig, CustomNodeInfo& info, const std::string& pipelineName, ModelManager& manager) { std::string libraryName = nodeConfig["library_name"].GetString(); auto status = manager.getCustomNodeLibraryManager().getLibrary(libraryName, info.library); if (!status.ok()) { @@ -350,7 +354,7 @@ Status processCustomNodeConfig(const rapidjson::Value& nodeConfig, CustomNodeInf return StatusCode::OK; } -Status processPipelineConfig(rapidjson::Document& configJson, const rapidjson::Value& pipelineConfig, std::set& pipelinesInConfigFile, PipelineFactory& factory, ModelManager& manager) { +static Status processPipelineConfig(rapidjson::Document& configJson, const rapidjson::Value& pipelineConfig, std::set& pipelinesInConfigFile, PipelineFactory& factory, ModelManager& manager) { const std::string pipelineName = pipelineConfig["name"].GetString(); if (pipelinesInConfigFile.find(pipelineName) != pipelinesInConfigFile.end()) { SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated pipeline names: {} defined in config file. Only first definition will be loaded.", pipelineName); diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index 6cc7e35e16..75eaa82c83 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -31,13 +31,12 @@ #include #include -#include "custom_node_library_internal_manager_wrapper.hpp" -#include "customloaders.hpp" -#include "filesystem.hpp" #include "global_sequences_viewer.hpp" +#include "metric_config.hpp" #include "model.hpp" -#include "pipeline.hpp" +#include "modelconfig.hpp" #include "pipeline_factory.hpp" +#include "status.hpp" namespace ovms { @@ -45,9 +44,11 @@ const uint32_t DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS = 10000; extern const std::string DEFAULT_MODEL_CACHE_DIRECTORY; class Config; -class IVersionReader; +class CNLIMWrapper; +class CustomLoaderConfig; class CustomNodeLibraryManager; class MetricRegistry; +class FileSystem; struct FunctorSequenceCleaner; struct FunctorResourcesCleaner; /** @@ -164,6 +165,8 @@ class ModelManager { * */ std::unordered_map servedModelConfigs; + // TODO we should either dispose this member or at least keep it as a pointer + // so we do not disclose details about ModelConfig to all modelmanager users /** * @brief Retires models non existing in config file diff --git a/src/modelversionstatus.hpp b/src/modelversionstatus.hpp index 69456ef3b3..87045e14c2 100644 --- a/src/modelversionstatus.hpp +++ b/src/modelversionstatus.hpp @@ -21,7 +21,7 @@ #include -#include "modelconfig.hpp" +#include "modelversion.hpp" // note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. diff --git a/src/node.cpp b/src/node.cpp index 3f3c252974..4695b5a5ba 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -26,10 +26,12 @@ #include "profiler.hpp" #include "status.hpp" #include "tensorinfo.hpp" +#include "tensormap.hpp" const uint64_t DEMULTIPLY_LIMIT = 10'000; namespace ovms { +Node::~Node() = default; static std::string demultiplyCountSettingToString(std::optional demultiplyCount) { if (!demultiplyCount) { diff --git a/src/node.hpp b/src/node.hpp index 924679f60b..2058f4f9a0 100644 --- a/src/node.hpp +++ b/src/node.hpp @@ -25,12 +25,10 @@ #include #include "aliases.hpp" -#include "nodesession.hpp" #include "nodesessionresult.hpp" #include "pipelineeventqueue.hpp" #include "precision.hpp" #include "shape.hpp" -#include "status.hpp" #include "tensormap.hpp" namespace ovms { @@ -38,6 +36,10 @@ namespace ovms { using TensorNames = std::vector; using session_key_t = std::string; +class NodeSession; +class NodeSessionMetadata; +class Status; + class Node { protected: std::string nodeName; @@ -57,7 +59,7 @@ class Node { public: Node(const std::string& nodeName, std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}); - virtual ~Node() = default; + virtual ~Node(); const std::string& getName() const { return this->nodeName; } diff --git a/src/node_library_utils.cpp b/src/node_library_utils.cpp index bf4889191e..e631b563e4 100644 --- a/src/node_library_utils.cpp +++ b/src/node_library_utils.cpp @@ -18,6 +18,7 @@ #include #include "ov_utils.hpp" +#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { diff --git a/src/node_library_utils.hpp b/src/node_library_utils.hpp index 86c55a893b..0bb764bb99 100644 --- a/src/node_library_utils.hpp +++ b/src/node_library_utils.hpp @@ -24,11 +24,11 @@ #include "node_library.hpp" #include "precision.hpp" #include "shape.hpp" -#include "status.hpp" #include "tensormap.hpp" namespace ovms { +class Status; class TensorInfo; CustomNodeTensorPrecision toCustomNodeTensorPrecision(ov::element::Type_t precision); diff --git a/src/nodeinfo.hpp b/src/nodeinfo.hpp index 20a4c2b3fe..8c4e6c368c 100644 --- a/src/nodeinfo.hpp +++ b/src/nodeinfo.hpp @@ -26,13 +26,11 @@ #include "aliases.hpp" #include "modelversion.hpp" #include "node_library.hpp" -#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { -class ModelManager; -class Pipeline; +class Status; using pipeline_connections_t = std::unordered_map>; using parameters_t = std::unordered_map; diff --git a/src/nodeinputhandler.cpp b/src/nodeinputhandler.cpp index 4ee038e6ac..33cdb53853 100644 --- a/src/nodeinputhandler.cpp +++ b/src/nodeinputhandler.cpp @@ -16,6 +16,8 @@ #include "nodeinputhandler.hpp" #include "logging.hpp" +#include "status.hpp" +#include "tensor_utils.hpp" namespace ovms { diff --git a/src/nodeinputhandler.hpp b/src/nodeinputhandler.hpp index 12665794dd..2b4b192afd 100644 --- a/src/nodeinputhandler.hpp +++ b/src/nodeinputhandler.hpp @@ -23,11 +23,11 @@ #include #include "session_id.hpp" -#include "status.hpp" -#include "tensor_utils.hpp" #include "tensormap.hpp" namespace ovms { +class Status; +class TensorWithSource; // This class encapsulates input tensor gathering and preprocessing before node execution. // It is resposible for gathering multiple tensors into one (in case of demultiplexers) diff --git a/src/nodesession.cpp b/src/nodesession.cpp index 0450536014..b22a4f0b6c 100644 --- a/src/nodesession.cpp +++ b/src/nodesession.cpp @@ -19,6 +19,7 @@ #include "logging.hpp" #include "nodeinputhandler.hpp" #include "nodeoutputhandler.hpp" +#include "status.hpp" #include "tensor_utils.hpp" #include "timer.hpp" @@ -33,7 +34,7 @@ Status NodeSession::setInput(const std::string& inputName, TensorWithSource& ten return inputHandler->setInput(inputName, tensor, shardId); } -std::unique_ptr createNodeInputHandler(uint32_t inputsCount, const CollapseDetails& collapsingDetails) { +static std::unique_ptr createNodeInputHandler(uint32_t inputsCount, const CollapseDetails& collapsingDetails) { if (collapsingDetails.collapsedSessionNames.size() == 0) { return std::make_unique(inputsCount); } else { diff --git a/src/nodesession.hpp b/src/nodesession.hpp index cb73f845ce..5f8bc1e72f 100644 --- a/src/nodesession.hpp +++ b/src/nodesession.hpp @@ -20,11 +20,11 @@ #include #include "nodesessionmetadata.hpp" -#include "status.hpp" namespace ovms { struct NodeInputHandler; struct NodeOutputHandler; +class Status; class TensorWithSource; template class Timer; diff --git a/src/nodestreamidguard.hpp b/src/nodestreamidguard.hpp index 310de11a9d..ced1709c01 100644 --- a/src/nodestreamidguard.hpp +++ b/src/nodestreamidguard.hpp @@ -19,6 +19,7 @@ #include namespace ovms { +class OVInferRequestsQueue; class ModelMetricReporter; class OVInferRequestsQueue; diff --git a/src/ov_utils.cpp b/src/ov_utils.cpp index cd443c0512..996487e563 100644 --- a/src/ov_utils.cpp +++ b/src/ov_utils.cpp @@ -26,6 +26,7 @@ #include "logging.hpp" #include "profiler.hpp" +#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { @@ -83,7 +84,7 @@ std::optional getLayoutFromRTMap(const ov::RTMap& rtMap) { return std::nullopt; } -void insertSupportedKeys(std::set& aggregatedPluginSupportedConfigKeys, const std::string& pluginName, const ov::Core& ieCore) { +static void insertSupportedKeys(std::set& aggregatedPluginSupportedConfigKeys, const std::string& pluginName, const ov::Core& ieCore) { const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); try { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validating plugin: {}; configuration", pluginName); diff --git a/src/ov_utils.hpp b/src/ov_utils.hpp index 2f52005066..29aeda3cb3 100644 --- a/src/ov_utils.hpp +++ b/src/ov_utils.hpp @@ -24,10 +24,9 @@ #include #include "modelconfig.hpp" -#include "status.hpp" namespace ovms { - +class Status; class TensorInfo; Status createSharedTensor(ov::Tensor& destinationTensor, ov::element::Type_t precision, const ov::Shape& shape); diff --git a/src/pipeline.cpp b/src/pipeline.cpp index f252d2d1d2..38b9599cdb 100644 --- a/src/pipeline.cpp +++ b/src/pipeline.cpp @@ -21,10 +21,13 @@ #include #include +#include "execution_context.hpp" #include "logging.hpp" #include "node.hpp" +#include "nodesession.hpp" #include "pipelineeventqueue.hpp" #include "profiler.hpp" +#include "status.hpp" namespace ovms { @@ -60,7 +63,7 @@ void printNodeConnections(const std::string& nodeName, const std::string& source SPDLOG_LOGGER_DEBUG(dag_executor_logger, ss.str()); } -void setFailIfNotFailEarlier(ovms::Status& earlierStatusCode, ovms::Status& newFailStatus) { +inline static void setFailIfNotFailEarlier(ovms::Status& earlierStatusCode, ovms::Status& newFailStatus) { if (earlierStatusCode.ok()) { earlierStatusCode = newFailStatus; } diff --git a/src/pipeline.hpp b/src/pipeline.hpp index 27d0792e40..99c10dfe0c 100644 --- a/src/pipeline.hpp +++ b/src/pipeline.hpp @@ -22,18 +22,15 @@ #include #include "aliases.hpp" -#include "execution_context.hpp" -#include "status.hpp" namespace ovms { +class ExecutionContext; class ServableMetricReporter; +class Node; class Node; -template -class EntryNode; -template -class ExitNode; +class Status; void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs); diff --git a/src/pipeline_factory.cpp b/src/pipeline_factory.cpp index 824f4dfabe..3ae6d80a9d 100644 --- a/src/pipeline_factory.cpp +++ b/src/pipeline_factory.cpp @@ -16,10 +16,13 @@ #include "pipeline_factory.hpp" #include "logging.hpp" +#include "model_metric_reporter.hpp" #include "modelmanager.hpp" +#include "nodeinfo.hpp" #include "pipeline.hpp" #include "pipelinedefinition.hpp" #include "prediction_service_utils.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/pipeline_factory.hpp b/src/pipeline_factory.hpp index cba5cd5ee6..a4cf11c695 100644 --- a/src/pipeline_factory.hpp +++ b/src/pipeline_factory.hpp @@ -30,14 +30,14 @@ #pragma GCC diagnostic pop #include "kfs_grpc_inference_service.hpp" #include "nodeinfo.hpp" -#include "status.hpp" namespace ovms { class ModelManager; +class NodeInfo; class Pipeline; class PipelineDefinition; -class MetricRegistry; +class Status; class PipelineFactory { std::map> definitions; diff --git a/src/pipelinedefinition.cpp b/src/pipelinedefinition.cpp index 47ae0aafcf..2a4a8eadac 100644 --- a/src/pipelinedefinition.cpp +++ b/src/pipelinedefinition.cpp @@ -20,15 +20,22 @@ #include #include "custom_node.hpp" +#include "custom_node_library_internal_manager_wrapper.hpp" #include "dl_node.hpp" #include "entry_node.hpp" #include "exit_node.hpp" #include "logging.hpp" +#include "model_metric_reporter.hpp" +#include "modelinstance.hpp" #include "modelmanager.hpp" #include "node_library_utils.hpp" +#include "nodeinfo.hpp" +#include "nodestreamidguard.hpp" +#include "ov_utils.hpp" #include "pipeline.hpp" #include "pipelinedefinitionunloadguard.hpp" #include "prediction_service_utils.hpp" +#include "status.hpp" namespace ovms { @@ -45,6 +52,17 @@ Status toNodeKind(const std::string& str, NodeKind& nodeKind) { return StatusCode::PIPELINE_NODE_WRONG_KIND_CONFIGURATION; } +PipelineDefinition::PipelineDefinition(const std::string& pipelineName, + const std::vector& nodeInfos, + const pipeline_connections_t& connections, + MetricRegistry* registry, + const MetricConfig* metricConfig) : + pipelineName(pipelineName), + nodeInfos(nodeInfos), + connections(connections), + reporter(std::make_unique(metricConfig, registry, pipelineName, VERSION)), + status(this->pipelineName) {} + Status PipelineDefinition::validate(ModelManager& manager) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of pipeline: {}", getName()); ValidationResultNotifier notifier(status, loadedNotify); @@ -1044,11 +1062,11 @@ const tensor_map_t PipelineDefinition::getOutputsInfo() const { return copy; } -std::shared_ptr applyDemultiplexerShapeForTensor(const std::shared_ptr& tensorInfo, int32_t demultiplyCount) { +static std::shared_ptr applyDemultiplexerShapeForTensor(const std::shared_ptr& tensorInfo, int32_t demultiplyCount) { return tensorInfo->createCopyWithDemultiplexerDimensionPrefix(demultiplyCount ? Dimension(demultiplyCount) : Dimension::any()); } -std::shared_ptr createOutputTensorInfoForPipeline(const std::string& mappedName, const std::shared_ptr& tensorInfo, const Shape& gatherShape, bool isConnectionFromDemultiplexer) { +static std::shared_ptr createOutputTensorInfoForPipeline(const std::string& mappedName, const std::shared_ptr& tensorInfo, const Shape& gatherShape, bool isConnectionFromDemultiplexer) { std::shared_ptr newOwnedTensorInfo; if (gatherShape.size() == 0) { newOwnedTensorInfo = std::make_shared(*tensorInfo); @@ -1065,7 +1083,7 @@ std::shared_ptr createOutputTensorInfoForPipeline(const std::string& return newOwnedTensorInfo; } -Status updateInputsInfoWithNodeConnection(tensor_map_t& inputsInfo, const TensorInfo& tensorInfo, const std::string& alias) { +static Status updateInputsInfoWithNodeConnection(tensor_map_t& inputsInfo, const TensorInfo& tensorInfo, const std::string& alias) { auto newTensorInfo = std::make_shared(alias, tensorInfo.getPrecision(), tensorInfo.getShape(), tensorInfo.getLayout()); auto it = inputsInfo.find(alias); if (it != inputsInfo.end()) { @@ -1088,7 +1106,7 @@ Status updateInputsInfoWithNodeConnection(tensor_map_t& inputsInfo, const Tensor } template -Status updateInputsInfoWithNodeConnections(tensor_map_t& inputsInfo, const Aliases& specificDependencyMapping, Extractor extractor) { +static Status updateInputsInfoWithNodeConnections(tensor_map_t& inputsInfo, const Aliases& specificDependencyMapping, Extractor extractor) { for (const auto& [alias, realName] : specificDependencyMapping) { auto status = updateInputsInfoWithNodeConnection(inputsInfo, extractor(realName), alias); if (!status.ok()) { diff --git a/src/pipelinedefinition.hpp b/src/pipelinedefinition.hpp index 8bc5c57273..556d65950e 100644 --- a/src/pipelinedefinition.hpp +++ b/src/pipelinedefinition.hpp @@ -31,21 +31,22 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop #include "aliases.hpp" -#include "custom_node_library_internal_manager_wrapper.hpp" #include "kfs_grpc_inference_service.hpp" -#include "model_metric_reporter.hpp" #include "modelversion.hpp" #include "nodeinfo.hpp" #include "pipelinedefinitionstatus.hpp" -#include "pipelinedefinitionunloadguard.hpp" -#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { - +class CNLIMWrapper; +class MetricConfig; +class MetricRegistry; class ModelManager; -class Pipeline; +class ServableMetricReporter; class NodeValidator; +class Pipeline; +class PipelineDefinitionUnloadGuard; +class Status; class PipelineDefinition { friend NodeValidator; @@ -107,12 +108,7 @@ class PipelineDefinition { const std::vector& nodeInfos, const pipeline_connections_t& connections, MetricRegistry* registry = nullptr, - const MetricConfig* metricConfig = nullptr) : - pipelineName(pipelineName), - nodeInfos(nodeInfos), - connections(connections), - reporter(std::make_unique(metricConfig, registry, pipelineName, VERSION)), - status(this->pipelineName) {} + const MetricConfig* metricConfig = nullptr); template Status create(std::unique_ptr& pipeline, const RequestType* request, @@ -159,8 +155,8 @@ class PipelineDefinition { ServableMetricReporter& getMetricReporter() const { return *this->reporter; } protected: - virtual Status updateInputsInfo(const ModelManager& manager); - virtual Status updateOutputsInfo(const ModelManager& manager); + Status updateInputsInfo(const ModelManager& manager); + Status updateOutputsInfo(const ModelManager& manager); public: const tensor_map_t getInputsInfo() const; diff --git a/src/pipelinedefinitionstatus.hpp b/src/pipelinedefinitionstatus.hpp index 20421422d1..036e0581df 100644 --- a/src/pipelinedefinitionstatus.hpp +++ b/src/pipelinedefinitionstatus.hpp @@ -22,11 +22,8 @@ #include #include -#include - #include "logging.hpp" #include "modelversionstatus.hpp" -#include "status.hpp" namespace ovms { diff --git a/src/pipelineeventqueue.hpp b/src/pipelineeventqueue.hpp index 85ab741d65..c44d261ed7 100644 --- a/src/pipelineeventqueue.hpp +++ b/src/pipelineeventqueue.hpp @@ -17,6 +17,7 @@ #include +#include "session_id.hpp" #include "threadsafequeue.hpp" namespace ovms { diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index 43850a7e0b..ba897107df 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -26,6 +26,7 @@ #include "kfs_grpc_inference_service.hpp" #include "modelconfig.hpp" #include "profiler.hpp" +#include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" namespace ovms { @@ -445,7 +446,7 @@ Status RequestValidator diff --git a/src/prediction_service.cpp b/src/prediction_service.cpp index b64c011857..c4cd618d4b 100644 --- a/src/prediction_service.cpp +++ b/src/prediction_service.cpp @@ -29,9 +29,11 @@ #include "execution_context.hpp" #include "get_model_metadata_impl.hpp" +#include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" #include "ovinferrequestsqueue.hpp" +#include "pipeline.hpp" #include "prediction_service_utils.hpp" #include "profiler.hpp" #include "servablemanagermodule.hpp" diff --git a/src/profilermodule.hpp b/src/profilermodule.hpp index 726a30f1b4..d71a5a220c 100644 --- a/src/profilermodule.hpp +++ b/src/profilermodule.hpp @@ -17,11 +17,11 @@ #include #include -#include "config.hpp" #include "logging.hpp" #include "module.hpp" namespace ovms { +class Config; class Profiler; #ifdef MTR_ENABLED class ProfilerModule : public Module { diff --git a/src/rest_parser.cpp b/src/rest_parser.cpp index d55717b8ab..f3d97bccd8 100644 --- a/src/rest_parser.cpp +++ b/src/rest_parser.cpp @@ -19,6 +19,7 @@ #include #include "rest_utils.hpp" +#include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" namespace ovms { @@ -311,7 +312,7 @@ bool TFSRestParser::setDimOrValidate(tensorflow::TensorProto& proto, int dim, in } } -bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { +static bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { if (!isBinary(value)) { return false; } @@ -321,7 +322,7 @@ bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { } template -bool addToTensorContent(tensorflow::TensorProto& proto, T value) { +static bool addToTensorContent(tensorflow::TensorProto& proto, T value) { if (sizeof(T) != DataTypeSize(proto.dtype())) { return false; } @@ -330,7 +331,7 @@ bool addToTensorContent(tensorflow::TensorProto& proto, T value) { } template -bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& value) { +static bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& value) { if (value.IsDouble()) { return addToTensorContent(proto, static_cast(value.GetDouble())); } @@ -350,7 +351,7 @@ bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& return false; } -bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { +static bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { if (value.IsDouble()) { proto.add_half_val(value.GetDouble()); return true; @@ -375,7 +376,7 @@ bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) return false; } -bool addToIntVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { +static bool addToIntVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { if (value.IsDouble()) { proto.add_int_val(value.GetDouble()); return true; diff --git a/src/rest_parser.hpp b/src/rest_parser.hpp index 4a955e9ca8..1ef421b2ca 100644 --- a/src/rest_parser.hpp +++ b/src/rest_parser.hpp @@ -28,10 +28,10 @@ #include "kfs_grpc_inference_service.hpp" #pragma GCC diagnostic pop -#include "status.hpp" #include "tensorinfo.hpp" namespace ovms { +class Status; /** * @brief Request order types diff --git a/src/rest_utils.cpp b/src/rest_utils.cpp index 2ab7eb1e42..14c726f959 100644 --- a/src/rest_utils.cpp +++ b/src/rest_utils.cpp @@ -29,6 +29,7 @@ #pragma GCC diagnostic pop #include "precision.hpp" #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" +#include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" #include "timer.hpp" @@ -48,7 +49,7 @@ enum : unsigned int { namespace ovms { -Status checkValField(const size_t& fieldSize, const size_t& expectedElementsNumber) { +static Status checkValField(const size_t& fieldSize, const size_t& expectedElementsNumber) { if (fieldSize == 0) return StatusCode::REST_SERIALIZE_NO_DATA; if (fieldSize != expectedElementsNumber) @@ -201,7 +202,7 @@ Status makeJsonFromPredictResponse( return StatusCode::OK; } -Status parseResponseParameters(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { +static Status parseResponseParameters(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { if (response_proto.parameters_size() > 0) { writer.Key("parameters"); writer.StartObject(); @@ -228,7 +229,7 @@ Status parseResponseParameters(const ::inference::ModelInferResponse& response_p return StatusCode::OK; } -Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer) { +static Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer) { if (output.parameters_size() > 0) { writer.Key("parameters"); writer.StartObject(); @@ -256,24 +257,24 @@ Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTens } template -void fillTensorDataWithIntValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithIntValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } template -void fillTensorDataWithUintValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithUintValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } template -void fillTensorDataWithFloatValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithFloatValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Double(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } -Status parseOutputs(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { +static Status parseOutputs(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { writer.Key("outputs"); writer.StartArray(); diff --git a/src/rest_utils.hpp b/src/rest_utils.hpp index 9b5880085d..c4cedd3a19 100644 --- a/src/rest_utils.hpp +++ b/src/rest_utils.hpp @@ -23,13 +23,13 @@ #pragma GCC diagnostic pop #include "rest_parser.hpp" -#include "status.hpp" namespace inference { class ModelInferResponse; } // namespace inference namespace ovms { +class Status; Status makeJsonFromPredictResponse( tensorflow::serving::PredictResponse& response_proto, std::string* response_json, diff --git a/src/sequence.cpp b/src/sequence.cpp index 3b3cabaf5c..02cb750c25 100644 --- a/src/sequence.cpp +++ b/src/sequence.cpp @@ -17,6 +17,9 @@ #include +#include "ov_utils.hpp" +#include "status.hpp" + using namespace InferenceEngine; namespace ovms { diff --git a/src/sequence.hpp b/src/sequence.hpp index 2c6157e5ed..2a0616ff06 100644 --- a/src/sequence.hpp +++ b/src/sequence.hpp @@ -25,11 +25,10 @@ #include #include -#include "ov_utils.hpp" -#include "status.hpp" - namespace ovms { +class Status; + using sequence_memory_state_t = std::unordered_map; using model_memory_state_t = std::vector; diff --git a/src/sequence_manager.cpp b/src/sequence_manager.cpp index bf61396fb5..1625c7b166 100644 --- a/src/sequence_manager.cpp +++ b/src/sequence_manager.cpp @@ -19,6 +19,8 @@ #include #include "logging.hpp" +#include "sequence_processing_spec.hpp" +#include "status.hpp" namespace ovms { diff --git a/src/sequence_manager.hpp b/src/sequence_manager.hpp index a72460c5b3..4bcdb8f9a9 100644 --- a/src/sequence_manager.hpp +++ b/src/sequence_manager.hpp @@ -23,8 +23,6 @@ #include "modelversion.hpp" #include "sequence.hpp" -#include "sequence_processing_spec.hpp" -#include "status.hpp" namespace ovms { @@ -32,6 +30,9 @@ const uint32_t NO_CONTROL_INPUT = 0; const uint32_t SEQUENCE_START = 1; const uint32_t SEQUENCE_END = 2; +class SequenceProcessingSpec; +class Status; + class SequenceManager { private: uint32_t maxSequenceNumber; diff --git a/src/serialization.cpp b/src/serialization.cpp index 0a2edaf3db..f0a3e6636b 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -20,7 +20,7 @@ namespace ovms { -Status serializePrecision( +static Status serializePrecision( tensorflow::TensorProto& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -59,7 +59,7 @@ Status serializePrecision( return StatusCode::OK; } -Status serializePrecision( +static Status serializePrecision( ::inference::ModelInferResponse::InferOutputTensor& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -99,7 +99,7 @@ Status serializePrecision( return StatusCode::OK; } -Status serializeShape( +static Status serializeShape( tensorflow::TensorProto& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -124,7 +124,7 @@ Status serializeShape( return StatusCode::OK; } -Status serializeShape( +static Status serializeShape( ::inference::ModelInferResponse::InferOutputTensor& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -149,7 +149,7 @@ Status serializeShape( return StatusCode::OK; } -void serializeContent(std::string* content, ov::Tensor& tensor) { +static void serializeContent(std::string* content, ov::Tensor& tensor) { OVMS_PROFILE_FUNCTION(); // We only fill if the content is not already filled. // It can be filled in gather exit node handler. diff --git a/src/servablemanagermodule.cpp b/src/servablemanagermodule.cpp index eb4403a5d4..32787061b5 100644 --- a/src/servablemanagermodule.cpp +++ b/src/servablemanagermodule.cpp @@ -22,6 +22,7 @@ #include "logging.hpp" #include "metric_module.hpp" #include "modelmanager.hpp" +#include "server.hpp" namespace ovms { diff --git a/src/servablemanagermodule.hpp b/src/servablemanagermodule.hpp index 0372dea251..8f78414129 100644 --- a/src/servablemanagermodule.hpp +++ b/src/servablemanagermodule.hpp @@ -16,11 +16,12 @@ #pragma once #include -#include "server.hpp" +#include "module.hpp" namespace ovms { class Config; class ModelManager; +class Server; class ServableManagerModule : public Module { protected: diff --git a/src/server.cpp b/src/server.cpp index 06f40887ff..6b85862ddc 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -57,14 +57,12 @@ const std::string GRPC_SERVER_MODULE_NAME = "GRPCServerModule"; const std::string HTTP_SERVER_MODULE_NAME = "HTTPServerModule"; const std::string SERVABLE_MANAGER_MODULE_NAME = "ServableManagerModule"; const std::string METRICS_MODULE_NAME = "MetricsModule"; -} // namespace ovms -using namespace ovms; namespace { volatile sig_atomic_t shutdown_request = 0; } -void logConfig(const Config& config) { +static void logConfig(const Config& config) { std::string project_name(PROJECT_NAME); std::string project_version(PROJECT_VERSION); SPDLOG_INFO(project_name + " " + project_version); @@ -101,19 +99,19 @@ void logConfig(const Config& config) { SPDLOG_DEBUG("sequence cleaner poll wait minutes: {}", config.sequenceCleanerPollWaitMinutes()); } -void onInterrupt(int status) { +static void onInterrupt(int status) { shutdown_request = 1; } -void onTerminate(int status) { +static void onTerminate(int status) { shutdown_request = 1; } -void onIllegal(int status) { +static void onIllegal(int status) { shutdown_request = 2; } -void installSignalHandlers(ovms::Server& server) { +static void installSignalHandlers(ovms::Server& server) { static struct sigaction sigIntHandler; sigIntHandler.sa_handler = onInterrupt; sigemptyset(&sigIntHandler.sa_mask); @@ -133,8 +131,6 @@ void installSignalHandlers(ovms::Server& server) { sigaction(SIGILL, &sigIllHandler, NULL); } -static const int GIGABYTE = 1024 * 1024 * 1024; - ModuleState Module::getState() const { return state; } @@ -347,3 +343,4 @@ int Server::start(int argc, char** argv) { } return EXIT_SUCCESS; } +} // namespace ovms diff --git a/src/server.hpp b/src/server.hpp index 1d0f11844a..87f2c5bec0 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -14,7 +14,6 @@ // limitations under the License. //***************************************************************************** #pragma once -#include #include #include #include diff --git a/src/shape.cpp b/src/shape.cpp index b4d557d141..a63d437e7e 100644 --- a/src/shape.cpp +++ b/src/shape.cpp @@ -23,6 +23,7 @@ #include #include "logging.hpp" +#include "status.hpp" #include "stringutils.hpp" namespace ovms { diff --git a/src/shape.hpp b/src/shape.hpp index e8b504815a..5f32bcd604 100644 --- a/src/shape.hpp +++ b/src/shape.hpp @@ -21,9 +21,8 @@ #include -#include "status.hpp" - namespace ovms { +class Status; using dimension_value_t = std::int64_t; diff --git a/src/statefulmodelinstance.cpp b/src/statefulmodelinstance.cpp index d2279afd77..8a237bff13 100644 --- a/src/statefulmodelinstance.cpp +++ b/src/statefulmodelinstance.cpp @@ -22,8 +22,10 @@ #include "executingstreamidguard.hpp" #include "logging.hpp" #include "model_metric_reporter.hpp" +#include "modelconfig.hpp" #include "predict_request_validation_utils.hpp" #include "profiler.hpp" +#include "sequence_processing_spec.hpp" #include "serialization.hpp" #include "timer.hpp" diff --git a/src/statefulmodelinstance.hpp b/src/statefulmodelinstance.hpp index 9a8b68c9ed..4f1e3ddbbb 100644 --- a/src/statefulmodelinstance.hpp +++ b/src/statefulmodelinstance.hpp @@ -20,15 +20,13 @@ #include #include "global_sequences_viewer.hpp" -#include "modelconfig.hpp" #include "modelinstance.hpp" #include "sequence_manager.hpp" namespace ovms { - class MetricRegistry; class MetricConfig; - +class ModelConfig; class StatefulModelInstance : public ModelInstance { static const std::set SPECIAL_INPUT_NAMES; diff --git a/src/tensorinfo.cpp b/src/tensorinfo.cpp index 06ddffd743..fbd569cf5c 100644 --- a/src/tensorinfo.cpp +++ b/src/tensorinfo.cpp @@ -177,11 +177,11 @@ std::shared_ptr TensorInfo::createCopyWithDemultiplexerDimensionPref copy->influencedByDemultiplexer = true; copy->shape.emplace(copy->shape.begin(), dim); copy->layout = this->getLayout(); - auto batchPosition = copy->layout.find(BATCH_DIMENSION_LETTER); + auto batchPosition = copy->layout.find(Layout::BATCH_DIMENSION_LETTER); if (batchPosition != std::string::npos) { - copy->layout.replace(batchPosition, 1, std::string(1, UNDEFINED_DIMENSION_CHAR)); + copy->layout.replace(batchPosition, 1, std::string(1, Layout::UNDEFINED_DIMENSION_CHAR)); } - copy->layout = std::string(1, BATCH_DIMENSION_LETTER[0]) + copy->layout; + copy->layout = std::string(1, Layout::BATCH_DIMENSION_LETTER[0]) + copy->layout; return copy; } diff --git a/src/test/custom_loader_test.cpp b/src/test/custom_loader_test.cpp index 792077b5b4..d1690145d1 100644 --- a/src/test/custom_loader_test.cpp +++ b/src/test/custom_loader_test.cpp @@ -32,6 +32,7 @@ #include "../model.hpp" #include "../model_service.hpp" #include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" #include "../modelversionstatus.hpp" #include "../prediction_service_utils.hpp" diff --git a/src/test/demultiplexer_node_test.cpp b/src/test/demultiplexer_node_test.cpp index 79ebcf5cd6..c3075b8d7c 100644 --- a/src/test/demultiplexer_node_test.cpp +++ b/src/test/demultiplexer_node_test.cpp @@ -20,8 +20,12 @@ #include "../dlnodesession.hpp" #include "../gathernodeinputhandler.hpp" #include "../logging.hpp" +#include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../node.hpp" #include "../nodeinputhandler.hpp" +#include "../nodestreamidguard.hpp" +#include "../ov_utils.hpp" #include "test_utils.hpp" using namespace ovms; diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index d3ca2559ac..ea76004b33 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -31,6 +31,7 @@ #include "../pipelinedefinition.hpp" #include "../prediction_service_utils.hpp" #include "../status.hpp" +#include "../stringutils.hpp" #include "test_utils.hpp" using namespace ovms; diff --git a/src/test/ensemble_flow_custom_node_tests.cpp b/src/test/ensemble_flow_custom_node_tests.cpp index e497e54532..525dcf2517 100644 --- a/src/test/ensemble_flow_custom_node_tests.cpp +++ b/src/test/ensemble_flow_custom_node_tests.cpp @@ -20,6 +20,12 @@ #include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + #include #include @@ -31,8 +37,12 @@ #include "../exit_node.hpp" #include "../metric_registry.hpp" #include "../model_metric_reporter.hpp" +#include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../node_library.hpp" #include "../node_library_utils.hpp" +#include "../nodestreamidguard.hpp" +#include "../pipeline.hpp" #include "../pipelinedefinition.hpp" #include "../precision.hpp" #include "../stringutils.hpp" diff --git a/src/test/ensemble_metadata_test.cpp b/src/test/ensemble_metadata_test.cpp index 9c5d1ca2e7..48a04c4165 100644 --- a/src/test/ensemble_metadata_test.cpp +++ b/src/test/ensemble_metadata_test.cpp @@ -19,6 +19,8 @@ #include "../custom_node_library_manager.hpp" #include "../entry_node.hpp" #include "../exit_node.hpp" +#include "../model_version_policy.hpp" +#include "../modelinstance.hpp" #include "../pipeline_factory.hpp" #include "../pipelinedefinition.hpp" #include "test_utils.hpp" diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp index 7ad062801a..ddf1788f46 100644 --- a/src/test/ensemble_tests.cpp +++ b/src/test/ensemble_tests.cpp @@ -31,6 +31,7 @@ #include "../model_metric_reporter.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" +#include "../nodestreamidguard.hpp" #include "../pipeline.hpp" #include "../pipeline_factory.hpp" #include "../pipelinedefinition.hpp" diff --git a/src/test/gather_node_test.cpp b/src/test/gather_node_test.cpp index ee6224d121..8c760a9de2 100644 --- a/src/test/gather_node_test.cpp +++ b/src/test/gather_node_test.cpp @@ -32,6 +32,8 @@ #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../nodeinputhandler.hpp" +#include "../nodestreamidguard.hpp" +#include "../ov_utils.hpp" #include "../pipeline.hpp" #include "../pipeline_factory.hpp" #include "../pipelinedefinition.hpp" diff --git a/src/test/get_model_metadata_validation_test.cpp b/src/test/get_model_metadata_validation_test.cpp index 033b2378de..ead4575163 100644 --- a/src/test/get_model_metadata_validation_test.cpp +++ b/src/test/get_model_metadata_validation_test.cpp @@ -17,6 +17,7 @@ #include #include "../get_model_metadata_impl.hpp" +#include "../status.hpp" class GetModelMetadataValidation : public ::testing::Test { protected: diff --git a/src/test/get_pipeline_metadata_response_test.cpp b/src/test/get_pipeline_metadata_response_test.cpp index b6438bf4bb..00b448c0b3 100644 --- a/src/test/get_pipeline_metadata_response_test.cpp +++ b/src/test/get_pipeline_metadata_response_test.cpp @@ -23,6 +23,7 @@ #include #include "../get_model_metadata_impl.hpp" +#include "../model_metric_reporter.hpp" #include "../pipelinedefinition.hpp" #include "test_utils.hpp" diff --git a/src/test/kfs_rest_parser_test.cpp b/src/test/kfs_rest_parser_test.cpp index 6e8621751e..c6777f8aed 100644 --- a/src/test/kfs_rest_parser_test.cpp +++ b/src/test/kfs_rest_parser_test.cpp @@ -17,6 +17,7 @@ #include #include "../rest_parser.hpp" +#include "../status.hpp" using namespace ovms; diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index 7edf0211dc..cd8c8c1914 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -24,6 +24,7 @@ #include "../http_rest_api_handler.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" +#include "../status.hpp" #include "../version.hpp" using ovms::Config; diff --git a/src/test/layout_test.cpp b/src/test/layout_test.cpp index f38304ebc3..73ece4a5f3 100644 --- a/src/test/layout_test.cpp +++ b/src/test/layout_test.cpp @@ -19,6 +19,7 @@ #include #include "../layout.hpp" +#include "../status.hpp" using namespace ovms; diff --git a/src/test/model_cache_test.cpp b/src/test/model_cache_test.cpp index 56cda2100f..3aefa0c992 100644 --- a/src/test/model_cache_test.cpp +++ b/src/test/model_cache_test.cpp @@ -20,6 +20,7 @@ #include #include "../modelconfig.hpp" +#include "../modelinstance.hpp" #include "../modelmanager.hpp" #include "test_utils.hpp" diff --git a/src/test/model_service_test.cpp b/src/test/model_service_test.cpp index 290778c32c..88425e3056 100644 --- a/src/test/model_service_test.cpp +++ b/src/test/model_service_test.cpp @@ -27,6 +27,7 @@ #include "../execution_context.hpp" #include "../model_service.hpp" +#include "../model_version_policy.hpp" #include "../modelmanager.hpp" #include "../modelversionstatus.hpp" #include "../pipelinedefinition.hpp" diff --git a/src/test/modelinstance_test.cpp b/src/test/modelinstance_test.cpp index 9affd7a20d..f1577348d1 100644 --- a/src/test/modelinstance_test.cpp +++ b/src/test/modelinstance_test.cpp @@ -23,6 +23,7 @@ #include "../get_model_metadata_impl.hpp" #include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" #include "test_utils.hpp" using testing::Return; diff --git a/src/test/modelmanager_test.cpp b/src/test/modelmanager_test.cpp index 9c075dec75..386287bd24 100644 --- a/src/test/modelmanager_test.cpp +++ b/src/test/modelmanager_test.cpp @@ -18,6 +18,7 @@ #include "../cleaner_utils.hpp" #include "../config.hpp" +#include "../custom_node_library_internal_manager_wrapper.hpp" #include "../localfilesystem.hpp" #include "../logging.hpp" #include "../model.hpp" diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index ad8a368d48..b501e37fd5 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -29,6 +29,7 @@ #include "../deserialization.hpp" #include "../executingstreamidguard.hpp" #include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../prediction_service_utils.hpp" #include "../sequence_processing_spec.hpp" #include "../serialization.hpp" diff --git a/src/test/rest_utils_test.cpp b/src/test/rest_utils_test.cpp index daa1f5efcb..3be7f1742b 100644 --- a/src/test/rest_utils_test.cpp +++ b/src/test/rest_utils_test.cpp @@ -18,6 +18,7 @@ #include "../logging.hpp" #include "../rest_utils.hpp" +#include "../status.hpp" #include "test_utils.hpp" using namespace ovms; diff --git a/src/test/sequence_manager_test.cpp b/src/test/sequence_manager_test.cpp index 6ecad716c3..e34c465e89 100644 --- a/src/test/sequence_manager_test.cpp +++ b/src/test/sequence_manager_test.cpp @@ -21,6 +21,7 @@ #include #include "../sequence_manager.hpp" +#include "../sequence_processing_spec.hpp" #include "../status.hpp" #include "stateful_test_utils.hpp" diff --git a/src/test/server_test.cpp b/src/test/server_test.cpp index 25db48a29a..50eef8225c 100644 --- a/src/test/server_test.cpp +++ b/src/test/server_test.cpp @@ -26,6 +26,7 @@ #include "../localfilesystem.hpp" #include "../logging.hpp" #include "../model.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" #include "../node_library.hpp" #include "../prediction_service_utils.hpp" diff --git a/src/test/stateful_modelinstance_test.cpp b/src/test/stateful_modelinstance_test.cpp index 33973c34b6..229fe3f3ff 100644 --- a/src/test/stateful_modelinstance_test.cpp +++ b/src/test/stateful_modelinstance_test.cpp @@ -29,6 +29,7 @@ #include "../executingstreamidguard.hpp" #include "../get_model_metadata_impl.hpp" #include "../global_sequences_viewer.hpp" +#include "../modelinstanceunloadguard.hpp" #include "../ov_utils.hpp" #include "../sequence_processing_spec.hpp" #include "../serialization.hpp" diff --git a/src/test/tfs_rest_parser_column_test.cpp b/src/test/tfs_rest_parser_column_test.cpp index 99e9a06d7a..13667569fe 100644 --- a/src/test/tfs_rest_parser_column_test.cpp +++ b/src/test/tfs_rest_parser_column_test.cpp @@ -16,6 +16,11 @@ #include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#pragma GCC diagnostic pop + #include "../rest_parser.hpp" #include "test_utils.hpp" diff --git a/src/test/tfs_rest_parser_row_test.cpp b/src/test/tfs_rest_parser_row_test.cpp index 04c8af13b3..b519b805ea 100644 --- a/src/test/tfs_rest_parser_row_test.cpp +++ b/src/test/tfs_rest_parser_row_test.cpp @@ -16,6 +16,11 @@ #include #include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#pragma GCC diagnostic pop + #include "../rest_parser.hpp" #include "absl/strings/escaping.h" #include "test_utils.hpp" diff --git a/tests/requirements.txt b/tests/requirements.txt index f217ccd70a..4ffb229bff 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ boto3==1.10.2 +cppclean>=0.13 cpplint==1.4.5 docker==5.0.3 grpcio==1.34.0 From 4af52bbe5e31740f0526f9143818179f636ec5a5 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Fri, 14 Oct 2022 09:40:45 +0200 Subject: [PATCH 023/130] Fixed implementation and doc mismatch (#1472) --- docs/metrics.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 6ee01f4670..a69d6c936d 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -31,9 +31,9 @@ Default metrics | Type | Name | Labels | Description | | :--- | :---- | :---- | :---- | | gauge | ovms_streams | name,version | Number of OpenVINO execution streams | -| gauge | ovms_current_requests | api,interface,method,name,version | Number of inference requests currently in process | -| counter | ovms_requests_success | name,version | Number of successful requests to a model or a DAG. | -| counter | ovms_requests_fail | name,version | Number of failed requests to a model or a DAG. | +| gauge | ovms_current_requests | name,version | Number of inference requests currently in process | +| counter | ovms_requests_success | api,interface,method,name,version | Number of successful requests to a model or a DAG. | +| counter | ovms_requests_fail | api,interface,method,name,version | Number of failed requests to a model or a DAG. | | histogram | ovms_request_time_us | interface,name,version | Processing time of requests to a model or a DAG. | | histogram | ovms_inference_time_us | name,version | Inference execution time in the OpenVINO backend. | | histogram | ovms_wait_for_infer_req_time_us | name,version | Request waiting time in the scheduling queue. | From d275453ad1a8761f04be8af9cf93f5fbe68815f9 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 14 Oct 2022 16:56:15 +0200 Subject: [PATCH 024/130] Ignore error (#1476) * Ignore error --- Dockerfile.redhat | 3 +-- Dockerfile.ubuntu | 4 ++-- Makefile | 12 +++++++++--- lib_search.py | 4 +++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index f0591c1161..1882967ca5 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -181,8 +181,7 @@ COPY check_coverage.bat /ovms/ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; wget https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16-1.noarch.rpm &&\ yum install -y lcov-1.16-1.noarch.rpm &&\ bazel coverage --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ - ./check_coverage.bat && exit $? + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 97a3f4fde8..cdc41b58a8 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -222,8 +222,8 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: COPY check_coverage.bat /ovms/ # Test Coverage RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; bazel coverage --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ - ./check_coverage.bat && exit $? + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" + RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Makefile b/Makefile index e3b6292d6e..74ac38c686 100644 --- a/Makefile +++ b/Makefile @@ -89,7 +89,7 @@ OVMS_CPP_IMAGE_TAG ?= latest PRODUCT_NAME = "OpenVINO Model Server" PRODUCT_VERSION ?= "2022.2" -OVMS_CPP_CONTAINTER_NAME ?= server-test +OVMS_CPP_CONTAINTER_NAME ?= server-test$(shell date +%Y-%m-%d-%H.%M.%S) OVMS_CPP_CONTAINTER_PORT ?= 9178 TEST_PATH ?= tests/functional/ @@ -225,13 +225,19 @@ endif # Ci build expects index.html in genhtml directory get_coverage: - @echo "Copying coverage report from build image to genhtml..." + @echo "Copying coverage report from build image to genhtml if exist..." @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) bash - @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . + @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . 2>/dev/null @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) + $(MAKE) check_coverage +check_coverage: + @echo "Checking if coverage is above threshold..." + @docker run $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) ./check_coverage.bat | grep success + test_checksec: @echo "Running checksec on ovms binary..." + @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) || true @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) bash @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms_release/bin/ovms /tmp @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) diff --git a/lib_search.py b/lib_search.py index 388e7839d1..22f32175db 100644 --- a/lib_search.py +++ b/lib_search.py @@ -83,6 +83,7 @@ def check_dir(start_dir): 'azure_sdk.patch', 'bazel-', 'check_coverage.bat', + 'genhtml', 'clang-format', 'client_requirements.txt', 'cppclean_src', @@ -158,7 +159,8 @@ def check_func(start_dir): 'bazel-', 'boost.LICENSE.txt', 'c-ares.LICENSE.txt', - 'check_coverage.bat' + 'check_coverage.bat', + 'genhtml', 'clang-format', 'client_requirements.txt', 'docx', From 5c716a07e9a808a93b52732292f1524a403ae7ca Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Tue, 18 Oct 2022 06:20:49 +0200 Subject: [PATCH 025/130] commands added (#1477) Co-authored-by: ngrozae --- extras/nginx-mtls-auth/README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/extras/nginx-mtls-auth/README.md b/extras/nginx-mtls-auth/README.md index f474b05d86..d5c6fc4246 100644 --- a/extras/nginx-mtls-auth/README.md +++ b/extras/nginx-mtls-auth/README.md @@ -1,7 +1,7 @@ # Securing Model Server with NGINX {#ovms_extras_nginx-mtls-auth-readme} Clone the OpenVINO Model Server repository and enter the NGINX directory: -``` +```bash git clone https://github.com/openvinotoolkit/model_server.git cd model_server/extras/nginx-mtls-auth ``` @@ -23,11 +23,36 @@ WARNING: Please follow [security considerations for containers](../../docs/secur ## Quick Start 1. Ensure you have `openvino/model_server` image available. This could be one from official releases or a local one. + +```bash +docker pull openvino/model_server:latest +``` + 2. Run `./build.sh` to build NGINX image extra layer. + +```bash +./build.sh +``` + 3. Run `./generate_certs.sh` script. It will generate self-signed certificates (for testing only - follow your organization process for requesting and generating the server and client certificates). + +```bash +./generate_certs.sh +``` + 3. In terminal 1, execute `./start_secure_model_server.sh` script. It will download sample model and start the container. + +```bash +./start_secure_model_server.sh +``` + 4. In terminal 2, execute `./test_grpc.sh` or `./test_rest.sh`. Those will try to connect to mentioned above container and use our example python client to test the system. +```bash +./test_grpc.sh +./test_rest.sh +``` + NOTE: Please ensure that your proxy setting are correct, both during model download and during `docker build` operation - adjust build.sh if needed. NOTE: When you generate the server certificate, make sure its CommonName in the certificate Subject match the DNS name of the OVMS endpoint used by the clients. From 5525e5e7e2ca2959872db45f3d5207c72e1be0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Tue, 18 Oct 2022 09:36:59 +0200 Subject: [PATCH 026/130] Grafana dashboards (#1453) --- deploy/grafana_dashboard.json | 903 ++++++++++++++++++++++++++++++++++ docs/backend_performance.png | Bin 0 -> 54588 bytes docs/metrics.md | 26 +- docs/service_performance.png | Bin 0 -> 94181 bytes 4 files changed, 926 insertions(+), 3 deletions(-) create mode 100644 deploy/grafana_dashboard.json create mode 100644 docs/backend_performance.png create mode 100644 docs/service_performance.png diff --git a/deploy/grafana_dashboard.json b/deploy/grafana_dashboard.json new file mode 100644 index 0000000000..3037571cf2 --- /dev/null +++ b/deploy/grafana_dashboard.json @@ -0,0 +1,903 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Service Performance", + "type": "row" + }, + { + "datasource": {}, + "description": "Number of requests being processed by the model per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "RPS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "sum(rate(ovms_requests_success{method=~\"ModelInfer|Predict\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", + "legendFormat": "RPS", + "range": true, + "refId": "A" + } + ], + "title": "Throughput [RPS]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Latency averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_latency" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "((sum(rate(ovms_request_time_us_sum{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / 1000) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))", + "legendFormat": "mean_latency", + "range": true, + "refId": "A" + } + ], + "title": "Mean Latency [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Value of latency for quantiles [0.75, 0.90, 0.99], meaning the latency that has NOT been exceeded by 75%, 90% and 99% of the requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by(le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "legendFormat": "quantile==0.75", + "range": true, + "refId": "quantile=0.75" + }, + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "hide": false, + "legendFormat": "quantile==0.90", + "range": true, + "refId": "quantile=0.90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "hide": false, + "legendFormat": "quantile==0.99", + "range": true, + "refId": "quantile=0.99" + } + ], + "title": "Latency Quantile [ms] (q=0.75, 0.90, 0.99)", + "type": "timeseries" + }, + { + "datasource": {}, + "description": "Distribution of the latencies across the buckets", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 27, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / scalar((sum by (le) (rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))) * 100)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency Distribution [%]", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "10": true, + "18": true, + "32": true, + "58": true, + "104": true, + "188": true, + "340": true, + "612": true, + "1101": true, + "1983": true, + "3570": false, + "2294682": false, + "4130428": true, + "7434771": true, + "13382588": true, + "24088659": true, + "43359586": true, + "78047255": true, + "140485060": true, + "252873108": true, + "455171596": true, + "819308872": true, + "1474755971": true, + "+Inf": true, + "Time": true + }, + "indexByName": {}, + "renameByName": { + "10": "10 us", + "18": "18 us", + "32": "32 us", + "58": "58 us", + "104": "104 us", + "188": "108 us ", + "340": "340 us ", + "612": "612 us", + "1101": "1 ms", + "1983": "2 ms", + "3570": "3 ms", + "6426": "6 ms", + "11568": "11 ms", + "20822": "21 ms", + "37481": "37 ms", + "67466": "67 ms", + "121439": "121 ms", + "218591": "218 ms", + "393464": "393 ms", + "708235": "708 ms", + "1274823": "1 s", + "2294682": "2 s", + "4130428": "4 s", + "7434771": "7 s", + "13382588": "13 s", + "24088659": "24 s", + "43359586": "43 s", + "78047255": "78 s", + "140485060": "140 s", + "252873108": "252 s", + "455171596": "455 s", + "819308872": "819 s ", + "1474755971": "1474 s" + } + } + } + ], + "type": "bargauge" + }, + { + "datasource": {}, + "description": "Apdex score calculated as:\n(num_samples < 0,t1> + num_samples < t1,t2> / 2) / num_samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) +\r\n ((sum(rate(ovms_request_time_us_bucket{le=\"218591\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) - \r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 2)\r\n) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", + "legendFormat": "apdex_score", + "range": true, + "refId": "A" + } + ], + "title": "Apdex Score ( t1=67ms, t2=218 ms )", + "type": "timeseries" + }, + { + "datasource": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 31, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "count_values(\"pod\", ovms_streams{name=\"$model_name\", version=\"$model_version\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Number of replicas", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 2, + "panels": [], + "title": "Backend Performance", + "type": "row" + }, + { + "datasource": {}, + "description": "Time of inference execution, averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "inference_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 22, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(sum(rate(ovms_inference_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_inference_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", + "legendFormat": "inference_time", + "range": true, + "refId": "A" + } + ], + "title": "Mean Inference Time [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Time of a request waiting for the inference execution, averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "queue_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 23, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(sum(rate(ovms_wait_for_infer_req_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_wait_for_infer_req_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", + "legendFormat": "queue_time", + "range": true, + "refId": "A" + } + ], + "title": "Mean Time of Request Waiting For Inference [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Number of requests being currently processed by the model server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 25, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "sum(ovms_current_requests{name=\"$model_name\", version=\"$model_version\"})", + "legendFormat": "active_requests", + "range": true, + "refId": "A" + } + ], + "title": "Currently Processed Requests", + "type": "gauge" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "", + "name": "model_name", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "1", + "value": "1" + }, + "hide": 0, + "name": "model_version", + "options": [ + { + "selected": true, + "text": "1", + "value": "1" + } + ], + "query": "1", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "gRPC", + "value": "gRPC" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "interface", + "options": [ + { + "selected": true, + "text": "gRPC", + "value": "gRPC" + }, + { + "selected": false, + "text": "REST", + "value": "REST" + } + ], + "query": "gRPC,REST", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OpenVINO Model Server - Model Metrics - github", + "uid": "Bdb-6G4Vk-github", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/backend_performance.png b/docs/backend_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..beddffb917b306d05e3b9da96ca1394c712998b4 GIT binary patch literal 54588 zcmeFYbySpX_b+}Uf=U{MNSc&@f`Bv%0)ljdFe)Gt14uV00xD8c(jZ*}(m5cgbSN<} zv~&zTAVZz&#^-syzx6x+p0&@NKG$@l76n-l+ivU_vwO?=SwpFG?4cw6U}_-b*7W#`zi zyy2`>%TOcXP#^ceL;N(BC#%1#j!@ZWiC9hLL)Bg9lX6nj2^qlk(o4pA?U1&Twyb(o?G-)*If<<^mT*EWTxs3Yq6_*wVB?w zYQ1INUiBR-cbst7^xc+e-FU<|Haqh$(_5pIxHPF6|M;|>c7fTge9ZKpf{aNM$Jlbb z!JNr{J!NR1oTQ}F<8n!NWmG^n9$Ph4%vQ{kCfvU6kh)pth65&Cd+8>3;8NJ6EOR%^ zbMsL%O|eMHPWwWPOlsDzD!zO+{D)F;?8{D#8ogj-IWn|L>R+LOExI z9Rk~5dmY`>9rrXuB@OYsS}T7AEK)DvmseQc(z==g5rxz8Rii_@kToo*vTb?gtNCa;L>pnWu|}8y%yQ$ovLnha^l04;ia@ zco+F;sR{N)jfk)!TZuLAsSMwQm~MROBF2g{+k`xqyD&H+Dmq?s&G2IW1+ z<7Pd46qQT6XNJk?!5e8mogUpYGmFA{e;jeUF_Ns~(S+Hu&vQ~Hn8PY+d~wNUJM+BFMfV?a+X-NG&2TObe_XoD z-ZTzUb_cv3Y(ZADWHEYGF63c(LjC3QBpp`mrd%Y4E>XQK=+W_yz#W*ZSdoNdJ{*#r z@}#pez;4_1JSk`z%Bh=FFVIbnDtPX*ZTBco>Vs9q>&AqB^TvCQ-#ort*%!+TwV9s} zB^mH`see{paP=GR0b;4kmW4g(@V_wj9sWik98);RQCR0}6hOh;dnkqH%_T;f2 zKFi(Xp_C+2JI?g5a%p91WAOVLM`Ph?2!C-GR+z~qK&B=yOGw^&Dxx7+z_c;AsT1lY z6Yigvp!IfQtTHDt`>{Gq+`YY7d!{Os|u{GLj7iS*F*=dd&TU{Ein4TuXJ5V=&7+_T=WFPE=SFg}Fzr>FB z?rEu-g*`l#ANFy!iv2@*w3*(cjHl)i|cZ;x%iI_$I zl(TVuPV&@}r+etf_UwG;wxAd#uD3PmOz4$h$IzRVUk7i5G!*z;6*f_hvJ953y}@2n z_Ojl(PCxj`vQZ_2)%2_hV_)|fDq-7OA@t~w<`q4g+TW9Q zbZ2jiJMoCeZB{ab{h+NZJC((Z^XiOq*QUQ4_nT5!dywGD!GT;A&g(|Cz8bQOHx7tD zGS}E^(aWnF@L03YD>HiCq|E)H95tEavofDH_9i|K?LEYl<7T9SI!*HH%O3f^dP6o5 z-|R|)tS~eF@;mLzE%;mVw$W2__a~?0F=HQNSdrlov(*Wv>t=Npy(`vfV7 z|9?M)NLQZu?{^L7{*QMeC;#ugEQM0I#NpsK_f2{q^}n#hX%%E!${F$G$aozex@Q6D zWwAhKlj}zlN!ojZuU-NDG3q!Y6s5irF<$@keu_&vhwr#+kyudnV#qP9%+b&=QvXCo zlLTAsiBB%1U1oxDFXmoUf`1@=XMG?hyxKqB@?tjX)*~S@RrsfNRN7+B1?)AS(Z|i) zxAUrA46SsNH~iV*-`))}YEu6Db-U>4fOu_2(FXAbKn{lY)P_R`U|!HuCI0iWwd?{-c<*!uh67 zO8Za|Gqs~k%Fk=Z=a^eUR6hJ`$&4sXn6$fwIy z1T$(0w^y~>w&WC;;0xJ%x=Q2_xrd&c)#9aU^~%XNJvC6DngtU^^TtV`3B3byVamQ3 zC`{Vy*Pk^;`Leq6$N!GR>P1`I;=!{`Gwb zZ~m40*54mG@z}QRMj(GV&Rt^21qt5S$eQo2BP!t*8HP6X$=d!`%Y)nJIcRK*qa_5g z+IwtvT@f@d#P!5wU$SN6>M{ ztOTj6B6U+)dDv7Z26I*Bld~>wyG(rUcZ`|g@EME)b}f57|FNKPjidF`=2ho_a;f^P zsjvOHO@)m0px_T)8p~u)i<=oKtC(3d5xXxH$jT;YCFc-tieB}Y`QVPYbN)ht+59*$ zws0@sI!@%FV(IW!uSX6uQQgw%y|B0JTVO^o(!xtw<= zea@f>Q*@M{AY8aqonVkU-YoRlKi+?mFt;$f9ag>`UeSRY7ax7uy)0gU!aEHKa6KD$ zbtD&#;Pm9-OQak(XTCHyU%pyEao{1%imK85JemwWF?mQKd=(>myL35<1d9_*TRZ8y zRYKOBm#gPlUe((hTwysFDBg;a3^pjkT>%8^j`g$?pnnM?P6K$g7)-;C(IDP~g@U=%-O=4JU3a7B-d1ia9 zm|=v+=C219xn+m*>Nf1RGJD&gbYH(U?4)~qB>@90+iF4g>LY1Hhn5hhp`QQg(V zICH}ZZD-i0B^xxH%o|$IlSuweG9j-wQghM!?eYZ}fO&`hFaVc1Gs-mk2<){!!a7EM z5)%tsb&`+G6MUvyyPSqP&z1F*2~WSnofEttDVh_{J8bN>{m_B}?vAC;DH|fLp;bU2^jkgIJX_f`pV#5ZLei5A%nmXf4^^%{*uZ;B$ zVsv-{c`w%c9OU1Llj4g^8}Jy$y9|`(<|et{{QTn%*52`Z%G$cm3YoFkW2vwu&t zJrL>ZyXm2VwxNriqZYncXsCIc!GEnwP@@F@P^7TDZq07qVsedsLAVgjo3-t-?167x zyCa(DxU0d+E2(AJ&14gu*+efR#n_DEe;=KynQIp6W+UitS@wa|#?vhnJMi|z{Dzr` z1L3Lu)5j(H)&axji85uXv^RCWxZ@l3hkE;-)9~ibf5{~0iufU~g$T&{s;){OKPPF$ zgyC>3mo!fjyMzB&wz_1WExHj`0}%6gGSkTzU+V0!PFKAmlO;nku-lYsVeQcyt@7QD zwTm|HQj2T1GvkiTR8uOh$?HC-{VVuEY5uwgXZ9ub_*1%2q_NCSE?v z#L-*zbA{2-j&Js=$=+PM>%mZv{p`e#$|LWyqQ-xcO2*n7k@jt#wxSiQZ4aj^<@VCX z3V)(%4ri2Z(8=tLEveO2Gfq)4vJ;2?#P@2aCmUGcrRu7Jib0ySEUjR#EX9=rKH&&`win3g=z zEuuG^D;d&^iw}pZ(T)DaIXU{yC@K=A+f8X}m*R61Q;)ojwjWX8>xb$m56SCPQ#Acu*Ha>-K zt?HN7Uw1^aPkFHRD?C}+q{oa?U$LC**`D0tXG(YWlxv(59%eo8rbOB<*pJ3-Zs(DP zI{BRNRUW!Bk`6iw5f8$kev=V!h=9?~x(2hPe3{CaG+dz$) zf!pd!zx43GihDSW@HI*?Gw&{Rf+$wh^H}C>OcT_8fkag>P*r!bEpLt49r??@%*AmF zwZ`FJh8BSmL_CUl|W!R)B6Z^}W9I^}4dIQ%<4IWS@neeJElO~qs3#GOd z)%kln8!cAB+U)v!8~|@?rLm({Z@d0>E!s~8=!_liJc~p$UA1quwsnZv)@q+u%p1SD zm!iacp^P^;&6tacJE%hPW>!+!{pOGb9g{$LtNU~Al6U;Slm%oD*@8As6KTteygT4J zt~K#|L?$>Dy)xorG*+Rv^GF;sI&7zD*#6**aA82EXvm&%$dUy40c!KMNo{B~dT|B- zdzc2F6!Nm(c?`YeR-Ih2S5;k4FEKE2dsP)G$|?{D6V@jl;*yjO0FAV%M}s?6d3 z$LLmROrp%j_1`>|3zRN#b;+n+ap7x(pp~#?c!ZG_O)qtRpLV#wyYFRGqMYSb({DCf zALCLzaM4MF{o$wR0dpoz{P+G|kESx6%56(TJAyMJ*P^%cD8$788^Wp5^eS!Feak(^ zaxP->65l@dTaUsc!Np(uA|Y!nOqX5pJZZ^V9G4EgD#!cKi(aSupN}pTs`Qe$c#S_B z6;NMVmsC+=eA(?df0{o{Rf(~=Ze&IxXuIMK0JQoJ05s+p&=nRI!PuQ5b#kd%-P~s* zg^5lQsyoNYW9slEB&KLs`49|d+x_^Um-9h|>)$KA>Y{ibv~x_E;M&r`OUFXL#v=ZoGZ4&FJdW4%k-tE9y^=SqZu+9GM=NH z`$ElY$Sl<-%~##ZyWHPAA5H znE8wM(>bePWA+D0Sti44f~@El#(|>FbC<60CoJUeE$JAg%C!ZYXpL`7dsryxqqBWc zursySGFf<9t?Y}Jlk24(?r?m_dWm+kUhf-Yciv~?A(*rQrOBO{#S&^IlcxI?;+{hz zF)2iRJixi)J2yNiZ;KD@h|>R`$NA|XI*Fha-S!4>GqTx ztC*3YEhywenV^O%27jdRNs15gUkT!#6qEpEd|>^NQFre_?E$Fd_^Z%e8G}Y;FrXJz z*!WMq@6r6iOrpK>`Md7$O>$2fT23WczUdD@WSyxtS&(*9Y-02S>o409R#b!2I4%yH=(rsKyeJqIb@z^79 zn`!hqssqsj!i9C0!3<>eW9!;Ku@kJZT)N|c*|~KHXD!gv+sEEXq@uOYWk?%)i!&}5jae~qnI;27E{nr?DZ~a;g{=gdk-9b zHVQNIF4G@Z@)I=k36II@oo9yTrXOdCWXam zsDrqf&sAblasK)z z{!ZZ?79oJZYVi7a4xU9Jg4~5i_e`p_%A^vkiZU-smfw2DcrC`MFeyqoP7t}*Bs}8k zvtu|}>CFfgh9w z2+3!mQL&=5tryU`odzP0i7dzT&7@RC;hL2;6F3!|9w*Wp~1i3 zFWO@Nm-o^8c-}_im<@ui;06EvsjQ;wzt7I+^8G6zCGX9Daudgn$qVoXDDlm|XYfth zy78}wt!tP6C9T-vT>mAlr0<+LBkXkQcuUxD-nKGl`0t*E{O=-@ZQ&nM@!daEqk`yP zPEh*FKRe5hA%8cSVgKwmebcZ0-Fja8XBYbaL_pAATZj&^Fg&^VDS~fk?AdyBNLb@n zZ4J>}6)yHMF2BrzsLLg`5tEzFZ7qa7vp!`Hv#r;}{Li13Oxm^Z1b&<1rfTH_%aYHh z{XZ@Vx<;PRR7pY!AgYYx;xGAdqQrHcqUtGu)IoM8GFY{J*<(DC(8Q`R z8cv=5{HXi>r0J#@|IwiB`Aqc8RZa$eZG_$p-`y>g`mH(lpli*BPv|nm!^xo{ZupJ8 zvG|NjP8K1R6v5?Msi`Pqhf9FG^^JhR{P>;(GWY%?)qB!hKX}5<;dk28oy<+UypA|G zhtao)I~Grk@yi?D5dF6Nj=_d1DW8Waf>SGuPSH6@mGpUDmL1-U6q4qGy^zqD%jY1T z%4wn3k>{d~A z&uwk3N`22JS>&rvik3(^{8oYfhd52Dg*_U`*YpZ7%V3l?g13+>YB;amY;)Nji6nuK zAORnNNk%y+t+8WL(4{>xxLcv~RdW2&tXN-%9Lrlv!Lb z5sdoPkl>kStB)Xo!aU&%3S#Q`nYP^XL)yAGFe_9o&4I)lOT_nfm95mKQ&VVYmzFpk zBxcHN+Z!oVE|$k$N=Kl6u?%&e({-|eMUMy<;&Hddii(}~1H0v#W=TEkO1 zk{jkb(W!tDzs3p@QiBlbv}=cx^7}xQ2=&c+E&&(M=7i@u1yLfl-JzvKpA!YM?Mm!F zWDb@Hl#$G}^4u7~cpNy5`75YeN?q+C%50GQA#k(Vs(g z1XRTUkZr?m=A!d2>0Cor=zQw!@cwz}l2;Bn&s4_1L|^{{1qxWU4=h{bxpcYiM!Q_g z+kibYmEL+K)oYr4Vl}60@dK|AgL|-F$YN&9YTN2r`19d;Vd6j9)vs>CQg? zZt|%U8*X_9GMcPf+uvupZ%z!Oiq(fEthHSHpbMc2XX>`J_p(GTgFx9iQ5kU z0Em@$JRR-SN(1ix<_DjXhPY(&JL^Su6FQdgIAW;?HxHkiWv6^nDTzjNC+E$7^Htq5 zEUF4`F3hCmy7gWybQ|;*ra}Oj(uf>_y;hN02rnuuG>&*<V-y5t-D8tr`t`ARThdDGA*XNt9$w`0tbMHgGP^yctxz6uPJvmvWmL)I#BpJ z2)sq&`bWRE{r|k@-+PGAe)D{$*oF-91*beKrHvR9DSYf*&c~L_a;A+HyESQ2d~hJE zrl_=4Mx_x7)<#d@bhs6ib&Dg7ZaJn}ral^&0~6@UaI?PXv0mEmq2WOJIP&9Fc)(b1fbXJhrOyS*$ zbm1TFA1#$Lh=|>0bjvdS&eYFs|8mGr{S86!8hfEMsi|A(4g=KS1!qB{;B1aCpU3)_ zeSA1LRtInPl|IHM4JifRg%hPI3oP!jr`c7rk>bFN)L=CM8)UK-IiKSiL8$}p=(xiK zH5kESqA#0~*xNtwugc6Yl2LA_=edXEA>IS7_3gL^T>G|7^K97d3KmNLxIORB)axnz zc3T3-ul_3=p5OChPe8o#;2{pqt@bY&GaD5b?ztU>Nj@r#A7`O#OMjYQvv3Whf(dVM zeuxKOI@PjEt>D!jsNpa%hh3xknS)nw@~3l;jzL`07um&noQEc?!J}bS03UO+{JS!S zYLcS))aP)DWp2Zd2Sy#QfT*$KgOA3(xbWk$Kt-_oI zhV$y=iJ&f7pj)hbe~jVtm{xg^gWZB8#T{R>U($E!Z3X`OYsVY-2lbxTejQjRx)e-{>C@1XUJJ^OWrmvcrW$B0p3-O zt8EKUmf#Odg732Ne=_iRHdtGevC?uXKQH#AQ{Xs;PII1dK#2%4hdV+S&GG(K6pw&R zXJ{P$a;oM*rXWYqjToyXn!6W(+wU@h@G|{u9UA!|J33a(7<3-qpDAa1e1M={I`w(h zV?l5#kKtcz*S{Jx-DwoHF?QMO?mTH{5LYSwOhPcwCE*>c%ZE5`vyR_%IkvlcmedCZ zjBS^1TgH#~)2Yr&6&A`C$10wNnBe1&01L~JJeNE=EsFpU3qtALW!w-?w)=$oLpBhw&DIpanS1cHQAuyM}QlosPIl zXyA+3xAc)V^lPYt4U+^6)9|40g1_)1ge|H-AG41VvXfct7~Ux_zkACI?gq3L!l7~CH@XRuh}6?s??oI-df z3TxG$ulzB0NKfH4m2uZv-9;epz3J0b9rE_ivL<3HfR$&E)1hl)+&MX2=XWw*lqZP0 zRPYr4DS+v8xM#UL*kE0F@&;cFcGg7?(HoFkPmsdiKfN%y9;Kk8g5EAU2`NZ|yigU+BtM)@^oK;BRg2EVt<&L8n)5IG_Os8(alzkDz&xi*fya@rz2 z^IRL=LP`_zorc3>5A;;Jwp=k%r3=Ioj(;}v)DTu|o1o*D#=_Vlo^d5J-Kg%>@bLW^ z=Pcy1vx6r&`->N(=)Iv6&^2nH@nA$QU-FPz>x<62Igl$sVPI7D6s&~=tVI~n5X-vm zhiLtpu`sMm{V!6>qEL~YgRJtkR}#~%oYF*WQLs&nwY)@1KkK1=5YmzopQ$%eD$=5z zgnu%?RsnY>pu681m|fLHf}CG3!LC`CNsLzG>q3sUQ9i0YuXhw$!aMY7W93 zmu5?sW>eyhw@$r)vF5Pp%nak+UWdu_Mf>0r3d~~NhBw{N&(b)!a5~p6yKQU)iFu7d zP%DhE?G(%C{@%8kLiVk%%+EZI)W%s&@dU+=>QKV3L=YZqg0MAbwQdXKKhF}eRHN@m zrN^Rg?WqTUOYmB)Z~gXz-E2Mqry*&d3Tk^5hq-#yq3#y_}b-u7U?tkOF+6NP$lq!D6g+xn3fddiiW(99q{u z4o;edg_O!lOYm@dKL}*OS;d zf8y}e2Vc4j#n92^vT64+1TQF@-n;Bs~r=DD4h%ct^G3>&}aZ zHfmm5Ghb3X=TM4ONSi8(9F9zMI(k_c1|xi(tB66{HJ4LM`$ZP{<{D#;*ao_f*?W9B z>s~}mtmb?+yLo@9XtiXgHA`9;nUvcSH(enDVVx?b-LM2k!XA3KPPylzJ)c@-c>c_# z-Z~q}XAtQtUYDv7{fILQ{oy^KxDZCt7Z^!$T9YEk;RiWV8?S`=+D*AStX~}H<(4PK zG<>@*_f`MX7uD|Vz(u~Y%=cWfP9*fA?XBEB+gG8K9T+oZT3&9u+lsqh@si-XxvFT- zTO44ZiF^>DmGaPQh`ukT4!)c94mP<^c+?cA3Xg>_ANu?~VikgowGVDP%xN?5o0wEi znAW|QRqrLwk;L zJK3Eaz6<^Yeo0WE!!Ck@KOufz4dLyjaUn=j%6fger|^HMNC1q8KwI+zQs zWYCW^!6?>YLlCZdigkCO41G0Zj7+}l=87#hr%8c&q+r{7=A!p89;bz91@X(&V6?SPz!pSyODpM{h_pGRJf z&go+fas-z!-v$G9KPvUd9J#Az2+g-&^O0bQ)GgO-OSqlU7bA4fD+jJnUT}a~@HhN9 zF9tk{9E>@ml_FA)Baf|}Qu$MaM1 zK2ql*=gPI(=I?z6GUdF6;M{!tOuE1PjRLVW%7YOVWeG7b(Z%-ssr=IxAV3@Bmo0U6 zdPeh&`+*!$Tci%uV+C8_CX3s1lUdSuAF*?h=E5YVYWnB!8GdSr063t=K|o1g`ha+; zv1lmS7vN(f{1h+s(7RO)T`Q)SeVXSN-SgPSDZX%`ktV2z8y3eDib6$7G4-DEvDs;y zRXh_}l$05vjsT>vGy)d(K3Lxs?&}m8_p7vJme@7QW=J?se=upIrEhKP?4G~U7lG&? zNI*U0ur$|#p!S)PtAvT}g2VkMYG^NcX*tsOIebQM2p>V3afb&@jTf#s-WZKY;I?Um7Fu?@kvrb5GV*8^906FtS zFcM7Q;y&sC$N(U zei*?*-nO$!-ctGXL0?P?Xh|L6fPa{t*%9lsQ(x&VYCvmEm51)-xU08wGSSRq|2ICy zGDuMPS=>!%o&mgiE0loRP2M4WEX9mt4JDZwBiJu0Il`%<2=?#CZ2aXLws*NVw-00) zPgmaqV;9^2T(5V`E~kdzZqU9ib|VtWsTxfzO$}?es^zu2oT&&VI1hQ`A?Uh|>$%xp zX~^ZLnuAOM!L9zsCxB=9P&MP*$MmO4h{n+m9(w=OGRuW`M!b2e3OGFR!}H2tXL&)T zW(b@4xlII_@&kX&GioD*`0@dZfKN^7J=k@9Q96rsXwTt02j}mWqGC z_-)KeMDg?MJ@R@t23maI4{%$3M(0kY;-aC{0I8n}#%@cKp<=6Y+Lf#nL-matbZZ+% zXr1@fa-(I$(wU&`^Bs%7o34iz&!i?zTI4z^IucMmMB|N9cnP171cHZ z%NuYOF!{1&jV3*Y~4 zrup*_C#xpZ$(x3jZ38jqTLZgHDu=J5WaThP#4C^Yn&TD0GMn9ba81#)`a*hi+%tB* z@xu*GTAHhla3z(LqRw!H8<+m=U_QF%3q%~E`$Zf8{bhNCcI*{pcEHfL=^y8)^ z*XH)20^@m6<))wAFx!0j@O^l`&+AhAr91catHg7t*c{-e9dk*=%F`c@iJa{r^;g+^ zprnQxXAEQH0;w$gcwhy87XkH9M_sUq0^OR0-~B8LP}(z*H5^rA_i<$>a%O*==v(5R zq!xqt;SOB3;>ELf``g))9FCU+pKbOJbLa3qWrQbRURaR4$nJH1+tttvn4Hj?zXoIa zO;qS7#I_TLEL}+@X^*Vuv7|Y~PXh<^O{sCdRVIq#qV2D;5m=rgUX}{dYdIX$D{!Oh zm6G&XbJZ^88M?$sM@h_m}+X(ZtVOu;Q9Nkm*Ve z(eW>#<|#};zd^!w6N4ebdNM0N8as=yYP!!3H!>eAU48xMpL*DEOK)xjm1537Ht<~Q z&Yh!lY!c0Mn0LRvHUi> z=H|kHnB^3oyh-e^QRbg;0nEd*G^W){Cm~@tSi3+Y0&bUKD)*4_O^Nr>?Vi z7uHEO_zkY_&O+Y(;{xcQI66Zsm#K*>gXx*7U=j(w)4N}Nd6 z(FF!7gAK$08z`V*npVSOO&2@Sv(k4dUQ<{~b@B{Gb7_D6y(oJtp3b6>5*m38jJ#nw z6nTDk>#gJ+$Ja6C=d2rHp10u;mO+M3A08Q%3fn~@UNp&wS-m)reLi*|L8fX-Q8pHe z%69me|5&mR8Udq>bS6Nx>Ne^0DwyAcxerw zxv!1=zK2~pF~Ws0%;s54Xq@5>FEsTi#gIZOz)EbrqZl|VIQA<`=s`J#~A4qi?avBS0RYp{}v0`BYa;qn0O05*g=q^2dV;{3f}>UG_}4y2u_ zJ=B=mZ0{|DMgqYxZ+N)s`+Iuov9e%e$xJbi__ENlKVZE)fL;)#&y(6arCumXYFs|( ziuj#>o@4kFQCP>zrb*G!Nne;*{FllJ>5YOB2~-@yQ|<7u?k*U9#54r z_SZj2N|$mAwr$n^v{_VQdjs1ZZVjqAScH_0EA3lnA=`1)Z&Q4|#uvhJ03VC}-F3#b zn$C9^ys|}ByD4Z!m3om_R%{Rd%Jcvk}Q421Q2Lb}i2ezvs zgL5K?h6qb1_@mrK`==ac2^``^Bzj}2s?plzO8dQ+~IJvyi;RI}*8?bdcm&3nrWHe=_9pHYZETeN? z6VA`W{P`d;(B%zf$97BUFybfaTW4*TKVCUl=Wr)A-;4GVB;ZIZ@R8!-BV7X(zAG^b z3;pcsQ_uuP5e2?ifQO*d+zgWPi=m#`l;?1{AmXmoo4(z4ha3UpozJtj%-hV7;DjND zR6y&;7SVCh_@>W`-cxU#@xueYT9HLvH_lSSl3>F7RyI2()8;U`uR_#Z5pE6Rs^vT+ zplVG`s~P%8XYQ5dkSr^rN{}RcAElye>Rb&oJale%j zPyn9rWuHMjyj8h?5r2-tr_``O?w0R)j9!_$z!s*2E&TqBFpa2XL1#S+b==dN`0K}u zy)d{$hRuY(%VW7Me}D0{tXQtdnnUa6rWQ{Rz1!{s?1}C!*ffl=X((9S?3MjJon=|C zVSi@HpZK`S*Y}v*R}Ta~n*{s{aobkA<5eKg6ja=HN^PqTe4jz~f!?anzBKlESKd-0 z0K3Z(cGtb4bhSh8`>a>6KUare7rPsX0Hx&NMv4Xg_1Gsvs@B>AseaOT&b>X`Ra(e8 zHW4>(Tp9E>^guDy9qUhLwFT}*8mQ=I@&3*#SG{KsHLGXwu?MWBLnA~@a;e}_kBS})Co9F)e z8DF?fv8%y3v`>2le!%WeeMwn1R0FWSb{f`BC!e4}63^9F3XE$ zX4WY```$PF0PTDbIC8@G8fC~J4qIuFeg%ciZ?y6 zSGrNPhx}&chQORvolzTmLSCO<2p5jif|2MWoLn@(XelkX5ls;e*9SjY zF)oS_jPWVoZ!PV719a}rQBACA(A~4(ofTYsoFCya4Vc67?9wt(>%R;f+Z08|Hhd!_ z9qul|Ua*5ctnJSXyFM|ILNOqOk>TdLEUZ3ZCyw_OBMSX7JDr7Ld`!2lk6<2#s>x`3N@k@x%MZbgE>9(Dx={bLyP?+NxIHoH=Ag|Q%hAeJMx zpENx~Zvj-YH}%-Z-tP{nMQfyWCfQ&h$QjH;4M;HOg#sS0_bNg9rwaPT8PK0|!nA)Y zoQBe}IsGi-ec;2%24k5@iej@YU)@i-hYI|06o5v~qtIvu1+Z<^^ui!nh z?=)B@gxw}J^)UTK5YQ+&GRI2)eSr2GkYo5X)o%>8L@@s%tUk_=Jf_FV zdgoN>tItOJyI)hpDgHyZbY$rzDL@H5FD!1wNZUA?Z)E5UkJNXtD;i z4LT1uQF5mr8Soy@OI+9o;5LJXWcj^eZ)hQWj(dR4!+EN8?wu#J4Ghr06RpA-j$rKp z23l`6xtGIWn2VJ1-2$=c`whS#>>OwvwPta9?}W5UgNAkvPLWp-3q>T(VSt!BY|UxF znnmOQ>Ne}LCjPjr^n;->2!H*F01V!{0KAV?l@HV%CXSumm)-fa-{n(K2QY>-!PV&N9l?d8(30YpMD7 zZS=$YXoOaKmD}#Ghlgyx5?8}*$}_Dl-yB>w0*$`m+7P8?DADn?EtM=#C9I2&B1^$; zkpY*1KDqoJIEVm9R^PlCTGDiIeFB;Nf*$5?;2vt5xF~R0yuN&}BtD_`1agn4feLu} zu95oFSNpg(lED5u)Fc2V@i6ys2zJF2mR~u!JJ;0cxv=+|eyYM7Q$-zKO;RG~%%oo! zS`X(4(1akRph1|&$}+rNCOamUPECuGCMg6rZa@qSGJ%S-1=Gp8j1~J<8YScCB_v!S zz_Q}SFcsKNJ*U&^g&ZatjA$m(Jmu#3=mF{RI`!_*Moz{? z*FCSI7pHv0=_1d_?6RD`dj_@%Tv9RB>jNerQDtGD{G`g=*$-o%fC5{cTl(%17fS$d zNsGduExw@CK10k{Sh3qhBzT<%=m1}c=y8Fu|Ilka4y@&{0Ej;d?{ex_?ZQsZdIfGm z93x+kp5(Yox8nj;WRG``G~Tj$L^F>I{HanPoHEg{lNA?(bb)uvQa>~IvF?I9ccUG> zw!XwU7H*Ct$%$a^TZTO4>jM6fE~;TN8ugo zF)G8rcmbE%u2L3X8M7HeDM|9^JeCSt09P>@a-933heby`@!O!DHlzZurr5%nDkSc` z`7bOxID^rL{CWhqEl`F#^~82@{FA3oPWeR1h0N>3PlSQV62K5bm?bE}{;!v~6vROF2Tzhc0q>oAz&4VGIthlRxm{y52W;JPV(Du5u`B;n<(t^Atk~E5 zPZe^?aZti55Pj}&^es)hlAC%D&%us`b8Hn0oyn<$M_vI=z@p&p!Lcb|tH}b34xfSX z0oKjnG=XsFjlo_Kz25ZvA)Kaxh|KYIwH$A^O? z4U%vg49?{{0!wOOMgM{*fC(kI5m<$t9re25F5D9d>6Q9{`eHdu*$}&Nkb(W&&D+;o zU+oM66@DyE{neFSWS*D2&6svmeXMUhEMOl-r{ecYL->kp>DOn~{T1FfXVR=naikNE z0L6{`X-*G#L1YaP8#Am$r}C82bklRMqE9w(#t=!bv69 zxjNtU{Z5|P7L{rh5pTrQx>b;54Y=7=h9mWL1fA5x9eLY=Fg^qP|&R}NIYM1!RWt3^fGe6iE)4J6{yt{{2*VQg=bmRZMJ4^ z0T(oik`DI-zCv(|9Wm?-b)`%8Rs%%-aXIz3zMj`c{4e(2E3B!l3mZk<=oSQQh;$VZ z0RidKtrQ_35PDPT2~8mM0LlgnQlvvD(n;vOgMt(((g}nnT>=5=9nM_1_21v!e{r7Y z+?;h|LgLC?vyA$Vchr=(#;bR8@!3-Tl%lbt|2+ZtW`UZg?EvcxttT6!Q>(XC5One^ z0l*R*i_;ApWR=k1fj{lB>5Z;$KNSfF+Ladye;$pU0mKsla6_K)R+=7^&#+i|3{4J3 z*ZHB=+TbJ292GFB^1{%$N+Pz(0K?ATsq;?U+|^^Ekf_^&7Me=s*D|G9o3YFzwJS%RH42+n2kqXYvJ_nXqkj$#NL^imnxo?~e=bUh?nj#p~B`YHc= zn-x`%@z_FJmqppEXR|5>rXOvLh+iAybFCEF!qI5jm<_mUC2-(2uz;J>5-i`9p>=`s zqB-Yci44@sG?59~y&QfDi^G$H9&#~@MRcUSB`tecdH5%l_;bQZ-l^?-1R`#l<4S{k`GXe8BixRkqHw_SBk@{5d9I zSuc1#s^ne)I-Fo^S^%Z5!NF%?1^diS-SR?uZBM>!x&K<*o&GRYvh+ zaIV?`0Qs||Y93Q@T#MeZU9WVefwyoEwM;F#K2lpy@oZ1cSRK~WyO}$t!^_G-(G2`V z@#+LGF_z#Cz2hsBr?{ptNX54KC-`KEh){C>Dznk%7L{ro5K&t%E0j$mg);mSXY z0#ZdStU;*1_?+w!rsV6U0~wEV^@nx2jGRuNnM}O@h&E%y!*~ME#~c;Mp~$f_?d7D^ zJa+AVzYF$?CV8BXV%)xT=BAv%YYJNwp8l0)KH_)MoGqmqR#ET8qJ5KA9hj&W&~ zMRtTMEEsyY0fGN#)VaCYVYN*7*SF`KOjI8=;v?A)o>*KGTglCZZ*37aR{kM>E9f`f zx{2Wu4`3o?j`mMSzx~e!Poh377wBvk#}~Y&t%r9 z51Fkg1!%_>tiLKzVH@NehFfd0T}uuSBKhwmC$u#C%C%XVpk@PJm)geyOau*SdDnRK zAVG0%fSll9Mo~$Z@D9?f2uQO|MF)VWVk317gTU@si-ON;yCn>T?qOvWJ=Mnf3^n^6 zZ_wwvGITwwZztC|Qt@N+LGm4qAA8vWsc44m$7S?KLjUN5sf$WwQWxImaZ%=YS!y55tqMaoBnq4mCb}@)FB{8jQp~?fZLxSS{1dB6<#BK0~|nOD#I$ zI3bhr2jL7E7G#D#Dezhq+m@AQi#r@<$4b?i=jdVQBOgBbJVn4ACzTtEm3Ma8qYb}50euARCf=9{vk^+|6yq?69GVBZ;s8HoeLj?qaKqjXC_ zwU<(dNVOF zQhuzwP``?07hO6gl>BjwP4l#<1f=_+oV5Nubbd_9eu#*pLTSZ}v&qjk7{Td5y$c^l z)^1t}A8O&O4Rv&-;NrSrA48J5ZRJfiTj`rVna(Z$0Ay9{ZY}F3yu^WZ%V;?)ptmGB z(ZEZZd^WKdThmBZp2 z`U>1CO#d*<)pLX&?ye{+{6#oK6Q-hf=Xw0W9bWUz35Tc3OYxGYjw(pLUj)X$J}+2H zFSpw_Thk&CLnN~R4f2&HlJ&8kEx z<&+oP2db5|L#`m$0qoI}Gin~kkMah}iNw|=5?J?bkOaPk3DFr(W4O0XAa%)E<5aza zvbh3~MX7|DW}bX}6Fiw4wJR5M=jK>!qh@I>KI%q~8Gw}cD=umC;q4d^D}~aX{vh@{FR&#eZ-PnuK2ip( zvZ3GX1anQw+V+Vw#iRnI+IgVhjgm%;f`rnE(YQ7i?*kgEZ>VG)X6j0 zYzVm{r5p#Jd!8$+ORUBtL*f!#drVwC8lcB?DY)d#4A}z|Ac0dk6hA%#s$xtVdfRqt z41~XMB(S_a@QfX^@|X>9V9;rpU5|Q+S=5e5@F-S$^{7iuelgC-Bv8JWl$&wIerRD7 z#wG#einL82OBJpwsRFJ3XnnnBUa{EPKjqcP(Ch$`!GHsbR~5bGxcvYs~uZkb_qSH#2MKu-f%~nsN#ch(l z2*UDoCmna5+|R4|^9--t5LS(G8)rnr@2P8A3aCZRz#=T2 zJ@igaZFp9EbV&=-=ou$&XxLMpf<#j8NQXetj3KEeKchwyogy^j ztVdw$bb#sJ1h$r^+x44_W~Q2D+v~z#PwzCITmki0Qnei*%iT=C#m|?&yQb1+sWWtm z4!Ie?uK97dr~|=`rp?7k-jPbaW*p$Mzu)DZg^4&2 zz8Ci%m?h64JwJrk?^uc_Sn)I`1r!kj|pK8wftpVJuU5n{o7!0toC|p-u1={#6Y6KxZQm|Z#*hC5D zrhPA@%uV=hu!GYfT{;*)o<mkcHw;b~QF%vouWx9D5^Q`H!J7 zVQ+m_+KZ)7yNyX3K_%ZTD(YaU_S_M}H0x0Ik@=SuP)hR)q_k;!Jq|BuN+nV0Eqrg( z3!=AH>Ima^61koG1ZU%=F=XCS7JhhLjQpOSA7-?GU%(c)z1C_wLpj>xW_NvxFk!}!%Po(xIo$T0-ta-E~Z-( z-xOgnZYF>&mhOv5s1V_t17xg(G(>>90Va$lhP(ZVRw>`akxu~G)C`jmERj1AK}9bW zD_{tZ?pT66eo;xpWKr4j&JMPu1K{>+CpZ}`hi>z5*FJpJBii!4+!;X?Qw&>q=ZAoH z&AMcAcJy3!nd;CD#fB6Ov@Rw5ZE<{h47C=_KXVA5ebNk^+_f!Ndxn1%|A4ro?Rxkm^*vQedV+?01~L>x4lrv$)h#{pjqJ7PB!pGH| zt6PS151y<%_Cs#MM1-Hi@i$t(teY3_mg{K7xBGW^yi@sQe<>{5-i)NdtY1~GKs(EM z?-V4!pE$d^uNAW;jD^LkzPWI6y4C%ZJBuo=-(WFXcIUponoxeGr)5<1SBj?+QSK@(c&XC) zkwn@!{(2}6AIcKz7!HNW*a)JVw;yJf) z+>nuVLV;*_Rrq9z8>lL_R)gP`0{(bq?$9)dNdrgnr!CFD(H@ks;#Bp69kLrEo@1A$ zBn#d^rh~{*;=cjBDQzkfBS!(XpmYHbs8L?#YoOcYCIN4ZYU?>H`BrV4P zIHBZIzU`?qheX3AD%$Tkn27RC(LW)Aiz8>zadnOW-trHhw_i6SUCgtToFK@r3F%s` zB!w!B&j4>|3=>k_mV&;}#b%AjL_cmCeIAJGEB=}!i%K)b>4S*njJV2dmN81iERDey>`mO z_{8?gLE@;9Pt5ZT8K-be&mtGGwr|OC$SFLhSfvLlW&AbVlQT1lM<_))iOZ_)K_w&6 zcVVJK>UYy*_cfFaGDlQixR=p)gyrIbdc%q<_CpnXv0QL4(2$(YJ`s@40nTN?doM;P z@_!LB1VDtcXdWcF0Nk$xaZtSg$VAd8&K{aO`&vaAMip@y?E>q`_12YKD{jF}JhN!H zn>!WP?M1ALgZ=7MysPiZ;D>>PTu@UjfGo=$;@5{tbtWMFEDR#WPRGh3eiB-Jz8~ zRN@D*SrAbLneQCZzY60aD1;~k#&+5Y4L6ryLCyO#nVxQ(>MNOHgR6pm><5(&Ug@bC z^VT>{2gSWP_u9$olYK!Qw(oMY3+1+^>L29G7zbsy#CBmIq5E$*N{f+Umbp$HLVihjiwdYv=0N;jo~AD7^r;btaidYD%o%s)n89YoCtlbae| z^{`fzP0i*Mp@0efJEOKa`#@tpBFBvB>vd~(DaUN31$!THJZ;6*BkSp${=ft;H6k6=D-KW36_2$$$u z!-rZyRb%*!P*KIf`;LEB^gkvZoKXUQ*EOM(ljd z&Bxt6aJl8i=w)a0C_y$d-aso!D*ll#j!VCfH2rgBZ!qzM5aRC>06&!yo;JHu@k(V+I5B(g@3M~l}F#PlkgWN&J}%>v9Ygy8>dD|wgWIh z-aT0vT%k0WNk-L31auGseCQwbX;seqKM#CZRaKp(rJi-;6$KC9h^CqHG7gBv{_j=h zh6o>$_IO&1lNPKtb4c`3ZWG&~tL+-6uhFo6+6dkn!dZAif;iRU#~FtmErXlJr#>qv z7LlA7iv!biRn+mPxFi{?GhmeoftWfNt#%`Yh{lzySIAZOM)|%&``oaF~jCzCFJje?mf7iWKq|n?S8W9*cK+ zNV9w3>^zp{P0BR!!?a+hyVBD%=CQ;L2U;kvMhgY}L+1Ge@%Fvsf=;)cR5Q;sJ8j?J zYNT5^Jzs;Z_A~{`VgoUipjX!pg4!u=pxhdnm%=?zG{2MJB>GD(C^*>=)4Gp)YeVi5 z?B$rb@*$2Acm(6qfIyQkRNmKz>w52cHZW`F=$SL`YtZHp2`MS!E^*RAGQ{5eeJatu zn3OKrOkA-n_d7KST3Z}sPX0=_fgByEhjFmJV|+?*vFe&mIr^|}>925geB+rGHPg=b z*#wca7z%QYI~mC2N9%)c$lJ{ZG9+XRz7!tJ0)Sj(h$D|b{#L3EB5E5!gIhlv#np`m)PB&xF$OM zURYp_!zerK>4!=m4`>FV%p~vR2YTnKG%AY)0CpIR?^QvDav>hhETqj7kSeDn{sF#E z*NlaR!R{@DxVA8l70w4%nR7UI5%HMC;Ilc$3AHNOvq#JqpIjwZs+mECwohJM6d#E_ z%MAzBr;z#0(J5BRy93Y8wN@E{XX9w#Mn4FNXKgkhu7@>hC{9mbI}z3^T$@RS=!U_f z=lNK%)}@SJV(yFdbu-)G@|-~`qR6^1>hnzZYm>Xmh;(P@)ULY?Zs`^8vZ+7`U~&+S zr|cH=y+pPKVTUg4_Yf(zlQdY%QifQEUI|zoL zBF?mN_p-3*SXCaR*Sq}6q)LLyJCHww2qo@xeIwYrE(J@;*&fvJ+?+Q{;r(vUR@c6T zww3GXnZm6L^i@Px>`%Pdthxs6e;M<+7?wNX5E#H2ibW|8Mq!T9tuF(Zl?J!ompduq zohpxwo4ujn1`#c!dWz#;nB|;9g*9tSosHCpjl*7j#d~>G=+}axI|9FdOY!fAC412xv!r|@>>h9N={JN%Jj?vgWP8(^HCjPV1 zS~GEP`yN^^tVWqx;Zw+G22!3}BjlRgepq^;ecQ(OkOuP4D7J^H2lCw+K^ zs%-xdRN$ZDVxQi z-lBEmy_H9bYsQ%k1_Vp4v_0el}fRLXH!R`ul!7S*IaDg);u({PdVI}52#U4?4#OPb!ANfv5*_yd9BF_P z=0ojr_{ZT79l6#A`v7_?841u67a;V6cv0T{n)d?t`9{9w-A>lZN^?9M&AQtLs^wt0t?l!Fy3{{G)E6y0fG|!`wiRW4Qs=(nwT^FYHi+ zurv6wXUq~bZ4;9hvRMd{JEpxLmm#4yY^Y(WdPtj#BoW!tucqeryD+ z#YyanOy`9{PW};kDm7NJ_NcD2E#Lo{!_sZMZC5cXGa!hd2P zo_HoO`?9oXH@)QtdsDG>W>gW^j8OSk6HM`w3s!ea1p6O+6*-{lEoLR_223*E>HyG} zT%Yd~Erv}u8(#T4ch-1Egv_$R-IY0H+zeP2!WJqXU zM=7IhMh-674qlEf1=!<#_0)7b2gq0C^y`+65=9wd1aA=mJu6snD$!HR}XH zf&i5!Fy(}wk5Yr`yF?L}eBY9e0KT_f^@7(qhMD-i5;n9yn2kEX3RaW(VpPfo1xx)f zEJiO%?|0Y$JU4(rT^WHk)8ID(#r6Y?^qYJi8fx;?l;)9V!3^0M{bfpKpRliB#VqNI zdtDiyRI{_O5@tym=Yo^D+~@>AXsp8cks6yEumD1`z_^wkD7uwi%20>wIEZ<{E_)(T zFlEtht@vz@Lbk6(_`kr( zsYLdbq2!(0jnoHj*KW?IRm%MHpQ~YHqT&pC!LF$op zRqt{R>1I$xgoVmtHcwcs>}j7qBoMOhH**Lngoq5JCNfck?`F%Jr|N&OkGhD*@gNyc z^jT3H0JE+MgJKrZ9FK=9txbrbKxl1iLr5P0ca`V~vl0HjjCK$-+d+dafg5ev<7tmz zun1(&eIaiA$b2`M9R$c%P9wGg%+yOI@%=eKb>?>Gib@vm+m|Tf!=jmG3>NRiXO_Bjo^M7M35m9#-Sd@{lzDPOJ9@71H$Ht59kU z@Hf`DV5Is-_Exm-O5cHVg6NaT18D0kHq|-!LyTL6*ccM5)p$}34+Hw%S$@gUDjYX14*#8hN;in_R|{$hANC% zJ3oHDN|HWRW^)A7g3tztdgP+W32uX}5G(Lp3hP>4ATcg>4LI6Gq!ZtHPuN$5%ySC8 z`?Pj32_Wc849K3If`tF#BEfz;>k(_`bqs}&S8;7OQ$zaw>jt|5I{9P9%Aav{#SnC6 zDbR^}0RA!kc)ig=cG#?Fcnbmd$}>p&J^stR?UiEFh85Pzmi8;T^MA|Hp7b0&AIvJG z(TtsD7jRDL=;|rZU1|V;nd#8_@VGkVdGyxu3EGta;ncv|eiJB16%XzG;=0f3>J^J= z6hgC$2YmGWUbS1tLyN4B2h+o^#)Qt&<<>sbyLsaQ1l77kmUjZ$Iik-%*|UhRdJpdI zDu+AQbmyyU^mNxd{}O=sARM8{PG7<5is@)gf`w1>L#*w-HWbAjM^)-PxZoYZ^{`M% z7J%7pu8QA|lyN(j2UUDQ0P#)gb?cZe7xebiC!Sk(Qx~jnwqytk9s2^Y*@4q<;(a=k zCH$x^=4*`w$cqRSd2$sx?Ps3J9%PU2K1WzR?j5M}W@fu#5*Tm6`-u{)WCKtgsi;d5 zf00chZ`6Xn46CT*$+0Qu3)Rnn&=qL2K5T!#d$Ey`q z5UTk`ymtEHIg8^zWPtRGh8&i8*F9Q+L+vFwn||Tf{lwT8c<1YPKp&so6NJeVcdr%E zC4p+P34u$+xr=3FPr;Ta8~)g7bj1t*S493^iLg){sdhyD>W~@j=qLbfjohkWHfZ~F{lql9dAbs1~$HY~!sOc|n69nJG>SdW1nHycql)nIWjG*uW zG=tM~UIk26V)s{#oFZBq>B%QL@T`FL;O)+NZ$_{+n*U})I0Im>+|J#?ZDvp!{tyS? zWZ~+Xt={_a$|s((#!+^4+dMLt$2u92-=qe=iWEw>2B@m z$YK3`ezK#YnWLJ$_fIY}@d5mw++Q(Ek0EH|dWk=jJIoJAACe|}Dk?>rguq=L*&urg z8Q#Os;d$BzYfm439ApTVDIuR)A<+VB%jl%V#xb%=ARoV*fN$w%+WoMiFV1U-)fJKG z;OKMrybe||m=hBp+5N)5XqU`e{o$n~q8Dn2%sUPe8k9h2I9#~`ad@?shJWRm+~z&` zX%K9NJ5bS$7AK1apRtV;FOajGpejxgz|K=$+C1ok+nbfh78`0`ki|ln6}_J$l-`(S zTP89NNKaBcVw-Oh7M>6CMC2p@Ub4XV1{*YLLP@ffQh;Px4XQ>2*Aefr{%2rnwGABt zmIfj}CzH5hB^CIn8^HYjdFcz+fqxo+BKCn$mE14Mg(3HYz*d3&CiS+o<~_X)7x4-g zyGH%pqrc^4=<>%u?ST+!;Od0F!3|28e_!5?AguoT4cY&Fm*4;E-g@`-$IYB_j68g& zC-`tjKlCvUN2}4Ek;#5EKKY`k@iZk<)7IqTPtK7DJ} zY8U$SSMxcjWExz-6rj;trbu(Y!KtZu1LxLP{4w}l?|;AZd*egjnH}Li6(q$a7bJB{ zZ`RUqiT%}4KSR44K*oUUF4r30FcX}EQq%eaZL4kgX&<57&Eubs+9=+6 z*=V+QOZ0{n`NcUuejt;U%06&3>qp#@;43NjW$KQ4+_769{`qkB-=%>*ggubK4__Yu z3UKc$nL~SJ>&rH0bvvovl<@?f;Ncv_2O_j06oRhu<=* zX$+X^RbDM3C({A~1-kFiFt-bm=I)7E`C&Lr$3_@U-Z@n+{w;Fs%1a7-^Mgou)Otj$ z%-{_ejHdMhU$v>9US#0l&0CKh%eK39CUr3|cV&HQa(Ar&V6+lEuMhKC z+Y`4t(RR~Qjf+D=r@_3H{pMeXCQmk}+|pu@o3dyK1ugGe4b&D&@RNA0x!ac|(H1N> z*MhsU2sXjlK&Vu#u+ZrNedwbipqH2^4*-=#!w2o?ttm9@&=d6z3oe-AFu1jd` z?lmQL=>#CwMybm!-z+> zrX64L)41i;J`U$MM{WKbMJ@)$4PVtzZ;Tl?G`UkUr$GD8AoBGk+(x*sk-Nc$BjZ~v zjcE3&k$6d-)EjjfBiD_jSg|tI8nrUEiyw&u^931crZhvbeY>V&tfz3t*p=73jtfJM zTS=}S$T3xc%bp{dTo#Cgf$Ya1_KFV7C2yJrnA$mfFrc%m}~E=@K6WjmroFC5ngI)}U8x(MGMkDmAvN zX#7CR3-r--$`X=d6yJEDN7phD*YTAdwP22!mL*kpt!^*_5{o>j=n7nSxl(Q&9m=WO z^_g$F9CTj<4i8`87s%ONs!JwIunkQdxAy3N;mjcO`fyi2RQNf6)jrgca-WHtiSm@~ zER7apuCGtQ8wXV9KbY{=sbDd3kAC0cOoS7tBoH{I|)9~IEFyo zG_hhGx3G{zmNYFC{`7Kwf33I<|758Bux2OLisn??q}BrI{4Kdy=Ly-t!QNXK8y1$y zvT>0|Oj@i@H`WMOqKkKvT=z0%ji}lgK-aG&`+#wD+g%s)AN&4HoI`W_Z1>>& zWlP7}J+!|uHa%&YYmu~h@9>49Nt`@p*vIBns5%ZxMdJ( zi`glyw96|Mym$?V)$Wi%Yf@Vq++goNNPcge8H{(?Z}DxlC;p@^#|p$xI?;rp@O`{{ zx~0a|jv|tA$TBbCp^Qo2wAaOHtkiR`GJ4|Jwlm zH2EfCDgHwBn;ufr(B+OKvGd8`m4M2GFhROHq`V;|jGS@540(T_&XBKHGH|AX;A>R5 zQxsEIqO&1+N!+PPH)|YOwU(0g!!ZXzFH?1E7_|BR9z6OFMnWa?(DMO{?#T8)|HUte zwdu|ViK))7tt6zUA^Mike$GL~+K_V<^?$bNb>UwROBrzBuJ}z@=W4`x>Pz)V9^KEz zO=0=t-?K!=(}Y(#bQQbdH#BmoJc-$z|4&eO%Uo%_Hz9h@Imo@2gM77&y%S#wp6amA$J-ehqPP3dN|E!qN( z6Qi~2okRe>>nzG+^%aH=<04|FId2hbx%5B$g77tg7R_6h9W3Ec&^E{p2_#J;P9ECy{etPP^crT1Fnzd1l+i?lYr@8VgssEDE+lYQ_W6A zEMe@ZP2)*_EtSmmo2w>O^qvg6v7Q4z>P1{9LJ+TuXMPBXArAI{R-`9R3K9v1YRQ64 z4ES>tf(ZlVvbcyvDiR|#`)h{%=YG*ar0@noe6zB{!&KS=S&5wdrN!1pq%325SClNq zP;JPmoY$YXlPY2Nu?wkdW6pukDC_~kF@5k9Vv1-tDNV77F7t|^HWCWLFCS><8bu_N zc4ez<)|ZOs?Pv1x4-X1~rb;HMEC%0oAMn(n7Um6$X?#5$OX$~CFC2oTTU-u3*;9zd z5M17H`Nw4nK;3eRyudJ_#V<<6caqE6jncT`Ph%9pjlZ93B<3!1F=<_37|!;C zLB)}2ww*h-^#Y9Y0o&noUrb|-MD~+)AoCix;ue|u*YbBbEyJb#FeiyrR6-%wmLGKZ4dn)+$HgphR6)nH^>4t^*>-As zYJ8^BM(890-ANU($dQ5N(T5T_gJa`oRY}ehS@dd6j5eO^nO7xs1%;?`DrgWf?@Ka1 zc1q4L%Iek4hcS5gvqm`d7ga>YhjLt#(_(0}^~#wPHx%biuc#WudQnBjdtR>^Ob@dji?MfjCSGappU0T={*GW|Bq-J!A*iGPE$gnlWbr0f?>yM%Um_7C*@i- zaOTOXAu=6s$@~8E4`j+g-yVcM02gTT{o9^}7|y{Z@9TMF@IkNtSh#5FqkDrtUHUEY z);J*{0DbErL}&!Al>50y!vrtyYW-gL9~wss5B}B(;!3~1HhlEB;Pp`o3jFI0f5ZLf zZlHJw`kK5iXai|5P6564{xgt&Z|nJcX8twpK92o2gDQ&uDe{oD~4X|K;%;3 z3a0q~>=pusfigxOFQ0SL>+pt$p8Wbf`hlqnNuD0PYEi!2pAT>?ccRPOl9Y5M6wEIJBohusaQ9Oedj8 z%G1~MMCT88Yuc7PDpn|l73ViwWFiU|bwOVnD2fBFUkkvL``u5CUH}=Tq%HjU%o)0b zC|kckJwuIW6BIg!!}fwE@Rw5&iisb)sY|K~8m(qy3Kf;+sB9hW-^d zP}W2S6rP&w2q+2NTMeRi2Sjx&x$yTiEdh!)Vda%(E#2yrCaa2>CiM=jrt6{)jwMnE zqV6SKz#{WyU)f03)-efJ?B(mVA+^vSckZvh&B+CO4XQW&GgvD{SdX~dXA5k(S76po zV|Ky%3WI}Va?Ei}7+ogiUxF{iDtdnS$oddNO%IEn%!~@QE^zmmR{(0SD}Rm^v<>qD zy7|M8R&`9@*e{at45Lxo@YzDq-oHCE0CBNjErO@h&u6u*29CzfBZ24n-!1B0#CaLc zWqc{%Lj}5TrYLj1ws=_IR*h~n(Yco)Ia4_^&mnrSl-Q`ahiDzkrj7qKGgQPz^BZTfs_bf4JYQLJR$eG&TNTi;lf#-+F@zCKu? z=>+byK#^M+SnzjVPu@;#trU}#Poj81@@MgxYyxz4a7%IQrA(QTedad8Lx__fbFa(n z4gaj8(Ams3ZFDbqj9`sI*2od7yNemgyj}hk4@X-m*zUw?l|Yhb%5X1OFx1SO&;FT3 zSM|Iy<}~}>v}WgK@yor!VYN7N6K*}2rH-5!pAMN0IInN!ZsPsysd5b`H=YAfRefE) zC=a-Q0J`IPht@3?AG{IQ=qrJ*qAvUImJ2^m^6?vqPSd^Ku+*|(;lL+fZW79vG@r$- zxnQ zHQ1!h$Alk5#8wT)_G<?N1!~L~Z)~jH?r6O>j?@v!%qA4h97*cFs;p)@(T7 zy58%6#pC1saJ5G+DlvigY};Rnp9-G~W1=evL{mg)PUEVv+) zUPN-~fF}f6TxJ(7?fu%g%H4i0zhW~MfO!iU)tzuM6J5d+WFdg6HxUSmw_ zS6kcHFQZ;+RrUT*qJ57O7WDTEcNPzdaYDJQ%SRVI4;i*E`h2D<494eVROvwKX)U_P zdcS87hm!1rSD$d`{kt`AUBW*=I>B|;aaHQcl5cmh?PT%X#N$D`XQ{((_AlROFb;ZB zlifA@Z4W@o`7U=;0^bEKmEhtI-cXdEXTVnF@J;S`?dzdwBZCh8Ut)zp|<@Bg~{D7&C2VB?~V8kBc$y8w^`

18!ZTV6V$}rn#*~$+q&KoL0bU<|ZuowRUgDH>;r9o{J?! zshBFKkgUC(3D4}F!L7CPtD~x$!4;8NlPAcA0kgxOUnNuT$}9Nk7K1ct7mc*=$<&5t zq^4!Z8i;#UfvhctUVrjNXgUmL)$2S{TU8?ENOqO z5IR_^G<}W?zUgUk%sQg7Hq!dGCyfHkBU}1gclIwR#AW6ssWE=L`l9!ZaR^J8tE(c&VJv4yJ(lA)o)g-IWIc7=q1znj!CVa z{!7h(C&QJ-#B~*P?)X}8px5foY>(Sj81W~@!#B;Qswv3+PZ$_H@!gY`OC4jas`6US zBs$(xSLqdxMKE-Ci8?DuVd7>(pE<~W<;kX+5lTi#!A5GZdE>s}0>v+IB5xiD{A$$o zFsof`CW60U2U8l8EPz_lQ0e{gteAJUjD3<2ME<`&=Iu`vEvTd!Z)g>0*GsM_r8;lN zD~2jOgJT%>%mRE>O4{P8{joaFS!`Dos~NG8wJI@*7gZg`MZR* zRZs1f1_oy`Ny+UlwIWE{>sVINNrerMG&0GlMvRF1q|ro}290!QeuyP{8o1j;_bz3M zEIpgoCi~R}2poVENZOROg;9l?&SxS`4ESC(12eUyod>1H&)ex4TlX}&t->LVUKvZK zTAIL|f#k``=sjs8FQwv{Z!*0-w3PfEPHWG*pwV2u)0RE6i(53Eze7g0`(IC^>`U-q zecL*}mY2g@k=xy>i8I+8bn??F_ef2g6^Z&Yi?KYV1~y@hLOJR5e8r11&=$DI*xdw3 zYXRXGU{mK`;udaLe%D0zbtZB$;hRgJsI73ihUR7o=;The@H>$PxhUW5T-nLhMudi# zu-LlC2J*dqY5tieebIWRKeH0S&w(=)%UozWKP_C)e=ITWk-=(jBbMMzH8;B+88u(5 zyBV#kt{%_^SHChp+L6;+jZ4dnb3SDH%Ds4pn(UM3f88Z&5#mf&!Fb(c9rTTwt{AQ| z__Ob2sbHUcFw(y+SlV&hNYDA3SfW`9h6a!H5gw3>&n@eTiA|<^AF430Y0t9JTw>v97a1BzHn+amTIjit4@@^H1U@$v(0C&$tDNX{rTt z4dDeM-&7+I0rd?u?Q~vwUxtkW;wzUUmccROZd(VazB}#xT6pQ8ft82QDmZ)98v=Ka>9gF zc!cp2Z9G4s7D3NP>Hq%5smqcM78dQFSGkMssc3OA#S|O6?H<&uoX3{u>WUcXvzHq5 zFHM?Y4E5R7`<41mTIAK{%)j_n?w(eDGHbDV5z#P7nYxTkoiYDHVEx6S3g-R@;Y5bX zBuEM`eu&_z(%%W9Oaa;f+;hloVnv}~aI3lp-}inxZARSM?&Rzn{UG&``c_uGn;0YA zRCFy{Hrfqq<>KkPt@Czo;XhekXmdVQCl?KCNqRlg3LLefm&#g@91iVQIzvhzeP}uD zV>%~uD;PC9%#6_Om#$V~jQZ?)4L=!MmLwNVrnws^S3kuoU)PNr7SUH-y)}-Rh>3884oiA$AO(+l7P3mn=7h^;8m4LBAP+n#eWX__qp0UkN% z{7ncy{GN8NEL<1!q9q2L+%w;1mT60K8+1&DR}lt-0eoLr8nFEn@Sffo+Fc2B|G#~b zejls6m+usOQoynohzCAZ_l;a#OfX(dKc4Vequ#VVn!VtleBfEo(1>H6GGpLpX24cX zN{F^Bg#{oT3mHBSZnddlqwZv&L6aVcY7^Y_mowdMwf3rO29DwSr^{FHgNVL1r;j;} z;HG-AaJqHu4Lx4KE(T+!Jv)D&zf2DFIs$p-n1q-2dQLX2m#g;l;yR*svL1PEkdeKG z8aMnNs(i8~COv8%Njx4ck7(&@Fsf9r-YlN|syilB)CWF$8cf!(&d} z?-4(7#H$I7`N(bze!gw`s9=qi?n|t4Pb_Yu?lHX(`c1i!Q;v$+7{8)O&E1T_3&kfV z!-Um~?_C~N2$*Q22%3N=ga`07vV1O|Svarb;!)$$N+E%UVsgJ@T98vBpBU0bJ5v$N zX!u4tZXu0MtEE8B=EL5*=U_%Y(-ur2DNRU%`EqV%72BtXMOQBm!vZGu9+x!jwcYaa zNE8bP-52t*&`!}HGyy_M4WJ98{Kfxx9e3ZIiILU@7GDdKK?bGXEk+}K&SH05JNl%V zyXnJq3lQgFpGP(-_%>Diq+Fc+)qHJx5GyGC05Mg#$daG&`tM$N<7Z1B3AE-hGW7op z{Uz`^+bqpc#}uy@*UU^6UuZ{zt{J980=8iL`GTcL*R%O)S(K;z&J7=O#KqaDN5&FpClKQNpHU{A=5A`Oux+5e zi3ir&<0g0uu)Z!*;ew}wctfZro2dNSSwrOF>}ButBfH`X zO&!de-2VH_|I^)<$3wmTZ|f*dWvLWO(Mp9V`#PN*>0~|jCA*Lr`))9;oTwBbhLMD3 z7;DB(5oMV{7{;1C#uA#b&dhUv^!=vu`#rDM^Z(OdlXB0!yqD{JUGMwzVS0_t)j0jL z6s%EO%inA*uA$+pgitOx^Qcy3)!3@n(5kIA<*x{92^fANo-%bYype-7b)7#vdp^lx zY$c+oZGNClhD{%O&pX?XF}=RhTSglD8BO2Dm-A}EPgsfEBhF0)?3SBq>J+p<}#)!{4@e~&D| zGz(QLxgnFBUGm(_85IC8m)Ss`KW1%`f9FA`<@Dxkaqt`sgV_7ED@;b5BPCLlCzP(j zrV2f5lZ`DHM4HxO4I@r)!;T|1y=f|yF@!~gakbF7=Xs4p)YQF3rav@Q1vNT+O!iNE zV1+tlLumX`9t)A~{f0OcP)2xkV zK6Vm1aHF`*kG8S%GI-)CNVuv1n@s*S0G>l!KjL)l5`1RtSploDH8}iYuj<-kKKsqE z_miDb@4Zr5s}@aBimYmxk<9tjbUfVM^+KRi=L>Ol?!1I#!Jx)DpD{Xa;!oF)0-cIs zYGeLt>)rMGT#BAm-HBm@sTwghV?HQkKLyX)f2SI>M!qA4NmN?vxYjFECI%g|%&@v- zZ|I&<5iXaW`95l5-Ox~&eRrcj#WZhGqIGtlMQNeIu++hlKcI0AA)cE4*+ZpYDx)w{ ztr7#ebx{uQl<3Z$ieB(Z?cHo$+a~4m*3)T$?;cUny(}lt3~VJ@EkBi;Z)F#{L?>MF zfCozY=mo~N8@&N;S_DmaO6#M30;!t${EA~sc3i4^YWe_%XZa03`)Ck%AZ@N$Jp+sM zCihx}*$;mkQtA=D-L}0bnO^@eGeS4wJQp~)x&WKOBz(;K$%(Y@PVcSV=6v|si9wS2 zn{UOcR-5VVhBoDhY?(C8Dx1Ytk|21zj~dg`7p@gBIO_8KFFs%zhksYeZwpP>@n(A| zQn#q(iFI9C`g`QH>YwDO6w$mZ=4aY2s9)r=)+@TaJaX+Tixyu%{fyCPCnhcL3l8w! z=nK+qA>@VSCSkt_dd&xo?sQ2h8w&BT>Mr5f(i|>Bpw6}~D+8S-c-P6Q!zPUqCC?Gi zg9<-a?xA!9^IO<( z96+LPyYQ59CYJDO5T@*_ZaWY~Sb^rUXPEp?ZO7X65K zT0W&beP;bP374(Juml8x)!+K${UU(P#@T<&6+{S!(+mhT>6s^cQv#;q zI%ab_9&-|$`0kTBZiE(hreqSjn_haBN7-Ker^ki$wPvi7 z5lKotPe*UHUeU86lajd=W0-U4EY$L#huH-^($FE??^XCrkKAL=KF)hG1Y(L1%QrS63w^6O)_t z+s-~pci!l`Fm;PunDo@w zliEm;$5t1_n~*5e+5IZ{bGa1}Y%S1Re1C)%VaKubsbXxa@r!4{viw(vzwVXH=_{sf zMDpCXTDXvAC!+`F*_?C>@)+Km{+?c}ZOWBv)gm6ZYUxRTM-_CB{) z+@bB7wZ`prVXEcZAUbz(lR1$o7Y zs#DPNInmv$pRh?9f7F>%$XvCEF>1!P)#sLl`OhBn%|G_eXRE(!D+e7JH7&+|b1>yn zl0P<>f*!J}II=45JR- z@tT_wz~u|QyR<#veIwTJqO^BDy=(FVW2At(-ntf&+LpgpVfgU1Jge(j>cZUjYAcVB z5lB*bax225vXwN3xPWhU8wQbaAl#r5_Ncl;;0a^!84k`-(6yZ?o+MuI;2m3WOG7Ks zI<9?X(#0u#WoO{5{Cp>S7q>aX{cYsX;O{Se<*=WP-FPSHc}{|52Z}yDVGSjLT+bY@$UM$b-V+{v z@ohd8S79ohla4)T+1zq7s+d~bG(oK`TkVxm9HW=yl#)`K_7)u7s(hie_QjVYJWxp| z+#GpC+{2{5kFx)W<@u3MbM+KIQ@26GeWmGY(jn5agjOBg`#9MW!UMyNnLm%0y?smY zk2v%=Ggm;>Lwja%g%h@MY3HxUrGIRrZ^Ua1U*mLbf5Sr(Z9^gAknThQW0T~yaWAv{ z&|IpPqy1M@{)29`!u>-e@_4kRs8LE`qq*noZX;f}zUPx_3YwOHqwXx&xxKZ0(Jq7t ztA%dNyQqF`;Tm6Qt4ez2W6I&@Sby3Dj&1r}+RSFw*G03Q)6UFzwHb(@|NbkHL^wjV5Q2VYHO9J|L`SK>FSC2 zL4SAsGTOYIk9BqOrq}5d=44ZQSD%f)yPbLi{)y4Z7efO#YOC9$>QQf_!NutPl^X$O z)=Tex+YisB+)!of=7~DOnP?l8xGW!loZL9VpCIZ&u?^Rj!#{P zN~;!UdDqNTut`Am4jUVke-bj)7?t(Jk145;i@nLEwvzU$(&IDiXNi4QJ!JGV7(F6h zs7J*%PP4?g6zh(_>a095iAfzgiY)f5;GDZ|BXB$9X9l;UO!s!tdVLoi3d(Z=MT2^M z9QqxzRj9hd`FZyfmYQ+!-=gi$aimNTx+wyj0XO=D0X%uWdGx9`n%Wy}*t6@U%}*0G z56Iih6+0~78r2U#lBD~wkw!FwLO}xqo&3z(=3@VOfVv|ZTkb!-D@8IboCFyVQOfX9 zHAN_-X4u+ajcq3Bzft?n_gcL} zRuCBENeJjqI%dCVGbee5s+n>`#b@K1tDAd%g|NYCZOJbPd~WkhUqFD>i#HvCqXN7h z)haJ&S>-+KFNL?)EB^$=a4icirFS_GqpX|BB?SDQ(jn}zyX0s=#JotT68bnADf#lB zBUXK}f7n;Mxux!wL^4ZMbO!WM77pK~sz-Wzc+9N#-VQna-+IH&!n|~)+-u{HxxZQk zcbby%d$>XRpHI_^e5jn(m$DutJ4+oFIbyXFJbENz^7^H-xmqBKBBH5FexAVtlxn`t z>RW@DsaP;k%(fl!=P^dF5Sy?nF^zSF`6L~^YOhfEi_Mqx!@)GWnGx6Co>EpZ_Tn1_ z@#L1Kf&yWvCXt()9=2ZA<||aNX0`1#>DlpeXnVjVf^DC0l+y8b*{GPp3UF_$i?gOQm zumSqp(xGFYLdXMfHUp{ccy^2PBk%n}1-QODeG?u?@Sr8on$mir%Sz@S<{#rF;^$0y z*k&v=IzRWmB9zN6QvecDSv&khbe=onRK75-f`g5=9T#G|3HByspu9PJ@=qg?$M+|n z5DH#l4?^%<5Csbpwoy~uNcDIxs%`41ede$rInHMzxxM`ckK~-Yy-&r;YxR$RZ@#)R zYmKF{8Tf6xa4s<9XQ_^fd-i$21ov=WUAajmV8YKmsx_O#SJ0XQoGE+8#6= z8JYZ0onEjOFG;CjKebU$EbZ(})fBJ|%*-jq1(-KmHjBrl+gG10!SkRxjrUl!dF(Y3 z$<9{zI4r_7Rf`g}fL(sJ?I%9ug<(O_`5tvx@k(N1k$Nr&n|>g_1(zmyA-o0* z$zv&=+VrKsrd7{sg*ET9w0x}LwhX+nYi#7Sjom;UYN^-}J3OxK+;+gPQYOk`WmU zaXC(P_oZ^2J-NNk#!Qj47J|F(uAPDEhq;dlnMFoS`p7e0SQm=M6v4C^c}|zSg@G_8quYQNZn2Th!CmXK?MXK^Pec>Tp-td7-bfswRpGwXTzCm5s{h-D%FR@T0|78 zDU89mrivX=Tm1@9w7)z^?HxEI3e)k_60%;OE=LTm54OPqtoQ2>qRyCdN(-~}QRN#LPeT*lE>3TDa`xVrjR=+vP{)#vHC#;~&Bz~e^&5_B45)BK5E zUgpV_mOj8nmawGyXN9udzGMmcC?xa}U^@Ve|MO!Im|MmeIYL?O6;H3YL z!?(l8p@HNbQ9mQoW$S*Gw+dXWMaxUSgW}?BZhxtFi1j%?6xE4eFpGSs10Tq71+L{{ zQvEKj#MJZ*`-hWYxB)LaqZjLn49!PY3p`G0QYJE&_y64aH}F#^&;Ts-kvVMBM1GrZ z&1Vy4%vgRE)!*R(s}#?5J#W^%SJi@RcMEW=6AsUadY?8Pj63NF>RgZeqs6D-5CJUKeVz{5WY$*X6usUP_6A%#M`*Gr!T(*Me zz+791EzbFN!%v9v(!+XlB*|+z)z38;50M%pAc~oG?=kpsB?`(jGlm~0^JaC3UG=P{n5%W~7uhKlgFW;JwC9Rg z&mrO*Z$$n3!h7k3nT94(-x}m}Pzi=R9G=DZIg|R-mS)lsu__etETrx19bf7$bHqf{ zQ%+35@$2E9w|Hy5dDZJ_W@dlp*!CKjh)JWM*nPb4O)`v8RNpSrX$6nbU8(S8Ef|}m zvPJ*(UlexP`W$}#_dsDXk0fs`D6C}T>swB7)_?wQqlR~fTK z+rg#ppVg zb5o7gmx`wZgdl;5pnP8MJV?2gVXH-@=*lIRbYhZpqdv94cHd8XaJ7YP1DRb3C_ zrE;;sWapH-1K*aiAPhueECZ*UzwBfEH^f>#Va)w)gf6(|kpHHugMtRAHTPq~Zkn-v zIfzcwXIRS}OTU==tO!LQg7so3Rvu%0xJNcorKniJ2p;7$lkv>JqND59f&LaAbc2k6Q_5(>SqK!TXoDVAfiyA zlP_uU57Z4lu@hq!30OHzZ$J!sEk;i1xL^6>Lz}~SFrUfK&9idD%;VK^bG_(li~JO; zQQ_HOi+J5?v*p{5b8 zRp%ys^Q8*K7DAJNOUB}OkL&+D{XOoWz}8XI;Y45jHW0$hL<~-Qrb=ukyoD&9WbI=n za=%4q1IgmBD1S%Pq+!s=cr+-BLPBIP(;Eu9)5<%b@cBaxdD$*69IpPT7WN|asfb*H zIE5UmFpRCv`W=*3dyvQ#041&$>Dev*Fv?fA+}!s*&b(gq@lS7f-(xL|KKXYDFZ{<% z9t3>PiL4)wtZ(96gBVMEu*dMS{bsd8Bsva2ZE#|Q*vo|<9?lrQFqf5_FriSZnP~!X zVWJ+yu<>pU6WWgPEpc?qfzunT(N^(74D77a-90FXPE2L(=5F*c(O% z3nPe;1ic<(9B%*F2*XU7!)N(*2l^^}sePbgQyzE}H(wZ-<+jt@b$j#SR0^IVAC2W) zErkh^$j_Bk>Ql8cnt)LN-*G*+SLo%ZWXt|Idxs9$(g3IYz)>6Jfujs6c^z+U|8L_-B$r5^F`J*KC?qt_}cy$Zknlgx-7MqG? zOp=l877Hm>BG4XzZHLJ4gXA$%zp6Ar%F1_1NNi8o&=wanr!AD@#vUcc5sAHAXwX+# z)KfuS&r3Fq;>J`7E&o8FXrRp zc(vkaJ0wnTyGG~Q=Za(~iiF?b6PXgS zn1GE51toFQ<-%*%-BT%WyubV+1j?mSt%TlP`6(V4D1NL3H-edVXV=~_TLbl+;dGu^ zDZerbiz-g>n;z)9JqkzM(&mWL#JauZ2-Zo3md>%g-qHMZ8*)pKJ4 zZcvikX(9CfY0h;`?{WGwP_Cuqf<^T@o(DM=OlFk&NbO?xy;noiHpw_%yY%f4o6^J^ z8OJ*_7K~1n$|Gjst}=VkU~jw~+y)7{O?|i54S1(K+#c^0*BU*a_BiwF3>1 zR%{~+EHpP8+Ld2c^gQg;24Dm~k4(~7RWI-q|6BSmHl?e%9rqtiQaoTPn@MG#sRx1i z5;$&hGj6s7)Th4CSXXU(VS`3fWR3!ZZToAUMyc-4azhlZHp7m#k&hy2=1WtW``T*y z3sxTBDsHbExk=H>(SS6X^@oT~Cs=Zs!V6kgx&ID{gWCr9c_l}?De`*L7XUQm zipaZPSo^vyQT*=6ur04Hp45Lo^Ch-q6`Yl9d|mVOkU4xb3`opH^0yysryFTYQ~&!- zyb*D{Uo(Q9+~nd}P}PBbkh<$TJjVkNhEv$(uv@O@#n>B8-a_0$&p*y|f<-y#KdzQD z5qTH7o!$lE0f^rf=ILT0?QMD@{vI|WfQ;a|EYLqR#4z#SLCg6(1TE&IUSea5cip33`&0@ln?LGbh`Mk8b+XaH73 z!+f1DzPs|vX_XFXr)36j+#6{x#EQ$xo7MHAr)89)k9#ClzQR7w<$j|NSRN=!IFubm zj*n}dY8pC-gyZCIuWi?i(xVhFEa$TK5iIsIKS;23u^ECMjdBwvIrXVW`Y&7{U}LoE*Hhcfx_5Qe z0znAc+Q(@anhH?9BO9kzdL5_8yQGJ=9kQ-jfPB9nC@{Q}V)NY2a8pmpQ?=LncngI0 zB^*8nI%+1wMx;CZEgSD#M??p9S|Nu+0*_M*eW|~_@Z7dQ`@x#W_M_p-Wsjy1jdEzZ z=k8LGgE9k;Ix~r9;{Gl;_#cCLF{gHqCDJr6{!w(rRf5oo|aa_iRVn#Y1oj|IbgX!^XWEVcO!Q2}F=Dh^Gcv?6Zxnx%(&P67Sz`uE5BF4{@nS{GK=X}o2iGMM|J7hg$nCg z$k`g)zgMFeI%QDC02oenk(NP827KQU57@PQ3&d1yps1{DV$RH6saNTl2MII;FVGrK z=%^jtJvfZnO}DrK8V4XF$RvPrsJ-jG6RN_y6jkw2I2U+F&35v;KV}O%0F>3g)L+2- zHYN~6ENl7ti_Npu&V>y<{OnK5{r}d{vQZPX340zcl>xF`J=wQEwzhH~d5*G=R7dmqtV@^6sQJ}=Gq@1PYXn0yov>*ggve<`g>4=&jpJ#NIfY_-0J@M(ROt9 z{xZu4H>t5E(1swAZ>%!Q$CDcZraH`f!)|(-=pJlq8RNv9H%@8ulHqWqu6S|*8=~*^ z%#nC{{>guWjEx2SH+6b%x!k{ez-xqa8rzBmG|*l#+Y}q4J&7nk^>T0y#aBvKX7g|l z9W07{!^!~Hxh71K1CQnSxNJSL+7kohSFY2nC)p)kS3TAoyxg z3t*Gmkm>it@K{W96{vm@;S&#L-!tOVY_8>5AQn|Cw@5?HiytY`ho`BM!KX}?z?YA~e5(7cum*Q?@sW!8o_z-!| zetA(6YBjsDVQ~@sU_-+mzCw4hK3M-mT%~w-f7eFh(#?9J;MFWcJSGk9c9QnK4v^V$>;4;d^C@> zX$bOXC1{R7P=}u7MmHQPol#v*jcZz&^IrO0VT@3spDcRFgI^1ampOc%Uo&OkQMFHK zg-h0arZ~&qhtE7{J36MDmL1l}B8rvXj{tr$IX*cadCGpH+`6g4{03)#M%4#`34?^Vi|>O2LOlOzyEb`Sl;`-kn3)cbu*l#;Vm_`PS+JqRc z0laR~z4KowiGos(4?Qk2S5X6KU5N)6lv3Pg@cyG!!Dn+_ulHPKj#BRNtwhExm~dTK z`98-}k19*p4E_$xr#_J9fzhz?ep63t^ybE>lhZsAVSrOT9*OK4#lGDp?4vxw*mjhSgR)4^4ZuoS(|$ zPBoLfR<@?OTY;*1ZtLjJKhF0xTgaQe(Nn(Cv09z(H7cYv)wNTa^-HSls8xHeD?ki8 zD{^dOW&}YLw^b1qghrTR5!1n7v#RZiZqq3$>H`nt4 zWfpF>snw)J&pG`et;M6H|DDMJX>7k4zMfk~nL8MxBg`|lkPZAn8>T991P!v)vwW!4 z1BwyD8Q92EBky`!DR1g@YyMfK=sf_Ku#XHd;TQ^aI#}d!_5Uq32fFIZTR#tx9$vN# zBuu{h)G{(T_#K2i##G{cbaXEE#_mKGBXo)J5VRG`_=ddBke3u!kuC4l={K zY1XPxh4H^)^sz+H7pV#GkQp;W%&M3=g8}U+Ob|`wzt{Fa?MPmYqyHEsc`M7%m_Wzj zYC9InEG~hj-bmg8k6ZLh)%Q5xrVZEsYXY{?^W=8)q>GBQuCOM32+kn(c#RPO9#9yb z2XKE5>`qM36K7R@8QM_Qx~oLW>P7~t#HOrCpyLLRbD}ga3VF~OJ(=TS2kVLdYv-?| z;41%i^zj|n?0q#-RAYuUIY#RvnAlC!MP*Mjxmb#L!Ln!=6HqvqSnfF~)f04m(>F0Ue zc9`w3`Dg@Yt*`zf4KSb^jns2P-NJ3Y@7TiO2W%b>M-w*XHG=_o051m14zm zQZZFW8y7O$EN&n!n!Dn2b8*pYouF4Gui6F~Qo3N4SR{uXW~f3_F8N_~AQd80#cG39 zgL!pT1xX=PfXn&8Yq5WHH{Pl14ZnpXKCBY7Y>?|G&HfI++-VS`%EIQ8@n~Rm%qrFJ z;{XSGRv&J29t?HRURs$1XZg0--MY!3@GR8C-V;m{5Qtqd7|2qhlmADRxH!;)uEPg@ zem_C1WKfbzRI+=j5}{HW@I3_#fSoE{h%6o@3UHCEWT!34GOPaJNlwUERBZwLE}(Stlm;z!g2X0##Oi!g-|7(DrS^U{5E0~u1s z4o08mC@54QuZMnUmoWi6(oad=;H>v=QncIZA@l9ao$%`;eN=%snABY$KBi;4Y)k}f zZBjlRUF)Qc(`>lvh!dwg){=o`IB6Qe&1+GP1UcUs;UnwyCBW!@;RVvaSCLagFN+*Y z4pOr6oe{BVy7Hk5glQ(zuD8I&*TgzBSikRzyeAki1G0y+{F>z%;>c8~7?Ep3FmJbF z2vhu<$?;hwz4t0^BozOsusrQhWs3nQAL1*dEPp`CunCwc#!H4#DN*HvGy@fWlwtqO ztiR1>N76XMcg%z3r>|ygBJeK)tv`ei*`cr=%R8gR3{|r#o^Fo@nQ9bM8B80Na&oyx z!atVMDobKo^P4*Qo`L*K`{qr$rK6|LAZ)9aoPk)^pUwVvu1xop*Gw8uf>NH6Ovvj7 zeHzcklTShtReRjh55~y+d^ma z3pvM~=>xUxti1#r^|iD47D>9sgQ1mWacD?=o`aK5Q^#5!cDjIGp;k0#`w5->?x~t1 z%@K_qPHii=R%USn@16<;w`9xn7Wyg4^GetaG5V{31v{~6NKLkFeC%crLgsNTE|sOMism`)(e}K5PudTvrV>Aob zca~#BG{)%G!gY=)%}W~-4;PWsQo%Y^FhO~}mH#$+#DBoD_}Fb$Vos(dOaZDX;x9| zdntJMPnQ~%PO)F*&8Go32KP5EWVgklQ?FT;j;#T8$h)2xG(Y(SlWI~@xOyMmb)c3N8pX7jt)AoI1eBVlsw<}G&gc|A3!9sA6^z8y+_K!z4X4A>*~gh zep3Ck%H@?J4TEF1OaEm|Oiu-vXvS)BnKSDhQ<}*f)$we+?E76RY~)+2hCMBy`ar@x zz$OEl{V>1ySBVm$&b>6o6wqW366spEt9}_ujlHrS>aveq0TlZdGQ08O)8YWA-FR3d zpmqyCp4;cpuVqehXr~PH2{(h5lt0F=xL)vgLajV5?VNH4J=BRbQhEL%sAu>;I`|%O zJ&neFGBh?Arjk;GQh%~fvx%4uWf3a=Y`xde&P_uL>HOYphjoz{sWKc$55HU!s$_m93^ud_~|}Lpw&7YE$zm zA066KHjc3U5>?tk!zL#6gc%hmN=7#2GupZypqH?jS-Jv~x(QyKnd3|z716%G!}82O zEi$eJ#3ot51%AB@(PLqAP`rWni?fFK2$;bmM=V=RP@M0;`@&up=bZ0K>4?Xx(M#%^ z)CFju^5{84AI!zrx7Asg4uRq{M`O8m%-bO@!pEgWMz(#S+p*wB2 z!Jj2}NoDcwn`^y7)9w{x#v!xuzAV+-s@o=6<&VVd1!clD_4c$7#Usk{c|1VDqiC># z_y-?|r0hmNao1y7u+c`B|8UtH?xi}kY18U@PdVo6FqGJWw^NgVM|B*~LPef9!)!Z@ zaqGu!XF3o*cJEQ})LZ#fUd#S;o2>O8`JnLRChxaw;U`X=N#56|dxwwh21J)m9;bN} zS$CxhIO+k9s=+jOO%>X-MSY=G7MXo~o58W}4RZ1oWY(8>%~*j4g;)#4!r2nydiOfM zQTo0Cw}{cf@Awq=U*rhx`UN@vbQLzXFG)Yw4xpvWPih_b66R+|=Ktrv^zDOJvHL_H zFm65O*deg;(b~gfwa2EI$OGUnll^@4#H<85xLk113tY1@IK#BM$V6Xc_IuCQih-@M zt5Lf+@MsifI9BR4Ca(d0MOvI`^*8h|?8`Qo^H5g>^yfeAWZ=_F=$!#uKf7JArGi19 vXSTpS^y$sst@8n&{-6I5b<887p2hp{r1zn39nr(kZ~vvK_h;S}ySx7f1Ep=9 literal 0 HcmV?d00001 diff --git a/docs/metrics.md b/docs/metrics.md index a69d6c936d..3421539af3 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -31,18 +31,20 @@ Default metrics | Type | Name | Labels | Description | | :--- | :---- | :---- | :---- | | gauge | ovms_streams | name,version | Number of OpenVINO execution streams | -| gauge | ovms_current_requests | name,version | Number of inference requests currently in process | +| gauge | ovms_current_requests | name,version | Number of requests being currently processed by the model server | | counter | ovms_requests_success | api,interface,method,name,version | Number of successful requests to a model or a DAG. | | counter | ovms_requests_fail | api,interface,method,name,version | Number of failed requests to a model or a DAG. | | histogram | ovms_request_time_us | interface,name,version | Processing time of requests to a model or a DAG. | | histogram | ovms_inference_time_us | name,version | Inference execution time in the OpenVINO backend. | -| histogram | ovms_wait_for_infer_req_time_us | name,version | Request waiting time in the scheduling queue. | +| histogram | ovms_wait_for_infer_req_time_us | name,version | Request waiting time in the scheduling queue. Indicates how long the request has to wait before required resources are assigned to it. | Optional metrics | Type | Name | Labels | Description | | :--- | :---- | :---- | :---- | | gauge | ovms_infer_req_queue_size | name,version | Inference request queue size (nireq). | -| gauge | ovms_infer_req_active | name,version | Number of currently consumed inference request from the processing queue. | +| gauge | ovms_infer_req_active | name,version | Number of currently consumed inference requests from the processing queue that are now either in the data loading or inference process. | + +> **Note**: While `ovms_current_requests` and `ovms_infer_req_active` both indicate how much resources are engaged in the requests processing, they are quite distinct. A request is counted in `ovms_current_requests` metric starting as soon as it's received by the server and stays there until the response is sent back to the user. The `ovms_infer_req_active` counter informs about the number of OpenVINO Infer Requests that are bound to user requests and are either loading the data or already running inference. Labels description | Name | Values | Description | @@ -214,3 +216,21 @@ DAG metrics The remaining metrics track the execution for the individual models in the pipeline separately. It means that each request to the DAG pipeline will update also the metrics for all individual models used as the execution nodes. + +## Visualize with Grafana + +With server metrics being scraped by [Prometheus](https://prometheus.io/) it is possible to integrate [Grafana](https://grafana.com/) to visualize them on the dashboards. Once you have Grafana configured with Prometheus as a data source, you can create your own dashboard or import one. + +In OpenVINO Model Server repository you can find [grafana_dashboard.json](https://github.com/openvinotoolkit/model_server/blob/develop/deploy/grafana_dashboard.json) file that can be used to visualize per model metrics like: +- Throughput [RPS] - number of requests being processed by the model per second. +- Mean Latency [ms] - latency averaged across all requests processed by the model in a certain timeframe. +- Latency Quantile [ms] - value of latency for quantiles [0.75, 0.90, 0.99], meaning the latency that has NOT been exceeded by 75%, 90% and 99% of the requests. +- Latency Distribution [%] - distribution of the latencies across the buckets. +- Mean Inference Time [ms] - time of inference execution, averaged across all requests processed by the model in a certain timeframe. +- Mean Time of Request Waiting For Inference [ms] - time of a request waiting for the inference execution, averaged across all requests processed by the model in a certain timeframe. +- Currently Processed Requests - Number of requests being currently processed by the model server. + +The dashboard works with three variables: `model_name`, `model_version` and `interface` that determine the model instance and interface (gRPC or REST) of interest. The `interface` value is ignored for panels with: `Mean Inference Time`, `Mean Time of Request Waiting For Inference`, `Currently Processed Requests` as they concern only backend performance and are interface agnostic. + +![Service Performance Metrics](service_performance.png) +![Backend Performance Metrics](backend_performance.png) \ No newline at end of file diff --git a/docs/service_performance.png b/docs/service_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..a680dce4d084f0f9cd7edb1d2f9857205d3b543a GIT binary patch literal 94181 zcmdpeXH-*L*KX|6u>jIRQACR*SAioRy~)>rKNp-e`LkzV!G2P`*CQ&>|r_v4GUxVN?@^ZHlCjF62D z4UG-`&j%P8lpFf11_}_S$Te-Rt+P(H4cX&&Tavk{r&P@5f8?b#-%)e=f?cc zF$lzC)(=bj_{FLCSz>|F_5U0<-e>gm!P`*%=ZHV!@inCAg?|qmO!}c2F8})x(!zxO z&!-=@Z=?s-U@Hm^X*avr(LpSlC~c$p30rtnB758Y4>^u48UCgDsAAYjdvmTx$WeG> zoglMSo#1`4{h;vsYdFyA5Qu+&2rUf3UCUG9ywng%$CB7kg2A^NJLAkJU6B>jqNE{)5XT`?;2|aI{66p{^PT^)=^MbC z!Pkr=di8Ge)Adf4`ZtHd^>3;isGB~$WWGWOkBt&Mk-Dp&WUlxf5F1_LrAauK=CF7= z_#eZF(!j*HZq8|@C|7&N#?h(zJcqJ4 zwJ)-2sZq{U#pWyhow=S-I@7ftGNiO?$mUlFi^TTe=yox+?$+>*yMmkXx&!(h4IZ+p z$U1ckGm#oejT{a=G(qKD^!;md(aa6zq|!RU9Fu#7T__IafKYsdS=VW)*)x-q9W!)4 zl8xu5PW{?v>9+@AJ;T^#=7O=JOW6j9O_k3r;$+Gyf?_RZDu*F6Sl&frmml*QLOLy~ zYOD1FT!NkO+Tt^lX)fb9RGRatpI0iK8VMdEi@j< zMx&4->?X5^^p@p9s-OuRf6pNcL^4)xQdfSfBeOp!zKuf~@o3BxR;`FN%193^Gb&Q2 z-HspNEq0 z@U@tyK3LohMyjgqOj`<_PtiXe?Tfa)dPQ8JZ>RKE$;n;iy)!1wJSe*`#`~=a_X_^xJq9~XA;)1V4mJ+tAx`p>fPR0QCZEw!k_+N$H1Qn*hIuS!o&z10FqD(23pOGPD?aTADH#@d{ ztlWU7)z8Um_(q>LQ>dY&!#E3uvXkW?GNz;3!exomGZYdGqg>=jVc9AE7vj*&oZkp&J#bV^b@N1W-> zrm^m=`*j?+uMLoK-jkM5nXoI`O}O^9s}esKr4k+HN2Za+Uedi5p)y=F zg6EY~Q*2)&`4dd3@!5l#zXH8{nWLH;b7qLS&N%t2BNVJU-xN%;R~)>v40G*JIJuRz^Q5^=s{_Pvoho8#rwoI~Hgyp`KX3ss1przc3i}X3Q4Ob4#BH38QPz6~e zIIRXZH5`g1KJhG)@fB6nY4Mg#3b%{t_8!aHW}aL*`04aL&>hGLiuy+S9OB)jabxl& zojHR0$~jYM)WpWIxuzo7>Zi)yJ9Z_2zYE=4A15rY0)w%?u2; z?UkqDa12ow``gVJo2>F@SEAKsEpbJkd@~L)k5-&&YTF4E`7`o_w^$uwhmRL{7Rz}# zP2M7K)5MNmdFvvm;SpY2sj9q{wdB>+NllGg9D2ne`?WQ$Q*XW>6Uky6N*h zuUm+d7rp^&uaZA%YuQcEQB!LyF7J5bT|jM+4h!;OX)Rh<;#o!Z zgJPy462FdE zBj}7b^-$EXJs$-(I+1%pjL0#%@tSP{if6{Lx{UM=94OqyFch!WEtv4Yw6R0AtYF?s zgv~zR{!8sqa?cwpZ&|c~bTq+z?7D?QL%Y2FXtp|-_kKs*^R+!qe=TP|z3-5@wH7Wt zkYbhaa&DR-erMTu?aik7lYMnBqun6Ns3&iYez_6gkZgF<;0d=HXJ7i+*wMpk{02TF=h+%M=w5HV>)${5SgCRHqcr>B+)1Z#Exl2*xRsoV z)ydgLO@?K5P1b__nP!EOW2_1*U90t)7!ixrNp$1VOnp!0cU9RFLujs{Al$6!Ux3F( z3^j%E-wEEt*HSR(^;k@|s@9M&X*&|II_tYj4mc!FKE5?-0jM}e*YEvI~ z(Ig4fq;8~`$nP(EG|<^RC5L@i)^K*@c{-Q2J^x{Rg<;KE_GmxlFR1~Y*6F0F`fCCq z&f+$e4z_?P(ccwR(5uHq87W?GvU;r;-iqVTP9|IpNT%n*1~tn+43u1jEAbdR{_%0N zC?ggt$}x5{eR#JdIU|p{Ve>>G5U2ghb~?TKFyW2VKqqJWX)3f>I0LYm4S+K;=BF|O zbCH%o>z>>6Lhjp$6tB$;tB9`4QfglLhNzg)QpLl(xvA1>+ z4g963QWoifrY>bQvl~Fj%%cbb;N#HZ(K;!H2~Y>NDtsFxKu29<6|`*MH<5!r}X??&IiQ`M$);k?&w7PcNO6 zNdHOxCBPCacVGhh*ec(;QGAfnle2l0wY$gkYieOn1aV)4Pg8L$J1W_w=YIrzKt>)k zFOQ!r4Vt&|V$7=xJ~EPI_{c(4ZN)8vxh1k&#YFLNVse%g+E>4~M{}gzQ8jgo#P}8& zG_(18_u0|K7zvIJ>hytp&;eOim(b7IE}GD z1IL;#E+8Fu-*jfsxvoge;`nGnt|PLdeXqTFx`g(`t18KI_b&!#J)@&?*`p2kIn+#8 zcv~shT^L`|*&M*RR*SYk7o48<-OwLxYNmj-|7+t({K+KGiuq^4tLsA0;`&X4zE$cK zpHw}x_3TLK$-LXZ=5ogrGQMp_zI5$~J<*oV@02q_SLm~mcP@peqytXwcGa{ z^`=8|0a9Ji4tfb1)q|#pfPt*hyEjK-X;gK|HcI0 z2S*M>*4|K?lCd_P&1?7_Remuqb~pOQZHB<}Z2ET^(auFDEqneRb&KdMTiSDLU+jrc z&aM}fMoACobj<9)M+>=cFbX*op*|0e3~oM-p+^Im*1omHW)j=3UGIYVG?NqP-cu@2 zGA1kJj%H@VzmeJ0XziKmsR-%`8=#q>|V;k%>tFJkD&rJ%`b{}ucm1rh@>KDh~eHLk=U2%qNAbE*kUi4>iuR_4lG~7{%1dB^TfsSz4P3h zqun@7Jy+8Q>)&1bF-NBsFHLeyO3g#jW>e+r$`S9TmlK@$#i`?>qcvP!({DhL1sghd z%>kddozPHPbo>yX*p|eT?wRNraS%exQMNY%?mn=4SS=_ zUN6&6iBKJrn=m!qmZx^Kr_?)JG4vgw9oz-n}{xf)$C zp+(7-xbbx+tdzhUAU^89Gd%e!&;`xwsHsLbS|KKua>&4CbCLRW<$p&Qz2Br9I-bf_> z%DhtlzZvrX4*d~cmthHVPM!T@zN-P7U7g_xD8aoXs(9kKHJEv=C*ALcHP})@smW|b z|BvQHuK!|Kx?spZ97_7`&p7ZybKXnFEzyB z!+ccSDfj8!-~5NfQ6B%dt)UY9FFFiTdhfqJ{jmKA|5JUz|GmQo|HIPxm)?N=Svm03 zZ+pZAcjsO!-G;^JsBSpPi*G5J4yF#|%W0gxK0Yx3PhylA*RhTWr{ zbY9Ww8ZJqDkyF^8&>>pAdv#;StiT8JcK1+-KUMgih(lWTZ0_z|9@8;Jp)s?Kvo_Zx z*MC*^#||%MSZ?%(4Eb;VH21`{MZY~*a{o0WOEF~>)yn>b;%ZH)%b}t*Pk0()P%|JU zXw36b$2!yKQ`Ayx^|WhunEbDl`j(!ZhN-ha5!~NY7%1I3J4kvU=vVqA`!>D4V8haO zk!Mimg3$=T#*E=l&4b`8a2xuPaAVhYVk2&3x%`8t3|n6)ShG4j@H!`d(>;I~CK8+z zx?Ytu8G37=6JxissJXO|r|sA&UEz{)GQpyc9i=X$%3qkFo?N^`IZjGp;sc9lI;Ax| zoDRx%!}tn~J$T-vwpXz9$co0>u3FaKkmOw^cmdbc}=DTV0$=b)TB1V3_R|sh6`e$@`vhm!*tUjNrF$c7eRZx?| zW)C6sT5)via=lTMCO7So@8RnzR`hgGQtMZp3zpGuN(P{JV~GU2F(*SA?}5SGg}Pv0 zXLIt2Pt2Tz`Pj|HYF4+;_I1(z&jvKVm1P>|yP|JX<8ZI)F1?juN3JC4n}SD(3QE>@Bmmds4Rtfl`ZVL(Nf>83?s@Wr{iEs+*uSC2 zgv1@Bz{P1!KG9XZ=NqYa|MlaRdkrSw;9g_Lay`I=z(j2``2CimTrCDIf`V1r=lg#Q z`5ylcwwe%lwJWkbUn*|VmqASnn{yVWNlKjq8Un(hXWfe1BB%FDy;n0oXakCYpTsoHFRW^ zpfKrSJBTU2kvIA_CL`;H#g7j%*Ar-bp0%j4V~(9{qd<>P+10X|D-5;_3Dr`dnwgX4|(hYd+ZiD@4`~2q@-2Zw{!jW~6mm_`?9{r2*P~nbsB(_jZm(wY@}VK{+q8DJXQK{*hs0srF+8 z5qnk-&=urmN~4;aJ#Bh++@uD^q0rQIIGoDdB)P^eCTABrJ$Ym={#A1w9-{ke%-((< z342vtZ>-@3GeF%f$gtcMO{QA78M7<|d79E7)fmXzMG*@u9X>Y0|q*`KbiZ4IyY)3++ZtNAGH4=l@Vv z1D#l>xbr(?Tog{l-Cej`R%-bxhMxt2Nxef-5sC}fTSMBDka=;?xD~hthq?llU%Xj! zFEwPda%`|hDLK7t3InxTKT>h!$bwpwwgIX472nssBzFbX0w1G>yLrYnQ+9jLkPnkB zp6+c#TA>fKhTOOnO=nGQ=2@4tdDENjUfZI6Su#>ExZ44MWK8B^ozF(*5pVxio2!OR zas24u2r2u<+U}0X+kS@S8s3b&p}j`ZYTr4NT6hPFG5PlFgLiA-2IKk5A3NTfk*C2? z9l;Qfny>lJxgj0+~asKga}92(_>w(d;&9l2gEHM_hxFH~L6 zYPuq&mZRt3{hid2sr>x!;YrFI!I@*v4W&Z2@jA*{Qj-FK1_m6~{7-jH?gqN_UNe!z5 zx)ix#bb#qxy@aco^A z)BwNrJIx^o{a-dGlS(fv%Fgz3$=3q@`4uzlDu%{3Gn86DyWZk2S1zmYc{C zss4a?lw<4i-@;1&OBhjxW%W$c3lgZA-eDP^i|GOL-(RZT4EXzs{ttl!>GkV|Gdg3F zMEsvip^*PvihM~5ZH-<|VF%3OA1hzloulm7t<=XOST_wf{FPXVIViA~#-_yf?DXRB z0Q^KN>`SR#0?om-T5k{oy9F7fXeNTPDO_MWFbJkiwfYwAkYRazVi>SIne}$7N>L+b zJDct^(t)b9bG6mb+`-Q_dCeqpx53qTl9On|l{jjuc;~pQhjT5Zh3SZ~B<@prQGSuZ z=4@V^`p1iqhW@^DO)uuxkJsNF5Z4>FI2?4fEXF^;;eO5JHL8p2uehGB8e)5lZWxAO z##d=DDi8_XI&xTlRDiATqv_sEmoz&)*N7Nflx@iS+mPW_Y@4BZUsC>wqio2SsmMmh zc=vntkMQEjfMp^c)bTVSbY6JMo4%H==6%nip?SVR$D@Q~W1+!{{%`I%ip9=&2^aCK z&=WptKS3n>fTQNxdhE)4!)q}{d7W_nh}q&ydI&@suo2*V62u*3qRGa3wx=&E%@6hT z&kS6sGc0-j8qe&KMG`>J98^+jD=F-O^lD03ZpiV6fDNItt<5#Es@QF!inGk@Y}*J%A{_YkO`HG+Mw*N3~1JVqOpS%a0Z z(x!}Y#|EY&L5En}5todYtU;U6@m^K!aWI{Jad5jw#)9th`b!$!rn&l5$+l@6_Dy>--{ewAS`o_GBU#KmY|YM?)>`Le%1 z0BuzhF07Qha!Sr(Qy+PVupg1-3%8R3vWF13b%XNN01Au8wS#o8alv%cHP37u<{rsw z`A@h=ZSqSG#ZB!tsk|^_ySTSob5?za?KM|F6Q|vc z-IrUd+&$7!0Q|Pnex$bG@lJ44xubhe`9_UCzquVrVZSoHV-tm0je|QMOY?q!6_nQA zQqh^y`qY1Xav17&I4c|2oB_( z$hpimZLf^Hu@9Qi8tg}HavfSf!vIu3R>g#I^Fe<7F}6Q&*++)RxKP*hT83qg_OqH^ z0?oU&#!upE38B-r5f+E}vqR)KbhKPK4L3IQbi*#Bz9}|bYTza{?1*US=abw!u5ia} zn}BseBDW*d;QfjpY-^~xz}ykbu5XoH_aB;0d&u-Sl}`37IC6@3a!bfo`u zI>>MOd0qEivO24*d|f$)y->$)waqHX+vd=TTD;QW5&f=qP02e0oAN8;ZyR;952ZEM zo){hvRvOskYWEy0xb;zMqN)oP;<`?W+TYsVX29wFRdR2v93ZD{*H84<(!`kfsmF$F zvurGE4UdEq_^k?IZn9M?d5cduu^B6eNcSq^ITtC0Dokf*5;_GjWPmc5JEn^TwxoeH zW|OtacbYqQ8R8O`C#?n~e~GFIJ8rDYNB?5NuM^~D_pA6Wsd~{`z9}TQklLZ=sdv(d z<2LMD8h`S$R=%=Hn|8WJkS)Wqh{)0$2_20ZYZh3`H+fvz(meM3Srp=ah3~w+P%)uV zb^nztdSb!L5QQ&V=Y=Km0bTXPpzEWTUb=4Dv=PboCT09OF-4`B)tN?QLi#vezbppW zl96>q&^!KW66kc{Qp!)1+I@|~-|C*N>jbNV#%Q5FnlOb{QMY8br_%^V`uu=A&`HpcpJ4{;A5>6N0iFpbEIkQl2cQvj~R<7|?D z7U7XOQ!)Z^wB4z$a|)jyVcj?BbES#HWcgrqlW z+<B(u4}9YHjtGw>Kbk(TzL3 zKdl@G#`&LjRHtap;JZqO!X091D!JCu^DL!Ze210W@x{~jDxZgo2S*rwjx!MIiNA16 z-i@#~l|tpo1aj#y8`6na*$dyHH%I`>raqh)ESr!TU?wMnybHjlThKyzU}<%D)Wq+a zjp`2Lx$WOIdyxvoLB~VuEwK~3VVp1XI`OE{y{&GpBjFl0xoI0{L`dJ%ei!n5af0Eb z&;srYOnp@dSEZt|T_fJ-A}x-e>s1T!-89`}#umUfx{|X?a!vKpdnEKWtlc5rv2T0Q z7;U}ZUiE$kZF*lrpZ`973jjeD9@7|B^Rop(^B$=+$F+uhp9!3ai@55OmHFH-`WI=| zYTdl;Ptx?MQz>(E+`_~JN0Whj&vyPGBd1`>Ec=j@1LmyUG9HF$S79J&PS@BZ!4?;_J@RZ`u#L=}EnruD9&2OH(_w0?|p?6%|UaVM?C+U8E3B-yNJ3!P}Wh@6Ky&J3;SLjbXUSTkPk3z-(}E#B5! z!}pa?ly=e{xzV}~7z<@$3TyYeT}0?Yi|-}drjtrRkDe%3`SM_zQxVN?Jq}2zVUB!e zcEX-oZWvs1NZmQQ9JfQZBY3G z$*3Rii)wRutV^Q8Tr-`$rXtQgUwFh|vnbZEy2rp_bit6)gquHPzx#>nGuv>r^Lpj^ z2%F7x1@e~?ux1H$&)`BO!4Im7%Qv^F%-%RlTHf{{gGY|+iE=v9$DV|nBqa50iDC47 zb>`|`9;1ZiUT3Ua@jOk3nsk|G4H~^BjW8<_e^ab&hA`jiKD{(_ag^)%J)+QyDB-=R zpkn)^Hh8H+e&(0umt({Rx2{8DGd&HL4x>3y>l!1Ca*gq_A2ctW#}W)s3QU?~x#RkG zkH$n3!{>tSF|`v0+rJ2avK_XwSTxq37Hf30uI@SdR%K)I720~Er)#EJhSE-ieWkcz z3;XTTj(4GtT2!n>zP0Gy_=SRU2E${Q9)b9NTK<)!wXBgO7jl?!{FCRD4x-Za_8z5dwi!MVLWW)BdUIH4!Jab3O? zFnCQ!-@(-FF-3E#B>5#{=Q-+G;VxY*9nSsDEU?_FseL>L&n_rEedn2c-f8+Noqw?| z{Q7y&!Ubje@?MjHBS_UhCJH9e1Wn1(ScuJFmWD|1pi*Af?wl=wCTS@~Y_Az^61}g}( zXLU{cC}5|*qX|$=!0o5+3eSfEjYW2?**`Epa_+@B2nYL(uIpeH0N<5-1j{S<%4z%~ z3uuQFJ^yK6%3N8&l>^UDXwaOeRp4z(1Mv3G&PIv`nzy{yK`~O%FxF2;7YxU2UKd}z|+jXc#1i~9Gol_()mHDuv_UB}h|KG&^ z51KzLz61~XA8!A}->t90)jucvZU5%-yCnYf#7-oC=zJ#XzfY7f|JgoZx%B76SN=ac z2>xb&_tbTPe>M>!iT~bOK>dBKFTe&aIE();lt72}Mq+ue{oVZLX7ViTtZEffJ}e?B z+(PwNzT9GqiDXXE5G`w=mpCo|IWVC_2-oS@h2+ex7`c1NG$Zx|mr!FI&rg!ZSmhNP+JTqejaKyY<8g za(kXT$6?Jfcd9r|X(LXJn9*>wMf@|@{9gSi_1gH>ffB~uI6Ba1K52Tu6LZ$ zR32H+px#I=_%37LiRQI%99D?(G}d1bTlU-`GKTo;f|EV&FXDFI$#z0(QS&V^R8r*< z@>`1SujmPw1O8Zp54WbJ?QIc|DLsIA{G94`?guOKB<@6^8>^`WzSV^-7CDF}$<4`i$M#?Tm>(S)A`f-&7x?^#@sT4XCoMfbYwz)fhTU zsITk;ZqG5%{bth@B~@c_VCDQR!BD!G<8IMOjMMr`P5(uQ5t8{=ilNqt28vj+FCzqt zx()G%%K%ID8(1oiI~r`#mYngQLrfy|BhIvG+cI4A&$$-Beap*{&?g1`e!ns=ufxsBR zsVQsu6iD?i@E2zPJVj+Ytg&j;o`hI^%_BG)l)^djuiTs6Dc71iYamWTY7k!41rwkK zm?3_MV05$eg?w081)uIZ!J-KziTZ43LrgFiA+YP9U=;-5YQ%BhG&iF@yNf#R{owo1 z2q95lqZGMZ{N%K7`IL23O{%Zoj4a5WJKuRhs0ESO2bTh7T-D6MAY|C>;45cQdKN_ebtriR ziW4bKWZqq8osTXT6uU2b&q5uL)c~g8A8g1<=ldct+zZ!R$MLccHGus(2v8Ys025Bn z%g0wZP4H2BwUUB?wEVmfC(=9~=;RauQ`Nc-^=j3oc0M6xyCV<_Fk^%F!R(5{b!$1h zEGw#JKc_`}6CZWHvvDD*(5K`Ag7MZ}# z3zwf|qDX=Df^AHM7qw21Pe^I> zJn#p>Qo(}b(VaGHg`*@OG6ca8)-AyElvX`!f6-!OBjEG0pG?2;P}<)A>Y2vE%CAOJ zO+YPS(47@N_ojUM^^3BK;IBy4;QK(PMJ6O)f&2i=6q-ZZBu=$S@liZH22YRbEz_im zW4?o_@niOpM?L>X0$M#$$n)mw64k$+w^_>x>{dV^po*V0aZ6>GAoKA>-E6nr4vm&Y zc6)`umx;KEv})I!Fu>F=!98s3_3x@IQn*uX=Pk6cG{hf!eqn`&e*U30GY?%2#iEi_ z&W5j=Ue7gH6k=3A`cwlCl?4I`EQ%}?uUH%I#uz@NZnu@+Pr3-XObf0zH0p~w&)#xc zlCyCCoRcLL%mt*E2QxEgz7xVaw!`39El zPq8S+P$2yl^?G8iEw4Rurr2-EEvCOE30!s+^xQ$tkEv*ag{Hf^)j3{$7a;EgfP*hX zVnSmo4EK~;9lom`r;R$CXeqJ2C@rQI`2fsy_!_90@^z6YLZNosP$7K?(*$Ux46z)Z z<#wZ?GUwK#DG2du9=>b8%JsD_;Mo$tK`n&?%5FJyc5z0A7QNW8y$Si@2>54YGvF+5 z`xil@0%?}p{&xUYs}`q84fU(65nJa1{Ny?p=mk=vrOeXxA5)!l-JY7j%mb@S)gk`k zfLsZ@I&LY_TDdR1(yfPeYLLj!qU_>;p1J7>cmNtW{GZtjz|EhbZwi@@c0@S$-jNYl(#9L8L*?NX|AP%PBKoA5e zkS%;?2I`G8P{@=McSp(WrMSN8G%lG^YrSc#Azfz!2xoE-+V~5M&QRKf6B#i>JVA)X0)x&i;!l~#+(n3snh}M$fA>p>xB@x; z3@pmU?tJbLIjx>53M#X`!2Rk4O7g;2D}owsyCYLc+_8W!d`$zF;Q@cB@3z4$Y+8KG#nSpOLu{sA1W zXQ2xJ2#mf!5qz76iU@K50k8c5xb`eo(|J>sU#5U-@Z!&cX?!Aas&7k8vokHQNCzNP z%x1<(LRo8{A4xJ?r{PyC-t++oe9+*(1+`7~=tbl$fP!Zu=$B7Y=`SOC$I4^zS+s&fmA4e1Y==uv!TN1MYc&Ga zVhS@0II&$| zNRoULCY}^d@>u%*__;*p4X-PZNEl$GAKZv$I=&?baH`j`xV~N#wr!s}8ya^smIs1v zP708HAwH4*iQl{YBPY^qb{xF(hi*OO2k^WJ>LP$IUCeiX&^meXGofrS%RP48Bjbw@tSD$-0Wdo@YunWNb|0srZ!9Sr zkB!^-$^Z&)-o*mkCvbyOEPzkx@)tf;+?(3J4N12IyxHHq0RQ$^N=M&*2EbD6sDWDoa<}rv1DJc}3VTq10is3Lvjd`NBT>uVfslVNxE~z_uIKZX$I}XiD>R!6Ew&z_0_eh^)4bo`2RYP zlp9MReyAFZIO_>q z>Vs^lih{M7vfZQw+QiDEYl|)OO0(l?=6lh)=F3w(Joar@3cP>l@*Hl)n;mDS_0-Y2 z_taf!G+$R~_jAM-#LQOcKH!+*zP0t#tVHw8nXLwHkUFuZLeT=c$67FX&^)?zU_j&M z$BLV7;R$-=0HB8&PBzP`taO*6ng>K6&Q_oYt^b7gx7n#G>f=3GurFA5dMAX{w0Nk- z&RNLvrs2 z;n~KQ8dfeqB1OUF6w_WIF4SiOGLm&G8;dG#qG;Z;kC6PYG+>ElTzpymUJ~RPs%>5px+=ESPgd|=Jb6*%P;zU+82 zutQt=;Toni9o#8^gSbk}^eWu*GJKYzn=_`!iE$TIjl5{!lrNVG22YD??#8Fx3ftEs%uX$nMy1H{l!Y6H3~g#HK5za@1lXKu(HajSyo~Nqt_c;)?1h1sab)c z>eAc!CR2z7PZty)G9dI~4~Bq-Eyf9cDxArwJ+W5G21#cJGbv8X2UzcfQ9HZ@t!OQ9^1($? zF^7-8$dYJgm#{f^v{RkL)}GP;@K_#L`hw^viYPX+)F?c?Nj1Uoa4dbNir=q&9uEw1 zG+xmH75^cR>q*d8Z5Nz&1L!Zmzz`+7_j}ZKbx4tUDjIzw}`vH?bu^<&H8T)i|`g9Ezm;P zQ#-~0Hi!!dfv}?5CaOLMDFVmJtTy5O4aI4R&Pt?+hGhKkePCz(Tnd<)$uOy@>?3&6 zF<*$PJSILJLzU5+^;7H!RiB{{wO6mCi&18o1#Vh#2?dM_i$iedo&KV?zn;K_sr6$c zX>5+(xnHW(g{HwMN2E88f?`X~m0LCUuJiFa<~XdeX`T^ts@l8c)Nqal?Ut*5BZw=X znp@qa1(gNqBrKh_i4N35iS}~t2mFSKV(aFX`S*=$_x-+J#we3zHQRxBr2%XbMenn4 zhr%D7Arjcn1eaikxi-xl)Os>AYTUo2RPn+*909KNm*h#Q>iJQMnO_0vIaXwQ^o?rUAiRQ(8!4KKbL# z6xlb)OtC@BrC-|_a~+v&*1$8JDpWRelg3Sf7t%FB*_NRub!s$jzF8z!J|G;9sCX>* z&$N!#tQCNJcYA1oxl&(m~gc zH#_h6v+cdaN>f;E162ooukW$*o5(u`QWJ9EWTjP~njFnsGpc$#WyF zo6MZfMMs2H0t6qDyw?OD$AM)iYa7OVp7WUgw@Df%SJ z0bvE%1|Wie2qNTK4_T5EFIAGj#QBUDMGt$s_?3JSSN-p7n`uo0#TykL?IeJ+9={`h7T@U=2<%`367aHYgbc zmTaRUBD&*b$q%UT;(g z(@zf>rT~(Ii}AM~N(J+tDzta-My(}VX5TZ@00U2hN%{md7Gb_3j#~v0UF!81t2KI> z{9s%`(${1oXaRmRIDf&;xq5}vB@V_}fQs3yro2rp)MNtr3KAQsS(u(g+tUu>Ypxj? zX=CN=RdRs*=g!n9iy69L{ z1sUi!ul7mlIMZ-nMd91t;^xiA)^Cx-TpVM*Osa9cDWV%fm%1_otA>$#@0o71jly{$ z*|eYqXCf1OvMqMylLiM{fV^`rdK1WEpBcc0;~Q9!pjJ}8eut{JlybZ0yJ^kLh9Dc) zer{f0p-60bJf6xMX-7SIV>iCA0hKv#9+2pF1!O&#p4c!9083``L3)1b)4u6JBF@-a0jZ}fqXF2;lD+>DCOn}Ui=gzON78AF#6KD{utbz%xH?~9EAgkek z<%DxFJ*$PjtkoAVOZq^syeiTVj=#xvt|JJTpb~vl+t^QphnW@Xv@*Fv&a*l6nr$6FGI0 z@7h4{l7kp?@X|7l{*j3%3lI+mLTb6x8^b{4Vd=>9(X^|CxiUHiRd&#CU>6;Hy?(>0)+Gw7&KK^uP_;2Feqv)Ak26Nxez!U1!$=H%%7~3$lz5{v|&0F z=plIkuNkC$?%N1(J2JF)@t-jZLNMTdOl1!+EDsogAY)NvN#Q=@*k3NI*{FB@`f%f# zX@JS)@7z2<^M@)kF<+LcS|NZ)f=g#_z7nU z%I`asQL#Jkf`Y2Y4jsC^^+lbYC6l#LPu|Y{9Y3dEZ*Sb;OVQ6R5X3W*3XlaL)ma6& zi(-gL3;HQSWagA$fmI1mmzQkZc^YCa!PlBNe^7p06g*2wQcq(rXg-EvauWdh56h! zAmB^|L0?meAWLP^^%02A8=;B~t8kdO3=la(aK<0<4R;3|rRoEwoGGm2~|Solwxcu~cwz zK-x6-dF0(P;OA+o1K?>Nca=bv#LE{#ejCu8yqKGP(@zEHsxwJ$BTE4CK*O!a7!XDY z3Io|0m!bE*t-n45*H1|dtu+_f#h^O+vYBLm6nK7gX@jSk1rb%PT!jF?HbMXq(h&B{ z7f=8E;gW#NCptCoLIVm9czln8M5V zOH|(xg@hAOn-$y<%_-HJ+yHNisstIDq7%GxcI2E~z5U^Pb@gRk_@7K~d7r+qocgYqTQ8C^@ z05paY5M$#21n-v41Gct6R@H+D(2XU($;sbae=?th61mQ}L~^r>a}<6nvIGpg%mK0k zey($Gj|?XaQi;c>qOjjGgU0NRC8Rv>PdQ&4rU2!De#(*#c%m^4uZq}jeeQuR5?04kjQ(8@K??2(|^1>5c+a(shYV4ec&c%A=6-g^f%xkY`W z5yeLHsEBks3L**uQUWMa6p;?nf=VX@RC?$tV4(_vv;a~QnzV!tih}eSN+L~q2`z+P z?tbui-uIii|K6Ez=H`zx9tiu{&)RFR@>{KTpg4BpxYZ$OLwmfQ=whLaV3$&r$@-MZ z#8&Y$uFipr>Pp95&rEAS<%y_+XmkXk(dX$#TtEvTrBEz>+6C<85X1_5jPBd;&3CD! z747C_Q%JB+dE_nwGiU@9I^~6kGC$c>O5F#m0N<8eTqVhAH9v3@WK%vk~@2awxUQbHf& zu&*tQYosDsV5>Y3+6f)Bld~62*h`%$GL-4|e~*Ab{rVPx_BaNgU=VQeflT#8?$iy?*1$#dZ3L5nM(FHEUl-Y#L)&~CVa455(tOZpOSB!_*Wr)8Ptzk}0Skf#Ief6W!e&gWjI|i9^T-#v=Azs9NI#} zF!;1gp~|Bi{Su=5&H)v$4<-x5`!Pp(OX_%O_8i;ofQ$#CHv+{>o4t|y6@_qK zzmI}d%%D{yp;b(s>yNGX@>NJz0g=7oRgy@7ZmAw|d2;uB`K9f;*7V8)AAnkdX+nq2g|-}P7`~x3M!i4ZI^mXWQ_00C)gO1- z#N`^Sluv~3v*1oSf1KXalbuf}B`}{TAiodu16*0iO<_;(vR7C%yt2#?^5wY$Q4G=( z%%0SDV~13$Cw0y|r%e?^MI~xYM+=kyO{4g01)H{>$O+ntQWsbKHP+^SabS|6) zRu4t5;1Hy9wL+vO-)7QvJ&MI2zx~r{BsgrWDG#>%C2-aVwK$=Zq@1)O4XbX1XcUGM zQtHnA2-y>WOjy7|4~i)g(<+GpkcZO>K@{&fYA0I5XNt}y_jrIyM(?ITq5{WQe^`b@ z*OrOq^edbLt7ZPXnq~H8rkL4s3V1;~7LPqu;h+jn3c{QEihK`InpO1=}!(d@y0`4V|Za5GR9Aa-x`vBo2X4KE$BFeaScUz66}v z?t?)HO<-kDtDN}(x7 zc|e`Mt)###@4(hz9|q{Jp_~Gr;k2Eau^OQM6Wdi}Qlt2WVzNTL#}UIa`>NF-k~I6& zzXy@2Oz8JeAYnqW8sJKQFv*k(MD$rV^I69dT~QOxo9G{PtPatQC!U3s>Wyw#&3zou ze?oDC`9RTv!^dgS&pY~+QY=Oefu#h1Iesm90+m>KzzE_SI0!Pyks9l#&JVguFKAcd zY$9aB&QT;_%0XFQyuE@ly(F@v06m~ArrRIb`9uA=dkSwSM|E+c7*~e}juu0astNbzOPOT2C#R%8g7$*Pnu zq#POwj#Ey;A6|}8w@bor!-k-YfR~>gJF;e#Pt@+`2?ywoAYDa$_9G56DY?=DCW z;DKaeJpj(m0L^q)CTH1MNR7xq+xa@uR8Mi5fK|hXmBXOzG;#S~+dqVYmXRYyiN?yb zLF6r4H)R(?O3*G2-G;;$7x;BYH>A+5fkN+{WHr$fy?=`!NA`lvX;y{yu=*mxxyO0v zs>u-h-Cq1@gYDl66SG}@2Y^?VhbhX?7eNqe4`2<&V@`{G?MzCyAv+{zuCa0tX;g^F z1O7+ear4Nd|Fg!|$y!y++*IEeDra=z8pu(oP6q6MMgpv#k_vcqefv8Hv=PX%nD&wn zS`jv(8`4Sobq95JQk3z@!97W{77rlN+>HuoBM?lt`zObSIq#Px@P>)3wv{R2f-X#eJf z+0@Dd(!l?Ql|om+fFzQCM`m@w17o5q+EDyW6Gh(V`ksHy(JwK5lod&4h9N*ML-l-v zooDy#JuCxfgv?wa##TP>UZ&N8h>l5%k-PrhEAj0DVZL?%+yV%r_9^TXH@ZCN>MGM3!av?XN;;9E zr~3#TA8|jA>jyMITpT6!?+Vy;>CB#8W3Y?cH97XF&y@Y}6j(VV9ZDfhNb0-ck?&zl z%wXcG7y?RR{Rh{ONVE+5%M3jWh_RkQwz({)%ZSzt#;LAaBCcmlw5Eh>aE}H#x{Krgin=Xk2xaq-MA#A=1ghQdY2B6%HbxaiCmIO!% zK)`vZZr7oFd$@GM1dy2!;IL)Dyl_^uJS6AX@&|}>D6pYRK-hVuPOFG|WL>jee(4`S zs9cq=2-r6OlHN#&eLr4N*xx)5QCi*30DQNbGB;0Ik(sN>`7f3V{Su>I1G@yj-zOGg zIi1Bf`kG}^@+k05AWaEqN14BP_?hghoYD_vmY+?oTl78fKeW$-|Gd;^!2=bFJPi0f+b00S>gfb64Glf-5D>_MXri_H z=~wwvaE~rB+8?Zjw!9X$@j%pikpY}BL)VwG^E134mf-OYA}_%G`IK#byt(0?K!RcQdN;-WRKa*BEe<{JuP@qz~{?)KRIAxD)4%^djk zMZVA?JD82*hYtfoe3A?Bq4imiS+JuqkO*CzC1AETJ*i|pb6D7+Tz3Bcd{ZdIuMke< z1&ozxzMVAiwM6vxAg~L4fR6WRfjpv9+Fw_Vi5}3Y9a2&Y1CkdyHI@0MFAymLZ7UD# zAL526p-7CfIjnCCzQ?*65B{7EAVe@pV6Dm-oDLgQ1DxJg*cviIFrg;*H^|O7I%W;_a+oU50OJ91fNJ~NfiLpYcrP3a zx3MlrmIGGcRM2g8&22H~S!+ z{uT(LW3CnX-^x*LD{{4fE zYlhyn6Vm_ld^(|;wILe9?UmRwM>2r>^Pm271(AggAf#n!4U9;J+d@f~>Gw>Gt-Hji zf&>pWPRY`2ugxgo&vP38`%UPCMEHL9txL-SP=sM4qzljKQ-yU?b^MX2XJ)2UNIB@E z4YobRhpWs2*1njdqhUY3)qH4F0ivfkL{COQ&uW36f)V!VO+C;7mxBZ5=g5E#m;v&R zzyUwb;e+w)z1N-f)Q&GRQmDyEJOF_1w|~L@p9^tD54JM0If284q5?l^59t|~?OQW^ z2uv?8`_~3etf-zVJ;OqD8hR=X=UxTusEc1RUrc1o6 zpCki4hwz6@j?Teh<)CAsPr@>{jsVEEX#@L4_O3G}Ec*CB7RGnThAyAkWQt;ebD$B~ zpU&$u?xQ>Kfq!Foe(v4GEsvPxW8&XsIAT{+CGPPaF{%Vo)f6kOFlK*XO zh@%@XWz0G@S}(t?RBysZtRLhfs#sq(DZ_9c3}7(ZPJF)9TJJFWt2p{Y6eURLq_D8O zuxM4nR?koV5^x3~EF77}LVfvxBf7LH0mDO4_d3_~$DP(Iabp!}JJ~#1jTjEG+ZLx$ zjH&H26)qN!3>nc1m+xpCt>)a@XD@wO3J3&~+S*Yk>~}$?(lK?QK0;3eIq17uprMof zQZZNk_Uy)!6U{6+rOxRmSTX&Nq#v~e#-1INxFaI;8WT2aRFXdUY`x}nUp5^a{DA*; z{|&R3>{m$?O|;}P7tgf_Hbvb}`!cD;Z`H{eD58wxCi4eG?TmpQw3S?CpbC_s22EbV zUc;>*aL@y7#x!S8(X&rM9@(P|BJ4#yI#}{f>dBz^%ziQ5AN%cXR>SUh>h9haKEGW= znX$%7?)qv)*@W)}vVMxnJl2g(ztbwCUy^UXHmdv8Q4j06WxIBP>fu;- zlrEPo^^fBE_q<<|S?&XCt>|oEEEMNDKhu+j^6cYwdzG1D(HTIRu{po|;6d)JjAqeR z4@vAIXHq%=cr^@BE?Zt6MT`tqxoTi%?{fNKp*E1XHd``Q zD%=-HhJ4kUOx|uhJ7p$5Txd*1%_bLnBFXY;*9~Hx6Y4&|tmoT{IANN=hLsztfO|!X z**+)Nu~J9pPBvFeiiw7dH2;uq^2w|icq@rL4Bs?UYQluaGcAJ?)`maPIh}E*w>7Zm zT@DlFp7SJj{P%+khv#szYPgR{!*rCq?T0`Pzs+g>cULLD2S!M+S`>f-J|u~B2pqWY z|5b!Ha^nDc?l}6=C2i^ZIB_E){z_km;Ig)(Xgp8>j4&*SgPs|>^TOGqNX+Uv8ARdC z+p9HV#16fhj=%NiM&PbBZ)YrPgXbTMH@u}jz$I1DC||S!et`Za)8(yAKbz*u*Rq!;ZS zwAZEm0&{J6PL5g!iQ$~6!XC{!wIoX@P37mT4hc`Y=@ise)KMhR*Kq@?-P%iAI9!O( zAD~QO$Cr)Tyo>SuUa55VS+%EI2m&A2(Yh;B0tp;FMlNwV%3$+%?)Tp)<=0QLJV)7R zC{9yj(sD5;O^$ZMe6 zxg3g&Hn3iQYOVaZcnB11E%N|P3Auk~+R)~HV_8ZQ%Ud8sduo|82}LmZ2kjihCa*>` z1t(`yv`N_ZeDUf*%S>j!O!v=UcR&tozn3kJF!8%mzKFc3b4xP?i2u(nER0~M@c{j* z76Qh?*suw5|7ijdWJ?}=aB2C$UFYHJ(IqQtq?ul*m<^ttVte~nwe#H(U5YdIy4Cm# zRP0I7dE0^NqZDUlM#z*0nMiVp*w4!kqQ&J~b8#GFX**NgXfJ&Up}A){cK#Tf!$z#% zrXsJ{`C&kI*i4$Cg^D6zUsfW8oaSE;uhm;(!qR%}GUW@UHH1<3AFg+Mw%t%vF{~WG z;65f98!x77CJKFn$&l39-)P^7=VXpP%kDrGAt6tPSl&@f-VUF0Qy?oUG^a<4e=({; z9Sy9s4#6AWr}<|d=jTc7XMPJgbWrlj;lcq$jST_7yZko|Zk>6=yLRFToX8bogHRQ% z8n0GQD4t1|T=uYkQ6a&LogV9L-@r1$b=ua+iv?yH^`N$-FSvW&ZjxPdKK4du;U*sY zOjsbX`jXNHRr_;)lWYj5?feFn@}8D^X{vG(!f3-a8IUTzx=K zF*2~-&ppshpgDqj8NKu5n=Pcxroy?Wo_VIv`D4xlM8wAJmGfYO9`Ia@+lrpj(tdw^ z?%sBZDzQqJ%OdM&%=Zcjn}o8Yx9$+5w0KuQdK(A}3Y(FRqssnCr6X7ic;{AmaWCAR zwC?IK2R*J)37@~Iv%a$-sD`~eG|+`t#Tx7&r#|*8m2n@3j9PBUsI>wR@=`|KWG;gjqf=$IonFZ{5+gl2p61GN!ifCP(B=RXvzMgf)7XJD%02T*H@&JS`OZV9r|gYn`?kuKFSct* z!V9yrlP9(+mJSy5WZN0a9z`AnP6p>Zy>aZmKl<1Y2prHe$9{wIfScxkhV2U z1i14Fij$VFh4uRHV&8bs1ILwW4wduWZ#ZQ|Q298!^;MXfCyOMfXROTN|JQ4%HuHV) z)=m{OT6{^Xxx|5DM3OPgn(Sz2U4Hr2mg%Xsg5>R8V3ARr-!3dUU_8#EZcMUH>wVjB z-Y;hZZq@<=$Z#yC_=vSFzRY8APH_TFg$5+nHP_q7BxG8~S<`|I#s$Yoc7C3V#F^bB zRIIR*dAQl=DQkl2$7l7YL11VShG8ET!KMBh+sQ%`Ud5z%`)2Pg`@%FkS&dnpDBb@*1NDB~F1xR2j%-&=2+dmq6 zg<3?>MR4d4=cF^ek_X78Pum1uocqeIczUl6Wg)zsN+^_ifFpOcl}#C$Oy4~ip#sDe zX-21N#*qCwWuF6FC;-#vS7SC#lC>tMHa&9C&!-*&_bK&vUJz8oiQ=rdF8yT?3ZOiN zl=1-XtND_C+(Lt@ePr{w-lyBZ|Q*5&hcBMR2n{_};-Pj}x} zjXNvduT2}eaz`xy(dTKl^kIaAOpOqNTQhrRcGF$}k?|bg)2PMXUaV7QjAM!`)h5fN zf(ZTx@h^A&z07~?ewBm4kpMZg*?)>)9QWvaw8Yob4Fv*+;bpbCNfSfQe7DRsv+>Oh zRSgrL zo@nk25%8jO^u&!XMr9~mQ0bE>e9*gTROa|a%12g^7$%ZBi_*|$9LBZciU|@ zc^C=m&0uLxb@!G2$9mQRNM(|ed~21e-Su}J(x|e|eMy!*Yd4m)<-29tZuudlf-T4xDPI`Nu)4nl6J*Xc@}u+)#;gYl$-1m^u{ zcj`J>-@y1#NbRB&K@Re}E&=8W*>kk%U-37sA94B`xapNh_?YDSi)NvWEmP#be3b*9 z-YkY-gsU{1QU-}xg%l0`$avuiK_V*FhJE;4bpCGw$@$WIH1oj+6>9EVj&lFGmYxm_ zV>1A|PXwNXUQ0M=l2--zoI}gCNYof2CVTXp!(PerJ;SAUG`HDC*g&qc|%{deCGiVt%<=G)t-BMl{*WcyF}{wYWVY@I}3yUKp92ggxF z(e840h;K)*22bNx3!p+aK%j$E_`Ze1kWm1#7XU_My`LTIX-jbi=Wz7$dD{b*&CA9q+?YzSJ?tZ0s+?yiF-X)?n*;1=9z>fjguErqm6GErKWykNA z!$I2S1i+RbQ3pw9Y8fsK?7T0`64J@7156&Y5_6f`&kS)CPCcl1uQ`pSh@$?R_Q*-) z>l+R%$bBXw6#U{(%!8m|nft_OV_S?8sDfuBdEQDboyguxVujE`x?B)w@PY#%w$%0O z%!q}u;xYHqi_r0w^PriT*nTGP5pBp8TJi(9Bzq_Fq1zS@?-~O4uT*@R#JuOmRjYMb zW;7gnX!Gi{(5KSh;@0{m%-P1Bb6@a4`<)%$F(*-M!6qAlXdoXXzp2e8QOf0-v69cG zHLsX#2w*`~cHaOX?CmAta#!?EjaiO1X)4B5`(q%41UzY zZ+h=&BO(2yn;elpe4&kUp(Dqa!>HyWMU;GcHy*WEwAu`0Lx61Q|3K3bdrw`?Z?0cK zbSvH1lRekc3^@4aKeIlPC~@V-(c?^4L&%Ltl}_S9zEt^%Vo+gJrduN$D}MTFvcPWO(P-f5twr zE|~a^kDjuEfK!E+5?R+_-jM4g4ir*MMUUg{Kw#^8M(NU0IgS^3SZ~7Z1$fb9BPBw1 zxZWK#;(IL@55vCCCMl}(imbYsZ3ZsfCyTNQJ@TB%iIAcH9WaP8gE)=%EQI78yl%G% ze5Try%x!q7NoY6klPtd%pHfoPu_2E{F#Dz-4?YRmhd)6s#($LA-(VjV6U1xv3i6GL zFVVlcgd}s{ePtf^jC$)aj(_&VZ{Y3u75ox)eld zp@`RA))pWLeD>hIpnqi+;Npm}DrOJQzN+6X&H+WLqO46*q+a${7S}>JQ`C(6F0g}i zC)BG4^v(Rwx{Xg<1l4~KJ@&#T=Ppdt>LK^D3#}Eu$T9O@x=CRcyVIJpcnA0ut(zN6 zQAnL2l34GKG*;I?t#6CR#){V%$Rkj94`N;7kV%jb`oP%hA-GGvF^$4MY3An~;AxsJ z@@lLWICU+8y#kY)l-UYD1%JCoV^YF>CwIL6|mEilsA~({|VNwwSNJiWiGa@Y-?z@z`9g((^7t`2d3 zY*PAB;Um)Is4##~1D2N(_Z-)w|HEe)W=%$*oYPnA&Em^?7hWb?Pxh3yf)ra{C^53k zBo!c!zz{Vm%hqE=J`xI_+fMENUWWMa(0Uy#mroN{7-Q2^XNqy!=?eQRo{VB78Ap`41`o1BWAf9ME8!n*KJR zA46tE4zM^wbwLisf%pRjaNB#HjwT={BwAPVD$h7q^NA)ng^zHPe}#QqLF-w@BX_ zec3dyCe+#WQML9to3o8_f86I`;hDEp$krM$U`0w6sSz6;Hj<+XH}kQkIMW$C3mwG& zP)d~b*v3u`9R8gm+rW8&59HFUfb96$7|>bppOLNLKwKA3jp9(>IVgdeuHr(3-d?^0 zXg}a%!t}}GeHUu;0q0NG+aO%j0a^hBYX@M=8N74g!K!{C>?QEEKDfvQQ@dZLEhX&k zwzN+M-O-`>$yDqhEt@gZFse<4=vEiY*OeBGCCluw-=E9`%@O&&9w6KnY8i5Lnxk$g z>@P&IoP&;3VKUnMWbz@U2;}Pg^56-Z(U!^@fPsaklvcWizRuqg6<4-Hz(_Phe|C$ zxesGmf`Mrp{tjqF)8(60ei%@-hWckx6u@5~N#{@I$44Bf0ho6^{ZJgLS3Pjzh_ViV zxgN0EOb0*FO3JV!o8f?;M#+osk}O4$JEizcF7G4n8y9bbzbLQa! zc_L8mv_ZKXZKWyXB$CW?6Kp_!ataQzGC>a5hi5Ur^k8dH_KPg%4UjZqZ}0s{y@#^% zb?s`e3E8<2dbodfP*3a)of@5kBG-n;%eKXZXcMZ;KCS#zpq?%@Jp^?) zaD*~^3q|st$^E+CxQFF|V#O{z+zQxR{%xJ^T}GEmfdP?U(?p=#%hg}gA8x~H5-1KR zxe410j00$9!e$+4oeP)$j2kx`LcH9ZHxJd1tsX^i|QC5^J=e<+FFT;LcQk)y2b+Yvr%cxVJ!MdxY^y92M5)FoPUiONcF$f zxMJlS6UN<+XNUa);g;%FPyh|GqfyEumskPDuyePzztrT#rgN3d;Zp6_7;gF?(3K}@ z>z{%m@+Cth`F3ab!+lMmjL}6MuJ#^D{oa;PqX9*sHS3{(LLy{+?r9ol0N0NeJr^mw zt%5Vow2APJ6#p4~cBDB}wP#?@$hxEEJ(+#9>kGg7d-BOQc9S2m_I!Vnqc-w?S7H+C zuK&ZY9$T?4&+gF4Q_m}DOh!xs1UEX&@Pd}t5+|h+1@!Da5N{(<++QC3CxDn8-p9FX zz0#oFRi)H2Xo1%ws9ItX3FmVMuYoX6JH|oUrgw7Dg(nigZIsGRCdh$+V%`P@GOlq* z`CnefeV2FbH7jg+G%uZ?-Y=x=KXw&l9e_q8;_u{Lj&`YEkT5X--JBrr-lYbq3mTb( z6n#}y6LEL6TT2ft^@mF|Xvo3Ui8xPNK!w=)GIH%7$_) zud~#ry>2l7iCxSu9nwp2M5q-gO@!};h6Nd3(3HGKz66`{1t^^XbPXbf9bSyttaG8{ zqOaOal6Eob=aC14@5M?;?sE+0R~M02^yQf9qbnk)$lus6cMkG@LGqv_RD-X6>HSej zOZcKV`@q5l2S>U#5SxcV0^D{NxLS|;bf7dLd&c>Pae@N(l^PST%jdzA`G#&Hgy%Iza36$>jPFZe!@`eT0Z^@P+aqqIa!wZI@Iv6Y$9K^8$`vvAh zNzMJSJq4g4jYY4717)`%R6p~Al=qvJN3b?_pu0Pu?A5}Lyw^fOUa5tWOu;yvXF6y# zP7yacT0kw_Jmh0$bxqxYp-{n6sS97aO}B?~9e$~fkd3BlPq?E2eoWe$Usl?^ydR{^ zN4uqxw)e{k1BrJSYF}WTNm#r+Kf5+9mXy}Q$Li3`6K~mQ(E!3u7r$-M!;IRD68%r? z`sI)m2}1u;B@x6!SawTQGq75qiw`)%E}g3IeOhxL1D}OCC&1dmfnB^;22;4QFeJQQ zip%Ry1n#4o%`e81BC|JU=Kh77FqnAZHR$#vYy2Rg_EoEZ6Fl(a4T;zc7dHjvqjx~K zNI=W$1!5ygC=wBP`arcjv?@JB3$gsfDkF#2m;~J>k%=h}Q;-{-Wc6ja!*)Lm(TVWI z`CskGW`QAqeSPNg(xE`8e1_}x#Dv9RgOb_8Wnd5FG0UdV4|xLOZb9)cgT|(}KL`25 zf3ktZy2cw?=61s{3K}-V)&K4RM088(VKQqwl~q5LuM`JaId8S$CgI!=7_ z&`+z=Y<{$yOaZI}efH)%aQkES{%7w$L*{@P{`>rYFam7m`f^iKOSrO^Nxi`2-e7w< z5gh?sjE0`dG`Qo?qa7s;O{Tvy!HF%KCt04UKD6^SiVE8sQooQ05(JMZ96iDR0P4F4 zK9s2kf96Tt*KzvslwTe6tOnFr#eUN!2A9TkK*@dJb4=|s&D zdN4V65hEV{A%;)z1x*utox|U@uWRr1{`8dvovGMd?zA?5NmSFSOFyS|I&LWzlrw-i z0$}B8?^5o+eOwvmUSkS)KQ^nu`N}XP0ToMRq|0&oKLakb2!9vz`5NEm@HqbFfOFN= za++fFb@hVs6aOw&^}iOo_8LQcL46^*9QtIerlT!rIN;we4VJR~ed1@M!|?eqrKY5S zQt_>0|31oObq{L*o0E%RNJ>wNb;yCpLeiGV_uTf*J5iP};8z%Q`{TkFvcu3D7q0-i4vhr%H zQLYKkpx{zdU$DHe_+6atPmwxeiT>?@^@63QeyzfpmDgfP>0tHMLx0e;cvIkdGhlTgzM3eA+v-5K0z-wD>i?x!AWD{*-rc}irGAVGsKus)|J;KT53KF9~a%I ze^=LxbXfd7Xx=lIprYt(7mMn0gO@yU3Fx7R9eV&+%)x9pCZR#B;Br47r*8lGF%pgR z8BpKSml4jV_^L#OO{g(1-K8-G#W%W4uTyRuc3vPEF6eHclw<*tlE*`Zu##vQmC*sV zoC}NVcA50%wRFm6%%+)hqbSc(4gMDAH zx>L#3Z$d>knxUI2pDILRLNATP zGn8fD@fl%HGQH`#j)Z61qe_cht{d|81^E`*vz1Rrb3-SHl`$<1*ODnStH#&KPF{n@ zw>OCHV@yuXEmc<-T8^go|67xYQQSKFuC5fzmM|M1j#eMtW`fN$_V?~_GjqE zOa6=`ORY{5b84ieS^o%mY_gp&+V8S^cGqc*ru)_MGZ-`*j|N`#y%y6T9&!Bv^o+~W z_wce&e@^=-5zA}QyE-9NW=3_K_6EPojV*H-U}tWD8yoMCbJ|(@N)U-=kPK+;!tI9{ znU^_J7gy23ob&NMQDAn%s#SsdJiH?07QAs5@eT~*Ai+Byf@vJF?mw9#{5V_j%Xs7* zch8x!0nLz#D*_ZA>-Qmw^0@ttSt!)Sr9^gR`d5>h?UWGnnt?*epU)X1;0Y zbCu5sY$(oYSic2aB*{#pE?_zoXnl zF@8BM`2w6L_SpmYN`Ys<*leF{%8ev@OIxp9!A7owH_T_q;q3CPG?;F2`!?(IqbSg*|qzs^YYdE-dE>`3h4>RjK+(eg8MBzpZ zhabn{5ut*eUDxx7`8%$Kl$jv!kh#GX*X&^P)wu%nOkdBB@)likOwkFnm_0J!%ZkI} z)jQg)z6OJmxl|=W!%Khj{ITNFOU@y+Yajw69Vw}_nfHk&q`4<4)w7)%XkIa(TO0R2 zNdz7ekRyf6nPDU}kBAyui)PJ6(g!aXHXc1>7*AleON6r*l6(bW)!h|s1yrCIt9gI>VVsy4nx3fXJi|t2 zo_(U@gEuE5M7C@`xhI!A_%puG`4(=xS=w{lSR^zvBZkQJ0qdhm`=&jUvCS>%_)B^q z&P_aaIFY1?%Df}SdKxznV~CeA^>@$Y>3f`$Ny-jR{-$Ob`?0r8OQV9-dy`yslef`r zV8sn5A!Bds7{cW`_i=m#S54%p1)jk+Dj`5}C$p~QQ_WZ0_IF?nRZMfu&OGK7LQIyR zn=PmY*lnB`c4e9ORu*xGxG^o6DJHqE@6<+0MzA#H&B+%v~ z5w_hM=`6}YIZClafo8@Zvlx&_eIj=?mRPHY$47^>Gv9B1mj0@(!13uTZIW-Q!GO(n zBo+UPXr(4QBW=1ubl&W4q##}j?#n6irk3`G=#!4X4Rf$gJNDdGJRMG75p@o>Irc`c z4rSB$u7!O>;u|Wc{+!-*Eb!#b=l=HHd0ee7wwq}}&def6LjeYDwU%Co8y9%c1}LYv zbhL4nUQIvZjsJ9W%@`@kJ(CjeIKjPX;F8~MFDWo%$s94WlsPuAzx|8E#+D6U_4=}{ z_;*{r<-$P|n>9mP@~h8oWy$cEW|~SC3aBu09mjltudvIAJ{3;Hp}O5?mbE}PLy%fu z2J()9@j1i`CK4u{uZ@8VP#!<@d&z*bZk^`-vh9+V604&h>fAmJDOZx25FoZI)&rvd$~2UXL1dYJIW3J-=(Q zax|#IuGI89=9R}w$v&a|YPKdH-M5Y6|*ST&dd-9IjW_j0(%{3gVbS!T( z5PoQ4hcYuO?QJPJ1b#+-gx@%`4US){Ml<(_n~m(2x`o8puB zmQz^&wrXd4wx;RIN+6TN({G>s_rD++RT5_}PMNKIvOV#_>C$pxKyk&@%~kAnA)DDn zOILD|qM^<$NA$dK^Fm=g6I_o+ch^e~Q&K?_&Ah5tLCvC|{!*JPUAN6?wphwG`m^3j z-DT=GeV=A_q?7P+A!c6pN%V^3_k2>v*f5Io^FG&eRCRobjAT6bKzE0?CY5@UJEm_g zJF`#hYg<7<4O#Ky&$gvWq=n-yz9KtWUL$iXJmn{*vw-*uwD@%H5`!s=gFJZP)>V50 z!Rg$V=C7<%gOPkCRU|D|t|gzhGv}f=`ZLTVdPWkPcmtxc3}P+q6a2RKuDIJATU`_4 z4U0^nVj0Fx)>(bcWS8c@?lS9^ndHW6wDYNatKo{sFRj__I=s}1{SN^U1GGbnm*xtumlpmLRh1(Sevcx-Pre*>sQFm| z%_yX2EEm-;CiU88Zc!Hu*3#Gv_E$mWG`_?zz$xqZ4QU2(j|4xK$_cG-;7igL zI`_E~rrqsDEH455P-)`4lQc8TItPrq0%*cQx@(f9ZHB06H>+6c` z72f7`UL9pzw?>5mainznhaE2ncyg4|(%qRs%Sl`n<;y&irYNBZ zTYNK3b#W8f=xewq4ipK2iI%b-v-fNcrQ}h9w+-951^kMW zPNNeF4@4R(UFH`<8d|4L#$lXg{KZ7o2;~Iqm@Ob681i&L=jO4pb9K(O2eeoQ&&tmS z9lWl}7t*jHT4ug`kM)z0(*YY@mXb7vr9f7*W>I$>JGnd!-^XvroVg8VwB9={zTA|D zZZ!S~+yYSiE!djrnR&+Mb44ZVCG2)AG(D(JT^^0|q5d|PSXs8yFIc3PuwZU?R|w83 ztT`i)h>{d23v|u_!Swdzc0MovHUg&3} z^@{05-AAqpz#THOtaGxlR+H8ZQ5~Tr<%GizS%xoeyz|c>{Jb_I>as(>X>XqOoO1JP zy!E>*Z&ga`bMEX!_q?N_Y#bD+t(RH0cHZybu!@Os%Ra7L`J79OY;&J`I6`|4kZ(t> zPDV#Ud}!zRLS8KrS;RV*X|5aUxsf;82cW3|u z#=v+k=+0MZze#(kNE;MI5^k35`YJW~sM9jED-IFkLZ_>yMT)^NVz*mFk2wD8T8(vT zSy0VZKzhuf7m!z0#av}~@QwIDVX><6Db5xw#?Qu>7_LrHqYtw&S@HM)CL!DK+C+OY z=V@VZ$1nZRHA)AQbS84vfQyR-3Q*41#Ih8h!Q$Eu7VAuK>UBFh>Y> z!EvH}9FY@t02+%KqQoP{^PUZjqag_mVV$}ewzsw$_UX@BAq#quz2h+kU?@5&+dZ2( zqGV>kR>Ge8D^fYE#BQsNTB^~Gsmw+Tj1QhQoSpGlG>ee79I> zNbZ;+l!#}ZUnU?7am{tkIaZzWyW@nWQB_KgyKr{+W=31kNjScjh4-ZZBnd;Ys3 zvYB=2zI{L2PngZ%Et|in(mM*3ua)-Iw6Ukb^U=vAMzrd3fTUPtWY9^lR%5p_zRb24 z#WoK_=ADhc7N?aIFX?}jktn0RaJogn)-HFN35T|YW@UZ8BRx@)F@Uv^bsoar0!1_U zR2J!TmaRMOMkapMo8Df9C2XnFMd)uaR4wEE#=Yqba2>U#WT6D%ntQC{xTGhvW4J%Cy9lTv!z}How+stY;jHDsAh$SL8+p zV}yDXxI-kscmT{1NG4VrmU6TOR&83{Y%6`H+x>~Ew-Lemf`|M_aHQEySM)!CEQAeB zf=iK&?W|6GL)WPn@a5$MSm6{V`lYMXIy*&(Sx*uMdrygIUxy_s?b$(cIQV8k&GP`f z!UPt?(z2YbvJLqnk7_0S`VtEo_H;z>?Q!`6Xjp0BFYEcluQJNykoJ7*M;8L)?P(R= zjEtKsZZRiySu`#pGOU??xNl5-I;)8Swb029rWn~TSt_~lX8fLym~PB+;rj%&KFiZ5 zcqI{^6~lZo%=RYz`e@Ub%-5vjTSl43}>%Swso=w~Qo%i+ZzW(2zz9D}6``u## z7yqsxmkvxp5L^Wxx%9fd>6Fkv-6c=_1&K+W$$!6m26Oi>3@s-OYKEto4?}U3~IIqKL3}F{t7Zonpi@)Elus7Y}lPo_@Ob++EA1x?KY*jy@7sG^3*;dROZDO zLvo4fQ`f{TFUKMeW4DT}!ISbtqjmI__PVEr?}M%^d(ID97lc@1_!A<;6$!CVw_G|L z*Apr)=3myTQvHmpNqu3HkvD(OA$gNW7iZimIvgskKcKWzo|gM_!$UvWP$@l=Z2e0@ zQm8ZmzW*-f#_W{Pu2;Ew%Y&o!*PAyG*Bv$P=IG~cA_MM_oQ?~hQNlc858JgC9vF!_ zmv2A_?(_-elI~K<+0M=w^IRp+tuMRgToqqQ?XSHkujB_r(Ho-@vpBNh6N=;_mq%k| z?uGG&gWFyaF$z!CY4si@D|$1fPDyb1*)WIM*%{q8f;*1wlqng}ICoU-b99urPhLx{}NM8>oqRXx0u zZ@QC^v|}RToL?Tv7IwCjX57`kn$)>e)a^Cf^KnD^#ietHMm%}0To2yW7M7`V9bDqO z*~8~E5zW|N@&B;)-EmE3+uAcawn20h0i~)ah_sEF zA_5{1q?6EW5PE+-uv_TPyR5P{jR;rv!3x_B1Ys zKK0bwD-|xaVcCu!N*z02wFSQkGF)?CgIgDM^?9nE(Y!^Z_~R*8>`eP+kKA-p?8p3HC%4xmFdGrj23At@YWzdqjK!O8 zTpX+&)Ccsmw-YBjZSuM9bYf#M1lOvC$zvTfYX5@kAHJet5;N+saxJa#RNCh%!#u?@ zIo{TY4OPtQoPe667=H}jI^_O?Wi)$u`9Vs6mjkz!BG=R{o6bfHLAXn#vscczU-%Sr zh_xaKI;<7-NUvkFCoEZzwSIn+rW@QGaew}UtHM}RO-|!Xs3@-*Ddf6Fj^16G%?vqh zXta^TaiukJUYGQ5aX3Fawm+F7d=@8XL3`A2Q)hH9%I`av_WP;h7mpp`-Scz2{L8#o z3mYf&g!M4##9?39$DRf1g+BkVc>-}T%xa}L^H*5Q{KokDxwj@DA~SOi*NvLX+C&ue zi=k!rU7OnN=(^BcHr@PsM$*ggD>+=Hy@hrK0v7;3mv21|e%_wuuc}4Uw5N-*J6-3;-$FC})-6Z?|_B2)e7XRp>SNpx?!*91^gq3Zy7X}QFkYBK!Px$D; z?&k7NR=Hc$){hf9RwE5YRmh{+wjHzvVs`cv$5J^}dHlJ6a`HZ_YPog$#pzONQ-<`r z?d7)C8==P7H$>WvHd4RR;5L<5eq}_-9X>rk}|Mmio z8wPd22Ya-i4iH6s)){JWY~QsOq&5c$iY%2Aao|58;`u~Y>fT6d)ko3s7XFtn<`2xZ**S9tU2Fr zNGsaYnD5Opkpecw=9A|pJ7KTy^$f&#@C!xBu*o*pJ$`Mq?+{ua8KIEiFv%EDLT|mVF-dg^-M! zRAqzeC;x=QjcPjO1+J{2E79cDBcG-(PR=g#`MtDim!Am>(Kq{SEs+0p^IcVNaFxxv zZqO_%`LJPzoa5qZ_v8FBOfTeZ8*A%#M-WoA?xW9^x}$bF9BgfAa2Ih?fV`=3nFjMW5qr1>V$kh{4#cZ!gU4E*7B2@r5fARJ^S@pdoPG;encffp^)+X} z7;DvvVerw3xCrQd&-=?vsnJ~eAk`(mo|)kAE-s%EY~BvlhDg@CX(o~D&l-QZ9mu^` zEZG>_zsFJCvfjVE?m?uXikmQ7gRd?3C(`eJ*Mj!ZHDsK4pX)g(jOkboX|m~ooXO$y z4Ohk%Nwgd4988Uv4yu4mBpPO95T({^9N}~C_0_VJ>o*g(wWUYn?aeB`_EIK%Iri2Y z@De8nt_@aN*Z$_<+|tmxnAR|#_ZJYIj;_BS%_%(A(U-HC9$!BeHN;ff$w+|366M{! z;H@wH%a`1K;snbP8~Moc(5FNBb`)yX2ftd{a(~`oDckqny_O{D;cLu0&yAnH`dHo~ zS_xB4KR9V4DQ1ja@rlI>@iMqKb098^M}8ZmSH9zugtNi zM=-E64p}sythLQOOLwHbC7!K+?BIiF#IV;%go?P%~fwYUArB%JAjbRh=>NuVJAyW4X%d#$5u~=T*vDIp;SWvTO&84iQQAhd}nr zp~u3z!^1ArXZ+?XxvhO@TSVzZ>y?7Yh`=oJekk`*1Yn85oXr#QpT1eb9oO(uM%(+j ze9>)ath_kUlO!)LZ!^dQ8uuw%?a(QU25)%56cOux=GNJxrG|22@ej4s^JgYvD)K}no7Qe7S zy3WRim}saGMOtx1%n)QiyV>g#9&pS4s{PEE-+nf*!7(&39#Q-cKv$U0vSp@=4;#*$ zC+Atx$4$moniS3wJh$FmbkOO)&fcs{n3s*5St-Nvwi$4$v%tTxS!HW0)~gvSs?Q8z z&5s;Zi;~Gv{Fv&AypaNFjM-4`ED!=)xp*-Z!!}X zPT1BaSopo#?VzFBH>OAMO>}@trHz3V#8Xt|9+B66g)LCxXEpQ9kZc0>i4c1y@C-6P z_7InR!Oso$WI~_IQ(PFa@=x2J6}ZleS2$Tf*Q=)B2*>9BlC51%4SSUmO&j-gs?fbP z@qUI?vN7VS-ctMEMyq@;i7U!1UHgYJDJ}FM{(&o4QTweeor9SUTfx~!a&H z>Q1&-6Z`Ynd&c%~%pUm&aj`9$Lv9UyaI#!Q8yu`fhkmSQHZ8lZToYX1h)&SaAYU%) zey+M~+eSjP_?-4-_$~`t_YX7u@%v>^;r@>ccPDKhY%45qCB~2EuyXM|%QF>?KFgEl zWTiwHHn~C9Au7`Ctjc2;nA~<~^4H{WQB1Gr0XHic^?E3x|D06>)%EJ32vJ)c?sbgJ zF9P8H_8B3t5qh`*Y;;!v*bmUaxk25g5z#Jcrg4ZNp9mH$rm?PerVpJY)fn!YuESgu zgQ-3Z_dXA#>pcOk$gKiB{^5~#P9=8~hs`dar{F%&X0d^(V` zG}6GWnLyd_P-F30x+&%^$dl(`wryg#4i6RLK!p>_$Zg@C3yf)c zsC$vLrS`gcVL!Im-5YLlNVHVrZk^DRARgm#oAG9_zJs#X6?oTjv+VXr2fj=xRcc}l z9&+q4%(=Q~X(L4KQwG9U-yhN64t%~Va192MRiEdn-s6cTD&*tf{_YuJ@X^Hf0q?PwKR26?tlYzS{2}+d$y0`OFJLy_l)_fD~md zsgdq={k6It=^&i$VGq9Kp7It)@_J^>Qx%>LkzzMhSBjwh`3u)C$Or_!U`5uW0$c{F z6>y_W>CKQw2NxVyT{0rOoghG^5FCqTheF^EzNVLksgAWgUz!!yKBf4%9l|G1jwuVk zgqAD*ORd4j|*|BKwR+*@iP|VP+xGIO3&xwD`(DOHgl0}Mzz2TM$Iu7=J#=VYwZPB}LYjTzR zD#|&5TfrRl5>L`vonGw2%8g?+3qXY*M*9NZsvhR#>Pe&v$*0T|xVDv8J`C&bM$uh4 zf}VR1CYwgp>0o0$dR#0k#-j4|`ojCJlU5udL!3>0w}oqj;6)xdd;TnD++{0>^3oco zc<5=5xw;MTImDF4W+22f{y9)83Me+GLya|H!ZM^qe{JLGso{y1gtFDv%6Ca_pqV=G zk;XlgKC$UP%KGYRU6(HhNE>72xWH<$Dv^6ubcs_qhL!|Tp^xd=ds#aUMHx0Rsf}7b zHh;wXd6?zNC|%vSeL$pK!6~K`T%$~o`xb1r4x%Y1M$1Q<3dlDF4oiAXSI8q(Y&d$ZEWi5Rw+w&XfATbdx*$8 zXiIy_UE{7#>Y{L+Ha$Bj;L*gxfz=#s?_}E_S`_?@3L-9fot5TA)CQAFhudRVhb^=DJ?6_ z(l4&~E&xbIHt#zBtHRDqnGTV$QLy*&g81`nYOu!JcZs3a75g2=8Ko4P zSC2#4e?Z&bV}3#R~aMpayiN4yu_Gq86)USUC7DE0y1x0s&sf0?y}{F zI>to$M6;gCV(ft`ip@xf_fIt_#HE8)MtRuF=0@_~a5;v{TpE!%0c3S6QGFIc!r9Qv9JMdcq7dI^;E^7j6#pLK`2K zTLP|ZvRz--mJz8r~ITH*UE~msjTue*c+*DP$(9FLP>x2AO?bV`;}T> zCw8l~rukO}W70mquBM#0iT=6`G|tuouRZ(f19*m;ieEa#qm5B8B`lG2SzF8DWSLJ~ zC_%0-(*0)4(6J0&Z>89D$($M=nZz)Sa}uiar*aO-IUM6$VI49v?&I^TGi^UoyL+>f z*z|brRm%3XD(TQGfF`(ezfAn~9f%si>rLpF(oD}7Qt1f+F1e3-pm-TlKKdUJ~og7TwOqb*G4LiZtuqzCN{a5kRW# zR*A~zec-M1FzfRv1+Mz;;1AsgqB}%HYu=zjZ7R}OD}BcWo!pO_SLO!*ZMOdXQ26WM z$N2T=5MMk<=iSs8M#Dsr;CUcyOM_MO_8!})5H+5+8a`}GA`ab(e2lpq!2(4Y`aM)_ z(yY))8eHb1?^l8vMtH9^&%Cg(s*!eAS_+^a%9c$Z&2zBJ^nTkQ%eaYSW(~fL3;hCq zljRY>hNxjZNCu<@jmDs!!`+LbHL=qhSW5B&>(1JvsE!CxNsfR`KuC}@PyHiYA#($p zOT){-)|*l_q7=fkG<~9)Bmk6KWdOjAbNm26yN(S1p8;Awky)Zqkq)HE%64)Z8 zj7EFZSz|N&*Da`YKxtG;EG`~R%nX$Bs)FmT#oqQe9*8aV)$?fG3Esx!t(*{(Hja!` z^H*&-ru?(AQHns8@U*Cg#v^Y-s?LBxc_NYk`iLvQZCS-K9SGG(pOH)_V%xlgq9GJG)aLv-e(c<%(n$4x5 zp%mGoUjvUvg?oq2aWaVIosMhN9PY+ew-2hfSV&IwG+VX_T^6UBi0V_eImK&iIwokf z_wh^wJeKBqvB)fO`jg6@VddIX{f*6~hUU2Ukk?Fs5NB*zSE?Fm>U{SX)=CQL@HXl7 z_D0f?quGT@9Up+cG%5PeZVb$IABdd_HO%ouZx!0{#I!0^t?VJ==^oHdD%Gu0L z@oQtn`ZO<9R}yxeS!|UK!>t5*qU>>2o*3Q0Xd_8$O1)KE;kIU-W)wDHT zTfPMlK2p^|*`#fCQlhLY^poNaB{qeFPLtR8@gEucfZj>R7Yi|8RwjIOtcznEP~tXfhU?UXlp3Uy8${pB4#OQW3L#2I4sAM`Ns+;uXe* zesJZxhxV6sq$;ldHWtOp(Z2bCl~{I3H2QVi0qm}|#!&R*h2pO)<-u=TQx0rR3glmG z4N(1J{-Rx`gdy;Zs`7t=sn3=F>kGdCP*&XQpMgBs*z_-YRQ}RCpq2dp@~1Tp_a{>U zHu|mPe=0)%&1LZX@_$;w>|g=oC!4@eW+-~OoO4U5MB^q?6a3&?_gennbka#+1)rt+ zu1(z|T5Lw7-vFJC9%>($slQrj0LxA^Ds2yY%HE8cpU8%0IqA68;N8a+P>ZjV{A~Id z4-&eX^f(+EA0Ikn_z5DLjuPt|@yDOdh)HzwLeDFl;&IH2v}YIth+}4tSAIZ!Y=1$R zgfam?-{jqqr_gOzW0smcInBElOQFstu~X}H?+-(JTd(sGj{9RjKgZ(bC7>2^F-yy62QZ!H$%{Mtx1p@#CucNBN0~S^4-v^kjCc@_o|N(Loa&=u{l%yQ93E5 zo8i3iSm%13lze>StQJ+^C4?GeUVv=W+lU2;%Zwxo>j`uc#%F|7SxoNkOWv}6(Ee_@ zZJ7zibb?)6m{_LhRz$9mLux+0=_FJ(#HgU8$*@(r!E&$Z=(#XmxdhqW;*Z1mFYuV9 z8ZroTqY+bK+*F9KBQEVRKFBd|)0T*Dm|4Y7Pwdl4vWq2CfnI>UKI^u{em5qHlXz25 z2-)h3cv1bNaGAn)a2D)#6uT>NwL!Z0i;d-SdBcu%dn%sAP4jAWo(IexDqZ%aB>_n8 zyD!#X_j_!&;LmN0|KiWxv&ZgG5I6Qfr%&<`Xl>j3cF1e|li>cMu^<~5IrB($Co;}= zd3naF-)U0-66;BWUpkbyeyk0E*Wh2Q*8f~%H*hVUQ(>ST1RAUnG?da*`K~$u9b9txJ z2%yw4c8m4pE-Kw*cSCM~n#it4BJf5HGZibFV(De_R=1dF_daCQK?K^2&s*c%=qhJq zlD!}T1?N}i8r9T|+^{@ShN>avkJZiM31tmU&Kt1qTB-BBIB$0qX;epj zD-n@=1Jsa%eTo!T2zcmO$I=?Uhpq1ifAK8oyWRD1(XU%E;zfYQ*!8tBc~F3fboI`E zvGSmHv;OQ&@)=vfTiKgU$JKm_yS}Ww1VUrFH*^)(+pyY=QlWYu0%A@Nt= z6P~;q%)A_o0>NNS6y=xpVjaq_KL$K)3j7$b+@oevk)y~HFq*?QO>dt5Nn0-UTC+DY zbc>&awPQhH*JjCOn`%M>zSX>ovat}B{Vqqf|GEJJFK(h~wKpO}LaG<~aH2_KD+d==*2_@3?%(5kv@`}8T}_!Ti!L1J zedL(~J4*bSa>hKflR$v4m^%jh{PfC~=O6^WH_8uqvLnt{{(3SM;NX~T%sIi!)!&}% zPZL}n222pS^Tlr_5bKD%QL2&@oujN3){iQ_n&)Y=zXbm#-3SD&W%+12cY{b_a0l)5 zrtzPB3c;NpNf!`4M?tq$lMSm*0Zc|km6BHW(yLHy;ZJE#@iE=P;x@zDv^B@cWtB%_ zO*;JqqaG#IS~SnEDn!fQIp74aG4|`yx4~3f|N2mF zsq)yBdK=xm15 z=;fEA6jadW9n@SqM}GsJL%Mu>4_>;f3YDsJYMu(W%cwf{*%y45xBRQH-j#EGXeW0M zxVkrgf?ZN!x&0JUCX84aQMLDVyX44>uopq_B$0E3AQOY>++0t~Z`LWuRMvXXQnsGDx=y{Oi;x^(x+j$tm1ax^|ECYr#;natcm}%d8z{`4H9dT zgSw#`mwmb{OI^y!XY7=WZFOpti1taWi1yh%UJ0Y1jHf$KJ%9Oj+#RUXCoL*GhZc-y z8#ygY!c*^+;MI+)Tkt}w$YIN}JLW>F;yLb~+I1lPOERslH!FugQ|M0-3CM?q<8|`G z9T|vq>>|@2S{jmQP*Fc$xx&I20YvD%ABZ*QDvFwWhfvuawcEY=v~VoxA)pb)J59gs zR=BrxE_i*q)_Hoan`Dg_*`E5+=@YD>sp>s)S}!g;o)i`p1nzt`6k>b>e~>$3;c5`A zrsuu)8JUW(Q@tP>qac;dSH4yS)QZ=ge;D+;GEbk-^vZi~6BJWx9@OI;*WM3te4$+1 zEX%L6mP_#48hwH;-Q)=C=%MSx3n78BO*3jjs=B*ly7$!{r_H>)CDfeOBW%D`t0ktW zqWuWm>ZAyP&y5G->%lENC`3iQePn&j3b!6xXnAvzH;yfoQ+EHyJ;5zHrDQoU96d9g zwroDlq#SP8>lU=to10j%_|{Gn6UX@iNkC(WP+}NNFdW&gje7T6T>z)~nzS|%z92}m zO#{#c5#on-DTKgdZGuYizXWvOz#jwEGmNehjpf)a8aD%%k^8lq~Z|s;Gvzw`&ZTh($oJXLpPVW!_j%( zL=zL(Kxw^?F?K7Fi|hVV5nrSFNBe-nKOfj615|BW@zO7+x<~v!jGU~NURYFiFZ2oe z2@}(I70UP^Q-J+)d_fbqE%?Q)MSVYRQ~cU%d2mIxvet^P9>F_eLq}`|(-|1C(EZ7d zwcYWT&CufR(KSRv$fsr63&tRtF@ICwqAZa_cC&z`!j;?OsM6U`w!kT{W zHRzm*By%K`tPXwh1^eA!4c03T$Q>KsGB#yxp3o{&pt3$j(=?*NsQ23u(QHvZsUNGD zj=Gr$r^^F{I> z)AHRhkVijO=1K3Q0Cz@3Pl#T#Mn_k(J|9+@{;I63cKtpl3ah9ql?i^{-ZQ$CF==02&eb%BhtR-NgJqQPJZlLj#! z@$U1&io@-EyE;eN%!q4BvWf9;=jTr(7JK$c)=-)bJ24*=`<5t_YH+`Z7G1mH;)v$Q zWM8w=na?c`GO`YjE@@af7JDMm5SVkf-^2gm_-;i~s1x>snd7;NR*Y=TZ`O!W?yo-f zXwWXEW`a?vyug<%^O7qxd6Lk$w|kbqpu&|Jo$VZj@Zs(YRB~Vl8kD2N2Sk1pJfb-; z*G@oHJoQThIH9k^nBPa+zg7u>Euowc`Zpl*pUOIbHva*Wg#(?nJ3uH+pM?t9-q0&L zkyr*8X)+Lw76yLUYiVGG^s2r-f7#90mahZo7URpKkuA#%o8V4KpF)Y46N!)=BiX)N zbWgXg+Kr*7|5&NnnPmMJ19H_5zv>TU$MWB6D{+fU{u)~D4X3^8x#J;+wjF>rD|(Xm zg!ZNa%@(2LVkBD;j5*T@!a+w9IkDjNt_;M7Z+QEE2Alt!PW}m)Pq_a5=;*Yv*PsD1 zd`^m@>U|=%9s)x{g3yrQ{~QDBC7YR=nTGee_k89*pWeu8oCP`!ggq6-pj>B%HE$i)8nf%#;NfDMShcP+LMfh@k^5@_%e8@7C=OwOQ;77qik{%`d5E7^9f+ri~y3lW?^T z5`u<=q9I}Zb-2H3_61;z0J{Gd=TWv}h#s-uTD#)M02|O8{kJ-xW@0FjBwuw0Nio10 zwlcX7w`FLM-j}1fGvGS~b_&SGsofgl=MpU!UH}#CV|l34j}^|!zh~k+=h+d_Zg7X? zS?qFfyRH99C1Mui>KWRrF`}~J3#Q-S`DrQ|;~g6aM6Az$CSt!)z)_`j>&Z_F$QAQ( z6O0F6zvYYZP0&8m#QDPQw!)Jf`#a4C1E|f03`s z>b07y`S_m#Sd`#P_wXwXel)V*Fg30b@&p}8el%S>0gY+jx8sK>2;4h_KISVu_JhU$ z?`ckxKT7S;BKdTHojfptt|tK(#0*VXYXh3PA88Bz-pjHUULYY)Hdbaw z^bt+Cwrlf0+dN zqo48oKnCDayfurQYS@5G%rw`J;RvAfHXiwM`K7#gwr5ZKAxcA}+ZT*pv96L9w($mc z&Y#a#^m~z8lr&-ESnPlF=lZ1Jh2#(Qo`r=5hcjVD=anG1mQhI>kWxkegH96baAluL z5+{taYkkW}c^x;*_1ZBfa_vtt+aP(*C?e(QHH~}H|JmQHwGYhI34_=KDW>+X7LHqam_isOf~@^Ar}b8TRa(`DnqQ}+cb z?idiCV0NT$!1kgd3n8KX|FI?P!_RIpP+Zqhd(UZM)9hWLvD(}Ms_~B5$n#e=RG$Qe zj;z7y?D4rHaVHXg1yazh(bcKuZ{VWOg?0)5P_ zMLYhsZTw)0l;vRjaN?X{4uU&BJdYmm)Fq><`IybzK%#Wzk2Ta1n zFpy1;wuJPRd(t*qM?*{1+nahm1yE@h0U~df{O5p#vH_Av$J?XNZYYb}lAR`?1)ZN? z30K@6#a&40{Bc*3heqqLUDEB2y&_*T{By1n!ftBwbO?DBiC=2qjEvCf)C@W(tOuhIGAxU=85_yOsOd4DN2mUHBCWw!gO@Zy6Yn)Vo;IfxYfb;&KC_x~x;Fb5Eu|IuAHXHz;3(x6oS?kxpD*8#RbyRN-a;$+b;n0= zPcMs>SuG2^9FX?a(E0|EaF8bsR(9bmaF0s%!kAg=EU5kI#$*V8o5sH4&Zk<+Q|Tcd z9b&432^uv+DU0)R2{DHA@a}i+lXIO*m$;`si%~%5u;j<<{`LMX*XZ`=_yxG2dIW$M zaF;%DsnJsL!_1(9s3`f3BEf`?{ldob^3?Ktjxn1 zF1*3C8v*N-K93_@m*nMo^cJNx({6(_qc_?IJnqev3n)M@hjMX-muP9F_$fmdJV>E> zUG@#0V0d^iX?(X}P>eDDE1`~gdblY!mVm0RSwNvugwTZrSp>T)BrR(Y+kcf4{id6D zsLtcGXyYvl@#iIR;5&jT1Ic2Ce`y|7L>~xSLJn{muGifWSy>PF`|P06Gjc6pnq}rg zHRx;xGd1}q$8!u+meqDa{NYQv|OlLSk&nhI;)Hs+n?BO?7UA>G-Pnw;0yEAOG zt9*&vvYHLK0D{I&B68ZA^=<%xYWyuwJFNuy@X2|HF+OFRxKS14L`sxarxDXEzu zsc!N32t7_iVz090vHL zk={l6>k%j3Kx$-MVZvy8z+$WVg@VD)W-HU(yNBy9*rw|m3v^cI;)^EPM%ea8C*!XA zb5$|bl;HD4b0k4IH)+17U20EdE9%K8;Y*YfXau&b=T7LCMSB}AM(&%)DD58%9&Rbn+4Pmb#==LwiF zrm;Y@>C`6a*i`sFs0PJdI#Ua7C`0Q$rj#PHGrR++cR|d}56bGq>Z_mZcPAlI&;2mG zf)2AIJ8>3`^8x+j{kGzKyYMo|c^QxXG)ugXtoBf1t6xhPeqdGLZaI~pe^&FT ze5>lI3&_6dvH&V-<1*#TB^b|qMKXVT`EeN4;*5EBl)oQ$!zL2-8>Y%yz6{wwcUv${ z7Vj45^y#^^jI=YFTvli68xL2Av&QTz-QZg5bKU1j0a@;vj&%^ru~5omlKs3l5g4t! zQc$BcEIS4Gn7k}EN>fBNZJlR^xYWv`Sdo9s%OS-ykZn}d;9^w+rKP#e!jvt3Rzoh& zi^%agVvOAxP;1MX{cuk)^Fk6l*GbLv==34IIo-Tr=7dx16FV~L-g8EZhwDB#O7(@J z8oF5CB9>Nk3MxTndn1a62IJ)%iq22!6%@LQ%Cw0c*{-H4&jY|}6Gsf3dsDVP{MG`z zzA|s&^!eo!+e=wVt1zyZ2FgoqtqLOF4oKHH+oa}&nda}|B*xi+HvbiXSz>dS3hAkNp0Lt!V*ze;e=FKlb`aEI}_0a)dw~%H>A&D(oa-rOBf-)7`J3e2&Jz5jfm>1%1RIJlre(hIVXKdfQc5CfI zBwom9X8%A?{+#}2H{@_p^*U;PWJE+DIx4cj_0FcQ!^5;zC2P$}L_@g_TeZf2|MW~# z?an0Wv}kN~=@SSXJyQsI&>w`px`nKiVo&A(s3b;`%6*<1bDqjAb&Ds-2&+L&BabzO z-9Jc8O;`QpF-pt}x=!Ck<<=h#m;B=vt>JkMeaOnWbFw)}27tb*&3Ez4uDd@#{{q$G z#Y?^!!nXG*B#M%emD6B9xU*1+L2`nT%r9urVPUQnrRV1-`If`C{KFSR1OBT0iLDP4 z=EROjy((Q-^SifuJx=v)P`#GBriefWA~s~3Jh2>q>E1U|9vIhs`!@p7*P4VNc1K`c zr5e=0zUN(w0?(Tcp*F*wBKT#{RG5$Bi+7Q)SrH>iuj~ zqurNn(wg%u3y@RrNqIjdP*A`mXeszonst@fH2!e)k79Aflb2+=PLd96Er%G>cxyFnWNO3Y^0}o(U2#JPVUqfnqHT zXE)D}Zn3k^_c!c`bKYHx>lc=v4|H$<)DV8tBy7Gx=jBju1we&z9xt--uxf7LFy`S8 zx6`f{wY3F;W4f8k%D| zw)95{YLZ*XFbU%C5Hj1`*eVgT4>%C=BLFjXxxf`8t2I5hs4+ELu1IHo03QJ>y2vEd zk~WI6G+5C$N0I0j!O@BpU7v-AhnrsTNRZ+WB)VK1D9EhWb~(-E*%v^CXEGZ?UjQCh z3YibmEA2~D;jCEl($ou;Qsk)h1vL#UPvu}r?ixQikG}&KeCBQgp~svvrff#;uiHHr zs;zEo;qv@2QnDW2twE?zA~2P;mxN|oQwaQ16-{W^CI@uDDf%%SnYRg%|vYGgc=;u3@|z* zqA447DEs+wJCsMIkQzOMxuA7!V!01o*d|rj1MlwVD;?>)a#-|*>*1V)2P)~KpgF#Q z@Xet;^h%y(nRnOZX7izUWQhvZzg=+k3sbAHv%2y}UCVYQb_TzqDMgpV>Z$w^r+o{z zx#4Q^MgoA2$(>dXXH-FcxeI7yjZj@>3Txw@#z77<+$JWm2Pm$|vx4HU=1Dq9!evi0 zWlP0ONk(wQhC!W%CU(*j;S8{$j} zkejeR|AXWXF#bCTyxu2lIs@UYN`_*aZIS6HUmTy0s~fH9O2g*g=J6{!a@hnELD+yu zO!{3Q<`;;KxuJeWJC+j@qBYAZ;~TnE`l1;Tp|o9pa~{8Y#-_Dtl#G{(5<`rPR}vyc zN6Y~BxQr-x=IC(yR7Bq6O}I`_ldWx>&Ka$GW%{?+JSPK4T0tSmYnQ@l6NfT?uV@NA zb~_Qau?8o3Ul3s60!zHAW>t+oSK|~nj0NOH%rm1ad`Z5Tk$On$J}pcnsJ`69fqiJf z)NeCVpP-#RJB;FC+KuO{ygc1kuxCLw&$i{7J=(^gMo z8}%TzQ43*A8(OrV9zjHhJ=rh(65MLKLan!p2vg4{*nI*Rn>5Lk_Q&sanyjv6DB=2k zd(+0n!oN7H$Y8(`eMgCVirmQo)|GrM8bpGuuz+cl1g+$(s>9JO-`tE0ge&5$?4<4# zCV<(40TnLw^M0TmFc5eYVI|JLix^O_mxbSX{2i1k{41mRDLII10Y>p~5QnggtEUGskgN8l za>6b~Z}?}CS~|wYSUvBzXH$<&ljNT-`)O1}gr8mm7lJkdR_VP{}sMw6*-$ZK@#N@}0F{53ZO@|N^e!}IoTntCcQS4$i9q6oBHxl+b2V7dpk zaIY_GAk@dh1ODeat+j(aTxDo&1_FRIU?K$>@@CuGjwc5Yh&hZ|fqEW?i!PClRY{># z_n^#8=G*X{*q5S1y^D};IJ|zdqM0DHM#B*oXbgVsf zrkNa4)tYSlM`3_O9_No9(%m=p`M%~ZUth$p1*sEYUOdCW0p>!uuN#J$ojfq5&+_1{ z8gDnr=CJy=CDy%rF%SKTGc$hl^5}cg`_r?a{a&Q!fpn^3wXd|JQTmN@9JZ}8W#BO; zaoS;1Vs$`Db zI{p#~Ji9;|QdV`;+HSIs_+;YV_qI`!x%6}J>XM|S8_yW|;DeB;O@tji0%rUh;lX|_ z#33p2VE3#tm*V15n6i0IX3ta(KyP+NG+JC!kv1Sg3M`6S>7qWj;^QBh*M`U@LQT-s z(r(ht9A9smzIqNOa3En`B|p$=*cDS8tT!7;8!fL9cL4Nyp`leCS8MNg?{eMJ#c9dq3_sJ8U(z&C*F_^|0+lg6>67_^a0w zil@8MGIU)?iL8bh@!=O z+%;|Lk)<-M$Gz4!l$5oEVO|5_!SG)oV-DQY#flcMZMZ}Zchm#~TrUfD!;EBHb=qv0 zjICNxm34%_DL5$lev9Wd2XTJ4POuAb`EQf>r$j{O&br^UL=7R{sy z)El}c>#V8+ZOev@DRUCAPcr22_6JC)Zd`uikA_|b0@Vf*qbKV@zNSgaC34-TqK%qE zHg^HiiHDoKs%36nX&uLEe9n|t}Ps0p+tBRdup9;_QgKIC;(?Gegql=I;%5vuqN{SF6enD_^Bm;V#xR`9u5Pu8`g7yJ_z;KD0R96)06}# z_RmWBgz@HNdjBxpBHkvPI>&d{z3ZKk+%IVH73T7~?^PR3L{o7}iNo#g;KZ!@8qhgW zO-JKc?OTdPduY7SxsLS0e&|3|O#ykUXk{uticV(UH&Qy8R|b1CzGlKB?Zq%@%2dmB zAc63(#h%)!n#7xLXA4r7gDEcG8HQGQP%lg=lU-+#LK`G=K-My1yL=Sv4yRl$8p_1?BHl8>zJ;im((&BZEi8C-qe%!EOg1f z`bp%C6613m>dUdFfcqr|<~$B|j9j~Oz{}O!b5tjtUj2L`ER=j{l;S=CXalzT3tg|Q z&*pEz8J~dD7kquphKatMNk)J7cl>A=2YU@XgJv}WZ;JY@!{%S;47UL4^pC(rf#B#| zYDz12C-d^xX(?H{K#o#M*#^t~oG5T>7|)AY@d44X`ims^TQtmm@xBJ6eP^hYDU0e2 zfqr;~3UHqJ3|H{b->t^hk9mUj7e4W}e0>R+cEqnPGA3QkQG2IAMG?fSLSrX3wgh5n zN*@5tMYucA<8Cc`@(OiYNO`NB=wsRkZkvTVpudMInuQhZnnZL=WmqfbvXA_R{)LDg z5D19BUtA(U@cPf~3V;OIS5u~$p`)kMdUvuvdTM#pUm0#ZcU4q%E*2eNYz*7V`AgzR z1)7wrXxC)5R8zdtWbFtD@cmLSS;V z)!XZ-J?l{8HI?jrK`}itabc2D-r+4fdPUXm;KC{FB>Ao0clPNgn2ae5pavCrfo-~7 zE6d&ZSW2MN%9LbdI3!myADnT!PCQA2sz?Ie>(f~EEOebCgcy4b*Qn^kI+WqxJ>8-% za=m`nn~^OJ1G*kg!|K(5^xqY9NJa_W-+}Tk$|QPZRB|8K?U@9)vaG)y4EP^dLSI{z zdSn)%7*c!tM#Y^|OSL|ZCuO(2VAZE=bF zXN*hBykSl4VMEL=olWnuvk<=7dhyWj`hufrR-VXr;xdb{t7NBhD6&_P{-r789s0RY zN#23koz~m{-0JqSeEoL-Wd?KZ1qgL2Q;)ApSx#e+7I{(1S`2J=s>5uqJde=#wr{>2 zIwg0Cs$w)6258`V{>3>9K+c7|05tSE!8Z8iH-W4!~bO50_2j052tS_us4PdgC%ojOThnHoOpVsS5%^W8oB z0yUID1?DZP8{7g+L<%wx9`=143!B$z0vIG)4@~)q#wrz2f{AJEC;}cypI_$ATWqFvEZE>L>DcQc}I^NOd(x;^~f^@+;>% z0OIO%c_?GK{B3`890-e>&97en3h*n)MFTKuq6q@&tj=68))i;%c(*@=a6?z z_pm&DWx(dF%Lcn-Ui&+q^qg=AtED>HXW8BcEa}6xU>k}|=cuP@p;Z#(Oon%5_ZdSN zcYtw_-f1}THNunX|;fRM)Th1FsgSFqu1oGD3GC^iNSKIJu zJHA~y!Kk-PDq@Vwin>vmihMU8EIeo@#1jlAXy$`i+v4cws==5rUS{kmttZ$qy zpDPlZe*1e2O+>o<6}AG@W!cM`vYjxhDOza1f4byf9H9Yprq4e}#SJY170%^ustnhC zOdviO8{;s5rdQ~(=rpr z&_8@q9RP$dHhHAu-QD|OXErNYGo)SHcBU4oMc~A-T=YcxHDHoBy-A+j5w`7wMcW7N z>FkH=oAqFo??$Y;YX0^8DMD|d@2c!**#;Ej>MIvNbZXH@e{y7 zhTbyR4&WnXkm3H^-rk4+J*Ek^{hLG#fP+({pVsH`#b5t7`-(ExijYJN2Ic(w{j5Bi z(`0X_&zAsU=mQZLaYWoV1n&Bn)kvBW_*Cy{c42!#O6r?lvzp7=+~Fxi-=t zu_7@162af&u_tg@t29XikK8%)4b+2q9qg_62bSxsdT`D6t>*J~DGQN)Qbx;5%hD(1mddK(z%On8jp{`Q9pk7Si&m%mQd4yFV|GiYNh_Pj@DO+cWQDOhN+JL*po4)h2-TmZ~oz5z0aD_?zg|w z&1?mHIJR#0Wac~B$24c@N^Q0da720|Mfh(Hh#kgik<1gRkqV3ZPihftEBbOXC+@8^B^@ghb~~WI;h(B`aWO$$)@YVYrpI@ zHu`?pK_{?kV+fi)Uqvhc35O%Ag7F1W;-!u6G%it^JI8)Hfuotsj7RB3eaE)nC&;Be z2W{>TzljEC>HXu9Bf3Gg%YWonn4=!4z-?K~AAb zWdrVCT20-ZV0vJdH$9h4Z{p-Hl2pmKATPdmkX!U0V3S{P7vw%nmAoTvIjvWPUWex z2@fAo6dKO&hh$%l3{@2tkP*4*L~O9!!<2m)6rk3ppivfm9-sLmi*T0rU716_rn|(v z5(_MtpGK0Td3P&6hhlZFNA4U9yF`bdb~2?sG#u2LN}imfQ)avYO%#*HJ6;t)`rdb} zgQvjB`**Bd-7pq#^!kpnF81dfM!QBz1>i#r3a(Q!B~xaG^9!y|MMnIc;Ziz1af3mZOUae((d|u$+2B8 zxAT<70+5+1IsSKH2}{MSHvqJ~UubmP%O~IQpBU#14D`k%tjzVi&t&(&*(;(^!T#^O*rY_(Up#wem(V!-C6XPwlvJYY|{&3y+)yj&Z=h)SF zcfw=$NP1$IK^E}{GYB4WIk*MFXjx6cN&UfSq>QvvLIeC_f;dtJ!SvIAt%=rCFkC-) z!?JNV^oucA5IRRc(z07|DdJqltrGX;tdrF#{{ADWh5FG~bOi zErJ0$ex34G(5!M)>k#>zXmBg; za{Ba$aZzmYU}KS4R{gU*Y}UQAh8wmpK;|U)3=P$IG8;X_KkwFX!N>SWxqqNn*%`9* zGu*=vTb;Jk75AG>0(>spBOM=E)$MHn7qVoek!?>!>p0@9#LGmrtr0Sg0pa zp|vM+Am3Mgrq2`i1{PWFMh64{Bd*naA>vyj_tPPh6DNTW=FauQ;4|tR{dv`~oA*E5 z-m$v2!A$k%>lmQn%tK1>nQ)bWZ^ku!k1rRF)`Y?~;C&VY!l@r_HT8;F2+SwY6QvW& zj(Up?B~ShYH5(wU9UNDF#j5A_&7TfZ%>vY+k1K2jRVq+9vQ<|C%3df)+3x}D7w!mc zpySZ7u!J_sotyHl8&!5x`vt;6xYQ&{6f|$tiMul=KBQk3NQ8Jf1%T8vI^@&oX3S-> zHu^muTcRK3;FNL+1i{U5G7&a67+mXB!9OhYt~50vfdEAg{TSTX?d+FFTY^OO&~ZX%cMjV`xw++y3S&%UKENDIFe+IfZH@@06RX z!WC4j$w9X*HM%^$e7QC^*W}vN*ppbng!;`#TS-=p0gSZNAq*u`1RFX*>?=rHu;A#H zyI8#-!n7W_@CC5upEm&N&v#xV8?HuvGmdikw`TUr2iOewf8xE|ku*%$>%s--voiv0 z3J%p_x7MlfV~mEVds337F+)>I+<(2W;;`w)HOPyhhQfP+gr|;egvzlCt_o4>O4R!} z1%YmbHU7E4AE1K`!6TZTuN`Fg4wI^0SLv3CEW(&P(Z7LHX4d8sff;JB7qeiM1CbZ1 z);o;F((+oS0#h?4#(vtZ7pHu{Hmcu>q`9nHxPO@45$S0f$eP%47XZEs!&%uQbttG; z*u)Z5P^XPp3xVMOX)n-UdZct>+|->~00y$R><-_Ia}yqhypXqWK~0gyvB^^Qu(ekT_9eFk}@{#$Xi^{0#8;^xs+;Nf!aj0|hS!C#nvA2NJaxo!}!8_Lxj za)eM&-!~z3cgMV8WzZBClxPG~&cX0FDN-COW%baSroRS~8_-Gcgkj0Nj!k1{U-*qc zch%9Y<$LSa8}Euvfdazq#QoNc<)fgMb8JdbE?{QWAWHOZ%`coAQR;Q-dKc4K1U*af z8C=#oqt4m+%G{Ou#{rT{Q&oP+mL zAuOmkgy$#~+(BCjC`L`$I?iF7X#53+(UPRX6VJ~6?LiQt9}i+ zCCr_`1=c+MJaB#WJ#XfE^h{1bHu5N44}OqO`e(tmj1FN)y`3n){lthq1&AE;zJ1v1 z&ZAEc&3)dpFqWI;nd_2E!cC*)wg9%V)vem1V+~#Yiu(RQKJOsDwk^P(-jQ2 zvTh*zO510(z(k5c$P3y}R%&82{No?44VN{}-?(?&tXEV%HQ0;Pe{*eO=7-Mj>fB?P z>zu3VVz+jyoz#0H4{;m1^JeV~+k(q%_o%ML=KtzQFw9I6so3BeI0Px10k)B2xK)h9 zP~}F`eqSt+9V&OD4r;Q(_T_Dauu4apl!%};DJlBL!QS2|zr4#%M$4=r_(Ta!P!r;2 z9pH*XgMR2ENxmnFiPUom@o#IW0C$86P51a*k}qxVM*Z<uNk^W3@4aPI2jmT^fyrPElX0ajIkYLZbY z*ol%ZHq?ZBMb0jJHQC}~Bn^xJ!pu8d>M{Oq`m!Lq5}nA+f;;#= z)W1IU$a=WM4P?cutRtcc`uPXR4XbOgp0b*ur!Vq0+thy!XBAjx{WJayWsRm;S8Ox2 z(mU=un~T3WI1Ln9^}M#Y8|g=x*)z2g5H#nmE923Uv|W2l9)iepc-*=91Vb% zG0YN4+Icm9tTP7$f;KWd#T#l6od+UX-K_39;2qx2ktQprfd5_g)=`+V0W87*mfJR& z&r&1i=^KgDeN=wh@AV?4P3&`iqo+WR-nja0*SX){*??fc3&A`{QhJV;Xt_x5ewn6( zrbDc^=v4h(u8&uI@#YNL(veJ*#2(;E)%#yetJeDzum6o&y6i~l$Q;{PozC}!z#6NKxHc8k{L24!OUR|vs_Qq)yds>jwteV{4-KOo zR7xWzpy>%qb2lbX<$jaD-%rOKl96Sua6TBx_YB+%i$2p{0h4DSDY3d+Tt2|I=6Xhn z7-~QDo)qOZ?(;^4T4l~0&Dk!R1;c-SAMeGyK~YovOM6n8B0Ko_s{G@E-v8w+#>T+b3Xi6qNN_#9 zYvWG3@Z~80p9yxuW_>~Py{5$cNZHtytRZES<>ndQe?43oXwb)kTkn->5E;Vf{a(EbNx!{=><3Bf9B|UZQ?Pd_AXl&FW<_9D4$I=;iJO5 zAUEF0Z%L-sq^)f04=$j#y^;O*86u;pal!4m58ZZxHaA;%%n=q+&uS(e2?P%>e{Buh zur=EL`fgs4FB=zA*ir4}1oGm9H>%6nxZFDeGoHc13j;RkcX+nY+2hdVB}~RpQiqol zEZ)y`hApYCrU^V|2{Koh?josH2C*P}M%~H{oA_!1(;&~|?_=s6<+X5o-=#Tvlo681 z5hpw-@n2V=c=IVTY6?2}L5FY+vsqQFNjt*h<15wAmK*dRX*At1Km0d48t|m-hPy$9 z8G7=Pd)0gbW+;8@lbQFVJLt!Y&TYNRq=` zl$>CrM@zHLvok@DLds*aS>xzapP|*XFVNyqQOR30p4Xw~T^@_A4xO$p>U?3iIu6aw zQX4h^KA8Zgrr%G_GrurHMyk3$1)uU7qw+_&uDl8)Sx>2^yVh7$;||dRB|T7U*he47 z8K=OMu2@oR!P~dfqib_+SZ_I@A5-<%zADq{fKBfIcC?&pI~%+kzXpJ8iCo`jN90ZL zyXja3l8wz^81V=j8C3M`1-2N9O97~I{HnFrb+cxoHYua4BLP`$YJHKf9 zR$$;}kAtGgJQa5J)1!#fyjxyIh#G0gsy?cew@cfseZVN^25Sg4hk7yiZFO?cUod-{ z=WL0&IF=}W;@RIwF|-S}aT4rffI;ZiOqtF8&@|ey;3k&15Z{j*ynN2;4lDAyf@&uP zGO*Zec#NVC2fb;$x_AkRH`{dnXgKn_D{WzMkZH(UXnLJSp|5tNWtD~IKuPWjPeTWu z-2SyPdAF26X;o0DF+aov|&XH_D} z=>qWSG;!RpjNHhfe~t3}3vHh6h!X*oBIxS^PJ9DexZ^+ka7@y|i_&>1a3P(hpjFtB z@d4ZP<xU@%?yi%H6Xm_;S!hxdBQd3k-iDp1b?ke8LVEng zHBgrpI1xKWk{|WhjD87L)WWYYhbUUDRjS$Zu_mxOZ;Y*qHnBvo{hElS4AE~bqurVL zfS>ByU$dD*hByrl#b>O=7Myrlyb)wJ3e~?TLiGw}A@!)iU?0@b%guaB+J)}H>clM% z_|v&jJ~ydeGZJQQQ7Ul9TI>be!ZCx^IPcXnmfyC{>i)&(xc678h+|ySsbET?(59)L za(WB~-{-N{3Jh2(93OYSl4>XbpN>_AW)|KZ;#RlO4e*@}txsOGlHfm8-CGm5lCWMs zxW=~62l8bVBJZc?q21T|4Jc9hQ&>e3wn1wvrj~f7tvFwPI%l?BoN?mK*@dgM`DiL1 z3uAMVjJU8#Baz)Ne=u>Mu~JDWc09#G2J-D$$I>8&@#i6enPF@TB8dd|ay@C8{7xt$ zp$2xt>P+$M;CcCV`xw-SOug{0Y-K2#nh5X*6HA26_W@@CCrd#luO_npsy4au%v{+w zHi3hyM7c9-xNyjvcXgBIQP+e0_Y3OFZL=YtWACx-5P?zE^D4>I<6siToC`gLQl=Pg zQ{FAU|1>w-^Jt70#>-hLSl08q{?0Tf7Oc(LtV;_iD8Z2@5NXv=wlsgnp*=?lSj8P=J>@2K zcZN3ZQQk;7NbtOuDSNLUjMa=a1;0xnxsGp6VCU#*)H&3V+3iw(B^v(RpLf`$`ZKGY z^xCnVWVsXRl}PTDNWO_1{7%!#W&c~VoV0u=1#)ADwC7DH?d*OpRXw%*@jZzO#g%*k zrA}%ds>G%6y@6g2pT|W-a|& z0F2)NrfRV6A?@7v{*Cn5a$jEIF-aWXq-wh4*4LTrDtzNVf4$$_lp8m*Qv0rc;*hz@ zRT1aNimyMn@H1wSUNg`x8sZD$A8o&EqIW@-s=eEG6ypis&CSZ~xjA%wXKGABQN!Ys zKUu1L!J!Wityp|`haojT@AI|qubpS>_q_K5;${u&X`aXlo>>G8lS_9Rk`^V~jG`OT zNpXJqS>29(H+uO@&NwqO$@Hg3Nx@B%dfqQlKc=)m;HA*HAmyei1uYNkfLc({D6;*X zg6_*T+OoDv)}|5P^Pn7lt8L@D-|QdHAH3f^WsQEm@I@5+{t88aU05Iy7z8B2bF^x# z&}oB90@CzKZpB+iS(P+bum9UuiCtNdA0%nnJ<$q_Mp%GgeX)6Dy)6DtH7kFjnGf01emEaQ9s1nKmZb9oJ$S|)wsiu4YUNGpJHVopOc??^^O4+1C&otD5I9_`S)x z`?VU6qOHHaEdszpCk9+kZoB>dDTS9ptK~459x7TajgjNdj2k+UjKtlsz= zG3d#YKM_wNM&q(S#1hxz0J!w_1iMhmMf3Eo+4l!M*ssDZif+j35Q`5h;zB+$zrdy1sbrkMQFF^^xFK4Vg3hipdMmFbo{ya*qX)?#JrN3jQA5M`F(Y|JNnAPq@= zb{gHa17Z=ic;?STnzy$OT{BCQsT?$SS1#{69un9F$PkGjb}{Q$_IEM0!oYuCUz8|1 z@=7WcRBdq*$-Ipdc{kR|hpW^q_4d&v{q{f;iToLp@17iOlLB?w&tn(Xe8;XbCXQVk zO?+XNOW!sE=%n6NFxqWAy^-g^qo=T%+C#3#2Nldu0!{#LDu+05&zmXF^3~JB;XU>A zH#|&mE1r7oN-zU@LK@e){Y?$R+$xPaB6_#$-LcNMuYW+)eg+ALHLiacGl^I<7K2+O zJX5!$?apyvf6<{2mPcvs# z(XWndS9lTzv?~vH{0)+Y5$E@*rMV?JX68mL-Dp;1UPxQ$xVK6!+&eT!wLlihe|0SJ zz^57I>EAO7#e6*_5VeheJ;Skzbk(;C4xXufhMcqUkF4Hx1Iz^tN;A;mvMv~{L#HRh zpeAIgMf!Q3LeG-_)*16Js3I(TOC(+ z$M)76Uc?`wEv{OvAI|06><2)d^|raEQaxHlV7q2@)%ou{3@|n)4|u!Ie2`dQCI2u7 zpg$=~QtRxE1eBu)c>nyfrDCs>4GYOhXH>Im)^r5&L7#4J*Aj9Yt>EC~gRUR!sk~2j zat)l6m&hmW3qcG28K)+77~6S=g+&O<26chU1aRI|fOOxd1d|}{a^`xtbCoRIX z}F{R55k7%Ue-)c}~dzeUkyTzK61rn^De_cgPIR7`^N_=Bye#z9WL_;USF)`yY? zAF4@Ckx;C#N{@E-Z>_Kko=@yzJzDdrbZbz7f)YmvlsIi;9(*gIO9 zi~In6;p)ld={!*p$a3^aW8LtThTGoZ1E5qmBE67x1U!r5xU*r#`bvH6Cu^QmkZEJm z_5k)7)V$hjnweC@Dm2kW}}p->fVDiqSM!)=3lmogjx1g z)gfQh>J>7;r`NZQ_>>uw8N>^-C-j~0cuSzkw`(>%$@2JCf|0p0O8^%@p~Qw1LF{9B ze{VW{*RCNQU`4;Ero7b>Piz_xZW2mJY1*TlC7a4BT4{7m^1RBg)%tC%!sQKVsy5}R zFUGdKt;5di>}vqn#8+bsE8QK4hZ{ZxA(2CEpd@f4Ypm{P@L@4l?++UP+Uyn1aZ~Ya zv>1^Wzv8T0(gvw@y>PlRwlrDy=kTaUyp6y%FwPHSuMf;h46J2}mge^4Q&ifwU?y`N#+r*q6J0z!)ZjcXG+ z>wz#<(5-{2nX*RURO3=DyHt}^>t6oaFovUrF;DT^RG+RhmSY$80F1DU-+{^EHM6%H z1ET|Sqm;Ch-bbwPmeEfiL$}JSU_QQ|3p;FsTL{<7@|LzO?^PYt8R`$PS*4~uWw@<& zjJ?4!FMnuP?D>Jk=5~WrVxdYr*d!m`RQ`Ri7dJyVM~s!_T>3A_r~LL=%IfDRq_1d4 zDkL7xaYfB>uq0|_dEpG0ZP613+MN9Uqq2zFqe{ z28^GeU0vdfHC-!WE}o0x(BIury6(|sDwc0JP>3r{823qengC-$&o3I(Z!QvO>5kGiKKAPV84SBYF_{)WO@2-<3 zC9W14YC3|c6RPAo0uArQukHNn*Pil7j;4vQC!m-u97mt(EHOBNcv9Q7lq9h1tEBh` z>G=LT3wIRkN|J3gUm_0yl^I+4}^$XQDE3r~Uf5-%Rs=&U@ z9KEFdL1=o$`o(Sk1>0k%?Ryl5^8p?z!J*wNSuyn9Y_NU(sPwvRq?64CzkQU+A27-^ zyLbUL1lV(TlC}xDhhg-qJL2^=UR@r{VpIc!;x9z0WXdS{Elw2BJM1bX8~NS*R+b`l zV1FRE)qBb*F6ru8Hs*z0Lx5CikjB-=Tm1B%C-Ey!@+wA4G(Z)iFNGC>3@gTtk6DFsiUs?v8`lHE(De&Fp9+zUry@F9{bXebEx|sDLRes;neK zvDPOclPVvbJH;Ii5YQ$}dlh?xW>_Y*zr=Ds#q%+oy-6@8@kg>3iANLYp)j3t6peL! zKd376-YHF_Oc7T$eT5QIkv~pbj37@dgWmV3m$&rg-&@tppuq?2ufnSWt3~Fr3%M?J z+1f<36J?n2Y`4dh#7kSbxhfas4>Ob8cHR?MT3)_#-vI30d}zkLV?QVff&v?7?mV#k zmbC6oI>ln)q3<@agIzDxD=h}~LMVFlz+x%8%li2dFut{t<1baRvtig7*5I^pVErS6qCy50uN)D>w{C zCFX*0PD@j#!W)&sOY&7ER|XbIPupauz98|M{GMwuwb}t(o&4G~EG9fwdZpxXH@4>1 zblChEZ@UI(%{l*V4Gw|rJ>S@14fuAr_+~bI(ImzR>fq|FJBU0tr|T5OOw8JQ67a>VQjL)Vm1{=}(-CpV%` z*Rm1)t#hhADNQ9%3r6$N#HxyNFFL4o)bAp67Y@CdOwObRssUAW8^1&OGWwL_t@vj= zsmM1KU&n+T+Qq1;x#wA&JNk0g=6%Mt5V%rJPoF(3E};;Iy*GSw#CkONtD^0&2W64X zhBQz>f>~4!vs?mZpH7q}my-3O*08$Vn+3&D-1+jt4ZZMxAMpp4q7rw+^8dp^MX&mP zmkiMnE-z_M62I`In9?mb9N72J&rVzHV;ezpFd2cf_7VbgMbz2D9A^ax^KSegc zzIiQg&L}x9KDwB=wU=h;QC$u$;(g zT(8S|E`~!mCC*__7JIBsWGfg^vjQo$rGrMfOs02R2s4}NM$Tp!My4pb+=b1aY8mSo zQHn=IJg_FqR~)kqhi6Y4Y5 zrtTjoyYV2i3z<=m=H|O3b+RQYgb*mNmKGy}DoNIoG1H=Zl806O&!wpPU-@a^Bwa}- za^)fa)zEwR_nCBw6@h3gb!AaY^bbejDn{&x_G>>uu-sNg`ny`A;sb%H+WS#^UyX-+g{f1xHlYiy2ph#Kvl)V+@ zp9pq9RtU)RNnV+@yW@kMc(~t=Q!ty6ezUX0b*@Tq@4L&1-&2@>)P;u{m5nFcFvoS` zt~CT98*qBhzkVp3D4jBwgSoRLF+7=cR;$h3mwjg{09EbLk)o7I*Yn+sk{HJnLYE%K z>K_E4EDhiv*`|0hr%^(f7xB)c!e#VVoKtV6&Y2PRXSkNiwmByj)>D8)b97{nTRc)zql~>;6h#X(rNv8!{}>FKi`q zD$43w?o@md==1 zkLZwl@>Nr(xJvg;0HC3T=>aD8qY8HKHNwkvEW(FgEGsmQfss0E>s}PZYMyxgzYj>F z_LuEEgawXJ@dm+pyb^^l{c0)kxp&`n6zG`cebuyjceTAZ87FohB}>*VgJTX_T=!2N zd<{w)VKj-)MdjMbWNp#K1aHV$!h#yXOP#QwezvU6_KYt2P8ukTt_`SJKF=~&zZRVP zEI=EK#iI)+qm&-Z=pu+vj7o4aX-a`W}~UJ6)N%bN8+* zjGO1nm#7@!rd?K(JHAhn|0E!=L|VNY-s~5~0b{GOh=e zHh+HpPEjE8&WGUoT6NQ79Y?N=NtCJ;E;kmmlTI2X;+&aY&*db=V&LpngHb@|U4u@p zZ%PF)GPUsup_Zd0BKmxoAua1@y3YFvetHFZcuEuGVi*SPCwvG<%L4$###7AcBn$AI zDINa&H2-FsMC>hs`-GlW{H9(B?1aC&9m-#SG5O*JTGH??2zi^VI~K4CYKr6$;PXql z;&i9fq*igZo*O%O!UHiT<+n~(i6ldDg|8gHgjaa}f;e?|?9KelL&u(qWG5B0kQH`% zpUtv(w9U7Iz68VGd0qUdqo@xqCo|s%O{h6MWVVJ}SXJ`y@8)V}GEqQ0Y zE|^q1j3f7y&b8llz`s9*m?-pjI4A3jyCR<=_5#lCs}f_*vSv#cq$p*gn#(-yp|Ovw zaSKkvau7SBfKzKn`Im8W!h<%Q-W8_ZcrEqGMw9bK>1AEom`7J(Y1FF(tEj{xiEkER z09~g=m8;QK_S+7#uJ#zlust_9N8REid&0gdv#ODg64&SFFM#I;uEFA`6U>YPSK-H| zuJ*m1ccZ|HAlI1?Q?<>Jp-`l!{B|v*ePtII*6*Ynm)WCbDSbU_Ncu zWY-xbTlpURQ55;D*}4@VGl^duDeb)@RAhLO(dM* z$A&#PVq861n;}_so|yY;3aU+l+`Rbw`YjP%mXfFbu%EK8wZTNEI@gdS z(}TLYvK@qVU1i9IzLGe-W3LBXKMpp;L5-;Oar+(*UhDUR3-alNJVef7L_*5h)k{V@ z(4BWK%}}g$UiOmUWqH0Mp+XLWqJ6d}xl8R|=lHI`E*o5dP>_y@Ba|!oBOlH);AKvErk{hG#FT^O}zM98nGl?Co&) zSwx=Y2`{L6?PAJ+$=PA>eY3AE2g07kZOki<_!brZ_~yS?4d6txxC=X3zi_{h#NooU z)nt$3CDd`Z55Q5Dm<`X6u2uB4^EzP0XoH^E+cJB}yfWxPj-P``uSJ~Xv7gF{VgN(r z^5N3hd+$r3<;W<4V#A0gjiLM*(6>&kj?Y);fAn{Ta~_%=4G>%g&4xfiL|cRr%1za} zL)hcK*pT=Jp)p}uA=273OX=1N7|Grj4K!b+|o`} zx`7&?GtVC+9F9cWdZow4uU1HLZJJClzEbt)s;y2mZZ|2bkzOBT?lL&G+qV$kV-%~Z zWZB~OhSB~982%bG<|T8VMe$ZcM+!G*mEGq1J0)YV4z`6)%I*BH0iDKYA_f#MaH=St zeaB!SlhSty_xYN24wbU(Hx9j&2;0E$r#$6>Y5f zQ1{hXIlSs_cypyA4^r#PRPr6>^X2{nDBiH-F0<3wOL_0iu)9_OGd1|pvEvC!WrRs4I=gH5r zaI#XS0)l&y+VLJR%RD4+&;$faeO~Jhq@ag{w?rDx1s7q`{~xihT6Q+6eqoBruRg)ixjGQK=I;T^41_t9F(O?(apC4m?PC5G%4mXJm+NV@M#Wpm^W$4#uyRr@ zKU-|Tka`UWj^~~Fa?rYc%C~N*mK($MK45-~avJgI4*q&3A@G=6c?coDK{nQ)aed~g zb6_ZP&>tQ{uY3OaLH_z($+dnHdkdl=x3)5&u_i<{oA~>+WxIyNvA;+!f`>f+ zzaBXaW}c&l*_#@7*uCPruRLO8QzO~fAE$`ECio*_YWf}x=^oaEsIks0qZ4WhMw*h- zZDqm`?<-|l*^Yso#zl3xrj3mk!!og>7GTpdenFHV)g~KD^3cz72*9uyh`?oCRn$~E zqOGS*JhmiYj^w8-APzw_kWxS05mtzJ32L1P=HnmN+x}sME3@;XOuEu}ys(up?r5f~ z^tUSNsT8M`hbb^Ty@17LJcHhZaBLeraNkq_+k*k@V8C|Sr34UXO@DKSlah@_?rdUG zA=%1LJZ}VTgO;C+Wk?R~#%7QaD{C0$6v7FbNdUUKWU{t*Eig>@O#JiQa;3QPDr9)-a{0u1K9+Z}KC@3TnvY?e`-d*JU0LE_1BQy3_#TyU8 z<)QeX+R*xVJqFNnut=>A!`|J;4n14tYB3+- z8GICXxGw+r2Jlk0&wib;=bKW0^Q`H@`oa6I7kEF~{{egKnu9B2cPBcizc$o@2}%yC zTPllwt79|F0RjFDqWRDkB}wp**GtMp%%I&Bmt)(Ni$B+_ydv#5gaB&(!X!uUjjK^6 zGT9z;u_DVJESneMv7k1NW!X(c5zbV2zqUpGIFaCnJgq%~{>Lq8u|?V`$Q=&zN(>soH_PDj&Rr6zf9N}rxLf0 zEvvf=CNCHqN5q&iZbld9Vu8BY&+ki0cuGEC4*SJLsH!l71WUGkl52w>OBb%>0!@(s ztnpF#0y$*d5|+yI=4%zH(JprHA}rZ|b}x6y`xXCG39{)+4{{^Iir96k9+$*ieOsV# zCVmT<3{=FQ&PZS0jzgH=9K;CRMJykG5$}c~_nDz+w`X;lgurr!@7a5-qs?95h;sL$ zgJ#j_Cs-$iwP#Yph%f2+=3fw#P%7AkUHTgnCeks~BbO8hQulK6G%wX$@7W=-m-@m& zk$Vj$k~o?!_*Q7?Wr8Va-j7N?Q^&chY;u74jb@Fxzg$YyQXqjNk5y z#br(#57JKa^>skrGlsi92w6gH=$v~Boeam*GbSy2K>k9S`6nH}Yz)0|bOcGoGapSLRxQ0e- z*LWUifw9CxJGmr|zhyCW#TrY02!dP4H;iRv4>}BHp(?V@j&KGL7EXf2G&f89q|IW; zR{}D8jHch)#nr#$6&dy@{`KSiKbkRC_iC)Bl04G58o5*y3p4VSw8rr=&|7@wld5E} z*M(lnYgy{&tpPoQxPi=}PVp#_6$#efGGoyK9Da>6$qZDXcn@|UB@&X>v#xB=J-`(U zrwIlid^8jwz>1W}SmJ%di`dWHw-45+ErxT`FMbfVzp;6+a6Q87{Eu@3L{cP+>yaU4 zxW&2=w3sxWxs+X??ne&@S|m@l|u3a6myKPC7W9%)VbeOjx5f-#2|R&%hA(Gw}RK4@$>{4 zw?RdgYD3n(m(TI?)fRne5~9B9gP+WmkzR2sdZG&mrStCGY8Qtf&cgH@arn%9{}Jz! z<#m3p;K1HaS}Rs=A}pmb4}G#{6BhJezpKqyyR?5>-2ZO{Y;8@+m*r2^ZitWmaLOn# zC0#;b>J51Nka2ll@9Y|(xHV+QLwv*w$$gT%zvK@pCVRBLz>aII_+{M1+7`T}PIL-}r|sdm@VVJA$`U@LUF%I0MpYa}OVhwu1>23fL=^8@!Wz_wVF8(T#cbc=J zi}s+F*>a7)15R>r{iI!?uApP$A>{d4Z~wC7)UVLjK$n+@$70Mizb{JGp4?3yu7+cK5$&ts5^ipi&q@aUvahvfm%rPrm-X*hY@2hoD_TIIWUd_ zf>2H>kHLgjTlxe)tR{Ex6#CoOeknP2=3@&#nzzkRRbnDZ#4@h0wbp6PSI)lwIHjyk zRh0hOO*E=5+mEKfT+3h2?g9PWrI&1sH6Lw`3;&(dzbUxxcru#- zx^JGZ()Mk5xwJy0dP@;!@_iddb?lrw%TRaejnyNcVWOPPU|5 zQ;FzK%*921Q44Me58560kHuX}93FgUf$wf;ZoBnD96MlpMFjTQNS0Cx4I)$AD+xht zf(fTo&p7;PbESG*9775(nLfJi1$sS=tToj$BgB7n`QlT*BKBF%Iw=d*?@%qzCuoVVmDrO7ES*z7^Zl=mN6k>ysm;%1wO z3MazxM-t5PctWgdDi|4*nM#Yo?)BCDwaK~mk=2Zw!QS=dBH2u}NcvNzF>fk)sACm* z-YVgefc_1~r6szN ztSLMi>5|;1t8X2O0f_pgws#d3wTa$)lpFDWwp!hFT z*M9Gy+AD8XGgpo%2PvmFyL>!*ox0(6X5y{<&~h-4--hP+0Yez>@X+q^&ShE95fj7k zBv+t%E&$vB3X?TJI65TwPila9bYOSzQHu(u;@3zM6{8=@9}Uc=LD$`*U^(IusRm*Z zZFd}l8(!&XeSP)r)g}v;VljJrJ5adYKO2a(cCyvZmhRB5Y^ij7Uql+}aCBNs7%*Fw zoRqA-#U3RU7e3GP8hLb@Ck<)hNu7#0TI|vhrlzgg**_m{QHnsFCmpnq|BVd&M;2x> zUAC*={xI(rpavV20B82+8oIiho<2Q4n|Rf~^E1qi?LN9f{ z&k5wXFpmcl`g{dRMg%8SRwp{SNjryE;zO#L7*tD?&YWcMq;3+|tcQ-)_P1`Lk976v z|F0JoBd(HcFnr{^Bcbc~6-GWd`B+ zCwGRqR?9ADbyR{55$E|U#`8I<&$zuJx!bY32tOefn9M~vwm9BdVyZ=~n~K5Vc-adrF5mf^&I*ul_>p;vVJ8 z(Ld?CK3%FDP`YVLNg59bl64hqmV0#;Y<3?yp9t2WIjY+2+2XNP+%iskL}^SPUXQmL0EWCJzvN=gYePO zwIoc$`k-6_@6$9e?CmVEIxs1-nYV=?e;+y}V_ExI&I6A$Kry>!zxRTS;UzrRNpc-o zar};n{KKnI(Xyyg|DZ*SE*ZE>#Dd;;p=aojKlc<`gK-ftF~fG|)l9{^hS`H&_*8dDOog{|VSuZa9sZ1qcj1d!tTwPJn^f!Ul`<}zT= z<>~yJuoeHhX?QBF5wPUN0Wpr{%E>L@M@`v%JS=n@*J&Oq% z`sie#A^(Z3%H@`7VaB{y*IUhQ|D#rWy5-(IyO5_&&pO1TAi_;9OLajd1yx?LrYC_{ zrpkI{Dxyy^%c5R|KTT}_QTh z*9OJEhlVayUa3ZQPg*U^GNphY^ua@87b=-%#ZETPZ1%TSaVsJqp)Q`CNw1@Jm!%~*SWkIHzQ0jnVUFV|$JirunRUKQn3 zag7KkWzT)uZ&@G8L8%AkdEb0R{Q54umJ&&JYL%+XNNj=BodmK0)}RkHF^~vk1O7p2 zkNmmP7yzpL&ttn=h0@fJH8GiQI2vPU$J$qzq4{BPuDyEyXn%Ef#%sNMytyJuB}eST zFN-F6giB4?(hyRi`5UV?=Vq%ogNuv+i2R&lIs1CMx4h%HJT6+VEh4d`zQf8cgXP|~ zYNJW25P9nutfh{#K5TYCwk{*OMXBzUXQ40lRn69IU94VYp+!R##PgMYxmQ`BD3ysIvW-th63 z&zLcFgR>?e|M4e0RlwRXb{m(2!)0JcJ7v@qFC*CSFNS@~=ZW~4J8TcYX)aL0{u=Ef zPx&?a*R9yKf`R6@>A0{68TAUOMRBfbhkAbdZy|hFXve)|%E3Kyn`1=_^GaeI} zq%F0P<%cVQ7fNs`zvfUWRFk||*p9u%GF$)85VDc^b{F+q=Pg);OHn`muZQ#RZw1P` zlbEMS?*IE$1%52x|F!N{@DFaQ&FNL5WyyvTz?4<5{p$;{>g9qUqc{+T@Lf#rZ zdXb2mtqg2@(vIx_weoyI_x8JXc(#6}Ve69=o%`nshIZ0*Ry4^4D+PR4Qc9Kvx<7AJ zLGX>=zr6iQrR>eWyY5x^X2`t2R`Zu_!vBaD&(|4`EfY;kSw;PgJdmd*<=*{e8F4lD z-wG{|?PWJ!McuE_EO{Kr{Pquw|H^7V3Z8$Tdw#5v=o5K7lX8dkccK4jL$Tw-CzQ(C zuCKO@5nw?g^jSEc<>QXFa*qiY%1BxM zgGbs8VC8Bf+N0_>uk}65#(;xX;fKyQE8TIKiMP|Qr;ez{rc~_rja=9y4gS=sUH5A; zAT9M#l8gI@DAY~ z8e$b`rR#fMPPKyHmVT^p{mkH<@k$H27q+lETn+Kp+KIXBNG}Cd0&8~_o%uL4AS!O| zpN3jHf8QV7*u0QRM=TC_vOn4{2=5%^J{s8i=Ax+0;7ycIP~h@Z+!$5V;v45d1t-k) zeTVsHRmq(HAbhr5@&&tEoATp^=Dg28R!2zHom8H9j14}>OOjI%?DzOs#P~9vK9RZ} z^3Di_!ou+@8n(Ta< z9&1SM)#3%>nj}RkLd>+FuAIWT{5dQjkko5=Ixi*AGONvel3f4oZ!P}11#dROLYNQhrv1yrY?pWb&gn#Z#kUkSpJubk50e8v$W|yDX$Fv6 z*%gR4moZchyT%^e%yWT?{XXUA2~>CTQo|(NDqNzWSt*P&Z}I-B~2uc$pf+$^(016W69i(?qq)31uy*ELc6bZdc3(^IVDug06K%_|^^n?;Z`2ypH$!V3C ze}|qY;{9#-aWL{ivKGg;V$X>SvVS)S(qml`ty z9#mT`lpsZ1_V$ApM5hfjuGHbVoSqUa6gja zu5<4MZrtn_;o954T+woXokNOLEZp289mI(tBLr%*Y^KW4F5x^RtodKEWb0OAxru9h zaYqOKQp%FV^uCdLk`Fr0=!luM`ke!4+lv{!!+sZdyx|$LIeGfa6Ol@4L1LpZB)Ov( z81ZnoyZ%gsiPijuC`qh@+YMy}+xHt@@V>s{>f$8m3~8G}2k|(NqV^te+O`Yy+C@Au)4d41hbQ*dsF`jvo`<6P{_m}BlfPlcpOR6Ji`6a!^+;KKGN?&tvGp6jgS~)lJ6;Ms6es5^|Jno| zThy^@Cpk5xwlhQc{lWJ7#N2@6^O%{)%FL*Y&LeF@ftSZCyz(C1Nhv_0WM%yfJ!v|< zsrl_-Q4n!=6`b^M`7?BXd*gq(N)lS{OpKjp!d0Gt#rA~GYv4nMVl&}&Inx=I4EQZNNn5`*&!h^8f86=Gg2ly;0b=%Il9MAWNMG|r+c!27(=)D> zt%Gc|gNX@5D1YIow&q4zy-9+)W*6-0`2N*&*-dy<=E*kncQkQ=1*AHE|HK}NZ12)e z{QHf&$<=m#btOOLy~)?jdo^Bb*`%{8`R995qdTN6;7Qw>=DVyUD-7-l_B{6e$*RWj z8p}c!^V)`GVO*sB_&?l9>;E@>&iAfF#INYYEc_3BhE(9-gz0~FK%#x0{YMxUOR6!- zIJg>TX?PHjBBr%Gw6Gc*P`q2yws@_=@jg#!(i?%G9kunZj42nPkwGw}LDYLBJ+(T1YSV@=&{(O-)p+ zckhmuIXXe|I@)b5h2K4*o{&6IS0Co+`C7@qD{P`{;qKak@fh}|xTMZ46qC+TU#&n4 zvXjNsY|F2-^If(P7FE7;=V)ZykZ~0s(X{tQU%1wcHY{J#`zvhy=%RPT_ayu4NET?m zxO3LByW>^&`!hQOjG3jeTpW!MK*v&s`D|Mmx@Jb;l>x@`$GhvLUn(-{w+*;5otBSe zNJUv>PLoH(MH)b75<6)~m7_QNNdL#hmNPioJgS8pP8Nf2jK&kBrrvy~8;reFvR)eM zfA|s&A0ckHZ5fPiXAr>|iEsS@;P>knZX&T7SgTTc976nu7P4zM|9K{vYuWy6q zdN5}ZxCq;U9Y*&yFSGZ6`{DX~%Pd1~c)T7~*mD1Ti4M(k!E*~s$X1l|tB z0`PBKFqB~Hp4`zZBrD&aetKiA9)DEr7TBZ0x_IU2lzWU@O=xVPzO#x>_603{@H}5I z^)&10vT&Et81|U?(f&(3meKr~ta1xt^kXf|oTsh@nQ@psC=Xy_*I`pODGPt*7~$W` z?iTWuCiy=O4+Jo}b@9oNGi zGHU=?opRpxqinOh3NXTU((iV_gU z?Q5cUWL`6t{!nw;z{hbWIDwI(_|N!o=S5rv%(gd*!Nn`z&H$(ba+wr-WB9nUEH?$8 zth@~B`jn=duRX$MaIRiHysIR5p*=M;lIIbUD@$i>W*whjITb7sXSgI@F%V+d z4tY|G&6WM^R)P_tEwmghS7Np9Zwe@I-RVAa$dqD5*=q!UioZ#AxU#S?9}BU-Z@bwF zcjc_qE%gWhyZu?xLz(inLMFi`fzc)3%YrRN;=>3QQ-rxp5aO<#_it z!UxpCHT_PSV3(fyj`&unsT`zwncbgGMuA676>f>m<5W`{u%?k&@=>zA-7A+H5!EGt zJA2q7k%Y00DZZlirjmoX2(-$VOkP}?t8vIc%Av1eE>VWfSvg$hgj5?oH4wFa30p}k3bfPxz)43=v=~6`>}>feu8NV) zA}IS78~-+}CPV86qpDMpBpd(h&~2Va3-s`r`5?aCiw6TOA?IB=);?Rj+?fElHxU>vNR^eT=dQ!AYVq- zz)WLb!fusqDtam*gt!cHN@~;dz3S41rV}@+l{l?#4B%L>;VS_0@k3*8$B6CTi`Al1 z+(YB-;y_+)JBfn5Xm>3mx=nS3AS~t~Po?+)IL<6AG!t*Md_Jt! z?@Erk)JGwhLGE?On_ayNm#eJo?Sk{Pdp?(hWYL|)@D?olO?TcJLm$fKrj_^6y_-mimMTTWo zeD`{nFMm$SSFD15)@1FL?aJxgli{js;s7`ptyj=C$e&rg1~VazAH z%P{Hbf|9d2g`D%%oiGn-vs>MLg*O^BlWFmf<<;Dkv-2n}Z++MCQ#XFLY1MVfpN93m z?BrVK1GQ~tS+&Y1v0M<>52MuOiQC22qMQa4x=s`mipfI^5(^Z1Q` zEgnoo3|mDHml#@4#`GAKV<_{?FIfYapM|rkFuFB4dxE8MRSP79&(3#J@~;YoNR#j@ zl?*2M1P#r%4SG*EP`z*R@UOD1&)jgXg8`E|y#5P+_G-ynhTfg!Q%T9=yDKelR@4g8 zG}zBb^a!P70BJl^#?)?`ioj8F^OGG>eC|bFmxNtfpAKl$N_Qkg6^n6mKn=F2EBx4j z&EhwYfH?a3R6Yk6*lk|jjV1xlwlkAPCHQWOmv-&2#iembKTdi#C}VQ7ZK3$A&r9E~ za%pweGZBapF_*cE@LV=J8MaL-Z94gIZ9-;)lxyziiF)&QjO=XO?7&v$IG#OH=&+FqW6eiD&3A7k63alPz$#KF%CZ*Z8}cQY^=Ij!MZNxj2G8s-}qV{K_awk28T*_BbB^1lP_9>_JTGEfU2}~}Ns`*E8 z-ixc{HNhq#6ANC#>y<-GLzK+M&K07rtK>4O<;ErluoszNkswWi zOFGrixGYRkvn=$-fytEMq}uAUT(|W+&GEpY#;(|uls7)B2fYb|Rl>Y%W{NE875S~5 z?x(z*x}T|Mbkzv%sJ6LIgb~e|^GXSCQ(|KVnWL$J_?rfrD)>qqMjGPi0VQ|QsA(^` ziS2m5xZ%%g8-$7N(JQwT$>KOPxd@;+5kL)*h5%)IJw>x-#y$NolYp3M}p44*sCTL(2KLI5}ioSh2tXrR9-r5S% zFmTDrxt@%P5{M^#gs^0pSkUbZd7XFRd1LzMuDQgpDju9t!VMK2oc>X=*ts+O3|{~8 zOL6ZGAcv5u3@SXN7Y+-9tIYdgB-7u72>J~SXYGq}qeqlSe_CF|FX~nRV8&+nu7d5td+H}9Pn~sgqJFUC@q$CILw#-X ztwp&daxKj+t8w%e!cw?Gi!thm`+ijgv<|JW)9=!2v33oET(W4dt=G42?ZxrQ-fEi! z`3bxJRpGXFY2p;H^YuRK^yDcHe7$dDj#7r)nN}E_tTgsd$|p+z<3xHc`{zd$MwH=XU%6(CBNrls0}0 z$PFcrhMei*N`9~_=C)d%7CK1D!c9fW4^7V&{+G%2!AJ!z*`AV(E?=qUaFgon{6$`9 zES-}*4z)80h7Bg^X{|5eFsO59tW!6wBS6jY_7GbDNAL9 zJr~MBp(21U>k{g{{-9+v8Ug%(W3*_DW3BV}(MOuTR&%_tPH5L6kW)hw#gS_^i z+S3c|EueefMXs(-e<8QjwI&i)-9ba7lVJWrF!KGD#q9SWz+->6lF=c8WJeKAn9Gww z-vI-s+{0qbCkidX5zzBWToooH-^Ne2$Mbv2`*J%rnYl2$|NCdd=!&yptJP6hg?6rL ziOfN}W^z|8k`(dcgKF&r;*~tReKtG z6r$9gN-fY@Q4BcnAxdb!sv5WTLLZG__;h!3?A}sx8tnEHVb5J6MAG2t&n;QLiYh$! zs2aEzEVJ2?n$A1YW!szOMeEY~ z66PqgrsbAO6YS+IOHZBn5}Gc}d)^{XS^Eb=Hud5%5YQnvQos%vHq(udLD(G*7}wRh znWOp^ukrsh4d+&+N)4Rp&^YK+ZuQW?=X*~~*aUXEX_ltm^2dIrFfXjj$%Sb5xyLy= z&oPv(OE9LyA+A_K;w2(i8#= zy>SJ#5U+6v+x;&4Y9Z7hm5TGRbCco?8Ik-n`(0*PLg8#vv*4U(D#=i4G!tE}piT|r z{euIo3Wk9zZX?V>@XoR+GXB8tyMj`q*DEUA^rxvzC|dz`EM%?Ard?`!^<}tmly)wi z$(0-2{;h9=gO%&W4rwd3csOP8rHt@FQHS12P4sU&?;x2fItxa0NXslRl>Mqx;$nFTg6Auo{PSe7I#!pn6uU;J{W9f9rj$np}v!C-Z*!%$Q*fMskp8 z-b5?29k4`4e=j}S0;|nTilEt!1Dg9tYi}_tahFpyTLbFDitPmOBh$$zDNGAT33=E{ zmdV{8YM1u%DxIB_m2Ec4KH%)Tra*!*ISiA{F-0)c9$JwPk&laAN=ko}p3eGqm=2Xt z4J;uiIiZ>D5<|K@_q6;vic(08y71$X#4S5D$;pbDc6M6;7)i%wVjMac5P-HJJByu{ zX7f5R(gE$GiXBXPZcc04KN9(2gNYK{l|jdza(xuR$BRC zM8!%QXfkm7gmYf@x1EJEz&Eod0jPASrhMWw?1dM@{JDNg`1htX<^keI@>VIzlx@Nd znjPcS#o4mLJ;?(?X!9Uay!5QnH^3<&_yQZ0ou*~DSdUa=;zzEB%dS&_-;aJcd(7Y9)YhSVZoIEM*!Zs@|=GyxA_H@2|e@%03X-?QK~Z2`sL{P?PiHgiT1XV8tcTRtsgxgm+|=! zmT_SWnjj0&r?L-x8GTO)-|AjkbtFrb{wi@6R1!41qex_NTs-`-#t?iz5JPmr88$ek zUFismqOv7kixlZ2izyF7i+1)>Mmeo@{=LvUzy7J3^XjLI@=-)rt zQR>kAp?NM}kT~g()Kd}pANm4KXxc9}GMcCV(4pjD$S*eRg=Bwdu-{L)NNl2qe#v-D z!=?WH?~|5^N^B#e#<)&!ZpkRP zmJ;`i6{W!UY7=%T1;QWZbO9vhCg6WY@{2Kv(c?>hrsnt4=<>jS=HfR`_N=0efAsZR z07@R)KZ^Ki5~NpMq=e)DwD|Qj`pyr)+!5JITIGIK|Eld@q~9a3vTTXWGvEIK4g%kH literal 0 HcmV?d00001 From 4d14531d0de716c28dc324a157a7b25f09f4eb36 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Tue, 18 Oct 2022 10:29:10 +0200 Subject: [PATCH 027/130] Fix binary inputs sample (#1475) * Fix binary inputs sample --- client/python/tensorflow-serving-api/samples/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/python/tensorflow-serving-api/samples/README.md b/client/python/tensorflow-serving-api/samples/README.md index eb214f6851..ceacc8d517 100644 --- a/client/python/tensorflow-serving-api/samples/README.md +++ b/client/python/tensorflow-serving-api/samples/README.md @@ -217,7 +217,7 @@ Classification accuracy: 100.00 Using binary inputs feature requires loading model with layout set to `--layout NHWC:NCHW`: ```bash -docker run -d --rm -e "http_proxy=$http_proxy" -e "https_proxy=$https_proxy" -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path gs://ovms-public-eu/resnet50 --port 9000 --rest_port 8000 --layout NHWC:NCHW +docker run -d --rm -e "http_proxy=$http_proxy" -e "https_proxy=$https_proxy" -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path gs://ovms-public-eu/resnet50-binary --port 9000 --rest_port 8000 --layout NHWC:NCHW ``` ```bash python grpc_predict_binary_resnet.py --help @@ -246,7 +246,7 @@ usage: grpc_predict_binary_resnet.py [-h] [--images_list IMAGES_LIST] - Usage example ```bash -python grpc_predict_binary_resnet.py --grpc_address localhost --model_name resnet --input_name data --output_name prob --grpc_port 9000 --images input_images.txt +python grpc_predict_binary_resnet.py --grpc_address localhost --model_name resnet --input_name 0 --output_name 1463 --grpc_port 9000 --images input_images.txt Start processing: Model name: resnet @@ -532,11 +532,11 @@ time standard deviation: 2.93 time variance: 8.56 Classification accuracy: 100.00 ``` -#### **Submitting gRPC requests with data in binary format:** +#### **Submitting REST requests with data in binary format:** Using binary inputs feature requires loading model with layout set to `--layout NHWC:NCHW`: ```bash -docker run -d --rm -e "http_proxy=$http_proxy" -e "https_proxy=$https_proxy" -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path gs://ovms-public-eu/resnet50 --port 9000 --rest_port 8000 --layout NHWC:NCHW +docker run -d --rm -e "http_proxy=$http_proxy" -e "https_proxy=$https_proxy" -p 8000:8000 -p 9000:9000 openvino/model_server:latest --model_name resnet --model_path gs://ovms-public-eu/resnet50-binary --port 9000 --rest_port 8000 --layout NHWC:NCHW ``` ```bash @@ -604,4 +604,4 @@ output shape: (1, 1000) 10 zebra 340 ; Correct match. Overall accuracy= 100.0 % Average latency= 27.4 ms -``` \ No newline at end of file +``` From ac3b36c4429dbfaed65fc30e24bdfdb8a109675a Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Thu, 20 Oct 2022 11:21:38 +0200 Subject: [PATCH 028/130] CVS-94091_doc_fixes (#1479) * export BASE_OS added * docs/dynamic_shape_custom_node.md updated Co-authored-by: ngrozae --- docs/dynamic_shape_custom_node.md | 7 +++++-- docs/host.md | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/dynamic_shape_custom_node.md b/docs/dynamic_shape_custom_node.md index 773f46b550..2ab82ba971 100644 --- a/docs/dynamic_shape_custom_node.md +++ b/docs/dynamic_shape_custom_node.md @@ -43,12 +43,15 @@ docker pull openvino/model_server:latest 3. Build the custom node: ```bash - make BASE_OS=ubuntu NODES=image_transformation + # replace to 'redhat` if using UBI base image + export BASE_OS=ubuntu + + make BASE_OS=${BASE_OS} NODES=image_transformation ``` 4. Copy the custom node to the `models` repository: ```bash - cp lib/ubuntu/libcustom_node_image_transformation.so ../../models/libcustom_node_image_transformation.so + cp lib/${BASE_OS}/libcustom_node_image_transformation.so ../../models/libcustom_node_image_transformation.so ``` #### Create Model Server Configuration File diff --git a/docs/host.md b/docs/host.md index bf92c83579..00e9fcda2a 100644 --- a/docs/host.md +++ b/docs/host.md @@ -26,13 +26,16 @@ Here is an example of this process: git clone https://github.com/openvinotoolkit/model_server.git cd model_server - + +# replace to 'redhat` if using UBI base image +export BASE_OS=ubuntu + # automatically build a container from source # it will also place a copy of the binary package in the `dist` subfolder in the Model Server root directory -make docker_build +make docker_build BASE_OS=${BASE_OS} # unpack the `tar.gz` file -cd dist/ubuntu && tar -xzvf ovms.tar.gz +cd dist/${BASE_OS} && tar -xzvf ovms.tar.gz ``` From e56da882fa92bf91594692e21b6f0e8f73d188a1 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 25 Oct 2022 15:29:45 +0200 Subject: [PATCH 029/130] Bazel 5.3.1 tensorflow 2.10.0 (#1483) * Bazel 5.3.1 tensorflow 2.10.0 --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 4 +-- WORKSPACE | 8 +++--- demos/common/cpp/Dockerfile | 2 +- external/net_http.patch | 51 ++++++++++++++++--------------------- 5 files changed, 29 insertions(+), 38 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 1882967ca5..58b9104a17 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -74,7 +74,7 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n xz && \ yum clean all # Set up Bazel -ENV BAZEL_VERSION 3.7.2 +ENV BAZEL_VERSION 5.3.1 WORKDIR /bazel RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index cdc41b58a8..7df50cc9b9 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -81,7 +81,7 @@ RUN apt update && apt install -y \ apt clean # Set up Bazel -ENV BAZEL_VERSION 3.7.2 +ENV BAZEL_VERSION 5.3.1 WORKDIR /bazel RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ @@ -128,7 +128,6 @@ RUN git clone https://github.com/aws/aws-sdk-cpp.git --branch 1.7.129 --single-b WORKDIR /awssdk/build RUN cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY=s3 -DENABLE_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DFORCE_SHARED_CRT=OFF -DSIMPLE_INSTALL=OFF -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 " .. RUN make --jobs=$JOBS -#RUN mv .deps/install/lib64 .deps/install/lib ####### End of AWS S3 SDK @@ -194,7 +193,6 @@ WORKDIR /ovms COPY .bazelrc WORKSPACE /ovms/ COPY external /ovms/external/ -ENV BAZEL_LINKLIBS=-l%:libstdc++.a RUN apt install -y python-is-python3 RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto diff --git a/WORKSPACE b/WORKSPACE index ec370e932c..02f2b112c5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -61,7 +61,7 @@ cc_library( git_repository( name = "tensorflow_serving", remote = "https://github.com/tensorflow/serving.git", - tag = "2.5.1", + tag = "2.10.0", patch_args = ["-p1"], patches = ["net_http.patch", "listen.patch"] # ^^^^^^^^^^^^ @@ -88,8 +88,8 @@ cc_library( load("@tensorflow_serving//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "cb99f136dc5c89143669888a44bfdd134c086e1e2d9e36278c1eb0f03fe62d76", - git_commit = "a4dfb8d1a71385bd6d122e4f27f86dcebb96712d", + sha256 = "bc4e9bbeb0136163f283ab8b695bec747cad738963e153ce3b7e414ebffe408f", + git_commit = "359c3cdfc5fabac82b3c70b3b6de2b0a8c16874f", patch = "tf.patch", repo_mapping = {"@curl" : "@curl"} ) @@ -102,7 +102,7 @@ load( "@org_tensorflow//tensorflow:version_check.bzl", "check_bazel_version_at_least" ) -check_bazel_version_at_least("3.7.2") +check_bazel_version_at_least("5.3.1") # Initialize TensorFlow's external dependencies. load("@org_tensorflow//tensorflow:workspace3.bzl", "workspace") diff --git a/demos/common/cpp/Dockerfile b/demos/common/cpp/Dockerfile index 94a4acffed..a7e770f192 100644 --- a/demos/common/cpp/Dockerfile +++ b/demos/common/cpp/Dockerfile @@ -66,7 +66,7 @@ RUN pip3 --no-cache-dir install \ --ignore-installed six # Set up Bazel -ENV BAZEL_VERSION 3.7.2 +ENV BAZEL_VERSION 5.3.1 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ diff --git a/external/net_http.patch b/external/net_http.patch index 5cff19c893..ebf46d5f09 100644 --- a/external/net_http.patch +++ b/external/net_http.patch @@ -1,9 +1,9 @@ diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD -index 4fe1c1be..601e8675 100644 +index 5beb2fb..e7b166c 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD -@@ -4,7 +4,7 @@ load("//tensorflow_serving:serving.bzl", "serving_proto_library", "serving_proto - +@@ -4,7 +4,7 @@ load("//tensorflow_serving:serving.bzl", "serving_proto_library") + package( default_visibility = [ - "//tensorflow_serving:internal", @@ -11,41 +11,34 @@ index 4fe1c1be..601e8675 100644 ], features = ["-layering_check"], ) -diff --git a/tensorflow_serving/util/net_http/server/public/BUILD b/tensorflow_serving/util/net_http/server/public/BUILD -index e7f96d98..905ab4e4 100644 ---- a/tensorflow_serving/util/net_http/server/public/BUILD -+++ b/tensorflow_serving/util/net_http/server/public/BUILD -@@ -2,9 +2,7 @@ - - package( - default_visibility = [ -- ":http_server_clients", -- "//tensorflow_serving:internal", -- "//tensorflow_serving/util/net_http:__subpackages__", -+ "//visibility:public" - ], - ) - diff --git a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -index a3b77b5..5e88ecf 100644 +index 36c925a..86a5ba9 100644 --- a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc +++ b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -@@ -104,7 +104,10 @@ bool EvHTTPServer::Initialize() { - NET_LOG(FATAL, "Failed to create evhttp."); +@@ -105,6 +105,11 @@ bool EvHTTPServer::Initialize() { return false; } -- + + std::size_t maxBodySize = 1024 * 1024 * 1024; + evhttp_set_max_body_size(ev_http_, maxBodySize); + std::size_t maxHeadersSize = 8 * 1024; + evhttp_set_max_headers_size(ev_http_, maxHeadersSize); ++ // By default libevents only allow GET, POST, HEAD, PUT, DELETE request - // we have to manually turn OPTIONS flag on - // documentation: + // we have to manually turn OPTIONS and PATCH flag on documentation: // (http://www.wangafu.net/~nickm/libevent-2.0/doxygen/html/http_8h.html) - evhttp_set_allowed_methods( - ev_http_, EVHTTP_REQ_GET | EVHTTP_REQ_POST | EVHTTP_REQ_HEAD | - EVHTTP_REQ_PUT | EVHTTP_REQ_DELETE | EVHTTP_REQ_OPTIONS); - evhttp_set_gencb(ev_http_, &DispatchEvRequestFn, this); +diff --git a/tensorflow_serving/util/net_http/server/public/BUILD b/tensorflow_serving/util/net_http/server/public/BUILD +index e7f96d9..89d1a9c 100644 +--- a/tensorflow_serving/util/net_http/server/public/BUILD ++++ b/tensorflow_serving/util/net_http/server/public/BUILD +@@ -2,9 +2,7 @@ + + package( + default_visibility = [ +- ":http_server_clients", +- "//tensorflow_serving:internal", +- "//tensorflow_serving/util/net_http:__subpackages__", ++ "//visibility:public", + ], + ) - return true; From b81a6adc35b238dbeb9752dd8536072d652c8ce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Tue, 25 Oct 2022 15:33:51 +0200 Subject: [PATCH 030/130] Remove deploy catalog and reference operator docs (#1484) --- deploy/README.md | 286 --- deploy/ovms/Chart.yaml | 20 - deploy/ovms/templates/_helpers.tpl | 48 - deploy/ovms/templates/aws_secret.yaml | 25 - deploy/ovms/templates/azure_secret.yaml | 25 - deploy/ovms/templates/deployment.yaml | 189 -- deploy/ovms/templates/service.yaml | 38 - deploy/ovms/values.yaml | 65 - docs/installations_kubernetes.md | 19 +- docs/metrics.md | 2 +- {deploy => extras}/grafana_dashboard.json | 1804 +++++++++--------- extras/openvino-operator-openshift/README.md | 4 +- extras/ovms-operator/README.md | 2 + 13 files changed, 912 insertions(+), 1615 deletions(-) delete mode 100644 deploy/README.md delete mode 100644 deploy/ovms/Chart.yaml delete mode 100644 deploy/ovms/templates/_helpers.tpl delete mode 100644 deploy/ovms/templates/aws_secret.yaml delete mode 100644 deploy/ovms/templates/azure_secret.yaml delete mode 100644 deploy/ovms/templates/deployment.yaml delete mode 100644 deploy/ovms/templates/service.yaml delete mode 100644 deploy/ovms/values.yaml rename {deploy => extras}/grafana_dashboard.json (96%) diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index b735aa1990..0000000000 --- a/deploy/README.md +++ /dev/null @@ -1,286 +0,0 @@ -# Helm Deployment {#ovms_deploy_helm_chart} - -To simplify deployment in Kubernetes, we provide a helm chart for installing OpenVINO Model Server in a Kubernetes cluster. -The helm chart is managing the Model Server instance which represents a kubernetes deployment and a -kubernetes service with exposed REST and gRPC inference endpoints. -This guide assumes you already have a functional Kubernetes cluster and helm -installed (see below for instructions on installing helm). - -The steps below describe how to setup a model repository, use helm to launch the inference server and then send -inference requests to the running server. - -## Installing Helm - -Please refer to [Helm installation guide](https://helm.sh/docs/intro/install). - -## Model Repository - -Model Server requires a repository of models to execute inference requests. That consists of the model files stored in a -specific structure. Each model is stored in a dedicated folder with numerical subfolders representing the model versions. -Each model version subfolder must include its model files. - -Model repository can be hosted in the cloud storage, Kubernetes persistent volume or on the local drives. - -Learn more about the [model repository](../docs/models_repository.md). - -For example, you can -use a Google Cloud Storage (GCS) bucket: -```shell script -gsutil mb gs://model-repository -``` - -You can download the model from [OpenVINO Model Zoo](https://storage.openvinotoolkit.org/repositories/open_model_zoo/2021.3/models_bin). and upload it to GCS: - -```shell script -wget https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin -P 1 -wget https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -P 1 -gsutil cp -r 1 resnet50-binary-0001.bin gs://model-repository/resnet -``` - -The supported storage options are described below: - - -### GCS - -Bucket permissions can be set with the _GOOGLE_APPLICATION_CREDENTIALS_ environment variable. Please follow the steps below: - -* Generate Google service account JSON file with permissions: _Storage Legacy Bucket Reader_, _Storage Legacy Object Reader_, _Storage Object Viewer_. Name a file for example: _gcp-creds.json_ -(you can follow these instructions to [create a Service Account](https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account) and download JSON) -* Create a Kubernetes secret from this JSON file: - - $ kubectl create secret generic gcpcreds --from-file gcp-creds.json - -* When deploying Model Server, provide the model path to GCS bucket and name for the secret created above. Make sure to provide `gcp_creds_secret_name` when deploying: -```shell script -helm install ovms-app ovms --set model_name=resnet50-binary-0001,model_path=gs://models-repository/model,gcp_creds_secret_name=gcpcreds -``` - -### S3 - -For S3 you must provide an AWS Access Key ID, the content of that key (AWS Secret Access Key) and the AWS region when deploying: `aws_access_key_id`, `aws_secret_access_key` and `aws_region` (see below). -```shell script -helm install ovms-app ovms --set model_name=icnet-camvid-ava-0001,model_path=s3://models-repository/model,aws_access_key_id=<...>,aws_secret_access_key=<...>,aws_region=eu-central-1 -``` - -In case you would like to use custom S3 service with compatible API (e.g. MinIO), you need to also provide endpoint -to that service. Please provide it by supplying `s3_compat_api_endpoint`: -```shell script -helm install ovms-app ovms --set model_name=icnet-camvid-ava-0001,model_path=s3://models-repository/model,aws_access_key_id=<...>,aws_secret_access_key=<...>,s3_compat_api_endpoint=<...> -``` - -### Azure Storage -Use OVMS with models stored on azure blob storage by providing `azure_storage_connection_string` parameter. Model path should follow `az` scheme like below: -```shell script -helm install ovms-app ovms --set model_name=resnet,model_path=az://bucket/model_path,azure_storage_connection_string="DefaultEndpointsProtocol=https;AccountName=azure_account_name;AccountKey=smp/hashkey==;EndpointSuffix=core.windows.net" -``` - -### Local Node Storage -Beside the cloud storage, models could be stored locally on the kubernetes nodes filesystem. -Use the parameter `models_host_path` with the local path on the nodes. It will be mounted in the OVMS container as `/models` folder. - -While the models folder is mounted in the OVMS container, the parameter `model_path` should refer to the path starting with /models/... and point to the folder with the model versions. - -Note that the OVMS container starts, by default, with the security context of account `ovms` with pid 5000 and group 5000. -If the mounted models have restricted access permissions, change the security context of the OVMS service or adjust permissions to the models. OVMS requires read permissions on the model files and -list permission on the model version folders. - -### Persistent Volume -It is possible to deploy OVMS using Kubernetes [persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/). - -That opens a possibility of storing the models for OVMS on all Kubernetes [supported filesystems](https://kubernetes.io/docs/concepts/storage/storage-classes/). - -In the helm set the parameter `models_volume_claim` with the name of the `PersistentVolumeClaim` record with the models. While set, it will be mounted as `/models` folder inside the OVMS container. - -Note that parameter `models_volume_claim` is mutually exclusive with `models_host_path`. Only one of them should be set. - -## Assigning Resource Specs - -You can restrict assigned cluster resources to the OVMS container by setting the parameter `resources`. -By default, there are no restrictions but that parameter could be used to reduce the CPU and memory allocation. Below is the snippet example from the [values.yaml](https://github.com/openvinotoolkit/model_server/blob/releases/2022/1/deploy/ovms/values.yaml) file: -```yaml -resources: - limits: - cpu: 8.0 - memory: 512Mi -``` -Beside setting the CPU and memory resources, the same parameter can be used to assign AI accelerators like iGPU, or VPU. -That assumes using adequate Kubernetes device plugin from [Intel Device Plugin for Kubernetes](https://github.com/intel/intel-device-plugins-for-kubernetes). -```yaml -resources: - limits: - gpu.intel.com/i915: 1 -``` - -## Security Context - -OVMS, by default, starts with the security context of `ovms` account which has pid 5000 and gid 5000. In some cases it can prevent importing models -stored on the file system with restricted access. -It might require adjusting the security context of OVMS service. It can be changed using a parameter `security_context`. - -An example of the values is presented below: -```yaml -security_context: - runAsUser: 5000 - runAsGroup: 5000 -``` -The security configuration could be also adjusted further with all options specified in [Kubernetes documentation](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) - -## Service Type -The helm chart creates the Kubernetes `service` as part of the OVMS deployment. Depending on the cluster infrastructure you can adjust -the [service type](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types). -In the cloud environment you might set `LoadBalancer` type to expose the service externally. `NodePort` could expose a static port -of the node IP address. `ClusterIP` would keep the OVMS service internal to the cluster applications. - - -## Deploy OpenVINO Model Server with a Single Model - -Deploy Model Server using _helm_. Please include the required model name and model path. You can also adjust other parameters defined in [values.yaml](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/deploy/ovms/values.yaml). -```shell script -helm install ovms-app ovms --set model_name=resnet50-binary-0001,model_path=gs://models-repository -``` - -Use _kubectl_ to see the status and wait until the Model Server pod is running: -```shell script -kubectl get pods -NAME READY STATUS RESTARTS AGE -ovms-app-5fd8d6b845-w87jl 1/1 Running 0 27s -``` - -By default, Model Server is deployed with 1 instance. If you would like to scale up additional replicas, override the value in values.yaml file or by passing _--set_ flag to _helm install_: - -```shell script -helm install ovms-app ovms --set model_name=resnet50-binary-0001,model_path=gs://models-repository,replicas=3 -``` - - -## Deploy OpenVINO Model Server with Multiple Models Defined in a Configuration File - -To serve multiple models you can run Model Server with a configuration file as described in [Config File](../docs/multiple_models_mode.md). - -Follow the above documentation to create a configuration file named _config.json_ and fill it with proper information. - -To deploy with config file stored in the Kubernetes ConfigMap: -* create a ConfigMap resource from this file with a chosen name (here _ovms-config_): -```shell script -kubectl create configmap ovms-config --from-file config.json -``` -* deploy Model Server with parameter `config_configmap_name` (without `model_name` and `model_path`): -```shell script -helm install ovms-app ovms --set config_configmap_name=ovms-config -``` -To deploy with config file stored on the Kubernetes Persistent Volume : -* Store the config file on node local path set with `models_host_path` or on the persistent volume claim set with -`models_claim_name`. It will be mounted along with the models in the folder `/models`. -* Deploy Model Server with parameter `config_path` pointing to the location of the config file visible in the OVMS container ie starting from -`/models/...` -```shell script -helm install ovms-app ovms --set config_path=/models/config.json -``` -## Using Model Server - -Now that the server is running you can send HTTP or gRPC requests to perform inference. -By default, the service is exposed with a LoadBalancer service type. Use the following command to find the -external IP for the server: -```shell script -kubectl get svc -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -ovms-app LoadBalancer 10.121.14.253 1.2.3.4 8080:30043/TCP,8081:32606/TCP 59m -``` - -The server exposes an gRPC endpoint on 8080 port and REST endpoint on 8081 port. - -The service name deployed via the helm chart is defined by the application name. In addition to that, the service -gets a suffix `-ovms`, in case the application name doesn't include `ovms` phrase. It avoids a risk of the service name -conflicts with other application. - -Follow the [instructions](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/image_classification/python) -to create an image classification client that can be used to perform inference with models being exposed by the server. For example: -```shell script -$ python image_classification.py --grpc_port 8080 --grpc_address 1.2.3.4 --input_name 0 --output_name 1463 -Start processing: - Model name: resnet - Images list file: input_images.txt -images/airliner.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 25.56 ms; speed 39.13 fps -Detected: 404 Should be: 404 -images/arctic-fox.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 20.95 ms; speed 47.72 fps -Detected: 279 Should be: 279 -images/bee.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 21.90 ms; speed 45.67 fps -Detected: 309 Should be: 309 -images/golden_retriever.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 21.84 ms; speed 45.78 fps -Detected: 207 Should be: 207 -images/gorilla.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 20.26 ms; speed 49.36 fps -Detected: 366 Should be: 366 -images/magnetic_compass.jpeg (1, 3, 224, 224) ; data range: 0.0 : 247.0 -Processing time: 20.68 ms; speed 48.36 fps -Detected: 635 Should be: 635 -images/peacock.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 21.57 ms; speed 46.37 fps -Detected: 84 Should be: 84 -images/pelican.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 20.53 ms; speed 48.71 fps -Detected: 144 Should be: 144 -images/snail.jpeg (1, 3, 224, 224) ; data range: 0.0 : 248.0 -Processing time: 22.34 ms; speed 44.75 fps -Detected: 113 Should be: 113 -images/zebra.jpeg (1, 3, 224, 224) ; data range: 0.0 : 255.0 -Processing time: 21.27 ms; speed 47.00 fps -Detected: 340 Should be: 340 -Overall accuracy= 100.0 % -Average latency= 21.1 ms -``` - -## Cleanup - -Once you've finished using the server you should use helm to uninstall the chart: -```shell script -$ helm ls -NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION -ovms-app default 1 2020-09-23 14:40:07.292360971 +0200 CEST deployed ovms-3.0.0 - -$ helm uninstall ovms-app -release "ovms-app" uninstalled -``` - - -## Helm Options References - -| Parameter | Description | Prerequisites | Default | -| ------------- |-------------|-------------|-------------| -| replicas | Number of k8s pod replicas to deploy | - | 1 | -| image_name | Change to use different docker image with OVMS | - | openvino/model_server:latest | -| config_configmap_name | Starts OVMS using the config file stored in the ConfigMap | Create the ConfigMap including config.json file | - | -| config_path | Starts OVMS using the config file mounted from the node local path or the k8s persistent volume | Use it together with models_host_path or models_claim_name and place the config file in configured storage path | - | -| grpc_port | Service port for gRPC interface | - | 8080 | -| grpc_port | Service port for REST API interface | - | 8081 | -| file_system_poll_wait_seconds | Time interval in seconds between new version detection. 0 disables the version updates | - | 1 | -| model_name | Model name, start OVMS with a single model, excluding with config_configmap_name and config_path parameter | - | - | -| model_path | Model path, start OVMS with a single model, excluding with config_configmap_name and config_path parameter | - | - | -| target_device | Target device to run inference operations | Non CPU device require the device plugin to be deployed | CPU | -| stateful | If set to any non empty value, enables stateful model execution | Model must be stateful | Stateless model execution | -| nireq | Size of inference queue | - | Set automatically by OpenVINO| -| batch_size | Change the model batch size | - | Defined by the model attributes | -| layout | Change layout of the model input or output with image data. NCHW or NHWC | - | Defined in the model | -| shape | Change the model input shape | - | defined by the model attributes | -| model_version_policy | Set the model version policy | - | {\"latest\": { \"num_versions\":1 }} The latest version is served | -| plugin_config | Device plugin configuration used for performance tuning | - | {\"CPU_THROUGHPUT_STREAMS\":\"CPU_THROUGHPUT_AUTO\"}| -| gcp_creds_secret_name | k8s secret resource including GCP credentials, use it with google storage for models | Secret should be created with GCP credentials json file | - | -| aws_access_key_id | S3 storage access key id, use it with S3 storage for models | - | - | -| aws_secret_access_key | S3 storage secret key, use it with S3 storage for models | - | - | -| aws_region | S3 storage secret key, use it with S3 storage for models | - | - | -| aws_secret_access_key | S3 storage secret key, use it with S3 storage for models |- | - | -| s3_compat_api_endpoint | S3 compatibility api endpoint, use it with Minio storage for models | | - | -| azure_storage_connection_string | Connection string to the Azure Storage authentication account, use it with Azure storage for models | - | - | -| log_level | OVMS log level, one of ERROR, WARNING, INFO, DEBUG, TRACE| - | INFO | -| service_type | k8s service type | - | LoadBalancer | -| resources | Compute resource limits | - | All CPU and memory on the node | -| node_selector | Target node label condition | - | All available nodes | -| annotations | Defined annotations to be set in the pods | - | None | -| security_context | OVMS security context | - | 5000:5000 | -| models_host_path | Mounts node local path in container as /models folder | Path should be created on all nodes and populated with the data | - | -| models_volume_claim | Mounts k8s persistent volume claim in the container as /models | Persistent Volume Claim should be create in the same namespace and populated with the data | - | -| https_proxy | Proxy name to be used to connect to remote models | - | - | diff --git a/deploy/ovms/Chart.yaml b/deploy/ovms/Chart.yaml deleted file mode 100644 index e4c93cb896..0000000000 --- a/deploy/ovms/Chart.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: v1 -description: OpenVINO Model Server -name: ovms -version: 4.0.0 diff --git a/deploy/ovms/templates/_helpers.tpl b/deploy/ovms/templates/_helpers.tpl deleted file mode 100644 index e2e4a70ff6..0000000000 --- a/deploy/ovms/templates/_helpers.tpl +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (c) 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "ovms.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ovms.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "ovms.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} \ No newline at end of file diff --git a/deploy/ovms/templates/aws_secret.yaml b/deploy/ovms/templates/aws_secret.yaml deleted file mode 100644 index cdafd61183..0000000000 --- a/deploy/ovms/templates/aws_secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2020-2001 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{{ if .Values.aws_secret_access_key }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "ovms.fullname" . }}-aws-secret -type: Opaque -data: - secret_access_key: {{ .Values.aws_secret_access_key | b64enc }} -{{ end }} \ No newline at end of file diff --git a/deploy/ovms/templates/azure_secret.yaml b/deploy/ovms/templates/azure_secret.yaml deleted file mode 100644 index a6b5f88de5..0000000000 --- a/deploy/ovms/templates/azure_secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -{{ if .Values.azure_storage_connection_string }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "ovms.fullname" . }}-azure-secret -type: Opaque -data: - connection_string: {{ .Values.azure_storage_connection_string | b64enc }} - {{ end }} \ No newline at end of file diff --git a/deploy/ovms/templates/deployment.yaml b/deploy/ovms/templates/deployment.yaml deleted file mode 100644 index 6578a547c5..0000000000 --- a/deploy/ovms/templates/deployment.yaml +++ /dev/null @@ -1,189 +0,0 @@ -# -# Copyright (c) 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "ovms.fullname" . }} - labels: - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: {{ template "ovms.chart" . }} - app: {{ template "ovms.fullname" . }} -spec: - selector: - matchLabels: - release: {{ .Release.Name | quote }} - app: {{ template "ovms.fullname" . }} - replicas: {{ .Values.replicas }} - template: - metadata: - annotations: -{{ toYaml .Values.annotations | indent 8 }} - labels: - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: {{ template "ovms.chart" . }} - app: {{ template "ovms.fullname" . }} - spec: - {{- if .Values.security_context }} - securityContext: -{{ toYaml .Values.security_context | indent 8 }} - {{- end }} - containers: - - name: ovms - image: {{ .Values.image_name }} - ports: - - containerPort: 8080 - - containerPort: 8081 - readinessProbe: - tcpSocket: - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 10 - {{- if or .Values.gcp_creds_secret_name .Values.aws_access_key_id .Values.aws_secret_access_key .Values.aws_region .Values.s3_compat_api_endpoint .Values.http_proxy .Values.https_proxy .Values.no_proxy .Values.azure_storage_connection_string }} - env: - {{- end }} - {{- if .Values.http_proxy }} - - name: http_proxy - value: {{ .Values.http_proxy }} - {{- end }} - {{- if .Values.https_proxy }} - - name: https_proxy - value: {{ .Values.https_proxy }} - {{- end }} - {{- if .Values.no_proxy }} - - name: no_proxy - value: {{ .Values.no_proxy }} - {{- end }} - {{- if .Values.gcp_creds_secret_name }} - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-creds.json - {{- end }} - {{- if .Values.aws_access_key_id }} - - name: AWS_ACCESS_KEY_ID - value: {{ .Values.aws_access_key_id }} - {{- end }} - {{- if .Values.aws_secret_access_key }} - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ template "ovms.fullname" . }}-aws-secret - key: secret_access_key - {{- end }} - {{- if .Values.aws_region }} - - name: AWS_REGION - value: {{ .Values.aws_region }} - {{- end }} - {{- if .Values.s3_compat_api_endpoint }} - - name: S3_ENDPOINT - value: {{ .Values.s3_compat_api_endpoint }} - {{- end }} - {{- if .Values.azure_storage_connection_string }} - - name: AZURE_STORAGE_CONNECTION_STRING - valueFrom: - secretKeyRef: - name: {{ template "ovms.fullname" . }}-azure-secret - key: connection_string - {{- end }} - args: [ - {{- if or .Values.config_configmap_name .Values.config_path}} - {{- if .Values.config_path}} - "--config_path", "{{ .Values.config_path }}", - {{- else }} - "--config_path", "/config/config.json", - {{- end }} - {{- else }} - "--model_path", "{{ .Values.model_path }}", - "--model_name", "{{ .Values.model_name }}", - "--target_device", "{{ .Values.target_device }}", - {{- if .Values.nireq }} - "--nireq", "{{ .Values.nireq }}", - {{- end }} - {{- if .Values.plugin_config }} - "--plugin_config", "{{ .Values.plugin_config }}", - {{- end }} - {{- if .Values.batch_size }} - "--batch_size", "{{ .Values.batch_size }}", - {{- end }} - {{- if .Values.layout }} - "--layout", "{{ .Values.layout }}", - {{- end }} - {{- if .Values.shape }} - "--shape", "{{ .Values.shape }}", - {{- end }} - {{- if .Values.model_version_policy }} - "--model_version_policy", "{{ .Values.model_version_policy }}", - {{- end }} - {{- if .Values.stateful }} - "--stateful", - {{- end }} - {{- end }} - "--log_level", "{{ .Values.log_level }}", - {{- if .Values.file_system_poll_wait_seconds }} - "--file_system_poll_wait_seconds", "{{ .Values.file_system_poll_wait_seconds }}", - {{- end }} - "--port", "8080", - "--rest_port", "8081"] - {{- if or .Values.gcp_creds_secret_name .Values.config_configmap_name .Values.models_host_path .Values.models_volume_claim}} - volumeMounts: - {{- end }} - {{- if .Values.gcp_creds_secret_name }} - - name: gcpcreds - mountPath: "/secret" - readOnly: true - {{- end }} - {{- if .Values.config_configmap_name }} - - name: config - mountPath: "/config" - readOnly: true - {{- end }} - {{- if or .Values.models_host_path .Values.models_volume_claim }} - - name: models - mountPath: "/models" - readOnly: true - {{- end }} - resources: -{{ toYaml .Values.resources | indent 10}} - {{- if .Values.node_selector }} - nodeSelector: -{{ toYaml .Values.node_selector | indent 8 }} - {{- end }} - {{- if or .Values.gcp_creds_secret_name .Values.config_configmap_name .Values.models_volume_claim .Values.models_host_path }} - volumes: - {{- end }} - {{- if .Values.gcp_creds_secret_name }} - - name: gcpcreds - secret: - secretName: gcpcreds - {{- end }} - {{- if .Values.config_configmap_name }} - - name: config - configMap: - name: {{ .Values.config_configmap_name }} - {{- end }} - {{- if .Values.models_host_path }} - - name: models - hostPath: - path: "{{ .Values.models_host_path }}" - type: Directory - {{- end }} - {{- if and (.Values.models_volume_claim) (eq .Values.models_host_path "") }} - - name: models - persistentVolumeClaim: - claimName: {{ .Values.models_volume_claim }} - {{- end }} diff --git a/deploy/ovms/templates/service.yaml b/deploy/ovms/templates/service.yaml deleted file mode 100644 index a830fc2b9c..0000000000 --- a/deploy/ovms/templates/service.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -kind: Service -apiVersion: v1 -metadata: - name: {{ template "ovms.fullname" . }} - labels: - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - chart: {{ template "ovms.chart" . }} - app: {{ template "ovms.fullname" . }} -spec: - ports: - - port: {{ .Values.grpc_port }} - protocol: TCP - targetPort: 8080 - name: grpc - - port: {{ .Values.rest_port }} - protocol: TCP - targetPort: 8081 - name: rest - selector: - app: {{ template "ovms.fullname" . }} - type: {{ .Values.service_type }} diff --git a/deploy/ovms/values.yaml b/deploy/ovms/values.yaml deleted file mode 100644 index af6f83debe..0000000000 --- a/deploy/ovms/values.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2020-2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -replicas: 1 -image_name: "openvino/model_server:latest" -config_configmap_name: "" -# configmap name holding the content of ovms configuration file. Excluding with model_name and model_path -config_path: "" -# to be used when ovms config is not included in the configmap but is mounted on the container via models_host_path or models_volume_claim param -grpc_port: 8080 -rest_port: 8081 -file_system_poll_wait_seconds: 0 -model_name: "" -model_path: "" -layout: "" -target_device: "CPU" -nireq: "" -batch_size: "" -shape: "" -model_version_policy: >- - {\"latest\": { \"num_versions\":1 }} -plugin_config: >- - {\"CPU_THROUGHPUT_STREAMS\":\"CPU_THROUGHPUT_AUTO\"} -stateful: "" -gcp_creds_secret_name: "" -# k8s secret name holding gcp credential file gcp-creds.json -aws_access_key_id: "" -aws_secret_access_key: "" -aws_region: "" -s3_compat_api_endpoint: "" -azure_storage_connection_string: "" -log_level: INFO -service_type: "LoadBalancer" -# alternatively set to NodePort or ClusterIP -resources: {} - # limits: - # cpu: 8.0 - # memory: 512Mi - # gpu.intel.com/i915: 1 -security_context: {} - # runAsUser: 5000 - # runAsGroup: 5000 -annotations: {} - # sidecar.istio.io/inject: "true" -node_selector: {} - # kubernetes.io/arch: amd64 -models_host_path: "" -# local path on the host to be mounted as /models directory -models_volume_claim: "" -# persistent claim name to be mounted as /models directory. It is excluding with models_host_path. - - diff --git a/docs/installations_kubernetes.md b/docs/installations_kubernetes.md index a38b3bce32..1271a4d86d 100644 --- a/docs/installations_kubernetes.md +++ b/docs/installations_kubernetes.md @@ -1,20 +1,9 @@ # Deploy Model Server in Kubernetes {#ovms_docs_kubernetes} There are three recommended methods for deploying OpenVINO Model Server in Kubernetes: -1. [helm chart](../deploy/README.md) - deploys Model Server instances using the [helm](https://helm.sh) package manager for Kubernetes +1. [helm chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) - deploys Model Server instances using the [helm](https://helm.sh) package manager for Kubernetes -2. [Kubernetes Operator](../extras/ovms-operator/README.md) - manages Model Server using a Kubernetes Operator -3. [OpenShift Operator](../extras/openvino-operator-openshift/README.md) - manages Model Server instances in Red Hat OpenShift +2. [Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) - manages Model Server using a Kubernetes Operator +3. [OpenShift Operator](https://github.com/openvinotoolkit/operator/blob/main/docs/operator_installation.md#openshift) - manages Model Server instances in Red Hat OpenShift - -@sphinxdirective - -.. toctree:: - :maxdepth: 1 - :hidden: - - ovms_deploy_helm_chart - ovms_extras_ovms-operator-readme - ovms_extras_openvino-operator-openshift-readme - -@endsphinxdirective +For operators mentioned in 2. and 3. see the [description of the deployment process](https://github.com/openvinotoolkit/operator/blob/main/docs/modelserver.md) \ No newline at end of file diff --git a/docs/metrics.md b/docs/metrics.md index 3421539af3..647f1b52cf 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -221,7 +221,7 @@ It means that each request to the DAG pipeline will update also the metrics for With server metrics being scraped by [Prometheus](https://prometheus.io/) it is possible to integrate [Grafana](https://grafana.com/) to visualize them on the dashboards. Once you have Grafana configured with Prometheus as a data source, you can create your own dashboard or import one. -In OpenVINO Model Server repository you can find [grafana_dashboard.json](https://github.com/openvinotoolkit/model_server/blob/develop/deploy/grafana_dashboard.json) file that can be used to visualize per model metrics like: +In OpenVINO Model Server repository you can find [grafana_dashboard.json](https://github.com/openvinotoolkit/model_server/blob/develop/extras/grafana_dashboard.json) file that can be used to visualize per model metrics like: - Throughput [RPS] - number of requests being processed by the model per second. - Mean Latency [ms] - latency averaged across all requests processed by the model in a certain timeframe. - Latency Quantile [ms] - value of latency for quantiles [0.75, 0.90, 0.99], meaning the latency that has NOT been exceeded by 75%, 90% and 99% of the requests. diff --git a/deploy/grafana_dashboard.json b/extras/grafana_dashboard.json similarity index 96% rename from deploy/grafana_dashboard.json rename to extras/grafana_dashboard.json index 3037571cf2..7c778f64db 100644 --- a/deploy/grafana_dashboard.json +++ b/extras/grafana_dashboard.json @@ -1,903 +1,903 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 16, - "panels": [], - "title": "Service Performance", - "type": "row" - }, - { - "datasource": {}, - "description": "Number of requests being processed by the model per second", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "RPS" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 4, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "sum(rate(ovms_requests_success{method=~\"ModelInfer|Predict\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", - "legendFormat": "RPS", - "range": true, - "refId": "A" - } - ], - "title": "Throughput [RPS]", - "type": "gauge" - }, - { - "datasource": {}, - "description": "Latency averaged across all requests processed by the model in a certain timeframe", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "mean_latency" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "((sum(rate(ovms_request_time_us_sum{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / 1000) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))", - "legendFormat": "mean_latency", - "range": true, - "refId": "A" - } - ], - "title": "Mean Latency [ms]", - "type": "gauge" - }, - { - "datasource": {}, - "description": "Value of latency for quantiles [0.75, 0.90, 0.99], meaning the latency that has NOT been exceeded by 75%, 90% and 99% of the requests", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, sum by(le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", - "legendFormat": "quantile==0.75", - "range": true, - "refId": "quantile=0.75" - }, - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", - "hide": false, - "legendFormat": "quantile==0.90", - "range": true, - "refId": "quantile=0.90" - }, - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", - "hide": false, - "legendFormat": "quantile==0.99", - "range": true, - "refId": "quantile=0.99" - } - ], - "title": "Latency Quantile [ms] (q=0.75, 0.90, 0.99)", - "type": "timeseries" - }, - { - "datasource": {}, - "description": "Distribution of the latencies across the buckets", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "red", - "value": 25 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 27, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / scalar((sum by (le) (rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))) * 100)", - "format": "heatmap", - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Latency Distribution [%]", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "10": true, - "18": true, - "32": true, - "58": true, - "104": true, - "188": true, - "340": true, - "612": true, - "1101": true, - "1983": true, - "3570": false, - "2294682": false, - "4130428": true, - "7434771": true, - "13382588": true, - "24088659": true, - "43359586": true, - "78047255": true, - "140485060": true, - "252873108": true, - "455171596": true, - "819308872": true, - "1474755971": true, - "+Inf": true, - "Time": true - }, - "indexByName": {}, - "renameByName": { - "10": "10 us", - "18": "18 us", - "32": "32 us", - "58": "58 us", - "104": "104 us", - "188": "108 us ", - "340": "340 us ", - "612": "612 us", - "1101": "1 ms", - "1983": "2 ms", - "3570": "3 ms", - "6426": "6 ms", - "11568": "11 ms", - "20822": "21 ms", - "37481": "37 ms", - "67466": "67 ms", - "121439": "121 ms", - "218591": "218 ms", - "393464": "393 ms", - "708235": "708 ms", - "1274823": "1 s", - "2294682": "2 s", - "4130428": "4 s", - "7434771": "7 s", - "13382588": "13 s", - "24088659": "24 s", - "43359586": "43 s", - "78047255": "78 s", - "140485060": "140 s", - "252873108": "252 s", - "455171596": "455 s", - "819308872": "819 s ", - "1474755971": "1474 s" - } - } - } - ], - "type": "bargauge" - }, - { - "datasource": {}, - "description": "Apdex score calculated as:\n(num_samples < 0,t1> + num_samples < t1,t2> / 2) / num_samples", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "(\r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) +\r\n ((sum(rate(ovms_request_time_us_bucket{le=\"218591\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) - \r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 2)\r\n) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", - "legendFormat": "apdex_score", - "range": true, - "refId": "A" - } - ], - "title": "Apdex Score ( t1=67ms, t2=218 ms )", - "type": "timeseries" - }, - { - "datasource": {}, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 31, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "count_values(\"pod\", ovms_streams{name=\"$model_name\", version=\"$model_version\"})", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Number of replicas", - "type": "gauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 2, - "panels": [], - "title": "Backend Performance", - "type": "row" - }, - { - "datasource": {}, - "description": "Time of inference execution, averaged across all requests processed by the model in a certain timeframe", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "inference_time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 22, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "(sum(rate(ovms_inference_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_inference_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", - "legendFormat": "inference_time", - "range": true, - "refId": "A" - } - ], - "title": "Mean Inference Time [ms]", - "type": "gauge" - }, - { - "datasource": {}, - "description": "Time of a request waiting for the inference execution, averaged across all requests processed by the model in a certain timeframe", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "queue_time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 23, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "(sum(rate(ovms_wait_for_infer_req_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_wait_for_infer_req_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", - "legendFormat": "queue_time", - "range": true, - "refId": "A" - } - ], - "title": "Mean Time of Request Waiting For Inference [ms]", - "type": "gauge" - }, - { - "datasource": {}, - "description": "Number of requests being currently processed by the model server", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 25, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.1.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "czgrlG4Vk" - }, - "editorMode": "code", - "expr": "sum(ovms_current_requests{name=\"$model_name\", version=\"$model_version\"})", - "legendFormat": "active_requests", - "range": true, - "refId": "A" - } - ], - "title": "Currently Processed Requests", - "type": "gauge" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "", - "value": "" - }, - "hide": 0, - "label": "", - "name": "model_name", - "options": [ - { - "selected": true, - "text": "", - "value": "" - } - ], - "query": "", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": false, - "text": "1", - "value": "1" - }, - "hide": 0, - "name": "model_version", - "options": [ - { - "selected": true, - "text": "1", - "value": "1" - } - ], - "query": "1", - "skipUrlSync": false, - "type": "textbox" - }, - { - "current": { - "selected": true, - "text": "gRPC", - "value": "gRPC" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "interface", - "options": [ - { - "selected": true, - "text": "gRPC", - "value": "gRPC" - }, - { - "selected": false, - "text": "REST", - "value": "REST" - } - ], - "query": "gRPC,REST", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "OpenVINO Model Server - Model Metrics - github", - "uid": "Bdb-6G4Vk-github", - "version": 2, - "weekStart": "" +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Service Performance", + "type": "row" + }, + { + "datasource": {}, + "description": "Number of requests being processed by the model per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "RPS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "sum(rate(ovms_requests_success{method=~\"ModelInfer|Predict\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", + "legendFormat": "RPS", + "range": true, + "refId": "A" + } + ], + "title": "Throughput [RPS]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Latency averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_latency" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "((sum(rate(ovms_request_time_us_sum{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / 1000) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))", + "legendFormat": "mean_latency", + "range": true, + "refId": "A" + } + ], + "title": "Mean Latency [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Value of latency for quantiles [0.75, 0.90, 0.99], meaning the latency that has NOT been exceeded by 75%, 90% and 99% of the requests", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by(le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "legendFormat": "quantile==0.75", + "range": true, + "refId": "quantile=0.75" + }, + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "hide": false, + "legendFormat": "quantile==0.90", + "range": true, + "refId": "quantile=0.90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 1000", + "hide": false, + "legendFormat": "quantile==0.99", + "range": true, + "refId": "quantile=0.99" + } + ], + "title": "Latency Quantile [ms] (q=0.75, 0.90, 0.99)", + "type": "timeseries" + }, + { + "datasource": {}, + "description": "Distribution of the latencies across the buckets", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 27, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum by (le) (rate(ovms_request_time_us_bucket{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) / scalar((sum by (le) (rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])))) * 100)", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Latency Distribution [%]", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "10": true, + "18": true, + "32": true, + "58": true, + "104": true, + "188": true, + "340": true, + "612": true, + "1101": true, + "1983": true, + "3570": false, + "2294682": false, + "4130428": true, + "7434771": true, + "13382588": true, + "24088659": true, + "43359586": true, + "78047255": true, + "140485060": true, + "252873108": true, + "455171596": true, + "819308872": true, + "1474755971": true, + "+Inf": true, + "Time": true + }, + "indexByName": {}, + "renameByName": { + "10": "10 us", + "18": "18 us", + "32": "32 us", + "58": "58 us", + "104": "104 us", + "188": "108 us ", + "340": "340 us ", + "612": "612 us", + "1101": "1 ms", + "1983": "2 ms", + "3570": "3 ms", + "6426": "6 ms", + "11568": "11 ms", + "20822": "21 ms", + "37481": "37 ms", + "67466": "67 ms", + "121439": "121 ms", + "218591": "218 ms", + "393464": "393 ms", + "708235": "708 ms", + "1274823": "1 s", + "2294682": "2 s", + "4130428": "4 s", + "7434771": "7 s", + "13382588": "13 s", + "24088659": "24 s", + "43359586": "43 s", + "78047255": "78 s", + "140485060": "140 s", + "252873108": "252 s", + "455171596": "455 s", + "819308872": "819 s ", + "1474755971": "1474 s" + } + } + } + ], + "type": "bargauge" + }, + { + "datasource": {}, + "description": "Apdex score calculated as:\n(num_samples < 0,t1> + num_samples < t1,t2> / 2) / num_samples", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) +\r\n ((sum(rate(ovms_request_time_us_bucket{le=\"218591\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m])) - \r\n sum(rate(ovms_request_time_us_bucket{le=\"67466\", name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))) / 2)\r\n) / sum(rate(ovms_request_time_us_count{name=\"$model_name\", version=\"$model_version\", interface=\"$interface\"}[1m]))", + "legendFormat": "apdex_score", + "range": true, + "refId": "A" + } + ], + "title": "Apdex Score ( t1=67ms, t2=218 ms )", + "type": "timeseries" + }, + { + "datasource": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 31, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "count_values(\"pod\", ovms_streams{name=\"$model_name\", version=\"$model_version\"})", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Number of replicas", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 2, + "panels": [], + "title": "Backend Performance", + "type": "row" + }, + { + "datasource": {}, + "description": "Time of inference execution, averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "inference_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 22, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(sum(rate(ovms_inference_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_inference_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", + "legendFormat": "inference_time", + "range": true, + "refId": "A" + } + ], + "title": "Mean Inference Time [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Time of a request waiting for the inference execution, averaged across all requests processed by the model in a certain timeframe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "queue_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 23, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "(sum(rate(ovms_wait_for_infer_req_time_us_sum{name=\"$model_name\", version=\"$model_version\"}[1m])) / 1000) / sum(rate(ovms_wait_for_infer_req_time_us_count{name=\"$model_name\", version=\"$model_version\"}[1m]))", + "legendFormat": "queue_time", + "range": true, + "refId": "A" + } + ], + "title": "Mean Time of Request Waiting For Inference [ms]", + "type": "gauge" + }, + { + "datasource": {}, + "description": "Number of requests being currently processed by the model server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 25, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "czgrlG4Vk" + }, + "editorMode": "code", + "expr": "sum(ovms_current_requests{name=\"$model_name\", version=\"$model_version\"})", + "legendFormat": "active_requests", + "range": true, + "refId": "A" + } + ], + "title": "Currently Processed Requests", + "type": "gauge" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "hide": 0, + "label": "", + "name": "model_name", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "1", + "value": "1" + }, + "hide": 0, + "name": "model_version", + "options": [ + { + "selected": true, + "text": "1", + "value": "1" + } + ], + "query": "1", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "gRPC", + "value": "gRPC" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "interface", + "options": [ + { + "selected": true, + "text": "gRPC", + "value": "gRPC" + }, + { + "selected": false, + "text": "REST", + "value": "REST" + } + ], + "query": "gRPC,REST", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OpenVINO Model Server - Model Metrics - github", + "uid": "Bdb-6G4Vk-github", + "version": 2, + "weekStart": "" } \ No newline at end of file diff --git a/extras/openvino-operator-openshift/README.md b/extras/openvino-operator-openshift/README.md index 0f8204d6d9..11aaeca289 100644 --- a/extras/openvino-operator-openshift/README.md +++ b/extras/openvino-operator-openshift/README.md @@ -1,4 +1,6 @@ -# OpenShift Operator {#ovms_extras_openvino-operator-openshift-readme} +# [DEPRECATED] OpenVINO Operator is now maintained in a separate [repository](https://github.com/openvinotoolkit/operator#openshift-and-kubernetes-operator) + +## OpenShift Operator {#ovms_extras_openvino-operator-openshift-readme} The Operator installs and manages development tools and production AI deployments in an OpenShift cluster. It enables easy deployment and management of AI inference services by creating `ModelServer` resource. The Operator also integrates with the JupyterHub [Spawner](https://jupyterhub.readthedocs.io/en/stable/reference/spawners.html) in [Red Hat OpenShift Data Science](https://www.redhat.com/en/technologies/cloud-computing/openshift/openshift-data-science) and [Open Data Hub](https://opendatahub.io/docs.html). See [detailed instructions](#integration-with-openshift-data-science-and-open-data-hub) below. diff --git a/extras/ovms-operator/README.md b/extras/ovms-operator/README.md index 44c60d150a..2588c2b4b1 100644 --- a/extras/ovms-operator/README.md +++ b/extras/ovms-operator/README.md @@ -1,3 +1,5 @@ +# [DEPRECATED] OpenVINO Operator is now maintained in a separate [repository](https://github.com/openvinotoolkit/operator#openshift-and-kubernetes-operator) + ## Kubernetes Operator {#ovms_extras_ovms-operator-readme} This Operator is based on a [Helm chart](https://github.com/openvinotoolkit/model_server/tree/v2021.3/deploy) for OpenVINO Model Server. It supports all the parameters from the helm chart. From d51b9430afdbba31b71a276269bda33ef4118770 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 27 Oct 2022 10:37:07 +0200 Subject: [PATCH 031/130] Fix debug build (#1489) --- src/layout_configuration.cpp | 2 ++ src/layout_configuration.hpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/layout_configuration.cpp b/src/layout_configuration.cpp index 6a1a6085d7..d055475f99 100644 --- a/src/layout_configuration.cpp +++ b/src/layout_configuration.cpp @@ -24,6 +24,8 @@ namespace ovms { +const char LayoutConfiguration::LAYOUT_CONFIGURATION_DELIMETER = ':'; + LayoutConfiguration::LayoutConfiguration(const char* layout) : LayoutConfiguration(std::string(layout)) { } diff --git a/src/layout_configuration.hpp b/src/layout_configuration.hpp index 3444bc6c9c..856943d742 100644 --- a/src/layout_configuration.hpp +++ b/src/layout_configuration.hpp @@ -25,7 +25,7 @@ namespace ovms { class Status; class LayoutConfiguration { - static const char LAYOUT_CONFIGURATION_DELIMETER = ':'; + static const char LAYOUT_CONFIGURATION_DELIMETER; Layout tensor; Layout model; From 79aacc67930061ed0907d777acc4da74623a37f1 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Thu, 27 Oct 2022 16:08:49 +0200 Subject: [PATCH 032/130] Fix openvino installation link in accelerators.md (#1499) --- docs/accelerators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/accelerators.md b/docs/accelerators.md index 3f0be8657c..2aa9e187ad 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -64,7 +64,7 @@ Intel® HD Graphics, Intel® Iris® Graphics, Intel® Iris® Xe Graphics, and In Before using GPU as OpenVINO Model Server target device, you need to: -- install the required drivers - refer to [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_linux.html#step-5-optional-configure-inference-on-non-cpu-devices) +- install the required drivers - refer to [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_from_archive_linux.html#step-4-optional-configure-inference-on-non-cpu-devices) - start the docker container with the additional parameter of `--device /dev/dri` to pass the device context - set the parameter of `--target_device` to `GPU`. - use the `openvino/model_server:latest-gpu` image, which contains GPU dependencies From 3a409ff564f4fe8e00e21303d09b609470559f4f Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Wed, 2 Nov 2022 09:39:47 +0100 Subject: [PATCH 033/130] Reduce size of stream analysis docker image (#1459) * reduced size of image to 526MB --- demos/real_time_stream_analysis/python/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/real_time_stream_analysis/python/Dockerfile b/demos/real_time_stream_analysis/python/Dockerfile index 3bf77237d6..3cf35c4487 100644 --- a/demos/real_time_stream_analysis/python/Dockerfile +++ b/demos/real_time_stream_analysis/python/Dockerfile @@ -16,9 +16,9 @@ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND="noninteractive" -RUN apt -y update && apt install -y libopencv-dev python3-opencv python3-pip +RUN apt-get -y update && apt-get install -y --no-install-recommends libgl1 libglib2.0-0 python3-pip && rm -rf /var/lib/apt/lists/* COPY requirements.txt /real_time_stream_analysis/ -RUN pip3 install -r /real_time_stream_analysis/requirements.txt +RUN pip3 install --no-cache-dir -r /real_time_stream_analysis/requirements.txt COPY templates /real_time_stream_analysis/templates COPY use_cases /real_time_stream_analysis/use_cases COPY *.py /real_time_stream_analysis/ From 3edd6b6a6289e1ff7ab1ba01e840add9583365aa Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Wed, 2 Nov 2022 10:19:10 +0100 Subject: [PATCH 034/130] Post release KFS implementation cleanup (#1480) * Make OVMS use just one using directive with KFS gRPC structures * Move KFS/TFS utils out out precision.hpp & tensorinfo.hpp * Remove support for bs/shape=auto from DAG This is not allowed anyway by restrictions in configuration. * Move pipelindedefinitionstatus functions definitions into cpp * Increase coverage in PipelineDefinitionStatus * Increase coverage threshold * Move KFS grpc service into kfs_frontend --- check_coverage.bat | 6 +- cppclean.sh | 22 +- src/BUILD | 9 +- src/azurestorage.hpp | 1 + src/binaryutils.cpp | 10 +- src/cleaner_utils.cpp | 4 + src/cleaner_utils.hpp | 2 + src/deserialization.cpp | 4 +- src/deserialization.hpp | 14 +- src/dlnodesession.cpp | 45 +-- src/entry_node.cpp | 16 +- src/exit_node.cpp | 8 +- src/exitnodesession.cpp | 4 +- src/gatherexitnodeinputhandler.cpp | 30 -- src/gatherexitnodeinputhandler.hpp | 15 +- src/grpc_utils.cpp | 108 +++++++ src/grpc_utils.hpp | 23 ++ src/grpcservermodule.cpp | 2 +- src/grpcservermodule.hpp | 2 +- src/http_rest_api_handler.cpp | 26 +- src/http_rest_api_handler.hpp | 2 +- src/http_server.cpp | 111 ++++++- .../kfs_grpc_inference_service.cpp | 96 +++--- .../kfs_grpc_inference_service.hpp | 57 ++-- src/kfs_frontend/kfs_utils.cpp | 136 +++++++++ src/kfs_frontend/kfs_utils.hpp | 31 ++ src/model_service.cpp | 3 +- src/modelinstance.cpp | 6 +- src/modelinstance.hpp | 10 +- src/pipeline_factory.cpp | 4 +- src/pipeline_factory.hpp | 6 +- src/pipelinedefinition.cpp | 6 +- src/pipelinedefinition.hpp | 2 +- src/pipelinedefinitionstatus.cpp | 240 +++++++++++++++ src/pipelinedefinitionstatus.hpp | 275 +++--------------- src/precision.cpp | 97 ------ src/precision.hpp | 15 +- src/predict_request_validation_utils.cpp | 47 ++- src/prediction_service.cpp | 7 +- src/prediction_service_utils.cpp | 4 +- src/prediction_service_utils.hpp | 6 +- src/rest_parser.cpp | 6 +- src/rest_parser.hpp | 12 +- src/rest_utils.cpp | 13 +- src/rest_utils.hpp | 6 +- src/serialization.cpp | 13 +- src/serialization.hpp | 8 +- src/server.cpp | 2 +- src/status.cpp | 177 ----------- src/status.hpp | 34 --- src/tensorinfo.cpp | 25 +- src/tensorinfo.hpp | 10 - src/test/binaryutils_test.cpp | 10 +- src/test/deserialization_tests.cpp | 12 +- src/test/ensemble_mapping_config_tests.cpp | 4 +- src/test/ensemble_tests.cpp | 3 +- src/test/gather_node_test.cpp | 6 +- src/test/kfs_metadata_test.cpp | 6 +- src/test/kfs_rest_test.cpp | 28 +- src/test/metrics_flow_test.cpp | 14 +- src/test/model_service_test.cpp | 2 - src/test/pipelinedefinitionstatus_test.cpp | 62 ++++ src/test/predict_validation_test.cpp | 18 +- src/test/prediction_service_test.cpp | 10 +- src/test/rest_utils_test.cpp | 12 +- src/test/serialization_tests.cpp | 27 +- src/test/server_test.cpp | 10 +- src/test/test_utils.cpp | 25 +- src/test/test_utils.hpp | 37 +-- src/tfs_frontend/tfs_utils.cpp | 97 ++++-- src/tfs_frontend/tfs_utils.hpp | 12 +- 71 files changed, 1183 insertions(+), 1030 deletions(-) create mode 100644 src/grpc_utils.cpp create mode 100644 src/grpc_utils.hpp rename src/{ => kfs_frontend}/kfs_grpc_inference_service.cpp (81%) rename src/{ => kfs_frontend}/kfs_grpc_inference_service.hpp (55%) create mode 100644 src/kfs_frontend/kfs_utils.cpp create mode 100644 src/kfs_frontend/kfs_utils.hpp create mode 100644 src/pipelinedefinitionstatus.cpp diff --git a/check_coverage.bat b/check_coverage.bat index 5265d590fe..4543bcd83d 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -2,11 +2,11 @@ #Ubuntu #MIN_LINES_COV=74.9 -#MIN_FUNCTION_COV=86.5 +#MIN_FUNCTION_COV=87 #Rhel -MIN_LINES_COV=72.4 -MIN_FUNCTION_COV=73.2 +MIN_LINES_COV=73.2 +MIN_FUNCTION_COV=73.3 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` diff --git a/cppclean.sh b/cppclean.sh index 175a57867c..7e2bfffe7b 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -20,28 +20,36 @@ cppclean ./src/ 2>&1 | grep -v test > ${CPPCLEAN_RESULTS_FILE_SRC}; cppclean ./src/ 2>&1 | grep test > ${CPPCLEAN_RESULTS_FILE_TEST}; NO_WARNINGS=$(wc -l ${CPPCLEAN_RESULTS_FILE_SRC} | awk '{print $1}') NO_WARNINGS_TEST=$(wc -l ${CPPCLEAN_RESULTS_FILE_TEST} | awk '{print $1}') +NO_WARNINGS_FORWARD=$(grep "use a forward declaration instead" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l) +NO_WARNINGS_DIRECT=$(grep "not found in any directly #included header" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l) +NO_WARNINGS_NOTUSED=$(grep " not used$" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l) echo "Number of warnings:" ${NO_WARNINGS} echo "Number of warnings in tests:" ${NO_WARNINGS_TEST} +echo "Number of warnings about not using forward delares:" ${NO_WARNINGS_FORWARD} +echo "Number of warnings about not direct includes:" ${NO_WARNINGS_DIRECT} +echo "Number of warnings about not used: " ${NO_WARNINGS_NOTUSED} trap "cat ${CPPCLEAN_RESULTS_FILE_SRC}" err exit -if [ $(grep "use a forward declaration instead" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 6 ]; then - echo "Failed due to not using forward declarations where possible"; +if [ ${NO_WARNINGS_FORWARD} -gt 6 ]; then + echo "Failed due to not using forward declarations where possible: ${NO_WARNINGS_FORWARD}"; exit 1; fi -if [ $(grep "not found in any directly #included header" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 14 ]; then - echo "Failed probably due to not using static keyword with functions definitions"; +if [ ${NO_WARNINGS_DIRECT} -gt 14 ]; then + echo "Failed probably due to not using static keyword with functions definitions: ${NO_WARNINGS_DIRECT}"; exit 1; fi -if [ $(grep " not used$" ${CPPCLEAN_RESULTS_FILE_SRC} | wc -l ) -gt 14 ]; then - echo "Failed probably due to unnecessary forward include"; +if [ ${NO_WARNINGS_NOTUSED} -gt 2 ]; then + echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 183 ]; then +if [ ${NO_WARNINGS} -gt 178 ]; then + echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 else exit 0; fi if [ ${NO_WARNINGS_TEST} -gt 131 ]; then + echo "Failed due to higher than allowed number of issues in test code: ${NO_WARNINGS_TEST}" exit 1 else exit 0; diff --git a/src/BUILD b/src/BUILD index 824ec871bf..e8abbdca8c 100644 --- a/src/BUILD +++ b/src/BUILD @@ -110,10 +110,14 @@ cc_library( "gatherexitnodeinputhandler.hpp", "gcsfilesystem.cpp", "gcsfilesystem.hpp", + "grpc_utils.cpp", + "grpc_utils.hpp", "grpcservermodule.cpp", "grpcservermodule.hpp", - "kfs_grpc_inference_service.cpp", - "kfs_grpc_inference_service.hpp", + "kfs_frontend/kfs_grpc_inference_service.cpp", + "kfs_frontend/kfs_grpc_inference_service.hpp", + "kfs_frontend/kfs_utils.cpp", + "kfs_frontend/kfs_utils.hpp", "metric.cpp", "metric.hpp", "metric_config.cpp", @@ -171,6 +175,7 @@ cc_library( "pipeline.hpp", "pipelinedefinition.cpp", "pipelinedefinition.hpp", + "pipelinedefinitionstatus.cpp", "pipelinedefinitionstatus.hpp", "pipelinedefinitionunloadguard.cpp", "pipelinedefinitionunloadguard.hpp", diff --git a/src/azurestorage.hpp b/src/azurestorage.hpp index 3a8b6bd79b..9b26a12b9b 100644 --- a/src/azurestorage.hpp +++ b/src/azurestorage.hpp @@ -62,6 +62,7 @@ class AzureStorageAdapter { StatusCode CreateLocalDir(const std::string& path); bool isAbsolutePath(const std::string& path); std::vector FindSubdirectories(std::string path); + virtual ~AzureStorageAdapter() = default; protected: const std::string extractAzureStorageExceptionMessage(const as::storage_exception& e); diff --git a/src/binaryutils.cpp b/src/binaryutils.cpp index 72696df4e5..f60aed82f1 100644 --- a/src/binaryutils.cpp +++ b/src/binaryutils.cpp @@ -35,7 +35,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #pragma GCC diagnostic pop namespace ovms { @@ -219,7 +219,7 @@ static Status validateTensor(const std::shared_ptr& tensorInfo, } static Status validateTensor(const std::shared_ptr& tensorInfo, - const ::inference::ModelInferRequest::InferInputTensor& src) { + const ::KFSRequest::InferInputTensor& src) { OVMS_PROFILE_FUNCTION(); auto status = validateLayout(tensorInfo); if (!status.ok()) { @@ -318,7 +318,7 @@ inline static const std::string& getBinaryInput(const tensorflow::TensorProto& t return tensor.string_val(i); } -inline static const std::string& getBinaryInput(const ::inference::ModelInferRequest::InferInputTensor& tensor, size_t i) { +inline static const std::string& getBinaryInput(const ::KFSRequest::InferInputTensor& tensor, size_t i) { return tensor.contents().bytes_contents(i); } @@ -326,7 +326,7 @@ inline static int getBinaryInputsSize(const tensorflow::TensorProto& tensor) { return tensor.string_val_size(); } -inline static int getBinaryInputsSize(const ::inference::ModelInferRequest::InferInputTensor& tensor) { +inline static int getBinaryInputsSize(const ::KFSRequest::InferInputTensor& tensor) { return tensor.contents().bytes_contents_size(); } @@ -457,5 +457,5 @@ static Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Te } template Status convertBinaryRequestTensorToOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); -template Status convertBinaryRequestTensorToOVTensor<::inference::ModelInferRequest::InferInputTensor>(const ::inference::ModelInferRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +template Status convertBinaryRequestTensorToOVTensor<::KFSRequest::InferInputTensor>(const ::KFSRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); } // namespace ovms diff --git a/src/cleaner_utils.cpp b/src/cleaner_utils.cpp index 36fb8dbdb0..ad173c72ab 100644 --- a/src/cleaner_utils.cpp +++ b/src/cleaner_utils.cpp @@ -27,6 +27,10 @@ void FunctorSequenceCleaner::cleanup() { globalSequencesViewer.removeIdleSequences(); } +FunctorSequenceCleaner::~FunctorSequenceCleaner() = default; + +FunctorResourcesCleaner::~FunctorResourcesCleaner() = default; + FunctorResourcesCleaner::FunctorResourcesCleaner(ModelManager& modelManager) : modelManager(modelManager) {} diff --git a/src/cleaner_utils.hpp b/src/cleaner_utils.hpp index 535728ffb6..becfbe926a 100644 --- a/src/cleaner_utils.hpp +++ b/src/cleaner_utils.hpp @@ -23,6 +23,7 @@ struct FunctorSequenceCleaner { FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer); virtual void cleanup(); + virtual ~FunctorSequenceCleaner(); }; struct FunctorResourcesCleaner { @@ -30,5 +31,6 @@ struct FunctorResourcesCleaner { FunctorResourcesCleaner(ModelManager& modelManager); virtual void cleanup(); + ~FunctorResourcesCleaner(); }; } // namespace ovms diff --git a/src/deserialization.cpp b/src/deserialization.cpp index 34ee33bd24..536ee99354 100644 --- a/src/deserialization.cpp +++ b/src/deserialization.cpp @@ -50,7 +50,7 @@ ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.tensor_content().data()))); } -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, +ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo, const std::string& buffer) { OVMS_PROFILE_FUNCTION(); @@ -62,7 +62,7 @@ ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& re ov::Tensor tensor(precision, shape, const_cast(reinterpret_cast(buffer.data()))); return tensor; } -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, +ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo) { OVMS_PROFILE_FUNCTION(); ov::Shape shape; diff --git a/src/deserialization.hpp b/src/deserialization.hpp index c82bdd7f77..444a5d30bd 100644 --- a/src/deserialization.hpp +++ b/src/deserialization.hpp @@ -26,7 +26,7 @@ #include "tensorflow/core/framework/tensor.h" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #pragma GCC diagnostic pop #include "binaryutils.hpp" @@ -39,16 +39,16 @@ namespace ovms { ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, const std::shared_ptr& tensorInfo); -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, +ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo, const std::string& buffer); -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, +ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo); class ConcreteTensorProtoDeserializator { public: static ov::Tensor deserializeTensorProto( - const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo, const std::string* buffer) { OVMS_PROFILE_FUNCTION(); @@ -271,7 +271,7 @@ ov::Tensor deserializeTensorProto( template ov::Tensor deserializeTensorProto( - const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo, const std::string* buffer) { return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo, buffer); @@ -347,7 +347,7 @@ Status deserializePredictRequest( template Status deserializePredictRequest( - const ::inference::ModelInferRequest& request, + const ::KFSRequest& request, const tensor_map_t& inputMap, Sink& inputSink, bool isPipeline) { OVMS_PROFILE_FUNCTION(); @@ -357,7 +357,7 @@ Status deserializePredictRequest( try { const auto& name = pair.first; auto tensorInfo = pair.second; - auto requestInputItr = std::find_if(request.inputs().begin(), request.inputs().end(), [&name](const ::inference::ModelInferRequest::InferInputTensor& tensor) { return tensor.name() == name; }); + auto requestInputItr = std::find_if(request.inputs().begin(), request.inputs().end(), [&name](const ::KFSRequest::InferInputTensor& tensor) { return tensor.name() == name; }); if (requestInputItr == request.inputs().end()) { SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); diff --git a/src/dlnodesession.cpp b/src/dlnodesession.cpp index a0240ed923..bd3fd205f5 100644 --- a/src/dlnodesession.cpp +++ b/src/dlnodesession.cpp @@ -90,11 +90,9 @@ Status DLNodeSession::requestExecuteRequiredResources() { Status DLNodeSession::prepareInputsAndModelForInference() { OVMS_PROFILE_FUNCTION(); - std::optional requestedBatchSize = std::nullopt; - std::map requestedReshapes; - // Validate each tensor against its OV tensor info const auto& inputsInfo = this->model->getInputsInfo(); + Status status; for (const auto& kv : this->inputHandler->getInputs()) { const auto& name = kv.first; auto& tensor = kv.second; @@ -108,47 +106,12 @@ Status DLNodeSession::prepareInputsAndModelForInference() { return Status(StatusCode::INVALID_MISSING_INPUT, details); } auto& inputInfo = *it->second; - auto status = validate(tensor, inputInfo); - if (status.ok()) { - continue; - } - - // If precision is incorrect, perform conversion - if (status == StatusCode::INVALID_PRECISION) { - return status; - } - - // If batch size is incorrect, perform model batch size change if allowed (shape mode=auto or batch size=auto) - if (status == StatusCode::INVALID_BATCH_SIZE) { - if (this->model->getModelConfig().getBatchingMode() == Mode::AUTO) { - requestedBatchSize = tensor.get_shape()[0]; - } else if (this->model->getModelConfig().isShapeAuto(name)) { - requestedReshapes[name] = tensor.get_shape(); - } else { - return status; - } - } - - // If shape is incorrect, perform reshape if allowed (mode=auto) - if (status == StatusCode::INVALID_SHAPE) { - if (!this->model->getModelConfig().isShapeAuto(name)) { - return status; - } - requestedReshapes[name] = tensor.get_shape(); - } - } - if (requestedReshapes.size() > 0) { - auto status = this->model->reloadModel(std::nullopt, requestedReshapes, this->modelUnloadGuard); - if (!status.ok()) { - return status; - } - } else if (requestedBatchSize.has_value()) { - auto status = this->model->reloadModel(requestedBatchSize, {}, this->modelUnloadGuard); + status = validate(tensor, inputInfo); if (!status.ok()) { - return status; + break; } } - return StatusCode::OK; + return status; } Status DLNodeSession::validate(const ov::Tensor& tensor, const TensorInfo& tensorInfo) { diff --git a/src/entry_node.cpp b/src/entry_node.cpp index b30afffc94..b5684deb7b 100644 --- a/src/entry_node.cpp +++ b/src/entry_node.cpp @@ -91,7 +91,7 @@ Status EntryNode::isInputBinary(const std:: return StatusCode::OK; } template <> -Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const { +Status EntryNode<::KFSRequest>::isInputBinary(const std::string& name, bool& isBinary) const { auto it = request->inputs().begin(); while (it != request->inputs().end()) { if (it->name() == name) { @@ -146,7 +146,7 @@ const Status EntryNode::validate() { optionalInputNames); // Pipelines are not versioned and always reports version 1 } template <> -const Status EntryNode<::inference::ModelInferRequest>::validate() { +const Status EntryNode<::KFSRequest>::validate() { static const std::set optionalInputNames = {}; return request_validation_utils::validate( *request, @@ -157,15 +157,15 @@ const Status EntryNode<::inference::ModelInferRequest>::validate() { } template Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status EntryNode<::inference::ModelInferRequest>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status EntryNode<::KFSRequest>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); template Status EntryNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status EntryNode<::inference::ModelInferRequest>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status EntryNode<::KFSRequest>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); template Status EntryNode::fetchResults(TensorWithSourceMap& outputs); -template Status EntryNode<::inference::ModelInferRequest>::fetchResults(TensorWithSourceMap& outputs); +template Status EntryNode<::KFSRequest>::fetchResults(TensorWithSourceMap& outputs); template Status EntryNode::isInputBinary(const std::string& name, bool& isBinary) const; -template Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const; +template Status EntryNode<::KFSRequest>::isInputBinary(const std::string& name, bool& isBinary) const; template Status EntryNode::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); -template Status EntryNode<::inference::ModelInferRequest>::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); +template Status EntryNode<::KFSRequest>::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); template const Status EntryNode::validate(); -template const Status EntryNode<::inference::ModelInferRequest>::validate(); +template const Status EntryNode<::KFSRequest>::validate(); } // namespace ovms diff --git a/src/exit_node.cpp b/src/exit_node.cpp index fd88638196..d7ca765fe0 100644 --- a/src/exit_node.cpp +++ b/src/exit_node.cpp @@ -70,12 +70,12 @@ std::unique_ptr ExitNode::createNodeSession(const Nod return std::make_unique>(metadata, getName(), previous.size(), collapsingDetails, response); } -template Status ExitNode<::inference::ModelInferResponse>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status ExitNode<::KFSResponse>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); template Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status ExitNode<::inference::ModelInferResponse>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status ExitNode<::KFSResponse>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); template Status ExitNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status ExitNode<::inference::ModelInferResponse>::fetchResults(const TensorMap& inputTensors); +template Status ExitNode<::KFSResponse>::fetchResults(const TensorMap& inputTensors); template Status ExitNode::fetchResults(const TensorMap& inputTensors); -template std::unique_ptr ExitNode<::inference::ModelInferResponse>::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); +template std::unique_ptr ExitNode<::KFSResponse>::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); template std::unique_ptr ExitNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); } // namespace ovms diff --git a/src/exitnodesession.cpp b/src/exitnodesession.cpp index afd33b977b..f680b722e6 100644 --- a/src/exitnodesession.cpp +++ b/src/exitnodesession.cpp @@ -39,9 +39,9 @@ template ExitNodeSession::~ExitNodeSession() = default; template ExitNodeSession::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, tensorflow::serving::PredictResponse* response); -template ExitNodeSession<::inference::ModelInferResponse>::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ::inference::ModelInferResponse* response); +template ExitNodeSession<::KFSResponse>::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ::KFSResponse* response); template const TensorMap& ExitNodeSession::getInputTensors() const; -template const TensorMap& ExitNodeSession<::inference::ModelInferResponse>::getInputTensors() const; +template const TensorMap& ExitNodeSession<::KFSResponse>::getInputTensors() const; } // namespace ovms diff --git a/src/gatherexitnodeinputhandler.cpp b/src/gatherexitnodeinputhandler.cpp index 1de478dacc..22b1d7de1d 100644 --- a/src/gatherexitnodeinputhandler.cpp +++ b/src/gatherexitnodeinputhandler.cpp @@ -21,34 +21,4 @@ #include "status.hpp" namespace ovms { - -Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& bufferOut, const std::string& name, size_t size) { - OVMS_PROFILE_FUNCTION(); - tensorflow::TensorProto tensorProto; - auto [it, isInserted] = response->mutable_outputs()->insert(google::protobuf::MapPair(name, std::move(tensorProto))); - if (!isInserted) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); - return StatusCode::INTERNAL_ERROR; - } - it->second.mutable_tensor_content()->resize(size); - bufferOut = it->second.mutable_tensor_content()->data(); - return StatusCode::OK; -} - -Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& bufferOut, const std::string& name, size_t size) { - OVMS_PROFILE_FUNCTION(); - for (int i = 0; i < response->outputs_size(); i++) { - if (response->mutable_outputs(i)->name() == name) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); - return StatusCode::INTERNAL_ERROR; - } - } - auto* proto = response->add_outputs(); - proto->set_name(name); - auto* content = response->add_raw_output_contents(); - content->resize(size); - bufferOut = content->data(); - return StatusCode::OK; -} - } // namespace ovms diff --git a/src/gatherexitnodeinputhandler.hpp b/src/gatherexitnodeinputhandler.hpp index fd470cec5e..2cc1902b3e 100644 --- a/src/gatherexitnodeinputhandler.hpp +++ b/src/gatherexitnodeinputhandler.hpp @@ -21,25 +21,14 @@ #include #include "gathernodeinputhandler.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" - -#include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" -#include "src/kfserving_api/grpc_predict_v2.pb.h" -#pragma GCC diagnostic pop - +#include "kfs_frontend/kfs_utils.hpp" #include "logging.hpp" #include "profiler.hpp" #include "status.hpp" +#include "tfs_frontend/tfs_utils.hpp" namespace ovms { -Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& tensorOut, const std::string& name, size_t size); -Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& tensorOut, const std::string& name, size_t size); - template class GatherExitNodeInputHandler : public GatherNodeInputHandler { ResponseType* response; diff --git a/src/grpc_utils.cpp b/src/grpc_utils.cpp new file mode 100644 index 0000000000..b05fa8d159 --- /dev/null +++ b/src/grpc_utils.cpp @@ -0,0 +1,108 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "grpc_utils.hpp" + +#include +#include + +#include "status.hpp" + +namespace ovms { +const grpc::Status grpc(const Status& status) { + static const std::unordered_map grpcStatusMap = { + {StatusCode::OK, grpc::StatusCode::OK}, + + {StatusCode::PATH_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::FILE_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::NO_MODEL_VERSION_AVAILABLE, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_NOT_LOADED, grpc::StatusCode::INTERNAL}, + {StatusCode::JSON_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::MODELINSTANCE_NOT_FOUND, grpc::StatusCode::INTERNAL}, + {StatusCode::SHAPE_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, grpc::StatusCode::INTERNAL}, + {StatusCode::RESHAPE_ERROR, grpc::StatusCode::FAILED_PRECONDITION}, + {StatusCode::MODEL_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_SPEC_MISSING, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::MODEL_VERSION_INVALID_FORMAT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SIGNATURE_DEF, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, grpc::StatusCode::ABORTED}, + {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, grpc::StatusCode::FAILED_PRECONDITION}, + + // Sequence management + {StatusCode::SEQUENCE_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::SEQUENCE_ALREADY_EXISTS, grpc::StatusCode::ALREADY_EXISTS}, + {StatusCode::SEQUENCE_ID_NOT_PROVIDED, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_ID_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_TERMINATED, grpc::StatusCode::FAILED_PRECONDITION}, + {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, grpc::StatusCode::UNAVAILABLE}, + + // Predict request validation + {StatusCode::INVALID_NO_OF_INPUTS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_MISSING_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_BATCH_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_PRECISION, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_VALUE_COUNT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_CONTENT_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_MESSAGE_STRUCTURE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::UNSUPPORTED_LAYOUT, grpc::StatusCode::INVALID_ARGUMENT}, + + // Deserialization + + // Should never occur - ModelInstance::validate takes care of that + {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, + {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, + + // Inference + {StatusCode::OV_INTERNAL_INFERENCE_ERROR, grpc::StatusCode::INTERNAL}, + + // Serialization + + // Should never occur - it should be validated during model loading + {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, + {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, + + // GetModelStatus + {StatusCode::INTERNAL_ERROR, grpc::StatusCode::INTERNAL}, + + // Binary input + {StatusCode::INVALID_NO_OF_CHANNELS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::STRING_VAL_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::BYTES_CONTENTS_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, + }; + auto it = grpcStatusMap.find(status.getCode()); + if (it != grpcStatusMap.end()) { + return grpc::Status(it->second, status.string()); + } else { + return grpc::Status(grpc::StatusCode::UNKNOWN, "Unknown error"); + } +} +} // namespace ovms diff --git a/src/grpc_utils.hpp b/src/grpc_utils.hpp new file mode 100644 index 0000000000..a0d960e558 --- /dev/null +++ b/src/grpc_utils.hpp @@ -0,0 +1,23 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include + +namespace ovms { +class Status; + +const grpc::Status grpc(const Status& status); +} // namespace ovms diff --git a/src/grpcservermodule.cpp b/src/grpcservermodule.cpp index 55c779cdb6..17ac168c5b 100644 --- a/src/grpcservermodule.cpp +++ b/src/grpcservermodule.cpp @@ -32,7 +32,7 @@ #include #include "config.hpp" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" #include "model_service.hpp" #include "modelmanager.hpp" diff --git a/src/grpcservermodule.hpp b/src/grpcservermodule.hpp index cc825b7759..0be3eb39c2 100644 --- a/src/grpcservermodule.hpp +++ b/src/grpcservermodule.hpp @@ -20,7 +20,7 @@ #include -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "model_service.hpp" #include "module.hpp" #include "prediction_service.hpp" diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index e873a75138..efe0c6e6fa 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -32,7 +32,7 @@ #include "filesystem.hpp" #include "get_model_metadata_impl.hpp" #include "grpcservermodule.hpp" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "metric_module.hpp" #include "metric_registry.hpp" #include "model_metric_reporter.hpp" @@ -200,8 +200,8 @@ Status HttpRestApiHandler::processServerLiveKFSRequest(const HttpRequestComponen } Status HttpRestApiHandler::processServerMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - ::inference::ServerMetadataRequest grpc_request; - ::inference::ServerMetadataResponse grpc_response; + ::KFSServerMetadataRequest grpc_request; + ::KFSServerMetadataResponse grpc_response; Status gstatus = kfsGrpcImpl.ServerMetadataImpl(nullptr, &grpc_request, &grpc_response); if (!gstatus.ok()) { return gstatus; @@ -298,7 +298,7 @@ static Status convertStringToVectorOfSizes(const std::string& comma_separated_nu return StatusCode::OK; } -static Status parseBinaryInput(::inference::ModelInferRequest_InferInputTensor* input, size_t binary_input_size, const char* buffer) { +static Status parseBinaryInput(KFSTensorInputProto* input, size_t binary_input_size, const char* buffer) { if (input->datatype() == "FP32") { for (size_t i = 0; i < binary_input_size; i += sizeof(float)) { auto value = input->mutable_contents()->mutable_fp32_contents()->Add(); @@ -360,7 +360,7 @@ static Status parseBinaryInput(::inference::ModelInferRequest_InferInputTensor* #define CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE " contents is not empty. Content field should be empty when using binary inputs extension." -static Status validateContentFieldsEmptiness(::inference::ModelInferRequest_InferInputTensor* input) { +static Status validateContentFieldsEmptiness(KFSTensorInputProto* input) { if (input->datatype() == "FP32") { if (input->contents().fp32_contents_size() > 0) { SPDLOG_DEBUG("FP32" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); @@ -423,7 +423,7 @@ static Status validateContentFieldsEmptiness(::inference::ModelInferRequest_Infe return StatusCode::OK; } -static Status handleBinaryInputs(::inference::ModelInferRequest& grpc_request, const std::string& request_body, size_t endOfJson) { +static Status handleBinaryInputs(::KFSRequest& grpc_request, const std::string& request_body, size_t endOfJson) { const char* binary_inputs = &(request_body[endOfJson]); size_t binary_inputs_size = request_body.length() - endOfJson; @@ -476,7 +476,7 @@ static Status handleBinaryInputs(::inference::ModelInferRequest& grpc_request, c return StatusCode::OK; } -Status HttpRestApiHandler::prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::inference::ModelInferRequest& grpc_request, const std::optional& inferenceHeaderContentLength) { +Status HttpRestApiHandler::prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::KFSRequest& grpc_request, const std::optional& inferenceHeaderContentLength) { KFSRestParser requestParser; size_t endOfJson = inferenceHeaderContentLength.value_or(request_body.length()); @@ -504,7 +504,7 @@ Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& r std::string modelName(request_components.model_name); std::string modelVersionLog = request_components.model_version.has_value() ? std::to_string(request_components.model_version.value()) : DEFAULT_VERSION; SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", modelName, modelVersionLog); - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; timer.start(PREPARE_GRPC_REQUEST); using std::chrono::microseconds; auto status = prepareGrpcRequest(modelName, request_components.model_version, request_body, grpc_request, request_components.inferenceHeaderContentLength); @@ -519,7 +519,7 @@ Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& r } timer.stop(PREPARE_GRPC_REQUEST); SPDLOG_DEBUG("Preparing grpc request time: {} ms", timer.elapsed(PREPARE_GRPC_REQUEST) / 1000); - ::inference::ModelInferResponse grpc_response; + ::KFSResponse grpc_response; const Status gstatus = kfsGrpcImpl.ModelInferImpl(nullptr, &grpc_request, &grpc_response, executionContext, reporter); if (!gstatus.ok()) { return gstatus; @@ -571,8 +571,8 @@ Status HttpRestApiHandler::processMetrics(const HttpRequestComponents& request_c } Status HttpRestApiHandler::processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - ::inference::ModelReadyRequest grpc_request; - ::inference::ModelReadyResponse grpc_response; + ::KFSGetModelStatusRequest grpc_request; + ::KFSGetModelStatusResponse grpc_response; std::string modelName(request_components.model_name); grpc_request.set_name(modelName); if (request_components.model_version.has_value()) { @@ -604,8 +604,8 @@ void HttpRestApiHandler::convertShapeType(Value& scope, Document& doc) { } Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - ::inference::ModelMetadataRequest grpc_request; - ::inference::ModelMetadataResponse grpc_response; + ::KFSModelMetadataRequest grpc_request; + ::KFSModelMetadataResponse grpc_response; std::string modelName(request_components.model_name); grpc_request.set_name(modelName); if (request_components.model_version.has_value()) { diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 379d978f14..4649026d21 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -92,7 +92,7 @@ class HttpRestApiHandler { Status parseModelVersion(std::string& model_version_str, std::optional& model_version); static void parseParams(rapidjson::Value&, rapidjson::Document&); static std::string preprocessInferRequest(std::string request_body); - static Status prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::inference::ModelInferRequest& grpc_request, const std::optional& inferenceHeaderContentLength = {}); + static Status prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::KFSRequest& grpc_request, const std::optional& inferenceHeaderContentLength = {}); void registerHandler(RequestType type, std::function); void registerAll(); diff --git a/src/http_server.cpp b/src/http_server.cpp index df0fcf4962..9652aee899 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,114 @@ namespace ovms { namespace net_http = tensorflow::serving::net_http; +const net_http::HTTPStatusCode http(const ovms::Status& status) { + const std::unordered_map httpStatusMap = { + {StatusCode::OK, net_http::HTTPStatusCode::OK}, + {StatusCode::OK_RELOADED, net_http::HTTPStatusCode::CREATED}, + {StatusCode::OK_NOT_RELOADED, net_http::HTTPStatusCode::OK}, + + // REST handler failure + {StatusCode::REST_INVALID_URL, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_UNSUPPORTED_METHOD, net_http::HTTPStatusCode::NONE_ACC}, + {StatusCode::REST_NOT_FOUND, net_http::HTTPStatusCode::NOT_FOUND}, + + // REST parser failure + {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_PREDICT_UNKNOWN_ORDER, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INPUT_NOT_PREALLOCATED, net_http::HTTPStatusCode::ERROR}, + {StatusCode::REST_NO_INSTANCES_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INPUTS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_NO_INPUTS_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_OUTPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_PARAMETERS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_PROTO_TO_STRING_ERROR, net_http::HTTPStatusCode::ERROR}, + {StatusCode::REST_UNSUPPORTED_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, net_http::HTTPStatusCode::ERROR}, + + {StatusCode::PATH_INVALID, net_http::HTTPStatusCode::ERROR}, + {StatusCode::FILE_INVALID, net_http::HTTPStatusCode::ERROR}, + {StatusCode::NO_MODEL_VERSION_AVAILABLE, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_NOT_LOADED, net_http::HTTPStatusCode::ERROR}, + {StatusCode::JSON_INVALID, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MODELINSTANCE_NOT_FOUND, net_http::HTTPStatusCode::ERROR}, + {StatusCode::SHAPE_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, net_http::HTTPStatusCode::ERROR}, + {StatusCode::RESHAPE_ERROR, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MODEL_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_SPEC_MISSING, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SIGNATURE_DEF, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, net_http::HTTPStatusCode::NO_CONTENT}, + {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, net_http::HTTPStatusCode::PRECOND_FAILED}, + + // Sequence management + {StatusCode::SEQUENCE_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::SEQUENCE_ALREADY_EXISTS, net_http::HTTPStatusCode::CONFLICT}, + {StatusCode::SEQUENCE_ID_NOT_PROVIDED, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_ID_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_TERMINATED, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, net_http::HTTPStatusCode::SERVICE_UNAV}, + + // Predict request validation + {StatusCode::INVALID_NO_OF_INPUTS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_MISSING_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_BATCH_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_VALUE_COUNT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_CONTENT_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_MESSAGE_STRUCTURE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::UNSUPPORTED_LAYOUT, net_http::HTTPStatusCode::BAD_REQUEST}, + + // Deserialization + + // Should never occur - ModelInstance::validate takes care of that + {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, + {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Inference + {StatusCode::OV_INTERNAL_INFERENCE_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Serialization + + // Should never occur - it should be validated during model loading + {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, + {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, + + // GetModelStatus + {StatusCode::INTERNAL_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Binary input + {StatusCode::INVALID_NO_OF_CHANNELS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::STRING_VAL_EMPTY, net_http::HTTPStatusCode::BAD_REQUEST}, + }; + auto it = httpStatusMap.find(status.getCode()); + if (it != httpStatusMap.end()) { + return it->second; + } else { + return net_http::HTTPStatusCode::ERROR; + } +} + class RequestExecutor final : public net_http::EventExecutor { public: explicit RequestExecutor(int num_threads) : @@ -89,7 +198,7 @@ class RestApiRequestDispatcher { if (!status.ok() && output.empty()) { output.append("{\"error\": \"" + status.string() + "\"}"); } - const auto http_status = status.http(); + const auto http_status = http(status); for (const auto& kv : headers) { req->OverwriteResponseHeader(kv.first, kv.second); } diff --git a/src/kfs_grpc_inference_service.cpp b/src/kfs_frontend/kfs_grpc_inference_service.cpp similarity index 81% rename from src/kfs_grpc_inference_service.cpp rename to src/kfs_frontend/kfs_grpc_inference_service.cpp index 0b2261d579..ae6294aaef 100644 --- a/src/kfs_grpc_inference_service.cpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.cpp @@ -19,26 +19,28 @@ #include #include -#include "deserialization.hpp" -#include "execution_context.hpp" -#include "metric.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "ovinferrequestsqueue.hpp" -#include "pipeline.hpp" -#include "pipelinedefinition.hpp" -#include "pipelinedefinitionstatus.hpp" -#include "pipelinedefinitionunloadguard.hpp" -#include "prediction_service_utils.hpp" -#include "serialization.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "status.hpp" -#include "stringutils.hpp" -#include "tensorinfo.hpp" -#include "timer.hpp" -#include "version.hpp" +#include "../deserialization.hpp" +#include "../execution_context.hpp" +#include "../grpc_utils.hpp" +#include "../kfs_frontend/kfs_utils.hpp" +#include "../metric.hpp" +#include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" +#include "../modelmanager.hpp" +#include "../ovinferrequestsqueue.hpp" +#include "../pipeline.hpp" +#include "../pipelinedefinition.hpp" +#include "../pipelinedefinitionstatus.hpp" +#include "../pipelinedefinitionunloadguard.hpp" +#include "../prediction_service_utils.hpp" +#include "../serialization.hpp" +#include "../servablemanagermodule.hpp" +#include "../server.hpp" +#include "../status.hpp" +#include "../stringutils.hpp" +#include "../tensorinfo.hpp" +#include "../timer.hpp" +#include "../version.hpp" namespace { enum : unsigned int { @@ -49,7 +51,7 @@ enum : unsigned int { namespace ovms { -Status KFSInferenceServiceImpl::getModelInstance(const ::inference::ModelInferRequest* request, +Status KFSInferenceServiceImpl::getModelInstance(const KFSRequest* request, std::shared_ptr& modelInstance, std::unique_ptr& modelInstanceUnloadGuardPtr) { OVMS_PROFILE_FUNCTION(); @@ -66,8 +68,8 @@ Status KFSInferenceServiceImpl::getModelInstance(const ::inference::ModelInferRe return this->modelManager.getModelInstance(request->model_name(), requestedVersion, modelInstance, modelInstanceUnloadGuardPtr); } -Status KFSInferenceServiceImpl::getPipeline(const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, +Status KFSInferenceServiceImpl::getPipeline(const KFSRequest* request, + KFSResponse* response, std::unique_ptr& pipelinePtr) { OVMS_PROFILE_FUNCTION(); return this->modelManager.createPipeline(pipelinePtr, request->model_name(), request, response); @@ -95,7 +97,7 @@ ::grpc::Status KFSInferenceServiceImpl::ServerReady(::grpc::ServerContext* conte return grpc::Status::OK; } -Status KFSInferenceServiceImpl::getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager, ExecutionContext executionContext) { +Status KFSInferenceServiceImpl::getModelReady(const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response, const ModelManager& manager, ExecutionContext executionContext) { // Return in response true/false // if no version requested give response for default version const auto& name = request->name(); @@ -141,20 +143,20 @@ Status KFSInferenceServiceImpl::getModelReady(const ::inference::ModelReadyReque return status; } -::grpc::Status KFSInferenceServiceImpl::ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) { - return ModelReadyImpl(context, request, response, ExecutionContext{ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelReady}).grpc(); +::grpc::Status KFSInferenceServiceImpl::ModelReady(::grpc::ServerContext* context, const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response) { + return grpc(ModelReadyImpl(context, request, response, ExecutionContext{ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelReady})); } -Status KFSInferenceServiceImpl::ModelReadyImpl(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, ExecutionContext executionContext) { +Status KFSInferenceServiceImpl::ModelReadyImpl(::grpc::ServerContext* context, const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response, ExecutionContext executionContext) { (void)context; return this->getModelReady(request, response, this->modelManager, executionContext); } -::grpc::Status KFSInferenceServiceImpl::ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) { - return ServerMetadataImpl(context, request, response).grpc(); +::grpc::Status KFSInferenceServiceImpl::ServerMetadata(::grpc::ServerContext* context, const KFSServerMetadataRequest* request, KFSServerMetadataResponse* response) { + return grpc(ServerMetadataImpl(context, request, response)); } -Status KFSInferenceServiceImpl::ServerMetadataImpl(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) { +Status KFSInferenceServiceImpl::ServerMetadataImpl(::grpc::ServerContext* context, const KFSServerMetadataRequest* request, KFSServerMetadataResponse* response) { (void)context; (void)request; (void)response; @@ -163,11 +165,11 @@ Status KFSInferenceServiceImpl::ServerMetadataImpl(::grpc::ServerContext* contex return StatusCode::OK; } -::grpc::Status KFSInferenceServiceImpl::ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) { - return ModelMetadataImpl(context, request, response, ExecutionContext{ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelMetadata}).grpc(); +::grpc::Status KFSInferenceServiceImpl::ModelMetadata(::grpc::ServerContext* context, const KFSModelMetadataRequest* request, KFSModelMetadataResponse* response) { + return grpc(ModelMetadataImpl(context, request, response, ExecutionContext{ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelMetadata})); } -Status KFSInferenceServiceImpl::ModelMetadataImpl(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response, ExecutionContext executionContext) { +Status KFSInferenceServiceImpl::ModelMetadataImpl(::grpc::ServerContext* context, const KFSModelMetadataRequest* request, KFSModelMetadataResponse* response, ExecutionContext executionContext) { const auto& name = request->name(); const auto& versionString = request->version(); @@ -212,7 +214,7 @@ Status KFSInferenceServiceImpl::ModelMetadataImpl(::grpc::ServerContext* context return status; } -::grpc::Status KFSInferenceServiceImpl::ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) { +::grpc::Status KFSInferenceServiceImpl::ModelInfer(::grpc::ServerContext* context, const KFSRequest* request, KFSResponse* response) { OVMS_PROFILE_FUNCTION(); Timer timer; timer.start(TOTAL); @@ -223,18 +225,18 @@ ::grpc::Status KFSInferenceServiceImpl::ModelInfer(::grpc::ServerContext* contex auto status = this->ModelInferImpl(context, request, response, ExecutionContext{ExecutionContext::Interface::GRPC, ExecutionContext::Method::ModelInfer}, reporter); timer.stop(TOTAL); if (!status.ok()) { - return status.grpc(); + return grpc(status); } if (!reporter) { - return Status(StatusCode::INTERNAL_ERROR).grpc(); // should not happen + return grpc(Status(StatusCode::INTERNAL_ERROR)); // should not happen } double requestTotal = timer.elapsed(TOTAL); SPDLOG_DEBUG("Total gRPC request processing time: {} ms", requestTotal / 1000); OBSERVE_IF_ENABLED(reporter->requestTimeGrpc, requestTotal); - return status.grpc(); + return grpc(status); } -Status KFSInferenceServiceImpl::ModelInferImpl(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response, ExecutionContext executionContext, ServableMetricReporter*& reporterOut) { +Status KFSInferenceServiceImpl::ModelInferImpl(::grpc::ServerContext* context, const KFSRequest* request, KFSResponse* response, ExecutionContext executionContext, ServableMetricReporter*& reporterOut) { OVMS_PROFILE_FUNCTION(); std::shared_ptr modelInstance; std::unique_ptr pipelinePtr; @@ -252,7 +254,6 @@ Status KFSInferenceServiceImpl::ModelInferImpl(::grpc::ServerContext* context, c SPDLOG_DEBUG("Getting modelInstance or pipeline failed. {}", status.string()); return status; } - if (pipelinePtr) { reporterOut = &pipelinePtr->getMetricReporter(); status = pipelinePtr->execute(executionContext); @@ -260,33 +261,30 @@ Status KFSInferenceServiceImpl::ModelInferImpl(::grpc::ServerContext* context, c reporterOut = &modelInstance->getMetricReporter(); status = modelInstance->infer(request, response, modelInstanceUnloadGuard); } - INCREMENT_IF_ENABLED(reporterOut->getInferRequestMetric(executionContext, status.ok())); - if (!status.ok()) { return status; } - response->set_id(request->id()); return StatusCode::OK; } Status KFSInferenceServiceImpl::buildResponse( std::shared_ptr instance, - ::inference::ModelReadyResponse* response) { + KFSGetModelStatusResponse* response) { response->set_ready(instance->getStatus().getState() == ModelVersionState::AVAILABLE); return StatusCode::OK; } Status KFSInferenceServiceImpl::buildResponse( PipelineDefinition& pipelineDefinition, - ::inference::ModelReadyResponse* response) { + KFSGetModelStatusResponse* response) { response->set_ready(pipelineDefinition.getStatus().isAvailable()); return StatusCode::OK; } static void addReadyVersions(Model& model, - ::inference::ModelMetadataResponse* response) { + KFSModelMetadataResponse* response) { auto modelVersions = model.getModelVersionsMapCopy(); for (auto& [modelVersion, modelInstance] : modelVersions) { if (modelInstance.getStatus().getState() == ModelVersionState::AVAILABLE) @@ -297,7 +295,7 @@ static void addReadyVersions(Model& model, Status KFSInferenceServiceImpl::buildResponse( Model& model, ModelInstance& instance, - ::inference::ModelMetadataResponse* response) { + KFSModelMetadataResponse* response) { std::unique_ptr unloadGuard; @@ -335,7 +333,7 @@ KFSInferenceServiceImpl::KFSInferenceServiceImpl(const Server& server) : Status KFSInferenceServiceImpl::buildResponse( PipelineDefinition& pipelineDefinition, - ::inference::ModelMetadataResponse* response) { + KFSModelMetadataResponse* response) { std::unique_ptr unloadGuard; @@ -363,9 +361,9 @@ Status KFSInferenceServiceImpl::buildResponse( void KFSInferenceServiceImpl::convert( const std::pair>& from, - ::inference::ModelMetadataResponse::TensorMetadata* to) { + KFSModelMetadataResponse::TensorMetadata* to) { to->set_name(from.first); - to->set_datatype(from.second->getPrecisionAsKFSPrecision()); + to->set_datatype(ovmsPrecisionToKFSPrecision(from.second->getPrecision())); for (auto dim : from.second->getShape()) { if (dim.isStatic()) { to->add_shape(dim.getStaticValue()); diff --git a/src/kfs_grpc_inference_service.hpp b/src/kfs_frontend/kfs_grpc_inference_service.hpp similarity index 55% rename from src/kfs_grpc_inference_service.hpp rename to src/kfs_frontend/kfs_grpc_inference_service.hpp index 1e87553099..c94d80d1ac 100644 --- a/src/kfs_grpc_inference_service.hpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.hpp @@ -24,16 +24,29 @@ #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" #include "src/kfserving_api/grpc_predict_v2.pb.h" -namespace ovms { -class Status; - using inference::GRPCInferenceService; +using KFSServerMetadataRequest = inference::ServerMetadataRequest; +using KFSServerMetadataResponse = inference::ServerMetadataResponse; +using KFSModelMetadataRequest = inference::ModelMetadataRequest; +using KFSModelMetadataResponse = inference::ModelMetadataResponse; +using KFSRequest = inference::ModelInferRequest; +using KFSResponse = inference::ModelInferResponse; +using KFSTensorInputProto = inference::ModelInferRequest::InferInputTensor; +using KFSTensorOutputProto = inference::ModelInferResponse::InferOutputTensor; +using KFSShapeType = google::protobuf::RepeatedField; +using KFSGetModelStatusRequest = inference::ModelReadyRequest; +using KFSGetModelStatusResponse = inference::ModelReadyResponse; +using KFSDataType = std::string; +using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; +using KFSOutputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; + +namespace ovms { class ExecutionContext; class Model; -class ModelManager; -class ServableMetricReporter; class ModelInstance; class ModelInstanceUnloadGuard; +class ModelManager; +class ServableMetricReporter; class Pipeline; class Server; class Status; @@ -45,30 +58,30 @@ class KFSInferenceServiceImpl final : public GRPCInferenceService::Service { ModelManager& modelManager; public: - Status ModelReadyImpl(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, ExecutionContext executionContext); - Status ServerMetadataImpl(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response); - Status ModelMetadataImpl(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response, ExecutionContext executionContext); - Status ModelInferImpl(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response, ExecutionContext executionContext, ServableMetricReporter*& reporterOut); + Status ModelReadyImpl(::grpc::ServerContext* context, const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response, ExecutionContext executionContext); + Status ServerMetadataImpl(::grpc::ServerContext* context, const KFSServerMetadataRequest* request, KFSServerMetadataResponse* response); + Status ModelMetadataImpl(::grpc::ServerContext* context, const KFSModelMetadataRequest* request, KFSModelMetadataResponse* response, ExecutionContext executionContext); + Status ModelInferImpl(::grpc::ServerContext* context, const KFSRequest* request, KFSResponse* response, ExecutionContext executionContext, ServableMetricReporter*& reporterOut); KFSInferenceServiceImpl(const Server& server); ::grpc::Status ServerLive(::grpc::ServerContext* context, const ::inference::ServerLiveRequest* request, ::inference::ServerLiveResponse* response) override; ::grpc::Status ServerReady(::grpc::ServerContext* context, const ::inference::ServerReadyRequest* request, ::inference::ServerReadyResponse* response) override; - ::grpc::Status ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) override; - ::grpc::Status ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) override; - ::grpc::Status ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) override; - ::grpc::Status ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) override; - static Status buildResponse(Model& model, ModelInstance& instance, ::inference::ModelMetadataResponse* response); - static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelMetadataResponse* response); - static Status buildResponse(std::shared_ptr instance, ::inference::ModelReadyResponse* response); - static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelReadyResponse* response); - static void convert(const std::pair>& from, ::inference::ModelMetadataResponse::TensorMetadata* to); - static Status getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager, ExecutionContext executionContext); + ::grpc::Status ModelReady(::grpc::ServerContext* context, const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response) override; + ::grpc::Status ServerMetadata(::grpc::ServerContext* context, const KFSServerMetadataRequest* request, KFSServerMetadataResponse* response) override; + ::grpc::Status ModelMetadata(::grpc::ServerContext* context, const KFSModelMetadataRequest* request, KFSModelMetadataResponse* response) override; + ::grpc::Status ModelInfer(::grpc::ServerContext* context, const KFSRequest* request, KFSResponse* response) override; + static Status buildResponse(Model& model, ModelInstance& instance, KFSModelMetadataResponse* response); + static Status buildResponse(PipelineDefinition& pipelineDefinition, KFSModelMetadataResponse* response); + static Status buildResponse(std::shared_ptr instance, KFSGetModelStatusResponse* response); + static Status buildResponse(PipelineDefinition& pipelineDefinition, KFSGetModelStatusResponse* response); + static void convert(const std::pair>& from, KFSModelMetadataResponse::TensorMetadata* to); + static Status getModelReady(const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response, const ModelManager& manager, ExecutionContext executionContext); protected: - Status getModelInstance(const ::inference::ModelInferRequest* request, + Status getModelInstance(const KFSRequest* request, std::shared_ptr& modelInstance, std::unique_ptr& modelInstanceUnloadGuardPtr); - Status getPipeline(const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, + Status getPipeline(const KFSRequest* request, + KFSResponse* response, std::unique_ptr& pipelinePtr); }; diff --git a/src/kfs_frontend/kfs_utils.cpp b/src/kfs_frontend/kfs_utils.cpp new file mode 100644 index 0000000000..74a36bc690 --- /dev/null +++ b/src/kfs_frontend/kfs_utils.cpp @@ -0,0 +1,136 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "kfs_utils.hpp" + +#include +#include +#include +#include +#include + +#include "../logging.hpp" +#include "../profiler.hpp" +#include "../status.hpp" + +namespace ovms { +Precision KFSPrecisionToOvmsPrecision(const KFSDataType& datatype) { + static std::unordered_map precisionMap{ + {"BOOL", Precision::BOOL}, + {"FP64", Precision::FP64}, + {"FP32", Precision::FP32}, + {"FP16", Precision::FP16}, + {"INT64", Precision::I64}, + {"INT32", Precision::I32}, + {"INT16", Precision::I16}, + {"INT8", Precision::I8}, + {"UINT64", Precision::U64}, + {"UINT32", Precision::U32}, + {"UINT16", Precision::U16}, + // {"BYTES", Precision::??}, + {"UINT8", Precision::U8}}; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +size_t KFSDataTypeSize(const KFSDataType& datatype) { + static std::unordered_map datatypeSizeMap{ + {"BOOL", 1}, + {"UINT8", 1}, + {"UINT16", 2}, + {"UINT32", 4}, + {"UINT64", 8}, + {"INT8", 1}, + {"INT16", 2}, + {"INT32", 4}, + {"INT64", 8}, + {"FP16", 2}, + {"FP32", 4}, + {"FP64", 8} + // {"BYTES", }, + }; + auto it = datatypeSizeMap.find(datatype); + if (it == datatypeSizeMap.end()) { + return 0; + } + return it->second; +} + +const KFSDataType& ovmsPrecisionToKFSPrecision(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP64, "FP64"}, + {Precision::FP32, "FP32"}, + {Precision::FP16, "FP16"}, + {Precision::I64, "INT64"}, + {Precision::I32, "INT32"}, + {Precision::I16, "INT16"}, + {Precision::I8, "INT8"}, + {Precision::U64, "UINT64"}, + {Precision::U32, "UINT32"}, + {Precision::U16, "UINT16"}, + {Precision::U8, "UINT8"}, + {Precision::BOOL, "BOOL"}}; + // {Precision::BF16, ""}, + // {Precision::U4, ""}, + // {Precision::U1, ""}, + // {Precision::CUSTOM, ""}, + // {Precision::DYNAMIC, ""}, + // {Precision::MIXED, ""}, + // {Precision::Q78, ""}, + // {Precision::BIN, ""}, + // {Precision::I4, ""}, + // {Precision::UNDEFINED, "UNDEFINED"}}; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + static const std::string invalid{"INVALID"}; + return invalid; + } + return it->second; +} + +std::string tensorShapeToString(const KFSShapeType& shape) { + std::ostringstream oss; + oss << "("; + int i = 0; + if (shape.size() > 0) { + for (; i < shape.size() - 1; i++) { + oss << shape[i] << ","; + } + oss << shape[i]; + } + oss << ")"; + + return oss.str(); +} + +Status prepareConsolidatedTensorImpl(KFSResponse* response, char*& bufferOut, const std::string& name, size_t size) { + OVMS_PROFILE_FUNCTION(); + for (int i = 0; i < response->outputs_size(); i++) { + if (response->mutable_outputs(i)->name() == name) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); + return StatusCode::INTERNAL_ERROR; + } + } + auto* proto = response->add_outputs(); + proto->set_name(name); + auto* content = response->add_raw_output_contents(); + content->resize(size); + bufferOut = content->data(); + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/kfs_frontend/kfs_utils.hpp b/src/kfs_frontend/kfs_utils.hpp new file mode 100644 index 0000000000..5aabd9640d --- /dev/null +++ b/src/kfs_frontend/kfs_utils.hpp @@ -0,0 +1,31 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include + +#include "../precision.hpp" +#include "kfs_grpc_inference_service.hpp" + +namespace ovms { +class Status; +std::string tensorShapeToString(const KFSShapeType& tensorShape); + +Precision KFSPrecisionToOvmsPrecision(const KFSDataType& s); +const KFSDataType& ovmsPrecisionToKFSPrecision(Precision precision); + +size_t KFSDataTypeSize(const KFSDataType& datatype); +Status prepareConsolidatedTensorImpl(KFSResponse* response, char*& tensorOut, const std::string& name, size_t size); +} // namespace ovms diff --git a/src/model_service.cpp b/src/model_service.cpp index 1aa60b9be2..94e0679ded 100644 --- a/src/model_service.cpp +++ b/src/model_service.cpp @@ -31,6 +31,7 @@ #pragma GCC diagnostic pop #include "execution_context.hpp" +#include "grpc_utils.hpp" #include "modelinstance.hpp" #include "modelmanager.hpp" #include "pipelinedefinition.hpp" @@ -67,7 +68,7 @@ void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, ::grpc::Status ModelServiceImpl::GetModelStatus( ::grpc::ServerContext* context, const tensorflow::serving::GetModelStatusRequest* request, tensorflow::serving::GetModelStatusResponse* response) { - return GetModelStatusImpl::getModelStatus(request, response, modelManager, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelStatus)).grpc(); + return grpc(GetModelStatusImpl::getModelStatus(request, response, modelManager, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelStatus))); } Status GetModelStatusImpl::createGrpcRequest(std::string model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request) { diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 42b6effc69..da580804ee 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1135,7 +1135,7 @@ const Status ModelInstance::validate(const RequestType* request) { getModelConfig().getShapes()); } -template const Status ModelInstance::validate(const ::inference::ModelInferRequest* request); +template const Status ModelInstance::validate(const ::KFSRequest* request); template const Status ModelInstance::validate(const tensorflow::serving::PredictRequest* request); Status ModelInstance::performInference(ov::InferRequest& inferRequest) { @@ -1220,8 +1220,8 @@ Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestPr return StatusCode::OK; } -Status ModelInstance::infer(const ::inference::ModelInferRequest* requestProto, - ::inference::ModelInferResponse* responseProto, +Status ModelInstance::infer(const ::KFSRequest* requestProto, + ::KFSResponse* responseProto, std::unique_ptr& modelUnloadGuardPtr) { OVMS_PROFILE_FUNCTION(); Timer timer; diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index f6e3144e72..b3d33667c5 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -31,6 +31,7 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "model_metric_reporter.hpp" #include "modelchangesubscription.hpp" #include "modelconfig.hpp" @@ -39,11 +40,6 @@ #include "ovinferrequestsqueue.hpp" #include "tensorinfo.hpp" -namespace inference { -class ModelInferRequest; -class ModelInferResponse; -} // namespace inference - namespace ovms { class MetricRegistry; class ModelConfig; @@ -551,8 +547,8 @@ class ModelInstance { virtual Status infer(const tensorflow::serving::PredictRequest* requestProto, tensorflow::serving::PredictResponse* responseProto, std::unique_ptr& modelUnloadGuardPtr); - virtual Status infer(const ::inference::ModelInferRequest* requestProto, - ::inference::ModelInferResponse* responseProto, + virtual Status infer(const ::KFSRequest* requestProto, + ::KFSResponse* responseProto, std::unique_ptr& modelUnloadGuardPtr); ModelMetricReporter& getMetricReporter() const { return *this->reporter; } diff --git a/src/pipeline_factory.cpp b/src/pipeline_factory.cpp index 3ae6d80a9d..b2c34687e9 100644 --- a/src/pipeline_factory.cpp +++ b/src/pipeline_factory.cpp @@ -96,8 +96,8 @@ Status PipelineFactory::createInternal(std::unique_ptr& pipeline, } Status PipelineFactory::create(std::unique_ptr& pipeline, const std::string& name, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, + const ::KFSRequest* request, + ::KFSResponse* response, ModelManager& manager) const { return this->createInternal(pipeline, name, request, response, manager); } diff --git a/src/pipeline_factory.hpp b/src/pipeline_factory.hpp index a4cf11c695..d64e45a63b 100644 --- a/src/pipeline_factory.hpp +++ b/src/pipeline_factory.hpp @@ -28,7 +28,7 @@ #pragma GCC diagnostic ignored "-Wall" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "nodeinfo.hpp" namespace ovms { @@ -62,8 +62,8 @@ class PipelineFactory { public: Status create(std::unique_ptr& pipeline, const std::string& name, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, + const ::KFSRequest* request, + ::KFSResponse* response, ModelManager& manager) const; Status create(std::unique_ptr& pipeline, const std::string& name, diff --git a/src/pipelinedefinition.cpp b/src/pipelinedefinition.cpp index 2a4a8eadac..f27141f5ee 100644 --- a/src/pipelinedefinition.cpp +++ b/src/pipelinedefinition.cpp @@ -1376,10 +1376,10 @@ template Status PipelineDefinition::create( +template Status PipelineDefinition::create<::KFSRequest, ::KFSResponse>( std::unique_ptr& pipeline, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, + const ::KFSRequest* request, + ::KFSResponse* response, ModelManager& manager); } // namespace ovms diff --git a/src/pipelinedefinition.hpp b/src/pipelinedefinition.hpp index 556d65950e..6dc7f048bb 100644 --- a/src/pipelinedefinition.hpp +++ b/src/pipelinedefinition.hpp @@ -31,7 +31,7 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop #include "aliases.hpp" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "modelversion.hpp" #include "nodeinfo.hpp" #include "pipelinedefinitionstatus.hpp" diff --git a/src/pipelinedefinitionstatus.cpp b/src/pipelinedefinitionstatus.cpp new file mode 100644 index 0000000000..26f22738e9 --- /dev/null +++ b/src/pipelinedefinitionstatus.cpp @@ -0,0 +1,240 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pipelinedefinitionstatus.hpp" + +#include +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "modelversionstatus.hpp" + +namespace ovms { + +const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code) { + static const std::unordered_map names{ + {PipelineDefinitionStateCode::BEGIN, "BEGIN"}, + {PipelineDefinitionStateCode::RELOADING, "RELOADING"}, + {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED, "LOADING_PRECONDITION_FAILED"}, + {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, + {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, + {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, + {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; + return names.at(code); +} + +constexpr const char* INVALID_TRANSITION_MESSAGE = "Tried to conduct invalid transition."; + +PipelineDefinitionStateCode BeginState::getStateCode() const { + return code; +} +StateChanger BeginState::handle(const ReloadEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateChanger BeginState::handle(const ValidationPassedEvent& e) const { + return {}; +} +StateChanger BeginState::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper BeginState::handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper BeginState::handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} + +PipelineDefinitionStateCode ReloadState::getStateCode() const { + return code; +} +StateKeeper ReloadState::handle(const ReloadEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateChanger ReloadState::handle(const ValidationPassedEvent& e) const { + return {}; +} +StateChanger ReloadState::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper ReloadState::handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper ReloadState::handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} + +PipelineDefinitionStateCode AvailableState::getStateCode() const { + return code; +} +StateChanger AvailableState::handle(const ReloadEvent& e) const { + return {}; +} +StateKeeper AvailableState::handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper AvailableState::handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateChanger AvailableState::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateChanger AvailableState::handle(const RetireEvent& e) const { + return {}; +} + +PipelineDefinitionStateCode AvailableRequiredRevalidation::getStateCode() const { + return code; +} +StateChanger AvailableRequiredRevalidation::handle(const ReloadEvent& e) const { + return {}; +} +StateChanger AvailableRequiredRevalidation::handle(const ValidationPassedEvent& e) const { + return {}; +} +StateChanger AvailableRequiredRevalidation::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper AvailableRequiredRevalidation::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateChanger AvailableRequiredRevalidation::handle(const RetireEvent& e) const { + return {}; +} + +PipelineDefinitionStateCode LoadingPreconditionFailedState::getStateCode() const { + return code; +} +StateChanger LoadingPreconditionFailedState::handle(const ReloadEvent& e) const { + return {}; +} +StateKeeper LoadingPreconditionFailedState::handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper LoadingPreconditionFailedState::handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateChanger LoadingPreconditionFailedState::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateChanger LoadingPreconditionFailedState::handle(const RetireEvent& e) const { + return {}; +} + +PipelineDefinitionStateCode LoadingFailedLastValidationRequiredRevalidation::getStateCode() const { + return code; +} +StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const ReloadEvent& e) const { + return {}; +} +StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const ValidationPassedEvent& e) const { + return {}; +} +StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const RetireEvent& e) const { + return {}; +} + +PipelineDefinitionStateCode RetiredState::getStateCode() const { + return code; +} +StateChanger RetiredState::handle(const ReloadEvent& e) const { + return {}; +} +StateChanger RetiredState::handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateChanger RetiredState::handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper RetiredState::handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} +StateKeeper RetiredState::handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} + +PipelineDefinitionStatus::PipelineDefinitionStatus(const std::string& name) : + MachineState(name) {} +bool PipelineDefinitionStatus::isAvailable() const { + auto state = getStateCode(); + return (state == PipelineDefinitionStateCode::AVAILABLE) || + (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); +} +bool PipelineDefinitionStatus::canEndLoaded() const { + auto state = getStateCode(); + return isAvailable() || + (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || + (state == PipelineDefinitionStateCode::BEGIN) || + (state == PipelineDefinitionStateCode::RELOADING); +} +bool PipelineDefinitionStatus::isRevalidationRequired() const { + auto state = getStateCode(); + return (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || + (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); +} + +std::tuple PipelineDefinitionStatus::convertToModelStatus() const { + switch (getStateCode()) { + case PipelineDefinitionStateCode::BEGIN: + case PipelineDefinitionStateCode::RELOADING: + case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: + return { + ModelVersionState::LOADING, + ModelVersionStatusErrorCode::OK}; + + case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: + return { + ModelVersionState::LOADING, + ModelVersionStatusErrorCode::FAILED_PRECONDITION}; + + case PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION: + case PipelineDefinitionStateCode::AVAILABLE: + return { + ModelVersionState::AVAILABLE, + ModelVersionStatusErrorCode::OK}; + + case PipelineDefinitionStateCode::RETIRED: + return { + ModelVersionState::END, + ModelVersionStatusErrorCode::OK}; + + default: + return {}; + } +} +} // namespace ovms diff --git a/src/pipelinedefinitionstatus.hpp b/src/pipelinedefinitionstatus.hpp index 036e0581df..40bc7d1345 100644 --- a/src/pipelinedefinitionstatus.hpp +++ b/src/pipelinedefinitionstatus.hpp @@ -37,17 +37,7 @@ enum class PipelineDefinitionStateCode { RETIRED }; -inline const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code) { - static const std::unordered_map names{ - {PipelineDefinitionStateCode::BEGIN, "BEGIN"}, - {PipelineDefinitionStateCode::RELOADING, "RELOADING"}, - {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED, "LOADING_PRECONDITION_FAILED"}, - {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, - {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, - {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, - {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; - return names.at(code); -} +const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code); template class MachineState { @@ -72,9 +62,6 @@ class MachineState { void changeStateTo() { currentState = &std::get(allPossibleStates); } - void printState() const { - std::visit([](const auto state) { state->print(); }, currentState); - } PipelineDefinitionStateCode getStateCode() const { while (true) { try { @@ -157,248 +144,82 @@ struct StateKeeper { } }; -constexpr const char* INVALID_TRANSITION_MESSAGE = "Tried to conduct invalid transition."; - struct BeginState { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::BEGIN; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateChanger handle(const ValidationPassedEvent& e) const; + StateChanger handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateKeeper handle(const RetireEvent& e) const; }; struct ReloadState { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RELOADING; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateKeeper handle(const ReloadEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateKeeper handle(const ReloadEvent& e) const; + StateChanger handle(const ValidationPassedEvent& e) const; + StateChanger handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateKeeper handle(const RetireEvent& e) const; }; struct AvailableState { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateKeeper handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateKeeper handle(const ValidationPassedEvent& e) const; + StateKeeper handle(const ValidationFailedEvent& e) const; + StateChanger handle(const UsedModelChangedEvent& e) const; + StateChanger handle(const RetireEvent& e) const; }; struct AvailableRequiredRevalidation { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateChanger handle(const ValidationPassedEvent& e) const; + StateChanger handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateChanger handle(const RetireEvent& e) const; }; struct LoadingPreconditionFailedState { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateKeeper handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateKeeper handle(const ValidationPassedEvent& e) const; + StateKeeper handle(const ValidationFailedEvent& e) const; + StateChanger handle(const UsedModelChangedEvent& e) const; + StateChanger handle(const RetireEvent& e) const; }; struct LoadingFailedLastValidationRequiredRevalidation { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateChanger handle(const ValidationPassedEvent& e) const; + StateChanger handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateChanger handle(const RetireEvent& e) const; }; struct RetiredState { static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RETIRED; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } + PipelineDefinitionStateCode getStateCode() const; + StateChanger handle(const ReloadEvent& e) const; + StateChanger handle(const ValidationPassedEvent& e) const; + StateChanger handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateKeeper handle(const RetireEvent& e) const; }; class PipelineDefinitionStatus : public MachineState { public: - PipelineDefinitionStatus(const std::string& name) : - MachineState(name) {} - bool isAvailable() const { - auto state = getStateCode(); - return (state == PipelineDefinitionStateCode::AVAILABLE) || - (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); - } - bool canEndLoaded() const { - auto state = getStateCode(); - return isAvailable() || - (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || - (state == PipelineDefinitionStateCode::BEGIN) || - (state == PipelineDefinitionStateCode::RELOADING); - } - bool isRevalidationRequired() const { - auto state = getStateCode(); - return (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || - (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); - } - - std::tuple convertToModelStatus() const { - switch (getStateCode()) { - case PipelineDefinitionStateCode::BEGIN: - case PipelineDefinitionStateCode::RELOADING: - case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: - return { - ModelVersionState::LOADING, - ModelVersionStatusErrorCode::OK}; - - case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: - return { - ModelVersionState::LOADING, - ModelVersionStatusErrorCode::FAILED_PRECONDITION}; - - case PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION: - case PipelineDefinitionStateCode::AVAILABLE: - return { - ModelVersionState::AVAILABLE, - ModelVersionStatusErrorCode::OK}; - - case PipelineDefinitionStateCode::RETIRED: - return { - ModelVersionState::END, - ModelVersionStatusErrorCode::OK}; - - default: - return {}; - } - } + PipelineDefinitionStatus(const std::string& name); + bool isAvailable() const; + bool canEndLoaded() const; + bool isRevalidationRequired() const; + std::tuple convertToModelStatus() const; }; - } // namespace ovms diff --git a/src/precision.cpp b/src/precision.cpp index d19c739c59..ea68b9b358 100644 --- a/src/precision.cpp +++ b/src/precision.cpp @@ -85,103 +85,6 @@ Precision fromString(const std::string& s) { return it->second; } -Precision KFSPrecisionToOvmsPrecision(const KFSDataType& datatype) { - static std::unordered_map precisionMap{ - {"BOOL", Precision::BOOL}, - {"FP64", Precision::FP64}, - {"FP32", Precision::FP32}, - {"FP16", Precision::FP16}, - {"INT64", Precision::I64}, - {"INT32", Precision::I32}, - {"INT16", Precision::I16}, - {"INT8", Precision::I8}, - {"UINT64", Precision::U64}, - {"UINT32", Precision::U32}, - {"UINT16", Precision::U16}, - // {"BYTES", Precision::??}, - {"UINT8", Precision::U8}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { - static std::unordered_map precisionMap{ - {TFSDataType::DT_FLOAT, Precision::FP32}, - {TFSDataType::DT_DOUBLE, Precision::FP64}, - {TFSDataType::DT_HALF, Precision::FP16}, - {TFSDataType::DT_INT64, Precision::I64}, - {TFSDataType::DT_INT32, Precision::I32}, - {TFSDataType::DT_INT16, Precision::I16}, - {TFSDataType::DT_INT8, Precision::I8}, - {TFSDataType::DT_UINT64, Precision::U64}, - {TFSDataType::DT_UINT16, Precision::U16}, - {TFSDataType::DT_UINT8, Precision::U8}, - {TFSDataType::DT_BOOL, Precision::BOOL}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -size_t KFSDataTypeSize(const KFSDataType& datatype) { - static std::unordered_map datatypeSizeMap{ - {"BOOL", 1}, - {"UINT8", 1}, - {"UINT16", 2}, - {"UINT32", 4}, - {"UINT64", 8}, - {"INT8", 1}, - {"INT16", 2}, - {"INT32", 4}, - {"INT64", 8}, - {"FP16", 2}, - {"FP32", 4}, - {"FP64", 8} - // {"BYTES", }, - }; - auto it = datatypeSizeMap.find(datatype); - if (it == datatypeSizeMap.end()) { - return 0; - } - return it->second; -} - -const KFSDataType& ovmsPrecisionToKFSPrecision(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP64, "FP64"}, - {Precision::FP32, "FP32"}, - {Precision::FP16, "FP16"}, - {Precision::I64, "INT64"}, - {Precision::I32, "INT32"}, - {Precision::I16, "INT16"}, - {Precision::I8, "INT8"}, - {Precision::U64, "UINT64"}, - {Precision::U32, "UINT32"}, - {Precision::U16, "UINT16"}, - {Precision::U8, "UINT8"}, - {Precision::BOOL, "BOOL"}}; - // {Precision::BF16, ""}, - // {Precision::U4, ""}, - // {Precision::U1, ""}, - // {Precision::CUSTOM, ""}, - // {Precision::DYNAMIC, ""}, - // {Precision::MIXED, ""}, - // {Precision::Q78, ""}, - // {Precision::BIN, ""}, - // {Precision::I4, ""}, - // {Precision::UNDEFINED, "UNDEFINED"}}; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - static const std::string invalid{"INVALID"}; - return invalid; - } - return it->second; -} - ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision) { static std::unordered_map precisionMap{ {Precision::FP64, ov::element::Type_t::f64}, diff --git a/src/precision.hpp b/src/precision.hpp index 9137f948eb..3b6c834caf 100644 --- a/src/precision.hpp +++ b/src/precision.hpp @@ -19,15 +19,8 @@ #include -using KFSDataType = std::string; -namespace tensorflow { -enum DataType : int; -} - namespace ovms { -using TFSDataType = tensorflow::DataType; - enum class Precision { BF16, FP64, @@ -57,14 +50,8 @@ const std::string& toString(Precision precision); Precision fromString(const std::string& s); -Precision KFSPrecisionToOvmsPrecision(const KFSDataType& s); -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& s); - -size_t KFSDataTypeSize(const KFSDataType& datatype); - -const KFSDataType& ovmsPrecisionToKFSPrecision(Precision precision); +Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type); ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision); -Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type); } // namespace ovms diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index ba897107df..3377afdeb2 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -23,7 +23,8 @@ #include -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_utils.hpp" #include "modelconfig.hpp" #include "profiler.hpp" #include "status.hpp" @@ -41,17 +42,13 @@ struct RequestShapeInfo { const RequestTensorShapeType& getShape(); }; -using KFSRequestType = ::inference::ModelInferRequest; -using KFSInputTensorType = ::inference::ModelInferRequest_InferInputTensor; -using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; -using KFSShapeType = google::protobuf::RepeatedField; using TFSRequestType = tensorflow::serving::PredictRequest; using TFSInputTensorType = tensorflow::TensorProto; using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; using TFSShapeType = tensorflow::TensorShapeProto; template <> -dimension_value_t RequestShapeInfo::getDim(size_t i) { +dimension_value_t RequestShapeInfo::getDim(size_t i) { return tensor.shape()[i]; } template <> @@ -59,7 +56,7 @@ dimension_value_t RequestShapeInfo::getDim(siz return tensor.tensor_shape().dim(i).size(); } template <> -size_t RequestShapeInfo::getShapeSize() { +size_t RequestShapeInfo::getShapeSize() { return tensor.shape().size(); } template <> @@ -71,7 +68,7 @@ const TFSShapeType& RequestShapeInfo::getShape return tensor.tensor_shape(); } template <> -const KFSShapeType& RequestShapeInfo::getShape() { +const KFSShapeType& RequestShapeInfo::getShape() { return tensor.shape(); } @@ -126,7 +123,7 @@ Status RequestValidator -Status RequestValidator::validateRequestCoherency() const { +Status RequestValidator::validateRequestCoherency() const { if (!request.raw_input_contents().empty()) { for (auto& input : request.inputs()) { if (input.has_contents()) { @@ -142,7 +139,7 @@ Status RequestValidator -Status RequestValidator::validateNumberOfInputs() const { +Status RequestValidator::validateNumberOfInputs() const { size_t expectedNumberOfInputs = inputsInfo.size(); if (optionalAllowedInputNames.size() > 0) { @@ -165,7 +162,7 @@ Status RequestValidator -const std::string& RequestValidator::getCurrentlyValidatedInputName() const { +const std::string& RequestValidator::getCurrentlyValidatedInputName() const { return it->name(); } @@ -174,7 +171,7 @@ const std::string& RequestValidatorfirst; } template <> -const KFSInputTensorType& RequestValidator::getInputFromIt(const KFSInputTensorIteratorType& it) const { +const KFSTensorInputProto& RequestValidator::getInputFromIt(const KFSInputTensorIteratorType& it) const { return *it; } @@ -214,7 +211,7 @@ Status RequestValidator -Status RequestValidator::validateAndGetInput(const KFSRequestType& request, const std::string& name, KFSInputTensorIteratorType& it, size_t& bufferId) { +Status RequestValidator::validateAndGetInput(const KFSRequest& request, const std::string& name, KFSInputTensorIteratorType& it, size_t& bufferId) { it = request.inputs().begin(); bufferId = 0; while (it != request.inputs().end()) { @@ -264,8 +261,8 @@ Status RequestValidator -Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const KFSInputTensorType& proto) const { - RequestShapeInfo rsi(proto); +Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const KFSTensorInputProto& proto) const { + RequestShapeInfo rsi(proto); if (rsi.getShapeSize() != 1) { std::stringstream ss; ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); @@ -321,8 +318,8 @@ Status RequestValidator -Status RequestValidator::checkBinaryBatchSizeMismatch(const KFSInputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { - RequestShapeInfo rsi(proto); +Status RequestValidator::checkBinaryBatchSizeMismatch(const KFSTensorInputProto& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + RequestShapeInfo rsi(proto); if (proto.contents().bytes_contents_size() <= 0) { std::stringstream ss; ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); @@ -446,7 +443,7 @@ Status RequestValidator -Status RequestValidator::validateTensorContentSize(const KFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { +Status RequestValidator::validateTensorContentSize(const KFSTensorInputProto& proto, ovms::Precision expectedPrecision, size_t bufferId) const { size_t expectedValueCount = 1; for (int i = 0; i < proto.shape().size(); i++) { expectedValueCount *= proto.shape()[i]; @@ -540,7 +537,7 @@ Status RequestValidator -Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { +Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const KFSTensorInputProto& proto) const { // Network and request must have the same number of shape dimensions, higher than 0 const auto& shape = inputInfo.getShape(); if (proto.shape().size() <= 0 || @@ -571,8 +568,8 @@ Status RequestValidator -Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { - if (proto.datatype() != inputInfo.getPrecisionAsKFSPrecision()) { +Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const KFSTensorInputProto& proto) const { + if (proto.datatype() != ovmsPrecisionToKFSPrecision(inputInfo.getPrecision())) { std::stringstream ss; ss << "Expected: " << inputInfo.getPrecisionAsString() << "; Actual: " << proto.datatype() @@ -608,7 +605,7 @@ bool RequestValidator -bool RequestValidator::checkIfBinaryInputUsed(const KFSInputTensorType& proto, const std::string inputName) const { +bool RequestValidator::checkIfBinaryInputUsed(const KFSTensorInputProto& proto, const std::string inputName) const { if (proto.datatype() == "BYTES") { SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.contents().bytes_contents_size()); return true; @@ -689,9 +686,9 @@ Status validate(const TFSRequestType& request, const tensor_map_t& inputsInfo, c } template <> -Status validate(const KFSRequestType& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { +Status validate(const KFSRequest& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { OVMS_PROFILE_FUNCTION(); - return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); + return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); } } // namespace request_validation_utils } // namespace ovms diff --git a/src/prediction_service.cpp b/src/prediction_service.cpp index c4cd618d4b..0000925945 100644 --- a/src/prediction_service.cpp +++ b/src/prediction_service.cpp @@ -29,6 +29,7 @@ #include "execution_context.hpp" #include "get_model_metadata_impl.hpp" +#include "grpc_utils.hpp" #include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" @@ -110,7 +111,7 @@ grpc::Status ovms::PredictionServiceImpl::Predict( INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().requestFailGrpcPredict); } SPDLOG_INFO("Getting modelInstance or pipeline failed. {}", status.string()); - return status.grpc(); + return grpc(status); } ExecutionContext executionContext{ @@ -126,7 +127,7 @@ grpc::Status ovms::PredictionServiceImpl::Predict( } if (!status.ok()) { - return status.grpc(); + return grpc(status); } timer.stop(TOTAL); @@ -145,7 +146,7 @@ grpc::Status PredictionServiceImpl::GetModelMetadata( const tensorflow::serving::GetModelMetadataRequest* request, tensorflow::serving::GetModelMetadataResponse* response) { OVMS_PROFILE_FUNCTION(); - return getModelMetadataImpl.getModelStatus(request, response, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelMetadata)).grpc(); + return grpc(getModelMetadataImpl.getModelStatus(request, response, ExecutionContext(ExecutionContext::Interface::GRPC, ExecutionContext::Method::GetModelMetadata))); } const GetModelMetadataImpl& PredictionServiceImpl::getTFSModelMetadataImpl() const { diff --git a/src/prediction_service_utils.cpp b/src/prediction_service_utils.cpp index 92cc77dd98..edc653c2e2 100644 --- a/src/prediction_service_utils.cpp +++ b/src/prediction_service_utils.cpp @@ -31,7 +31,7 @@ using tensorflow::serving::PredictResponse; namespace ovms { -std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex) { +std::optional getRequestBatchSize(const ::KFSRequest* request, const size_t batchSizeIndex) { auto requestInputItr = request->inputs().begin(); if (requestInputItr == request->inputs().end()) { SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); @@ -49,7 +49,7 @@ std::optional getRequestBatchSize(const ::inference::ModelInferReques return Dimension(requestInput->shape()[batchSizeIndex]); } -std::map getRequestShapes(const ::inference::ModelInferRequest* request) { +std::map getRequestShapes(const ::KFSRequest* request) { std::map requestShapes; for (auto& it : request->inputs()) { shape_t requestShape; diff --git a/src/prediction_service_utils.hpp b/src/prediction_service_utils.hpp index 03d598f5da..6e5cfe24b0 100644 --- a/src/prediction_service_utils.hpp +++ b/src/prediction_service_utils.hpp @@ -23,14 +23,14 @@ #pragma GCC diagnostic ignored "-Wall" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "shape.hpp" namespace ovms { -std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex); +std::optional getRequestBatchSize(const ::KFSRequest* request, const size_t batchSizeIndex); -std::map getRequestShapes(const ::inference::ModelInferRequest* request); +std::map getRequestShapes(const ::KFSRequest* request); std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); diff --git a/src/rest_parser.cpp b/src/rest_parser.cpp index f3d97bccd8..6e83b22ec4 100644 --- a/src/rest_parser.cpp +++ b/src/rest_parser.cpp @@ -502,10 +502,10 @@ Status KFSRestParser::parseId(rapidjson::Value& node) { Status KFSRestParser::parseRequestParameters(rapidjson::Value& node){ PARSE_PARAMETER(requestProto)} -Status KFSRestParser::parseInputParameters(rapidjson::Value& node, ::inference::ModelInferRequest::InferInputTensor& input){ +Status KFSRestParser::parseInputParameters(rapidjson::Value& node, ::KFSRequest::InferInputTensor& input){ PARSE_PARAMETER(input)} -Status KFSRestParser::parseOutputParameters(rapidjson::Value& node, ::inference::ModelInferRequest::InferRequestedOutputTensor& output){ +Status KFSRestParser::parseOutputParameters(rapidjson::Value& node, ::KFSRequest::InferRequestedOutputTensor& output){ PARSE_PARAMETER(output)} Status KFSRestParser::parseOutput(rapidjson::Value& node) { @@ -560,7 +560,7 @@ Status KFSRestParser::parseOutputs(rapidjson::Value& node) { input->mutable_contents()->CONTENTS()->Add(value.TYPE_GETTER()); \ } -Status KFSRestParser::parseData(rapidjson::Value& node, ::inference::ModelInferRequest::InferInputTensor* input) { +Status KFSRestParser::parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor* input) { if (input->datatype() == "FP32") { HANDLE_VALUE(mutable_fp32_contents, GetFloat, IsNumber) } else if (input->datatype() == "INT64") { diff --git a/src/rest_parser.hpp b/src/rest_parser.hpp index 1ef421b2ca..e575876b70 100644 --- a/src/rest_parser.hpp +++ b/src/rest_parser.hpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wall" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #pragma GCC diagnostic pop #include "tensorinfo.hpp" @@ -220,20 +220,20 @@ class TFSRestParser : RestParser { }; class KFSRestParser : RestParser { - ::inference::ModelInferRequest requestProto; + ::KFSRequest requestProto; Status parseId(rapidjson::Value& node); Status parseRequestParameters(rapidjson::Value& node); - Status parseInputParameters(rapidjson::Value& node, ::inference::ModelInferRequest::InferInputTensor& input); - Status parseOutputParameters(rapidjson::Value& node, ::inference::ModelInferRequest::InferRequestedOutputTensor& input); + Status parseInputParameters(rapidjson::Value& node, ::KFSRequest::InferInputTensor& input); + Status parseOutputParameters(rapidjson::Value& node, ::KFSRequest::InferRequestedOutputTensor& input); Status parseOutput(rapidjson::Value& node); Status parseOutputs(rapidjson::Value& node); - Status parseData(rapidjson::Value& node, ::inference::ModelInferRequest::InferInputTensor* input); + Status parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor* input); Status parseInput(rapidjson::Value& node); Status parseInputs(rapidjson::Value& node); public: Status parse(const char* json); - ::inference::ModelInferRequest& getProto() { return requestProto; } + ::KFSRequest& getProto() { return requestProto; } }; } // namespace ovms diff --git a/src/rest_utils.cpp b/src/rest_utils.cpp index 14c726f959..59750f1278 100644 --- a/src/rest_utils.cpp +++ b/src/rest_utils.cpp @@ -27,6 +27,7 @@ #pragma GCC diagnostic ignored "-Wall" #include "tensorflow_serving/util/json_tensor.h" #pragma GCC diagnostic pop +#include "kfs_frontend/kfs_utils.hpp" #include "precision.hpp" #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" #include "status.hpp" @@ -202,7 +203,7 @@ Status makeJsonFromPredictResponse( return StatusCode::OK; } -static Status parseResponseParameters(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { +static Status parseResponseParameters(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer) { if (response_proto.parameters_size() > 0) { writer.Key("parameters"); writer.StartObject(); @@ -257,24 +258,24 @@ static Status parseOutputParameters(const inference::ModelInferResponse_InferOut } template -static void fillTensorDataWithIntValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithIntValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } template -static void fillTensorDataWithUintValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithUintValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } template -static void fillTensorDataWithFloatValuesFromRawContents(const ::inference::ModelInferResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { +static void fillTensorDataWithFloatValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) writer.Double(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } -static Status parseOutputs(const ::inference::ModelInferResponse& response_proto, rapidjson::PrettyWriter& writer) { +static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer) { writer.Key("outputs"); writer.StartArray(); @@ -437,7 +438,7 @@ static Status parseOutputs(const ::inference::ModelInferResponse& response_proto } Status makeJsonFromPredictResponse( - const ::inference::ModelInferResponse& response_proto, + const ::KFSResponse& response_proto, std::string* response_json) { Timer timer; using std::chrono::microseconds; diff --git a/src/rest_utils.hpp b/src/rest_utils.hpp index c4cedd3a19..91d115ec91 100644 --- a/src/rest_utils.hpp +++ b/src/rest_utils.hpp @@ -24,10 +24,6 @@ #include "rest_parser.hpp" -namespace inference { -class ModelInferResponse; -} // namespace inference - namespace ovms { class Status; Status makeJsonFromPredictResponse( @@ -36,7 +32,7 @@ Status makeJsonFromPredictResponse( Order order); Status makeJsonFromPredictResponse( - const ::inference::ModelInferResponse& response_proto, + const ::KFSResponse& response_proto, std::string* response_json); Status decodeBase64(std::string& bytes, std::string& decodedBytes); diff --git a/src/serialization.cpp b/src/serialization.cpp index f0a3e6636b..b7171b71b8 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include "serialization.hpp" +#include "kfs_frontend/kfs_utils.hpp" #include "ov_utils.hpp" #include "tfs_frontend/tfs_utils.hpp" @@ -60,7 +61,7 @@ static Status serializePrecision( } static Status serializePrecision( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + ::KFSResponse::InferOutputTensor& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { OVMS_PROFILE_FUNCTION(); @@ -84,7 +85,7 @@ static Status serializePrecision( case ovms::Precision::U16: case ovms::Precision::U8: case ovms::Precision::BOOL: - responseOutput.set_datatype(servableOutput->getPrecisionAsKFSPrecision()); + responseOutput.set_datatype(ovmsPrecisionToKFSPrecision(servableOutput->getPrecision())); break; case ovms::Precision::UNDEFINED: case ovms::Precision::MIXED: @@ -125,7 +126,7 @@ static Status serializeShape( } static Status serializeShape( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + ::KFSResponse::InferOutputTensor& responseOutput, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { OVMS_PROFILE_FUNCTION(); @@ -176,7 +177,7 @@ Status serializeTensorToTensorProto( } Status serializeTensorToTensorProto( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + ::KFSResponse::InferOutputTensor& responseOutput, std::string* rawOutputContents, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -219,7 +220,7 @@ std::string* ProtoGetter -::inference::ModelInferResponse::InferOutputTensor& ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createOutput(const std::string& name) { +::KFSResponse::InferOutputTensor& ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&>::createOutput(const std::string& name) { OVMS_PROFILE_FUNCTION(); for (int i = 0; i < protoStorage->outputs_size(); i++) { auto& tensor = *protoStorage->mutable_outputs(i); @@ -233,7 +234,7 @@ ::inference::ModelInferResponse::InferOutputTensor& ProtoGetter<::inference::Mod } template <> -std::string* ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createContent(const std::string& name) { +std::string* ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&>::createContent(const std::string& name) { OVMS_PROFILE_FUNCTION(); for (int i = 0; i < protoStorage->outputs_size(); i++) { auto& tensor = *protoStorage->mutable_outputs(i); diff --git a/src/serialization.hpp b/src/serialization.hpp index 3f214e7f21..ea4672b2a3 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -27,7 +27,7 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "profiler.hpp" #include "status.hpp" #include "tensorinfo.hpp" @@ -62,7 +62,7 @@ Status serializeTensorToTensorProto( ov::Tensor& tensor); Status serializeTensorToTensorProto( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + ::KFSResponse::InferOutputTensor& responseOutput, std::string* rawOutputContents, const std::shared_ptr& servableOutput, ov::Tensor& tensor); @@ -105,11 +105,11 @@ template Status serializePredictResponse( OutputGetter& outputGetter, const tensor_map_t& outputMap, - ::inference::ModelInferResponse* response, + ::KFSResponse* response, outputNameChooser_t outputNameChooser) { OVMS_PROFILE_FUNCTION(); Status status; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(response); + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(response); for (const auto& [outputName, outputInfo] : outputMap) { ov::Tensor tensor; status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); diff --git a/src/server.cpp b/src/server.cpp index 6b85862ddc..df763fb470 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -38,7 +38,7 @@ #include "grpcservermodule.hpp" #include "http_server.hpp" #include "httpservermodule.hpp" -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" #include "metric_module.hpp" #include "model_service.hpp" diff --git a/src/status.cpp b/src/status.cpp index f9d7cf73ed..d756da8e0b 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -260,181 +260,4 @@ const std::unordered_map Status::statusMess {StatusCode::INVALID_METRICS_FAMILY_NAME, "Invalid name in metrics_list"}, {StatusCode::METRICS_REST_PORT_MISSING, "Missing rest_port parameter in server CLI"}, }; - -const std::unordered_map Status::grpcStatusMap = { - {StatusCode::OK, grpc::StatusCode::OK}, - - {StatusCode::PATH_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::FILE_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::NO_MODEL_VERSION_AVAILABLE, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_NOT_LOADED, grpc::StatusCode::INTERNAL}, - {StatusCode::JSON_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::MODELINSTANCE_NOT_FOUND, grpc::StatusCode::INTERNAL}, - {StatusCode::SHAPE_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, grpc::StatusCode::INTERNAL}, - {StatusCode::RESHAPE_ERROR, grpc::StatusCode::FAILED_PRECONDITION}, - {StatusCode::MODEL_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_SPEC_MISSING, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::MODEL_VERSION_INVALID_FORMAT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SIGNATURE_DEF, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, grpc::StatusCode::ABORTED}, - {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, grpc::StatusCode::FAILED_PRECONDITION}, - - // Sequence management - {StatusCode::SEQUENCE_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::SEQUENCE_ALREADY_EXISTS, grpc::StatusCode::ALREADY_EXISTS}, - {StatusCode::SEQUENCE_ID_NOT_PROVIDED, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_ID_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_TERMINATED, grpc::StatusCode::FAILED_PRECONDITION}, - {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, grpc::StatusCode::UNAVAILABLE}, - - // Predict request validation - {StatusCode::INVALID_NO_OF_INPUTS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_MISSING_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_BATCH_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_PRECISION, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_VALUE_COUNT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_CONTENT_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_MESSAGE_STRUCTURE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::UNSUPPORTED_LAYOUT, grpc::StatusCode::INVALID_ARGUMENT}, - - // Deserialization - - // Should never occur - ModelInstance::validate takes care of that - {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, - {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, - - // Inference - {StatusCode::OV_INTERNAL_INFERENCE_ERROR, grpc::StatusCode::INTERNAL}, - - // Serialization - - // Should never occur - it should be validated during model loading - {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, - {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, - - // GetModelStatus - {StatusCode::INTERNAL_ERROR, grpc::StatusCode::INTERNAL}, - - // Binary input - {StatusCode::INVALID_NO_OF_CHANNELS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::STRING_VAL_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::BYTES_CONTENTS_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, -}; - -const std::unordered_map Status::httpStatusMap = { - {StatusCode::OK, net_http::HTTPStatusCode::OK}, - {StatusCode::OK_RELOADED, net_http::HTTPStatusCode::CREATED}, - {StatusCode::OK_NOT_RELOADED, net_http::HTTPStatusCode::OK}, - - // REST handler failure - {StatusCode::REST_INVALID_URL, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_UNSUPPORTED_METHOD, net_http::HTTPStatusCode::NONE_ACC}, - {StatusCode::REST_NOT_FOUND, net_http::HTTPStatusCode::NOT_FOUND}, - - // REST parser failure - {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_PREDICT_UNKNOWN_ORDER, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INPUT_NOT_PREALLOCATED, net_http::HTTPStatusCode::ERROR}, - {StatusCode::REST_NO_INSTANCES_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INPUTS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_NO_INPUTS_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_OUTPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_PARAMETERS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_PROTO_TO_STRING_ERROR, net_http::HTTPStatusCode::ERROR}, - {StatusCode::REST_UNSUPPORTED_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, net_http::HTTPStatusCode::ERROR}, - - {StatusCode::PATH_INVALID, net_http::HTTPStatusCode::ERROR}, - {StatusCode::FILE_INVALID, net_http::HTTPStatusCode::ERROR}, - {StatusCode::NO_MODEL_VERSION_AVAILABLE, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_NOT_LOADED, net_http::HTTPStatusCode::ERROR}, - {StatusCode::JSON_INVALID, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::MODELINSTANCE_NOT_FOUND, net_http::HTTPStatusCode::ERROR}, - {StatusCode::SHAPE_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, net_http::HTTPStatusCode::ERROR}, - {StatusCode::RESHAPE_ERROR, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::MODEL_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_SPEC_MISSING, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SIGNATURE_DEF, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, net_http::HTTPStatusCode::NO_CONTENT}, - {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, net_http::HTTPStatusCode::PRECOND_FAILED}, - - // Sequence management - {StatusCode::SEQUENCE_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::SEQUENCE_ALREADY_EXISTS, net_http::HTTPStatusCode::CONFLICT}, - {StatusCode::SEQUENCE_ID_NOT_PROVIDED, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_ID_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_TERMINATED, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, net_http::HTTPStatusCode::SERVICE_UNAV}, - - // Predict request validation - {StatusCode::INVALID_NO_OF_INPUTS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_MISSING_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_BATCH_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_VALUE_COUNT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_CONTENT_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_MESSAGE_STRUCTURE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::UNSUPPORTED_LAYOUT, net_http::HTTPStatusCode::BAD_REQUEST}, - - // Deserialization - - // Should never occur - ModelInstance::validate takes care of that - {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, - {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Inference - {StatusCode::OV_INTERNAL_INFERENCE_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Serialization - - // Should never occur - it should be validated during model loading - {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, - {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, - - // GetModelStatus - {StatusCode::INTERNAL_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Binary input - {StatusCode::INVALID_NO_OF_CHANNELS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::STRING_VAL_EMPTY, net_http::HTTPStatusCode::BAD_REQUEST}, -}; - } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index c2c665ef29..3ab452b81f 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -21,17 +21,8 @@ #include #include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/util/net_http/server/public/response_code_enum.h" -#pragma GCC diagnostic pop - namespace ovms { -namespace net_http = tensorflow::serving::net_http; - enum class StatusCode { OK, /*!< Success */ @@ -288,8 +279,6 @@ class Status { std::unique_ptr message; static const std::unordered_map statusMessageMap; - static const std::unordered_map grpcStatusMap; - static const std::unordered_map httpStatusMap; void appendDetails(const std::string& details) { ensureMessageAllocated(); @@ -357,20 +346,6 @@ class Status { return this->code != status.code; } - const grpc::Status grpc() const { - auto it = grpcStatusMap.find(code); - if (it != grpcStatusMap.end()) { - return grpc::Status(it->second, - this->message ? *this->message : ""); - } else { - return grpc::Status(grpc::StatusCode::UNKNOWN, "Unknown error"); - } - } - - operator grpc::Status() const { - return this->grpc(); - } - const std::string& string() const { return this->message ? *this->message : statusMessageMap.at(code); } @@ -378,15 +353,6 @@ class Status { operator const std::string&() const { return this->string(); } - - const net_http::HTTPStatusCode http() const { - auto it = httpStatusMap.find(code); - if (it != httpStatusMap.end()) { - return it->second; - } else { - return net_http::HTTPStatusCode::ERROR; - } - } }; } // namespace ovms diff --git a/src/tensorinfo.cpp b/src/tensorinfo.cpp index fbd569cf5c..e5e4c118d6 100644 --- a/src/tensorinfo.cpp +++ b/src/tensorinfo.cpp @@ -21,7 +21,7 @@ #include #include -#include "kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" namespace ovms { @@ -133,14 +133,6 @@ const std::string& TensorInfo::getPrecisionAsString() const { return getPrecisionAsString(precision); } -const std::string& TensorInfo::getPrecisionAsKFSPrecision(Precision precision) { - return ovmsPrecisionToKFSPrecision(precision); -} - -const std::string& TensorInfo::getPrecisionAsKFSPrecision() const { - return getPrecisionAsKFSPrecision(precision); -} - const std::string& TensorInfo::getStringFromLayout(const Layout& layout) { return layout; } @@ -246,21 +238,6 @@ std::string TensorInfo::shapeToString(const shape_t& shape) { return oss.str(); } -std::string tensorShapeToString(const google::protobuf::RepeatedField& shape) { - std::ostringstream oss; - oss << "("; - int i = 0; - if (shape.size() > 0) { - for (; i < shape.size() - 1; i++) { - oss << shape[i] << ","; - } - oss << shape[i]; - } - oss << ")"; - - return oss.str(); -} - std::shared_ptr TensorInfo::getUnspecifiedTensorInfo() { return std::make_shared("", Precision::UNDEFINED, Shape{}); } diff --git a/src/tensorinfo.hpp b/src/tensorinfo.hpp index 8fd7b07aa7..9a2c77ae08 100644 --- a/src/tensorinfo.hpp +++ b/src/tensorinfo.hpp @@ -175,13 +175,6 @@ class TensorInfo { */ const std::string& getPrecisionAsString() const; - /** - * @brief Get the Precision As String object representing KFS precision - * - * @return const std::string - */ - const std::string& getPrecisionAsKFSPrecision() const; - /** * @brief Get the string representation of TensorInfo object * @@ -191,8 +184,6 @@ class TensorInfo { static const std::string& getPrecisionAsString(Precision precision); - static const std::string& getPrecisionAsKFSPrecision(Precision precision); - /** * @brief Get the layout name from Layout * @@ -233,6 +224,5 @@ class TensorInfo { const std::optional getBatchSize() const; }; -std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); } // namespace ovms diff --git a/src/test/binaryutils_test.cpp b/src/test/binaryutils_test.cpp index ad98817eb2..aaeaa4e13c 100644 --- a/src/test/binaryutils_test.cpp +++ b/src/test/binaryutils_test.cpp @@ -41,7 +41,7 @@ class BinaryUtilsTest : public ::testing::Test { tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); tensor.set_dtype(tensorflow::DataType::DT_STRING); } - void prepareBinaryTensor(::inference::ModelInferRequest::InferInputTensor& tensor, std::unique_ptr& image_bytes, const size_t filesize, const size_t batchSize = 1) { + void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor, std::unique_ptr& image_bytes, const size_t filesize, const size_t batchSize = 1) { for (size_t i = 0; i < batchSize; i++) { tensor.mutable_contents()->add_bytes_contents(image_bytes.get(), filesize); } @@ -56,7 +56,7 @@ class BinaryUtilsTest : public ::testing::Test { readRgbJpg(filesize, image_bytes); prepareBinaryTensor(tensor, image_bytes, filesize); } - void prepareBinaryTensor(::inference::ModelInferRequest::InferInputTensor& tensor) { + void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor) { size_t filesize; std::unique_ptr image_bytes; @@ -68,13 +68,13 @@ class BinaryUtilsTest : public ::testing::Test { tensor.set_dtype(tensorflow::DataType::DT_STRING); tensor.add_string_val(input); } - void prepareBinaryTensor(::inference::ModelInferRequest::InferInputTensor& tensor, std::string input) { + void prepareBinaryTensor(::KFSRequest::InferInputTensor& tensor, std::string input) { tensor.mutable_contents()->add_bytes_contents(input); tensor.set_datatype("BYTES"); } }; -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types; TYPED_TEST_SUITE(BinaryUtilsTest, MyTypes); TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingBatchsize) { @@ -549,7 +549,7 @@ class BinaryUtilsKFSPrecisionTest : public ::testing::TestWithParam image_bytes; - ::inference::ModelInferRequest::InferInputTensor inferTensorContent; + ::KFSRequest::InferInputTensor inferTensorContent; }; class BinaryUtilsKFSValidPrecisionTest : public BinaryUtilsKFSPrecisionTest {}; diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index 2f0d839469..26e99ac605 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -29,19 +29,17 @@ #pragma GCC diagnostic pop #include "../deserialization.hpp" +#include "../kfs_frontend/kfs_utils.hpp" #include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" #include using TFTensorProto = tensorflow::TensorProto; -using KFSTensorProto = ::inference::ModelInferRequest::InferInputTensor; using TFPredictRequest = tensorflow::serving::PredictRequest; using TFPredictResponse = tensorflow::serving::PredictResponse; -using KFSPredictRequest = ::inference::ModelInferRequest; - using namespace ovms; using testing::_; @@ -133,7 +131,7 @@ class MockTensorProtoDeserializatorThrowingInferenceEngine { MOCK_METHOD(ov::Tensor, deserializeTensorProto, - (const ::inference::ModelInferRequest::InferInputTensor&, + (const ::KFSRequest::InferInputTensor&, const std::shared_ptr&, const std::string* buffer)); }; @@ -149,7 +147,7 @@ class MockTensorProtoDeserializator { } static ov::Tensor deserializeTensorProto( - const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo, const std::string* buffer) { return mock->deserializeTensorProto(requestInput, tensorInfo, buffer); @@ -308,7 +306,7 @@ class KserveGRPCPredict : public ::testing::TestWithParam instance; }; diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp index ddf1788f46..e310002d32 100644 --- a/src/test/ensemble_tests.cpp +++ b/src/test/ensemble_tests.cpp @@ -25,6 +25,7 @@ #include "../dl_node.hpp" #include "../entry_node.hpp" #include "../exit_node.hpp" +#include "../kfs_frontend/kfs_utils.hpp" #include "../localfilesystem.hpp" #include "../logging.hpp" #include "../metric_registry.hpp" @@ -82,7 +83,7 @@ class EnsembleFlowBothApiTest : public TestWithTempDir { preparePredictRequest(request, inputs_info_t{{customPipelineInputName, {shape, ovms::Precision::FP32}}}, requestData); } - void prepareRequest(const std::vector& requestData, KFSRequestType& request, const std::string& customPipelineInputName, const std::vector& shape = {1, DUMMY_MODEL_INPUT_SIZE}) { + void prepareRequest(const std::vector& requestData, KFSRequest& request, const std::string& customPipelineInputName, const std::vector& shape = {1, DUMMY_MODEL_INPUT_SIZE}) { request.Clear(); prepareKFSInferInputTensor(request, customPipelineInputName, std::make_tuple(shape, ovmsPrecisionToKFSPrecision(ovms::Precision::FP32)), requestData); } diff --git a/src/test/gather_node_test.cpp b/src/test/gather_node_test.cpp index 8c760a9de2..abdbdef750 100644 --- a/src/test/gather_node_test.cpp +++ b/src/test/gather_node_test.cpp @@ -307,10 +307,10 @@ TEST_F(TFSGatherExitNodeInputHandlerTest, TensorAlreadyExistsInProto) { class KFSGatherExitNodeInputHandlerTest : public GatherExitNodeInputHandlerTest { protected: - ::inference::ModelInferResponse response; + ::KFSResponse response; - ::inference::ModelInferResponse_InferOutputTensor* getPreparedTensor() { - ::inference::ModelInferResponse_InferOutputTensor* ptr = nullptr; + KFSTensorOutputProto* getPreparedTensor() { + KFSTensorOutputProto* ptr = nullptr; for (int i = 0; i < response.outputs_size(); i++) { auto* output = response.mutable_outputs(i); if (output->name() == tensorName) { diff --git a/src/test/kfs_metadata_test.cpp b/src/test/kfs_metadata_test.cpp index aae1a0a2fb..f6c5f07998 100644 --- a/src/test/kfs_metadata_test.cpp +++ b/src/test/kfs_metadata_test.cpp @@ -15,7 +15,7 @@ //***************************************************************************** #include -#include "../kfs_grpc_inference_service.hpp" +#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../modelversionstatus.hpp" #include "../pipelinedefinition.hpp" #include "mockmodelinstancechangingstates.hpp" @@ -128,7 +128,7 @@ class ModelMetadataResponseBuild : public ::testing::Test { std::shared_ptr> instance; std::unique_ptr model; - ::inference::ModelMetadataResponse response; + KFSModelMetadataResponse response; std::unique_ptr ieCore; void SetUp() override { @@ -274,7 +274,7 @@ class PipelineMetadataResponseBuild : public ::testing::Test { }; MockPipelineDefinitionGetInputsOutputsInfo pipelineDefinition; - ::inference::ModelMetadataResponse response; + KFSModelMetadataResponse response; ConstructorEnabledModelManager manager; public: diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index cd8c8c1914..e3c0dd711a 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -312,7 +312,7 @@ TEST_F(HttpRestApiHandlerTest, inferRequest) { TEST_F(HttpRestApiHandlerTest, inferPreprocess) { std::string request_body("{\"inputs\":[{\"name\":\"b\",\"shape\":[1,10],\"datatype\":\"FP32\",\"data\":[0,1,2,3,4,5,6,7,8,9]}],\"parameters\":{\"binary_data_output\":1, \"bool_test\":true, \"string_test\":\"test\"}}"); - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request), ovms::StatusCode::OK); ASSERT_EQ(grpc_request.model_name(), modelName); @@ -337,7 +337,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT8) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT8\",\"parameters\":{\"binary_data_size\":4}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -363,7 +363,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsBYTES) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"BYTES\",\"parameters\":{\"binary_data_size\":4}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -385,7 +385,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT16) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT16\",\"parameters\":{\"binary_data_size\":8}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -411,7 +411,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT32) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":16}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -437,7 +437,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT64) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT64\",\"parameters\":{\"binary_data_size\":32}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -463,7 +463,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsFP32) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"FP32\",\"parameters\":{\"binary_data_size\":16}}]}"; request_body.append((char*)values, 16); - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - 16); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -489,7 +489,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsFP64) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"FP64\",\"parameters\":{\"binary_data_size\":32}}]}"; request_body.append((char*)values, 32); - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - 32); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -515,7 +515,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsBinaryDataAndContentField) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"data\":[0,1,2,3,4,5,6,7,8,9], \"datatype\":\"INT8\",\"parameters\":{\"binary_data_size\":4}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request), ovms::StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY); } @@ -524,7 +524,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsBufferSmallerThanExpected) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":16}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_BINARY_BUFFER_EXCEEDED); } @@ -534,7 +534,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsInvalidBinaryDataSizeParameter) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":true}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_BINARY_DATA_SIZE_PARAMETER_INVALID); } @@ -544,7 +544,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT8BatchSize2) { std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[2,4],\"datatype\":\"INT8\",\"parameters\":{\"binary_data_size\":\"4, 4\"}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); @@ -570,7 +570,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsBinaryDataSizeStringParameterInvalid) std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[2,4],\"datatype\":\"INT8\",\"parameters\":{\"binary_data_size\":\"a, 4\"}}]}"; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_BINARY_DATA_SIZE_PARAMETER_INVALID); } @@ -580,7 +580,7 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsEmptyRequest) { std::string request_body = ""; request_body += binaryData; - ::inference::ModelInferRequest grpc_request; + ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::JSON_INVALID); } diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 4823efea46..97e048b14c 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -24,7 +24,7 @@ #include "../config.hpp" #include "../get_model_metadata_impl.hpp" #include "../http_rest_api_handler.hpp" -#include "../kfs_grpc_inference_service.hpp" +#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../metric_config.hpp" #include "../metric_module.hpp" #include "../model_service.hpp" @@ -256,8 +256,8 @@ TEST_F(MetricFlowTest, GrpcGetModelStatus) { TEST_F(MetricFlowTest, GrpcModelInfer) { KFSInferenceServiceImpl impl(server); - ::inference::ModelInferRequest request; - ::inference::ModelInferResponse response; + ::KFSRequest request; + ::KFSResponse response; for (int i = 0; i < numberOfSuccessRequests; i++) { request.Clear(); @@ -321,8 +321,8 @@ TEST_F(MetricFlowTest, GrpcModelInfer) { TEST_F(MetricFlowTest, GrpcModelMetadata) { KFSInferenceServiceImpl impl(server); - ::inference::ModelMetadataRequest request; - ::inference::ModelMetadataResponse response; + ::KFSModelMetadataRequest request; + KFSModelMetadataResponse response; for (int i = 0; i < numberOfSuccessRequests; i++) { request.Clear(); @@ -344,8 +344,8 @@ TEST_F(MetricFlowTest, GrpcModelMetadata) { TEST_F(MetricFlowTest, GrpcModelReady) { KFSInferenceServiceImpl impl(server); - ::inference::ModelReadyRequest request; - ::inference::ModelReadyResponse response; + ::KFSGetModelStatusRequest request; + ::KFSGetModelStatusResponse response; for (int i = 0; i < numberOfSuccessRequests; i++) { request.Clear(); diff --git a/src/test/model_service_test.cpp b/src/test/model_service_test.cpp index 88425e3056..d22e17e415 100644 --- a/src/test/model_service_test.cpp +++ b/src/test/model_service_test.cpp @@ -40,8 +40,6 @@ using namespace ovms; using TFSGetModelStatusRequest = tensorflow::serving::GetModelStatusRequest; using TFSGetModelStatusResponse = tensorflow::serving::GetModelStatusResponse; using TFSGetModelStatusInterface = std::pair; -using KFSGetModelStatusRequest = ::inference::ModelReadyRequest; -using KFSGetModelStatusResponse = ::inference::ModelReadyResponse; using KFSGetModelStatusInterface = std::pair; template (ModelVersionState::LOADING, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_EQ((std::tuple(ModelVersionState::AVAILABLE, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); + pds.handle(UsedModelChangedEvent(modelNotifyingDetails)); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); + ASSERT_EQ((std::tuple(ModelVersionState::AVAILABLE, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); + ASSERT_EQ((std::tuple(ModelVersionState::LOADING, ModelVersionStatusErrorCode::FAILED_PRECONDITION)), pds.convertToModelStatus()); + pds.handle(UsedModelChangedEvent(modelNotifyingDetails)); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION); + ASSERT_EQ((std::tuple(ModelVersionState::LOADING, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); + pds.handle(RetireEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); + ASSERT_EQ((std::tuple(ModelVersionState::END, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); +} diff --git a/src/test/predict_validation_test.cpp b/src/test/predict_validation_test.cpp index 0fb390a23d..7889867dfe 100644 --- a/src/test/predict_validation_test.cpp +++ b/src/test/predict_validation_test.cpp @@ -19,7 +19,7 @@ #include #include -#include "../kfs_grpc_inference_service.hpp" +#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../predict_request_validation_utils.hpp" @@ -42,7 +42,7 @@ class MockModelInstance : public ovms::ModelInstance { const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { return validate(request); } - const ovms::Status mockValidate(const ::inference::ModelInferRequest* request) { + const ovms::Status mockValidate(const ::KFSRequest* request) { return validate(request); } }; @@ -679,7 +679,7 @@ class KFSPredictValidation : public ::testing::Test { protected: std::unique_ptr ieCore; std::unique_ptr> instance; - ::inference::ModelInferRequest request; + ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; @@ -776,7 +776,7 @@ TEST_F(KFSPredictValidation, RequestWrongBatchSizeAuto) { TEST_F(KFSPredictValidation, ValidRequestBinaryInputs) { modelConfig.setBatchingParams("auto"); std::string inputName = "Binary_Input"; - ::inference::ModelInferRequest binaryInputRequest; + ::KFSRequest binaryInputRequest; auto input = binaryInputRequest.add_inputs(); input->set_name(inputName); @@ -802,7 +802,7 @@ TEST_F(KFSPredictValidation, ValidRequestBinaryInputs) { TEST_F(KFSPredictValidation, RequestWrongBatchSizeBinaryInputs) { std::string inputName = "Binary_Input"; - ::inference::ModelInferRequest binaryInputRequest; + ::KFSRequest binaryInputRequest; auto input = binaryInputRequest.add_inputs(); input->set_name(inputName); @@ -829,7 +829,7 @@ TEST_F(KFSPredictValidation, RequestWrongBatchSizeBinaryInputs) { TEST_F(KFSPredictValidation, RequestWrongBatchSizeAutoBinaryInputs) { modelConfig.setBatchingParams("auto"); std::string inputName = "Binary_Input"; - ::inference::ModelInferRequest binaryInputRequest; + ::KFSRequest binaryInputRequest; auto input = binaryInputRequest.add_inputs(); input->set_name(inputName); @@ -1003,7 +1003,7 @@ class KFSPredictValidationInputTensorContent : public ::testing::TestWithParam ieCore; std::unique_ptr> instance; - ::inference::ModelInferRequest request; + ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; @@ -1059,7 +1059,7 @@ class KFSPredictValidationInputTensorContentNegative : public ::testing::Test { protected: std::unique_ptr ieCore; std::unique_ptr> instance; - ::inference::ModelInferRequest request; + ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; @@ -1258,7 +1258,7 @@ class KFSPredictValidationPrecision : public ::testing::TestWithParam(tensorName, precision, ovms::shape_t{1, DUMMY_MODEL_INPUT_SIZE}, ovms::Layout{"NC"}); } - ::inference::ModelInferRequest request; + ::KFSRequest request; const char* tensorName = DUMMY_MODEL_INPUT_NAME; ovms::tensor_map_t mockedInputsInfo; }; diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index b501e37fd5..f8e1454e4f 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -28,11 +28,13 @@ #include "../deserialization.hpp" #include "../executingstreamidguard.hpp" +#include "../kfs_frontend/kfs_utils.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" #include "../prediction_service_utils.hpp" #include "../sequence_processing_spec.hpp" #include "../serialization.hpp" +#include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" using testing::Each; @@ -53,7 +55,7 @@ void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest, const std EXPECT_THAT(output, Each(Eq(1.))); } -ovms::Status getOutput(const KFSResponseType& response, const std::string& name, KFSOutputTensorIteratorType& it, size_t& bufferId) { +ovms::Status getOutput(const KFSResponse& response, const std::string& name, KFSOutputTensorIteratorType& it, size_t& bufferId) { it = response.outputs().begin(); bufferId = 0; while (it != response.outputs().end()) { @@ -189,7 +191,7 @@ class TestPredict : public ::testing::Test { ASSERT_EQ(0, std::memcmp(actualValues.data(), expectedValues.data(), expectedValues.size() * sizeof(float))) << readableError(expectedValues.data(), actualValues.data(), expectedValues.size() * sizeof(float)); } - static void checkOutputValues(const KFSResponseType& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { + static void checkOutputValues(const KFSResponse& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { KFSOutputTensorIteratorType it; size_t bufferId; auto status = getOutput(response, outputName, it, bufferId); @@ -259,7 +261,7 @@ void TestPredict::checkOutputShape(const TFSResponseType& response } template <> -void TestPredict::checkOutputShape(const KFSResponseType& response, const ovms::shape_t& shape, const std::string& outputName) { +void TestPredict::checkOutputShape(const KFSResponse& response, const ovms::shape_t& shape, const std::string& outputName) { auto it = response.outputs().begin(); size_t bufferId; auto status = getOutput(response, outputName, it, bufferId); @@ -1131,7 +1133,7 @@ TYPED_TEST(TestPredict, PerformInferenceDummyBatchSizeAny) { * 2. Do the inferences with (3, 10) shape, expect correct output shapes and precision */ -ovms::Precision getPrecisionFromResponse(KFSResponseType& response, const std::string& name) { +ovms::Precision getPrecisionFromResponse(KFSResponse& response, const std::string& name) { KFSOutputTensorIteratorType it; size_t bufferId; auto status = getOutput(response, name, it, bufferId); diff --git a/src/test/rest_utils_test.cpp b/src/test/rest_utils_test.cpp index 3be7f1742b..ce3220362d 100644 --- a/src/test/rest_utils_test.cpp +++ b/src/test/rest_utils_test.cpp @@ -645,9 +645,9 @@ TEST_F(TFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Column class KFSMakeJsonFromPredictResponseRawTest : public ::testing::Test { protected: - KFSResponseType proto; + KFSResponse proto; std::string json; - KFSOutputTensorType *output1, *output2; + KFSTensorOutputProto *output1, *output2; void SetUp() override { proto.set_model_name("model"); @@ -721,9 +721,9 @@ TEST_F(KFSMakeJsonFromPredictResponseRawTest, ErrorWhenNoOutputs) { class KFSMakeJsonFromPredictResponsePrecisionTest : public ::testing::Test { protected: - KFSResponseType proto; + KFSResponse proto; std::string json; - KFSOutputTensorType* output; + KFSTensorOutputProto* output; void SetUp() override { proto.set_model_name("model"); @@ -900,9 +900,9 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { class KFSMakeJsonFromPredictResponseValTest : public ::testing::Test { protected: - KFSResponseType proto; + KFSResponse proto; std::string json; - KFSOutputTensorType *single_uint64_val, *two_uint32_vals; + KFSTensorOutputProto *single_uint64_val, *two_uint32_vals; void SetUp() override { proto.set_model_name("model"); diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index a6e81de129..c5233fdc9a 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -35,12 +35,9 @@ #include using TFTensorProto = tensorflow::TensorProto; -using KFSTensorProto = ::inference::ModelInferResponse::InferOutputTensor; using TFPredictRequest = tensorflow::serving::PredictRequest; using TFPredictResponse = tensorflow::serving::PredictResponse; -using KFSPredictRequest = ::inference::ModelInferRequest; -using KFSPredictResponse = ::inference::ModelInferResponse; using namespace ovms; @@ -277,13 +274,13 @@ class KFServingGRPCPredict : public ::testing::TestWithParam { } const char* tensorName = "Input_PRECISION_1_3_1_1_NHWC"; ovms::tensor_map_t tensorMap; - ::inference::ModelInferResponse response; + ::KFSResponse response; }; TEST_F(KFServingGRPCPredict, ValidSerialization) { ov::Tensor tensor(ov::element::f32, shape_t{1, 3, 1, 1}); - KFSPredictResponse response; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(&response); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); auto status = serializeTensorToTensorProto(responseOutput, @@ -302,8 +299,8 @@ TEST_F(KFServingGRPCPredict, ValidSerialization) { TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { ov::Tensor tensor(ov::element::i32, shape_t{1, 3, 1, 1}); - KFSPredictResponse response; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(&response); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); auto status = serializeTensorToTensorProto(responseOutput, @@ -315,8 +312,8 @@ TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorPrecision TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorShape) { ov::Tensor tensor(ov::element::i32, shape_t{2, 3, 1, 1}); - KFSPredictResponse response; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(&response); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); auto status = serializeTensorToTensorProto(responseOutput, @@ -347,8 +344,8 @@ class SerializeKFSInferOutputTensor : public KFServingGRPCPredict { TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecision) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); - KFSPredictResponse response; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(&response); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); ov::Tensor mockTensor = std::get<1>(inputs); @@ -367,8 +364,8 @@ class SerializeKFSInferOutputTensorNegative : public SerializeKFSInferOutputTens TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldSucceedForPrecision) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); - KFSPredictResponse response; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(&response); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); auto status = serializeTensorToTensorProto(responseOutput, @@ -382,7 +379,7 @@ TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldSucceedF } TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { - KFSPredictResponse response; + KFSResponse response; ov::Core ieCore; std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); diff --git a/src/test/server_test.cpp b/src/test/server_test.cpp index 50eef8225c..8712606bb1 100644 --- a/src/test/server_test.cpp +++ b/src/test/server_test.cpp @@ -22,7 +22,7 @@ #include #include "../cleaner_utils.hpp" -#include "../kfs_grpc_inference_service.hpp" +#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../localfilesystem.hpp" #include "../logging.hpp" #include "../model.hpp" @@ -88,8 +88,8 @@ class ServingClient { } void verifyModelReady(const std::string& modelName, grpc::StatusCode expectedStatus = grpc::StatusCode::OK, bool ready = true) { ClientContext context; - ::inference::ModelReadyRequest request; - ::inference::ModelReadyResponse response; + ::KFSGetModelStatusRequest request; + ::KFSGetModelStatusResponse response; request.set_name(modelName); ASSERT_NE(nullptr, stub_); @@ -100,8 +100,8 @@ class ServingClient { void verifyServerMetadata(grpc::StatusCode expectedStatus = grpc::StatusCode::OK) { ClientContext context; - ::inference::ServerMetadataRequest request; - ::inference::ServerMetadataResponse response; + ::KFSServerMetadataRequest request; + ::KFSServerMetadataResponse response; ASSERT_NE(nullptr, stub_); auto status = stub_->ServerMetadata(&context, request, &response); ASSERT_EQ(status.error_code(), expectedStatus); diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 42f7dcc8b7..6845ff9b34 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -17,6 +17,7 @@ #include +#include "../kfs_frontend/kfs_utils.hpp" #include "../prediction_service_utils.hpp" #include "../tensorinfo.hpp" #include "../tfs_frontend/tfs_utils.hpp" @@ -26,7 +27,7 @@ using tensorflow::serving::PredictResponse; using ovms::TensorInfo; -void preparePredictRequest(::inference::ModelInferRequest& request, inputs_info_t requestInputs, const std::vector& data, bool putBufferInInputTensorContent) { +void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, const std::vector& data, bool putBufferInInputTensorContent) { request.mutable_inputs()->Clear(); request.mutable_raw_input_contents()->Clear(); for (auto const& it : requestInputs) { @@ -152,7 +153,7 @@ void checkDummyResponse(const std::string outputName, void checkDummyResponse(const std::string outputName, const std::vector& requestData, - ::inference::ModelInferRequest& request, ::inference::ModelInferResponse& response, int seriesLength, int batchSize) { + ::KFSRequest& request, ::KFSResponse& response, int seriesLength, int batchSize) { ASSERT_EQ(response.outputs_size(), 1); ASSERT_EQ(response.raw_output_contents_size(), 1); ASSERT_EQ(response.outputs().begin()->name(), outputName) << "Did not find:" << outputName; @@ -212,7 +213,7 @@ bool isShapeTheSame(const tensorflow::TensorShapeProto& actual, const std::vecto return same; } -extern bool isShapeTheSame(const google::protobuf::RepeatedField& actual, const std::vector&& expected) { +bool isShapeTheSame(const KFSShapeType& actual, const std::vector&& expected) { bool same = true; int a_size = actual.size(); if (a_size != int(expected.size())) { @@ -271,7 +272,7 @@ void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, c tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); } -void prepareBinaryPredictRequest(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize) { +void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); tensor->set_name(inputName); @@ -298,7 +299,7 @@ void prepareBinaryPredictRequestNoShape(tensorflow::serving::PredictRequest& req tensor.set_dtype(tensorflow::DataType::DT_STRING); } -void prepareBinaryPredictRequestNoShape(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize) { +void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); tensor->set_name(inputName); @@ -325,7 +326,7 @@ void prepareBinary4x4PredictRequest(tensorflow::serving::PredictRequest& request tensor.mutable_tensor_shape()->add_dim()->set_size(batchSize); } -void prepareBinary4x4PredictRequest(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize) { +void prepareBinary4x4PredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize) { request.add_inputs(); auto tensor = request.mutable_inputs()->Mutable(0); tensor->set_name(inputName); @@ -340,7 +341,7 @@ void prepareBinary4x4PredictRequest(::inference::ModelInferRequest& request, con tensor->mutable_shape()->Add(batchSize); } -::inference::ModelInferRequest_InferInputTensor* findKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name) { +::KFSTensorInputProto* findKFSInferInputTensor(::KFSRequest& request, const std::string& name) { auto it = request.mutable_inputs()->begin(); while (it != request.mutable_inputs()->end()) { if (it->name() == name) @@ -350,7 +351,7 @@ ::inference::ModelInferRequest_InferInputTensor* findKFSInferInputTensor(::infer return it == request.mutable_inputs()->end() ? nullptr : &(*it); } -std::string* findKFSInferInputTensorContentInRawInputs(::inference::ModelInferRequest& request, const std::string& name) { +std::string* findKFSInferInputTensorContentInRawInputs(::KFSRequest& request, const std::string& name) { auto it = request.mutable_inputs()->begin(); size_t bufferId = 0; std::string* content = nullptr; @@ -365,15 +366,15 @@ std::string* findKFSInferInputTensorContentInRawInputs(::inference::ModelInferRe } return content; } -void prepareKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name, const std::tuple& inputInfo, +void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data, bool putBufferInInputTensorContent) { auto [shape, type] = inputInfo; prepareKFSInferInputTensor(request, name, - {shape, TensorInfo::getPrecisionAsKFSPrecision(type)}, + {shape, ovmsPrecisionToKFSPrecision(type)}, data, putBufferInInputTensorContent); } -void prepareKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name, const std::tuple& inputInfo, +void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data, bool putBufferInInputTensorContent) { auto it = request.mutable_inputs()->begin(); size_t bufferId = 0; @@ -383,7 +384,7 @@ void prepareKFSInferInputTensor(::inference::ModelInferRequest& request, const s ++it; ++bufferId; } - ::inference::ModelInferRequest_InferInputTensor* tensor; + KFSTensorInputProto* tensor; std::string* content = nullptr; if (it != request.mutable_inputs()->end()) { tensor = &*it; diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 30639cfac5..64bc9fdbb7 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -35,7 +35,7 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop #include "../execution_context.hpp" -#include "../kfs_grpc_inference_service.hpp" +#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../metric_registry.hpp" #include "../modelmanager.hpp" #include "../node_library.hpp" @@ -132,13 +132,6 @@ constexpr const ovms::model_version_t UNUSED_MODEL_VERSION = 42; // Answer to t static const ovms::ExecutionContext DEFAULT_TEST_CONTEXT{ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::Predict}; -using KFSRequestType = ::inference::ModelInferRequest; -using KFSResponseType = ::inference::ModelInferResponse; -using KFSInputTensorType = ::inference::ModelInferRequest_InferInputTensor; -using KFSOutputTensorType = ::inference::ModelInferResponse_InferOutputTensor; -using KFSShapeType = google::protobuf::RepeatedField; -using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; -using KFSOutputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; using TFSRequestType = tensorflow::serving::PredictRequest; using TFSResponseType = tensorflow::serving::PredictResponse; using TFSInputTensorType = tensorflow::TensorProto; @@ -147,7 +140,7 @@ using TFSShapeType = tensorflow::TensorShapeProto; using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; using TFSOutputTensorIteratorType = google::protobuf::Map::const_iterator; using TFSInterface = std::pair; -using KFSInterface = std::pair; +using KFSInterface = std::pair; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" @@ -157,24 +150,24 @@ ovms::tensor_map_t prepareTensors( void preparePredictRequest(tensorflow::serving::PredictRequest& request, inputs_info_t requestInputs, const std::vector& data = {}); -::inference::ModelInferRequest_InferInputTensor* findKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name); -std::string* findKFSInferInputTensorContentInRawInputs(::inference::ModelInferRequest& request, const std::string& name); +KFSTensorInputProto* findKFSInferInputTensor(::KFSRequest& request, const std::string& name); +std::string* findKFSInferInputTensorContentInRawInputs(::KFSRequest& request, const std::string& name); -void prepareKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name, const std::tuple& inputInfo, +void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data = {}, bool putBufferInInputTensorContent = false); -void prepareKFSInferInputTensor(::inference::ModelInferRequest& request, const std::string& name, const std::tuple& inputInfo, +void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data = {}, bool putBufferInInputTensorContent = false); -void preparePredictRequest(::inference::ModelInferRequest& request, inputs_info_t requestInputs, const std::vector& data = {}, bool putBufferInInputTensorContent = false); +void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, const std::vector& data = {}, bool putBufferInInputTensorContent = false); void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); -void prepareBinaryPredictRequest(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize); +void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequestNoShape(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); -void prepareBinaryPredictRequestNoShape(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize); +void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string& inputName, const int batchSize); void prepareBinary4x4PredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize = 1); -void prepareBinary4x4PredictRequest(::inference::ModelInferRequest& request, const std::string& inputName, const int batchSize = 1); +void prepareBinary4x4PredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize = 1); template void prepareInvalidImageBinaryTensor(TensorType& tensor); @@ -185,7 +178,7 @@ void checkDummyResponse(const std::string outputName, void checkDummyResponse(const std::string outputName, const std::vector& requestData, - ::inference::ModelInferRequest& request, ::inference::ModelInferResponse& response, int seriesLength, int batchSize = 1); + ::KFSRequest& request, ::KFSResponse& response, int seriesLength, int batchSize = 1); template std::string readableError(const T* expected_output, const T* actual_output, const size_t size) { @@ -226,8 +219,8 @@ void checkIncrement4DimResponse(const std::string outputName, template void checkIncrement4DimResponse(const std::string outputName, const std::vector& expectedData, - ::inference::ModelInferRequest& request, - ::inference::ModelInferResponse& response, + ::KFSRequest& request, + ::KFSResponse& response, const std::vector& expectedShape) { ASSERT_EQ(response.outputs_size(), 1); ASSERT_EQ(response.raw_output_contents_size(), 1); @@ -348,8 +341,8 @@ static ovms::NodeLibrary createLibraryMock() { T::release}; } -extern bool isShapeTheSame(const tensorflow::TensorShapeProto&, const std::vector&&); -extern bool isShapeTheSame(const google::protobuf::RepeatedField&, const std::vector&&); +bool isShapeTheSame(const tensorflow::TensorShapeProto&, const std::vector&&); +bool isShapeTheSame(const KFSShapeType&, const std::vector&&); void readRgbJpg(size_t& filesize, std::unique_ptr& image_bytes); void read4x4RgbJpg(size_t& filesize, std::unique_ptr& image_bytes); diff --git a/src/tfs_frontend/tfs_utils.cpp b/src/tfs_frontend/tfs_utils.cpp index 31361c8416..2bc41e19d0 100644 --- a/src/tfs_frontend/tfs_utils.cpp +++ b/src/tfs_frontend/tfs_utils.cpp @@ -20,61 +20,64 @@ #include #include #include +#include #include "../logging.hpp" +#include "../profiler.hpp" +#include "../status.hpp" namespace ovms { -tensorflow::DataType getPrecisionAsDataType(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP32, tensorflow::DataType::DT_FLOAT}, - {Precision::FP64, tensorflow::DataType::DT_DOUBLE}, - {Precision::FP16, tensorflow::DataType::DT_HALF}, - {Precision::I64, tensorflow::DataType::DT_INT64}, - {Precision::I32, tensorflow::DataType::DT_INT32}, - {Precision::I16, tensorflow::DataType::DT_INT16}, - {Precision::I8, tensorflow::DataType::DT_INT8}, - {Precision::U64, tensorflow::DataType::DT_UINT64}, - {Precision::U16, tensorflow::DataType::DT_UINT16}, - {Precision::U8, tensorflow::DataType::DT_UINT8}, - // {Precision::MIXED, tensorflow::DataType::DT_INVALID}, - // {Precision::Q78, tensorflow::DataType::DT_INVALID}, - // {Precision::BIN, tensorflow::DataType::DT_INVALID}, - {Precision::BOOL, tensorflow::DataType::DT_BOOL} - // {Precision::CUSTOM, tensorflow::DataType::DT_INVALID} +TFSDataType getPrecisionAsDataType(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP32, TFSDataType::DT_FLOAT}, + {Precision::FP64, TFSDataType::DT_DOUBLE}, + {Precision::FP16, TFSDataType::DT_HALF}, + {Precision::I64, TFSDataType::DT_INT64}, + {Precision::I32, TFSDataType::DT_INT32}, + {Precision::I16, TFSDataType::DT_INT16}, + {Precision::I8, TFSDataType::DT_INT8}, + {Precision::U64, TFSDataType::DT_UINT64}, + {Precision::U16, TFSDataType::DT_UINT16}, + {Precision::U8, TFSDataType::DT_UINT8}, + // {Precision::MIXED, TFSDataType::DT_INVALID}, + // {Precision::Q78, TFSDataType::DT_INVALID}, + // {Precision::BIN, TFSDataType::DT_INVALID}, + {Precision::BOOL, TFSDataType::DT_BOOL} + // {Precision::CUSTOM, TFSDataType::DT_INVALID} }; auto it = precisionMap.find(precision); if (it == precisionMap.end()) { - return tensorflow::DataType::DT_INVALID; + return TFSDataType::DT_INVALID; } return it->second; } -std::string getDataTypeAsString(tensorflow::DataType dataType) { +std::string getDataTypeAsString(TFSDataType dataType) { switch (dataType) { - case tensorflow::DataType::DT_FLOAT: + case TFSDataType::DT_FLOAT: return "FP32"; - case tensorflow::DataType::DT_DOUBLE: + case TFSDataType::DT_DOUBLE: return "FP64"; - case tensorflow::DataType::DT_INT32: + case TFSDataType::DT_INT32: return "I32"; - case tensorflow::DataType::DT_INT8: + case TFSDataType::DT_INT8: return "I8"; - case tensorflow::DataType::DT_UINT8: + case TFSDataType::DT_UINT8: return "U8"; - case tensorflow::DataType::DT_HALF: + case TFSDataType::DT_HALF: return "FP16"; - case tensorflow::DataType::DT_INT16: + case TFSDataType::DT_INT16: return "I16"; - case tensorflow::DataType::DT_UINT16: + case TFSDataType::DT_UINT16: return "U16"; - case tensorflow::DataType::DT_UINT64: + case TFSDataType::DT_UINT64: return "U64"; - case tensorflow::DataType::DT_INT64: + case TFSDataType::DT_INT64: return "I64"; - case tensorflow::DataType::DT_BOOL: + case TFSDataType::DT_BOOL: return "BOOL"; - case tensorflow::DataType::DT_STRING: + case TFSDataType::DT_STRING: return "STRING"; default: return "INVALID"; @@ -95,4 +98,36 @@ std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape) return oss.str(); } +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { + static std::unordered_map precisionMap{ + {TFSDataType::DT_FLOAT, Precision::FP32}, + {TFSDataType::DT_DOUBLE, Precision::FP64}, + {TFSDataType::DT_HALF, Precision::FP16}, + {TFSDataType::DT_INT64, Precision::I64}, + {TFSDataType::DT_INT32, Precision::I32}, + {TFSDataType::DT_INT16, Precision::I16}, + {TFSDataType::DT_INT8, Precision::I8}, + {TFSDataType::DT_UINT64, Precision::U64}, + {TFSDataType::DT_UINT16, Precision::U16}, + {TFSDataType::DT_UINT8, Precision::U8}, + {TFSDataType::DT_BOOL, Precision::BOOL}}; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& bufferOut, const std::string& name, size_t size) { + OVMS_PROFILE_FUNCTION(); + tensorflow::TensorProto tensorProto; + auto [it, isInserted] = response->mutable_outputs()->insert(google::protobuf::MapPair(name, std::move(tensorProto))); + if (!isInserted) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); + return StatusCode::INTERNAL_ERROR; + } + it->second.mutable_tensor_content()->resize(size); + bufferOut = it->second.mutable_tensor_content()->data(); + return StatusCode::OK; +} } // namespace ovms diff --git a/src/tfs_frontend/tfs_utils.hpp b/src/tfs_frontend/tfs_utils.hpp index 578aa73892..d1982184f7 100644 --- a/src/tfs_frontend/tfs_utils.hpp +++ b/src/tfs_frontend/tfs_utils.hpp @@ -25,9 +25,15 @@ #include "../precision.hpp" +using TFSDataType = tensorflow::DataType; + namespace ovms { -tensorflow::DataType getPrecisionAsDataType(Precision precision); -std::string getDataTypeAsString(tensorflow::DataType dataType); +class Status; + +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& s); +TFSDataType getPrecisionAsDataType(Precision precision); +std::string getDataTypeAsString(TFSDataType dataType); + std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape); -// static std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); +Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& tensorOut, const std::string& name, size_t size); } // namespace ovms From e3640c76e7d5694de83caa8321c29118a712bc82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Fri, 4 Nov 2022 14:04:15 +0100 Subject: [PATCH 035/130] Explicit paddlepaddle installation in the demo steps (#1392) * init * revert bash->Bash * added explicit installation for missing dependencies * fixed opencv requirement Co-authored-by: bstrzele --- .../python/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/demos/segmentation_using_paddlepaddle_model/python/README.md b/demos/segmentation_using_paddlepaddle_model/python/README.md index d3121a629d..511fae56e4 100644 --- a/demos/segmentation_using_paddlepaddle_model/python/README.md +++ b/demos/segmentation_using_paddlepaddle_model/python/README.md @@ -9,8 +9,6 @@ As an example, we will use [ocrnet-hrnet-w48-paddle](https://github.com/openvino - Python 3.6 or newer installed -- [paddlepaddle](https://pypi.org/project/paddlepaddle/) Python package installed - ## Preparing to Run Clone the repository and enter segmentation_using_paddlepaddle_model directory @@ -20,6 +18,14 @@ git clone https://github.com/openvinotoolkit/model_server.git cd model_server/demos/segmentation_using_paddlepaddle_model/python ``` +Install [paddlepaddle](https://pypi.org/project/paddlepaddle/) and other packages required to export the model to deployable format +```bash +pip3 install paddlepaddle +pip3 install pyyaml +pip3 install scipy +pip3 install opencv-python +``` + You can prepare the workspace by just running ```bash From e1992a1a300dfbf40161b655dcd66500746021cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Mon, 7 Nov 2022 22:31:20 +0100 Subject: [PATCH 036/130] Remove deploy catalog references --- docs/home.md | 2 +- extras/openvino-operator-openshift/README.md | 2 +- extras/ovms-operator/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/home.md b/docs/home.md index dc87d69fc8..1f0b98960e 100644 --- a/docs/home.md +++ b/docs/home.md @@ -84,7 +84,7 @@ For more information on using Model Server in various scenarios you can check th * [Serving stateful models](stateful_models.md) -* [Deploy using a Kubernetes Helm Chart](../deploy/README.md) +* [Deploy using a Kubernetes Helm Chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) * [Deployment using Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) diff --git a/extras/openvino-operator-openshift/README.md b/extras/openvino-operator-openshift/README.md index 11aaeca289..f0dd30f6d2 100644 --- a/extras/openvino-operator-openshift/README.md +++ b/extras/openvino-operator-openshift/README.md @@ -54,7 +54,7 @@ Modify the [sample resource](https://github.com/openvinotoolkit/model_server/tre oc apply -f config/samples/intel_v1alpha1_ovms.yaml ``` -The available [parameters](../../deploy/README.md) are the same as above. +The available [parameters](https://github.com/openvinotoolkit/operator/blob/main/docs/modelserver_params.md) are the same as above. Note: Some deployment configurations have prerequisites like creating relevant resources in Kubernetes. For example, a secret with credentials, persistent volume claim or configmap with a Model Server configuration file. diff --git a/extras/ovms-operator/README.md b/extras/ovms-operator/README.md index 2588c2b4b1..6e70ba3d67 100644 --- a/extras/ovms-operator/README.md +++ b/extras/ovms-operator/README.md @@ -30,7 +30,7 @@ It can be done by editing the [sample resource](https://github.com/openvinotoolk kubectl apply -f config/samples/intel_v1alpha1_ovms.yaml ``` -The parameters are identical to [Helm chart](../../deploy/README.md). +The parameters are identical to [Helm chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms). Note: Some deployment configurations have prerequisites like creating relevant resources in Kubernetes. For example a secret with credentials, persistent volume claim or configmap with OVMS configuration file. From 6e2415c7f0235e75be9e813b6db9c286e6829d20 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Wed, 9 Nov 2022 14:51:29 +0100 Subject: [PATCH 037/130] =?UTF-8?q?=20Place=20outputs=20in=20corresponding?= =?UTF-8?q?=20field=20to=20this=20where=20inputs=20were=20found=E2=80=A6?= =?UTF-8?q?=20(#1497)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Place outputs in corresponding field to this where inputs were found in request --- src/exit_node.cpp | 2 +- src/exit_node.hpp | 6 +- src/modelinstance.cpp | 2 +- src/pipelinedefinition.cpp | 2 +- src/prediction_service_utils.cpp | 8 ++ src/prediction_service_utils.hpp | 2 + src/serialization.cpp | 54 +++++++++- src/serialization.hpp | 26 +++-- src/test/prediction_service_test.cpp | 17 +++- src/test/serialization_tests.cpp | 141 +++++++++++++++++++++++++-- 10 files changed, 231 insertions(+), 29 deletions(-) diff --git a/src/exit_node.cpp b/src/exit_node.cpp index d7ca765fe0..e8db9b744a 100644 --- a/src/exit_node.cpp +++ b/src/exit_node.cpp @@ -62,7 +62,7 @@ Status OutputGetter::get(const std::string& name, ov::Tensor& template Status ExitNode::fetchResults(const TensorMap& inputTensors) { OutputGetter outputGetter(inputTensors); - return serializePredictResponse(outputGetter, this->outputsInfo, this->response, getOutputMapKeyName); + return serializePredictResponse(outputGetter, this->outputsInfo, this->response, getOutputMapKeyName, useSharedOutputContent); } template diff --git a/src/exit_node.hpp b/src/exit_node.hpp index 0818163824..da4e5063b7 100644 --- a/src/exit_node.hpp +++ b/src/exit_node.hpp @@ -34,12 +34,14 @@ template class ExitNode : public Node { ResponseType* response; const tensor_map_t outputsInfo; + bool useSharedOutputContent; public: - ExitNode(ResponseType* response, const tensor_map_t& outputsInfo, std::set gatherFromNode = {}) : + ExitNode(ResponseType* response, const tensor_map_t& outputsInfo, std::set gatherFromNode = {}, bool useSharedOutputContent = true) : Node(EXIT_NODE_NAME, std::nullopt, gatherFromNode), response(response), - outputsInfo(outputsInfo) { + outputsInfo(outputsInfo), + useSharedOutputContent(useSharedOutputContent) { } // Exit node does not have execute logic. diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index da580804ee..f0a14a9026 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1263,7 +1263,7 @@ Status ModelInstance::infer(const ::KFSRequest* requestProto, timer.start(SERIALIZE); OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); + status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName, useSharedOutputContent(requestProto)); timer.stop(SERIALIZE); if (!status.ok()) return status; diff --git a/src/pipelinedefinition.cpp b/src/pipelinedefinition.cpp index f27141f5ee..444aa45dac 100644 --- a/src/pipelinedefinition.cpp +++ b/src/pipelinedefinition.cpp @@ -274,7 +274,7 @@ Status PipelineDefinition::create(std::unique_ptr& pipeline, nodeResources.at(info.nodeName))); break; case NodeKind::EXIT: { - auto node = std::make_unique>(response, getOutputsInfo(), info.gatherFromNode); + auto node = std::make_unique>(response, getOutputsInfo(), info.gatherFromNode, useSharedOutputContent(request)); exit = node.get(); nodes.emplace(info.nodeName, std::move(node)); break; diff --git a/src/prediction_service_utils.cpp b/src/prediction_service_utils.cpp index edc653c2e2..255215adae 100644 --- a/src/prediction_service_utils.cpp +++ b/src/prediction_service_utils.cpp @@ -97,4 +97,12 @@ std::map getRequestShapes(const tensorflow::serving::Predi return requestShapes; } +bool useSharedOutputContent(const tensorflow::serving::PredictRequest* request) { + return true; +} + +bool useSharedOutputContent(const ::inference::ModelInferRequest* request) { + return request->raw_input_contents().size() > 0; +} + } // namespace ovms diff --git a/src/prediction_service_utils.hpp b/src/prediction_service_utils.hpp index 6e5cfe24b0..41badb69d0 100644 --- a/src/prediction_service_utils.hpp +++ b/src/prediction_service_utils.hpp @@ -35,4 +35,6 @@ std::map getRequestShapes(const ::KFSRequest* request); std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); +bool useSharedOutputContent(const tensorflow::serving::PredictRequest* request); +bool useSharedOutputContent(const ::inference::ModelInferRequest* request); } // namespace ovms diff --git a/src/serialization.cpp b/src/serialization.cpp index b7171b71b8..f8667f452d 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -159,6 +159,39 @@ static void serializeContent(std::string* content, ov::Tensor& tensor) { } } +#define SERIALIZE_BY_DATATYPE(contents, datatype) \ + for (size_t i = 0; i < tensor.get_byte_size(); i += sizeof(datatype)) { \ + auto value = responseOutput.mutable_contents()->contents()->Add(); \ + *value = (*(reinterpret_cast((char*)tensor.data() + i))); \ + } + +static void serializeContent(::inference::ModelInferResponse::InferOutputTensor& responseOutput, ov::Tensor& tensor) { + OVMS_PROFILE_FUNCTION(); + if (responseOutput.datatype() == "FP32") { + SERIALIZE_BY_DATATYPE(mutable_fp32_contents, float) + } else if (responseOutput.datatype() == "INT64") { + SERIALIZE_BY_DATATYPE(mutable_int64_contents, int64_t) + } else if (responseOutput.datatype() == "INT32") { + SERIALIZE_BY_DATATYPE(mutable_int_contents, int32_t) + } else if (responseOutput.datatype() == "INT16") { + SERIALIZE_BY_DATATYPE(mutable_int_contents, int16_t) + } else if (responseOutput.datatype() == "INT8") { + SERIALIZE_BY_DATATYPE(mutable_int_contents, int8_t) + } else if (responseOutput.datatype() == "UINT64") { + SERIALIZE_BY_DATATYPE(mutable_uint64_contents, uint64_t) + } else if (responseOutput.datatype() == "UINT32") { + SERIALIZE_BY_DATATYPE(mutable_uint_contents, uint32_t) + } else if (responseOutput.datatype() == "UINT16") { + SERIALIZE_BY_DATATYPE(mutable_uint_contents, uint16_t) + } else if (responseOutput.datatype() == "UINT8") { + SERIALIZE_BY_DATATYPE(mutable_uint_contents, uint8_t) + } else if (responseOutput.datatype() == "FP64") { + SERIALIZE_BY_DATATYPE(mutable_fp64_contents, double) + } else if (responseOutput.datatype() == "BYTES") { + responseOutput.mutable_contents()->add_bytes_contents((char*)tensor.data(), tensor.get_byte_size()); + } +} + Status serializeTensorToTensorProto( tensorflow::TensorProto& responseOutput, const std::shared_ptr& servableOutput, @@ -176,8 +209,8 @@ Status serializeTensorToTensorProto( return StatusCode::OK; } -Status serializeTensorToTensorProto( - ::KFSResponse::InferOutputTensor& responseOutput, +Status serializeTensorToTensorProtoRaw( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, std::string* rawOutputContents, const std::shared_ptr& servableOutput, ov::Tensor& tensor) { @@ -194,6 +227,23 @@ Status serializeTensorToTensorProto( return StatusCode::OK; } +Status serializeTensorToTensorProto( + ::KFSResponse::InferOutputTensor& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + OVMS_PROFILE_FUNCTION(); + auto status = serializePrecision(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + status = serializeShape(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + serializeContent(responseOutput, tensor); + return StatusCode::OK; +} + template <> Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { OVMS_PROFILE_FUNCTION(); diff --git a/src/serialization.hpp b/src/serialization.hpp index ea4672b2a3..dae3ae7537 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -63,6 +63,11 @@ Status serializeTensorToTensorProto( Status serializeTensorToTensorProto( ::KFSResponse::InferOutputTensor& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor); + +Status serializeTensorToTensorProtoRaw( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, std::string* rawOutputContents, const std::shared_ptr& servableOutput, ov::Tensor& tensor); @@ -71,19 +76,13 @@ typedef const std::string& (*outputNameChooser_t)(const std::string&, const Tens const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo); const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo); -template -Status serializePredictResponse( - OutputGetter& outputGetter, - const tensor_map_t& outputMap, - ResponseType* response, - outputNameChooser_t outputNameChooser); -// partial template specialization template Status serializePredictResponse( OutputGetter& outputGetter, const tensor_map_t& outputMap, tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser) { + outputNameChooser_t outputNameChooser, + bool useSharedOutputContent = true) { OVMS_PROFILE_FUNCTION(); Status status; ProtoGetter protoGetter(response); @@ -101,12 +100,14 @@ Status serializePredictResponse( } return status; } + template Status serializePredictResponse( OutputGetter& outputGetter, const tensor_map_t& outputMap, ::KFSResponse* response, - outputNameChooser_t outputNameChooser) { + outputNameChooser_t outputNameChooser, + bool useSharedOutputContent = true) { OVMS_PROFILE_FUNCTION(); Status status; ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(response); @@ -117,7 +118,12 @@ Status serializePredictResponse( return status; } auto& inferOutputTensor = protoGetter.createOutput(outputInfo->getMappedName()); - status = serializeTensorToTensorProto(inferOutputTensor, protoGetter.createContent(outputInfo->getMappedName()), outputInfo, tensor); + if (useSharedOutputContent) { + status = serializeTensorToTensorProtoRaw(inferOutputTensor, protoGetter.createContent(outputInfo->getMappedName()), outputInfo, tensor); + } else { + status = serializeTensorToTensorProto(inferOutputTensor, outputInfo, tensor); + } + if (!status.ok()) { return status; } diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index f8e1454e4f..c95de52ffa 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -196,9 +196,20 @@ class TestPredict : public ::testing::Test { size_t bufferId; auto status = getOutput(response, outputName, it, bufferId); ASSERT_TRUE(status.ok()) << "Couln't find output:" << outputName; - float* buffer = reinterpret_cast(const_cast(response.raw_output_contents(bufferId).data())); - ASSERT_EQ(0, std::memcmp(buffer, expectedValues.data(), expectedValues.size() * sizeof(float))) - << readableError(expectedValues.data(), buffer, expectedValues.size() * sizeof(float)); + if (response.raw_output_contents().size() > 0) { + float* buffer = reinterpret_cast(const_cast(response.raw_output_contents(bufferId).data())); + ASSERT_EQ(0, std::memcmp(buffer, expectedValues.data(), expectedValues.size() * sizeof(float))) + << readableError(expectedValues.data(), buffer, expectedValues.size() * sizeof(float)); + } else { + auto& responseOutput = *it; + if (responseOutput.datatype() == "FP32") { + for (size_t i = 0; i < expectedValues.size(); i++) { + ASSERT_EQ(responseOutput.contents().fp32_contents()[i], expectedValues[i]); + } + } else if (responseOutput.datatype() == "BYTES") { + ASSERT_EQ(0, std::memcmp(&responseOutput.contents().bytes_contents(), expectedValues.data(), expectedValues.size() * sizeof(float))); + } + } } ovms::Status performInferenceWithRequest(const RequestType& request, ResponseType& response, const std::string& servableName = "dummy") { diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index c5233fdc9a..9177303e55 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -277,13 +277,13 @@ class KFServingGRPCPredict : public ::testing::TestWithParam { ::KFSResponse response; }; -TEST_F(KFServingGRPCPredict, ValidSerialization) { +TEST_F(KFServingGRPCPredict, ValidSerializationRaw) { ov::Tensor tensor(ov::element::f32, shape_t{1, 3, 1, 1}); KFSResponse response; ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); - auto status = serializeTensorToTensorProto(responseOutput, + auto status = serializeTensorToTensorProtoRaw(responseOutput, content, tensorMap[tensorName], tensor); @@ -297,13 +297,55 @@ TEST_F(KFServingGRPCPredict, ValidSerialization) { EXPECT_EQ(response.raw_output_contents()[0].size(), 12); } -TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { +TEST_F(KFServingGRPCPredict, ValidSerialization) { + ov::Tensor tensor(ov::element::f32, shape_t{1, 3, 1, 1}); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); + auto& responseOutput = protoGetter.createOutput(tensorName); + auto status = serializeTensorToTensorProto(responseOutput, + tensorMap[tensorName], + tensor); + ASSERT_EQ(status.getCode(), ovms::StatusCode::OK); + EXPECT_EQ(responseOutput.name(), tensorName); + EXPECT_EQ(responseOutput.datatype(), "FP32"); + EXPECT_EQ(responseOutput.shape(0), 1); + EXPECT_EQ(responseOutput.shape(1), 3); + EXPECT_EQ(responseOutput.shape(2), 1); + EXPECT_EQ(responseOutput.shape(3), 1); + EXPECT_EQ(responseOutput.contents().fp32_contents_size(), 3); +} + +TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorPrecisionRaw) { ov::Tensor tensor(ov::element::i32, shape_t{1, 3, 1, 1}); KFSResponse response; ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); + auto status = serializeTensorToTensorProtoRaw(responseOutput, + content, + tensorMap[tensorName], + tensor); + EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); +} + +TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { + ov::Tensor tensor(ov::element::i32, shape_t{1, 3, 1, 1}); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); + auto& responseOutput = protoGetter.createOutput(tensorName); auto status = serializeTensorToTensorProto(responseOutput, + tensorMap[tensorName], + tensor); + EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); +} + +TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorShapeRaw) { + ov::Tensor tensor(ov::element::i32, shape_t{2, 3, 1, 1}); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); + auto& responseOutput = protoGetter.createOutput(tensorName); + auto* content = protoGetter.createContent(tensorName); + auto status = serializeTensorToTensorProtoRaw(responseOutput, content, tensorMap[tensorName], tensor); @@ -315,9 +357,7 @@ TEST_F(KFServingGRPCPredict, NegativeMismatchBetweenTensorInfoAndTensorShape) { KFSResponse response; ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); - auto* content = protoGetter.createContent(tensorName); auto status = serializeTensorToTensorProto(responseOutput, - content, tensorMap[tensorName], tensor); EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); @@ -341,7 +381,7 @@ class SerializeKFSInferOutputTensor : public KFServingGRPCPredict { } }; -TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecision) { +TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecisionRaw) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); KFSResponse response; @@ -349,7 +389,7 @@ TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecis auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); ov::Tensor mockTensor = std::get<1>(inputs); - auto status = serializeTensorToTensorProto(responseOutput, + auto status = serializeTensorToTensorProtoRaw(responseOutput, content, std::get<0>(inputs), mockTensor); @@ -359,16 +399,32 @@ TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecis << "should succeed"; } +TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecision) { + ovms::Precision testedPrecision = GetParam(); + auto inputs = getInputs(testedPrecision); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); + auto& responseOutput = protoGetter.createOutput(tensorName); + ov::Tensor mockTensor = std::get<1>(inputs); + auto status = serializeTensorToTensorProto(responseOutput, + std::get<0>(inputs), + mockTensor); + EXPECT_TRUE(status.ok()) + << "Supported OV serialization precision" + << toString(testedPrecision) + << "should succeed"; +} + class SerializeKFSInferOutputTensorNegative : public SerializeKFSInferOutputTensor {}; -TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldSucceedForPrecision) { +TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoRawShouldFailedForPrecision) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); KFSResponse response; ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); auto& responseOutput = protoGetter.createOutput(tensorName); auto* content = protoGetter.createContent(tensorName); - auto status = serializeTensorToTensorProto(responseOutput, + auto status = serializeTensorToTensorProtoRaw(responseOutput, content, std::get<0>(inputs), std::get<1>(inputs)); @@ -378,6 +434,21 @@ TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldSucceedF << "should fail"; } +TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldFailedForPrecision) { + ovms::Precision testedPrecision = GetParam(); + auto inputs = getInputs(testedPrecision); + KFSResponse response; + ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(&response); + auto& responseOutput = protoGetter.createOutput(tensorName); + auto status = serializeTensorToTensorProto(responseOutput, + std::get<0>(inputs), + std::get<1>(inputs)); + EXPECT_EQ(status, ovms::StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION) + << "Unsupported OV serialization precision" + << toString(testedPrecision) + << "should fail"; +} + TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { KFSResponse response; ov::Core ieCore; @@ -403,6 +474,58 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { EXPECT_EQ(40, response.raw_output_contents(0).size()); } +TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithuseSharedOutputContent) { + KFSResponse response; + ov::Core ieCore; + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); + ov::InferRequest inferRequest = compiledModel.create_infer_request(); + ovms::tensor_map_t tenMap; + std::shared_ptr tensorInfo = std::make_shared( + DUMMY_MODEL_INPUT_NAME, + ovms::Precision::FP32, + ovms::Shape{1, 10}, + Layout{"NC"}); + tenMap[DUMMY_MODEL_OUTPUT_NAME] = tensorInfo; + ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); + inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); + OutputGetter outputGetter(inferRequest); + auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName, true); + ASSERT_TRUE(status.ok()); + EXPECT_EQ(DUMMY_MODEL_INPUT_NAME, response.outputs(0).name()); + EXPECT_EQ("FP32", response.outputs(0).datatype()); + EXPECT_EQ(1, response.outputs(0).shape(0)); + EXPECT_EQ(10, response.outputs(0).shape(1)); + EXPECT_EQ(0, response.outputs(0).contents().fp32_contents_size()); + EXPECT_EQ(40, response.raw_output_contents(0).size()); +} + +TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithsharedInputContentsNotUsed) { + KFSResponse response; + ov::Core ieCore; + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); + ov::InferRequest inferRequest = compiledModel.create_infer_request(); + ovms::tensor_map_t tenMap; + std::shared_ptr tensorInfo = std::make_shared( + DUMMY_MODEL_INPUT_NAME, + ovms::Precision::FP32, + ovms::Shape{1, 10}, + Layout{"NC"}); + tenMap[DUMMY_MODEL_OUTPUT_NAME] = tensorInfo; + ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); + inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); + OutputGetter outputGetter(inferRequest); + auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName, false); + ASSERT_TRUE(status.ok()); + EXPECT_EQ(DUMMY_MODEL_INPUT_NAME, response.outputs(0).name()); + EXPECT_EQ("FP32", response.outputs(0).datatype()); + EXPECT_EQ(1, response.outputs(0).shape(0)); + EXPECT_EQ(10, response.outputs(0).shape(1)); + EXPECT_EQ(10, response.outputs(0).contents().fp32_contents_size()); + EXPECT_EQ(0, response.raw_output_contents_size()); +} + INSTANTIATE_TEST_SUITE_P( Test, SerializeKFSInferOutputTensor, From 41b3f2382fc4be666991c19c35f06dcb9b581c80 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 10 Nov 2022 12:43:12 +0100 Subject: [PATCH 038/130] Nonblocking coverage check (#1491) * Nonblocking coverage check --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 74ac38c686..432f666d27 100644 --- a/Makefile +++ b/Makefile @@ -227,10 +227,11 @@ endif get_coverage: @echo "Copying coverage report from build image to genhtml if exist..." @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) bash - @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . 2>/dev/null + @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . || true @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) - $(MAKE) check_coverage - + @if [ -d genhtml/src ]; then $(MAKE) check_coverage; \ + else echo "ERROR: genhtml/src was not generated during build"; \ + fi check_coverage: @echo "Checking if coverage is above threshold..." @docker run $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) ./check_coverage.bat | grep success From 93d8a26e793980813e20714a183bf25cc014bf14 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 14 Nov 2022 13:15:46 +0100 Subject: [PATCH 039/130] Tf downgrade to 2.6.5 zlib update. (#1518) * Tf downgrade to 2.6.5 zlib update. --- Dockerfile.redhat | 6 ++++ Dockerfile.ubuntu | 6 ++++ WORKSPACE | 6 ++-- external/tf.patch | 22 ++++++++++++++ lib_search.py | 2 ++ .../bazel_rules_apple.patch | 30 +++++++++++++++++++ 6 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 third_party/build_bazel_rules_apple/bazel_rules_apple.patch diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 58b9104a17..b0ae1967ad 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -151,7 +151,13 @@ COPY external /ovms/external/ COPY third_party /ovms/third_party/ RUN alternatives --set python /usr/bin/python3 + RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework + +# Patch apple build scripts +RUN cp -rf /ovms/third_party/build_bazel_rules_apple/bazel_rules_apple.patch $(bazel info output_base)/external/build_bazel_rules_apple/ +RUN cd $(bazel info output_base)/external/build_bazel_rules_apple/ && patch -p1 < bazel_rules_apple.patch + RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto RUN cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 7df50cc9b9..59dde21952 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -194,7 +194,13 @@ COPY .bazelrc WORKSPACE /ovms/ COPY external /ovms/external/ RUN apt install -y python-is-python3 + RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework + +# Patch apple build scripts +RUN cp -rf /ovms/third_party/build_bazel_rules_apple/bazel_rules_apple.patch $(bazel info output_base)/external/build_bazel_rules_apple/ +RUN cd $(bazel info output_base)/external/build_bazel_rules_apple/ && patch -p1 < bazel_rules_apple.patch + RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto # Copy example clients into build image for static analysis diff --git a/WORKSPACE b/WORKSPACE index 02f2b112c5..7189ba9af6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -61,7 +61,7 @@ cc_library( git_repository( name = "tensorflow_serving", remote = "https://github.com/tensorflow/serving.git", - tag = "2.10.0", + tag = "2.6.5", patch_args = ["-p1"], patches = ["net_http.patch", "listen.patch"] # ^^^^^^^^^^^^ @@ -88,8 +88,8 @@ cc_library( load("@tensorflow_serving//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "bc4e9bbeb0136163f283ab8b695bec747cad738963e153ce3b7e414ebffe408f", - git_commit = "359c3cdfc5fabac82b3c70b3b6de2b0a8c16874f", + sha256 = "fd687f8e26833cb917ae0bd8e434c9bd30c92042361c8ae69679983d3c66a440", + git_commit = "15198b1818bd2bf1b5b55bf5b02bf42398d222fc", patch = "tf.patch", repo_mapping = {"@curl" : "@curl"} ) diff --git a/external/tf.patch b/external/tf.patch index b80d525dd1..d144ed97ce 100644 --- a/external/tf.patch +++ b/external/tf.patch @@ -15,3 +15,25 @@ index b5521b1752..feaa8d1b91 100644 case 1: h += static_cast(data[0]); h *= m; + +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index be571aaf1f8..a8765b08bd4 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -706,12 +706,12 @@ def _tf_repositories(): + tf_http_archive( + name = "zlib", + build_file = "//third_party:zlib.BUILD", +- sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", +- strip_prefix = "zlib-1.2.11", ++ sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", ++ strip_prefix = "zlib-1.2.13", + system_build_file = "//third_party/systemlibs:zlib.BUILD", + urls = [ +- "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz", +- "https://zlib.net/zlib-1.2.11.tar.gz", ++ "https://storage.googleapis.com/mirror.tensorflow.org/zlib.net/zlib-1.2.13.tar.gz", ++ "https://zlib.net/zlib-1.2.13.tar.gz", + ], + ) + diff --git a/lib_search.py b/lib_search.py index 22f32175db..ce9ddbb682 100644 --- a/lib_search.py +++ b/lib_search.py @@ -110,6 +110,7 @@ def check_dir(start_dir): 'tf.patch', 'tftext.patch', 'vehicle_images.txt', + 'bazel_rules_apple.patch', ] exclude_directories = ['/dist/', 'extras/ovms-operator', 'extras/openvino-operator-openshift', 'release_files/thirdparty-licenses'] @@ -184,6 +185,7 @@ def check_func(start_dir): 'tf.patch', 'tftext.patch', 'zlib.LICENSE.txt', + 'bazel_rules_apple.patch', ] exclude_directories = ['/dist/', 'extras/ovms-operator'] diff --git a/third_party/build_bazel_rules_apple/bazel_rules_apple.patch b/third_party/build_bazel_rules_apple/bazel_rules_apple.patch new file mode 100644 index 0000000000..fa2e07aaee --- /dev/null +++ b/third_party/build_bazel_rules_apple/bazel_rules_apple.patch @@ -0,0 +1,30 @@ +diff -uraN a/apple/internal/rule_factory.bzl b/apple/internal/rule_factory.bzl +--- a/apple/internal/rule_factory.bzl 2020-04-03 18:27:03.000000000 +0000 ++++ b/apple/internal/rule_factory2.bzl 2022-10-19 15:31:40.437342517 +0000 +@@ -230,7 +230,7 @@ + + def _common_binary_linking_attrs(rule_descriptor): + deps_aspects = [ +- apple_common.objc_proto_aspect, ++# apple_common.objc_proto_aspect, + apple_resource_aspect, + framework_import_aspect, + swift_usage_aspect, +@@ -250,7 +250,7 @@ + """, + ), + "bundle_loader": attr.label( +- aspects = [apple_common.objc_proto_aspect], ++# aspects = [apple_common.objc_proto_aspect], + providers = [[apple_common.AppleExecutableBinary]], + doc = """ + This attribute is public as an implementation detail while we migrate the architecture of the rules. +@@ -258,7 +258,7 @@ + """, + ), + "dylibs": attr.label_list( +- aspects = [apple_common.objc_proto_aspect], ++# aspects = [apple_common.objc_proto_aspect], + doc = """ + This attribute is public as an implementation detail while we migrate the architecture of the rules. + Do not change its value. \ No newline at end of file From bc044cf0e1635a2aaf8561030e3956ab342c4671 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 14 Nov 2022 13:21:43 +0100 Subject: [PATCH 040/130] Remove ovms_test step (#1510) * Remove ovms_test step --- Dockerfile.redhat | 18 ++++++++++-------- Dockerfile.ubuntu | 9 ++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index b0ae1967ad..172e8e2e2d 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -182,19 +182,21 @@ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; sed -i -e "s ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ +RUN wget https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16-1.noarch.rpm &&\ + yum install -y lcov-1.16-1.noarch.rpm + COPY check_coverage.bat /ovms/ -# Test Coverage -RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; wget https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16-1.noarch.rpm &&\ - yum install -y lcov-1.16-1.noarch.rpm &&\ - bazel coverage --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" + +# Test coverage and unit tests in one layer +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test && exit 0; fi ;\ + bazel coverage --test_output=streamed --combined_report=lcov //src:ovms_test &&\ + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ + bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so -RUN cd /ovms/src/example/SampleCpuExtension/ && make - -RUN bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test +RUN cd /ovms/src/example/SampleCpuExtension/ && make COPY ${ovms_metadata_file} metadata.json diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 59dde21952..58e03b6136 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -225,17 +225,16 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: COPY check_coverage.bat /ovms/ # Test Coverage -RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else exit 0 ; fi ; bazel coverage --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test && exit 0; fi ;\ + bazel coverage --test_output=streamed --combined_report=lcov //src:ovms_test &&\ + genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ + bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test - RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && make -RUN bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test - ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json From 01851e59c77cd3ceb086460e5be6607cf40e008d Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Wed, 16 Nov 2022 10:13:56 +0100 Subject: [PATCH 041/130] fixed missing links (#1520) --- docs/host.md | 2 +- src/example/SampleCpuExtension/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/host.md b/docs/host.md index 00e9fcda2a..f396bc9034 100644 --- a/docs/host.md +++ b/docs/host.md @@ -50,4 +50,4 @@ Refer to [Running Model Server using Docker Container](./docker_container.md) to > **NOTE**: > When AI accelerators are used for inference execution, additional steps may be required to install their drivers and dependencies. Learn more about it -> Learn more about it on [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_linux.html). +> Learn more about it on [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_from_archive_linux.html). diff --git a/src/example/SampleCpuExtension/README.md b/src/example/SampleCpuExtension/README.md index 679af818d2..c76b63341b 100644 --- a/src/example/SampleCpuExtension/README.md +++ b/src/example/SampleCpuExtension/README.md @@ -1,6 +1,6 @@ # CPU Extensions {#ovms_sample_cpu_extension} -Any CPU layer, unsupported by OpenVINO, can be implemented as a shared library. While loaded in OVMS as a cpu extension, it can help in executing the model. An example presented here is based on the code from in OpenVINO™ repository: [extension template](https://github.com/openvinotoolkit/openvino/tree/master/docs/template_extension/new). +Any CPU layer, unsupported by OpenVINO, can be implemented as a shared library. While loaded in OVMS as a cpu extension, it can help in executing the model. An example presented here is based on the code from in OpenVINO™ repository: [extension template](https://github.com/openvinotoolkit/openvino/tree/master/src/core/template_extension/new). It includes a demonstrative implementation of the Relu layer which can be applied on many existing public models. That implementation display in the model server logs information about the From 36939862d41c83d28e3cfcfadb908ea0853c7bb2 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Thu, 17 Nov 2022 09:02:29 +0100 Subject: [PATCH 042/130] drop remaining api1.0 (#1523) Co-authored-by: Adrian Tobiszewski --- Dockerfile.ubuntu | 3 ++- DockerfileMakePackage | 2 +- release_files/Dockerfile.redhat | 2 +- release_files/Dockerfile.ubuntu | 2 +- src/config.cpp | 2 +- src/modelconfig.cpp | 26 ++++++++++++++++--- src/modelinstance.cpp | 35 +++++++++---------------- src/modelmanager.cpp | 10 ++++---- src/ov_utils.cpp | 8 +++--- src/test/modelconfig_test.cpp | 45 +++++++++++++++++++++++++++++++++ src/test/modelinstance_test.cpp | 19 +++++++++++++- src/test/ov_utils_test.cpp | 2 +- 12 files changed, 114 insertions(+), 42 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 58e03b6136..03b2a650a6 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -153,7 +153,8 @@ RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; \ mkdir -p /opt/intel/openvino && \ ln -s /openvino/inference-engine/temp/opencv_*_ubuntu20/opencv /opt/intel/openvino/extras && \ ln -s /usr/local/runtime /opt/intel/openvino && \ - ln -s /openvino/scripts/setupvars/setupvars.sh /opt/intel/openvino/setupvars.sh + ln -s /openvino/scripts/setupvars/setupvars.sh /opt/intel/openvino/setupvars.sh && \ + ln -s /opt/intel/openvino /opt/intel/openvino_2022 ################## END OF OPENVINO SOURCE BUILD ###################### ################### TAKE OPENVINO FROM A BINARY RELEASE - buildarg ov_use_binary=1 (DEFAULT) ########## diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 145613921d..1a60b884e6 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -35,7 +35,7 @@ RUN mkdir -vp /ovms_release/lib/custom_nodes RUN if [ -d /ovms/src/custom_nodes/lib/${BASE_OS} ] ; then true ; else exit 0 ; fi ; cp /ovms/src/custom_nodes/lib/${BASE_OS}/*.so /ovms_release/lib/custom_nodes/ RUN cp /ovms/metadata.json /ovms_release/ -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cp -v /openvino/bin/intel64/Release/lib/plugins.xml /ovms_release/lib/ +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cp -v /openvino/bin/intel64/Release/plugins.xml /ovms_release/lib/ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp -v /opt/intel/openvino/runtime/3rdparty/hddl/config/* /ovms_release/lib/hddl/config/ || true RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp -vr /opt/intel/openvino/runtime/3rdparty/hddl/etc/* /ovms_release/lib/hddl/etc/ || true RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp -v /opt/intel/openvino/runtime/lib/intel64/plugins.xml /ovms_release/lib/ && cp /opt/intel/openvino/install_dependencies/* /ovms_release/deps/ diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index 4492942914..894fb2fcd0 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -45,7 +45,7 @@ ENV GPU=$GPU WORKDIR / RUN set -e ; \ set -x ; \ - microdnf install -y pkg-config; \ + microdnf install -y pkg-config ; \ if [ "$GPU" == "1" ] ; then \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 7a36d45fc0..2d42c9387d 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -81,7 +81,7 @@ COPY drivers /drivers RUN set -e ; \ set -x ; \ apt update -y ; \ - apt install -y curl; \ + apt install -y curl ; \ if [ "$GPU" == "1" ] ; then \ apt install -y libnuma1 ocl-icd-libopencl1; \ case $INSTALL_DRIVER_VERSION in \ diff --git a/src/config.cpp b/src/config.cpp index 662c512e02..fdba29260d 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -151,7 +151,7 @@ Config& Config::parse(int argc, char** argv) { cxxopts::value()->default_value("CPU"), "TARGET_DEVICE") ("plugin_config", - "A dictionary of plugin configuration keys and their values, eg \"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"1\\\"}\". Default throughput streams for CPU and GPU are calculated by OpenVINO", + "A dictionary of plugin configuration keys and their values, eg \"{\\\"PERFORMANCE_HINT\\\": \\\"LATENCY\\\"}\". Default is PERFORMANCE_HINT set to THROUGHPUT mode. ", cxxopts::value(), "PLUGIN_CONFIG") ("stateful", diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index 33eda09755..47b354b75e 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -289,11 +289,31 @@ Status ModelConfig::parsePluginConfig(const rapidjson::Value& node) { for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { if (it->value.IsString()) { - pluginConfig[it->name.GetString()] = it->value.GetString(); + if (((it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS")) && (it->value.GetString() == std::string("CPU_THROUGHPUT_AUTO"))) || ((it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS")) && (it->value.GetString() == std::string("GPU_THROUGHPUT_AUTO")))) { + pluginConfig["PERFORMANCE_HINT"] = "THROUGHPUT"; + SPDLOG_WARN("{} plugin config key is deprecated. Use PERFORMANCE_HINT instead", it->name.GetString()); + } else { + if ((it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS")) || (it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS"))) { + pluginConfig["NUM_STREAMS"] = it->value.GetString(); + SPDLOG_WARN("{} plugin config key is deprecated. Use NUM_STREAMS instead", it->name.GetString()); + } else { + pluginConfig[it->name.GetString()] = it->value.GetString(); + } + } } else if (it->value.IsInt64()) { - pluginConfig[it->name.GetString()] = std::to_string(it->value.GetInt64()); + if (it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS") || it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS")) { + pluginConfig["NUM_STREAMS"] = std::to_string(it->value.GetInt64()); + SPDLOG_WARN("{} plugin config key is deprecated. Use NUM_STREAMS instead", it->name.GetString()); + } else { + pluginConfig[it->name.GetString()] = std::to_string(it->value.GetInt64()); + } } else if (it->value.IsDouble()) { - pluginConfig[it->name.GetString()] = std::to_string(it->value.GetDouble()); + if (it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS") || it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS")) { + pluginConfig["NUM_STREAMS"] = std::to_string(it->value.GetDouble()); + SPDLOG_WARN("{} plugin config key is deprecated. Use NUM_STREAMS instead", it->name.GetString()); + } else { + pluginConfig[it->name.GetString()] = std::to_string(it->value.GetDouble()); + } } else { return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; } diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index f0a14a9026..849299aed6 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -563,9 +563,8 @@ uint ModelInstance::getNumOfParallelInferRequestsUnbounded(const ModelConfig& mo // nireq is set globally for all models in ovms startup parameters return ovmsConfig.nireq(); } - std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS); try { - numberOfParallelInferRequests = compiledModel->get_property(key).as(); + numberOfParallelInferRequests = compiledModel->get_property(ov::optimal_number_of_infer_requests); } catch (const ov::Exception& ex) { SPDLOG_WARN("Failed to query OPTIMAL_NUMBER_OF_INFER_REQUESTS with error {}. Using 1 nireq.", ex.what()); numberOfParallelInferRequests = 1u; @@ -659,21 +658,11 @@ void ModelInstance::loadCompiledModelPtr(const plugin_config_t& pluginConfig) { plugin_config_t ModelInstance::prepareDefaultPluginConfig(const ModelConfig& config) { plugin_config_t pluginConfig = config.getPluginConfig(); - // Do not add CPU_THROUGHPUT_AUTO when performance hint is specified. - bool isPerformanceHintSpecified = pluginConfig.count("PERFORMANCE_HINT") > 0; - if (isPerformanceHintSpecified) { + // By default, set "PERFORMANCE_HINT" = "THROUGHPUT"; + if ((pluginConfig.count("NUM_STREAMS") == 1) || (pluginConfig.count("PERFORMANCE_HINT") == 1)) { return pluginConfig; - } - // For CPU and GPU, if user did not specify, calculate CPU_THROUGHPUT_STREAMS automatically - if (config.isSingleDeviceUsed("CPU")) { - if (pluginConfig.count("CPU_THROUGHPUT_STREAMS") == 0) { - pluginConfig["CPU_THROUGHPUT_STREAMS"] = "CPU_THROUGHPUT_AUTO"; - } - } - if (config.isSingleDeviceUsed("GPU")) { - if (pluginConfig.count("GPU_THROUGHPUT_STREAMS") == 0) { - pluginConfig["GPU_THROUGHPUT_STREAMS"] = "GPU_THROUGHPUT_AUTO"; - } + } else { + pluginConfig["PERFORMANCE_HINT"] = "THROUGHPUT"; } return pluginConfig; } @@ -720,16 +709,16 @@ Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { SPDLOG_LOGGER_INFO(modelmanager_logger, "OVMS set plugin settings key: {}; value: {};", key, value.as()); } - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); - std::vector supportedConfigKeys; + auto supportedPropertiesKey = ov::supported_properties; + std::vector supportedConfigKeys; try { - std::vector supportedConfigKeys2 = compiledModel->get_property(supportedConfigKey).as>(); + auto supportedConfigKeys2 = compiledModel->get_property(supportedPropertiesKey); supportedConfigKeys = std::move(supportedConfigKeys2); } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}; Error: {}", targetDevice, supportedConfigKey, e.what()); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}; Error: {}", targetDevice, supportedPropertiesKey.name(), e.what()); return StatusCode::OK; } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}", targetDevice, supportedConfigKey); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}", targetDevice, supportedPropertiesKey.name()); return StatusCode::OK; } SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging model:{}; version: {};target device: {}; CompiledModel configuration", getName(), getVersion(), targetDevice); @@ -890,14 +879,14 @@ Status ModelInstance::loadModelImpl(const ModelConfig& config, const DynamicMode Status ModelInstance::setCacheOptions(const ModelConfig& config) { if (!config.getCacheDir().empty()) { if (!config.isAllowCacheSetToTrue() && (config.isCustomLoaderRequiredToLoadModel() || config.anyShapeSetToAuto() || (config.getBatchingMode() == Mode::AUTO))) { - this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), ""}}); + this->ieCore.set_property(ov::cache_dir("")); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has disabled caching", this->getName()); this->cacheDisabled = true; } else if (config.isAllowCacheSetToTrue() && config.isCustomLoaderRequiredToLoadModel()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model: {} has allow cache set to true while using custom loader", this->getName()); return StatusCode::ALLOW_CACHE_WITH_CUSTOM_LOADER; } else { - this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), config.getCacheDir()}}); + this->ieCore.set_property(ov::cache_dir(config.getCacheDir())); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has enabled caching", this->getName()); } } diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index bb8c50a3d4..099a8084dd 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -116,17 +116,17 @@ void ModelManager::logPluginConfiguration() { auto availableDevices = ieCore->get_available_devices(); SPDLOG_LOGGER_INFO(modelmanager_logger, "Available devices for Open VINO: {}", joins(availableDevices, std::string(", "))); auto availablePlugins = availableDevices; - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); for (const auto& plugin : availablePlugins) { - std::vector supportedConfigKeys; + std::vector supportedConfigKeys; + auto supportedPropertiesKey = ov::supported_properties; try { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging plugin: {}; configuration", plugin); - std::vector supportedConfigKeys2 = ieCore->get_property(plugin, supportedConfigKey).as>(); + auto supportedConfigKeys2 = ieCore->get_property(plugin, supportedPropertiesKey); supportedConfigKeys = std::move(supportedConfigKeys2); } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", plugin, supportedConfigKey, e.what()); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", plugin, supportedPropertiesKey.name(), e.what()); } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", plugin, supportedConfigKey); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", plugin, supportedPropertiesKey.name()); } for (auto& key : supportedConfigKeys) { std::string value; diff --git a/src/ov_utils.cpp b/src/ov_utils.cpp index 996487e563..9fd533c496 100644 --- a/src/ov_utils.cpp +++ b/src/ov_utils.cpp @@ -85,16 +85,16 @@ std::optional getLayoutFromRTMap(const ov::RTMap& rtMap) { } static void insertSupportedKeys(std::set& aggregatedPluginSupportedConfigKeys, const std::string& pluginName, const ov::Core& ieCore) { - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); + auto prop = ov::supported_properties; try { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validating plugin: {}; configuration", pluginName); - std::vector pluginSupportedConfigKeys = ieCore.get_property(pluginName, supportedConfigKey).as>(); + std::vector pluginSupportedConfigKeys = ieCore.get_property(pluginName, prop); std::set pluginSupportedConfigKeysSet(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); aggregatedPluginSupportedConfigKeys.insert(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); } catch (std::exception& e) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", pluginName, supportedConfigKey, e.what()); + SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", pluginName, prop.name(), e.what()); } catch (...) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", pluginName, supportedConfigKey); + SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", pluginName, prop.name()); } } diff --git a/src/test/modelconfig_test.cpp b/src/test/modelconfig_test.cpp index f655c662ec..f185a12158 100644 --- a/src/test/modelconfig_test.cpp +++ b/src/test/modelconfig_test.cpp @@ -478,6 +478,51 @@ TEST(ModelConfig, plugin_config_invalid) { auto actualPluginConfig = config.getPluginConfig(); } +TEST(ModelConfig, plugin_config_legacy_cpu) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"CPU_THROUGHPUT_STREAMS\":\"CPU_THROUGHPUT_AUTO\"}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["PERFORMANCE_HINT"], "THROUGHPUT"); +} + +TEST(ModelConfig, plugin_config_legacy_cpu_num) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"CPU_THROUGHPUT_STREAMS\":5}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["NUM_STREAMS"], "5"); +} + +TEST(ModelConfig, plugin_config_legacy_cpu_str) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"CPU_THROUGHPUT_STREAMS\":\"5\"}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["NUM_STREAMS"], "5"); +} + +TEST(ModelConfig, plugin_config_legacy_gpu) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"GPU_THROUGHPUT_STREAMS\":\"GPU_THROUGHPUT_AUTO\"}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["PERFORMANCE_HINT"], "THROUGHPUT"); +} + +TEST(ModelConfig, plugin_config_legacy_gpu_num) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"GPU_THROUGHPUT_STREAMS\":5}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["NUM_STREAMS"], "5"); +} + TEST(ModelConfig, mappingInputs) { ovms::ModelConfig config; ovms::mapping_config_t mapping{ diff --git a/src/test/modelinstance_test.cpp b/src/test/modelinstance_test.cpp index f1577348d1..525c75d640 100644 --- a/src/test/modelinstance_test.cpp +++ b/src/test/modelinstance_test.cpp @@ -809,7 +809,7 @@ TEST(CpuThroughputStreamsNotSpecified, DefaultIsSetForCPU) { config.setTargetDevice("CPU"); config.setPluginConfig({}); ovms::plugin_config_t pluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); - EXPECT_EQ(pluginConfig.count("CPU_THROUGHPUT_STREAMS"), 1); + EXPECT_EQ(pluginConfig.count("PERFORMANCE_HINT"), 1); } TEST(CpuThroughputStreamsNotSpecified, NotSetForHeteroCPU) { @@ -844,3 +844,20 @@ TEST(CpuThroughputStreamsNotSpecified, NotSetWhenPerfHintSpecified) { pluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); EXPECT_EQ(pluginConfig.count("CPU_THROUGHPUT_STREAMS"), 0); } + +TEST(CpuThroughputNotSpecified, AffinityWithoutHint) { + ovms::ModelConfig config; + config.setPluginConfig({{"AFFINITY", "NUMA"}}); + ovms::plugin_config_t pluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); + EXPECT_EQ(pluginConfig.count("PERFORMANCE_HINT"), 1); + EXPECT_EQ(pluginConfig.count("AFFINITY"), 1); +} + +TEST(CpuThroughputNotSpecified, AffinityWithNumStreams) { + ovms::ModelConfig config; + config.setPluginConfig({{"NUM_STREAMS", "4"}, {"AFFINITY", "NUMA"}}); + ovms::plugin_config_t pluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); + EXPECT_EQ(pluginConfig.count("PERFORMANCE_HINT"), 0); + EXPECT_EQ(pluginConfig.count("AFFINITY"), 1); + EXPECT_EQ(pluginConfig.count("NUM_STREAMS"), 1); +} diff --git a/src/test/ov_utils_test.cpp b/src/test/ov_utils_test.cpp index 7d24cdc8e0..ec3c62563d 100644 --- a/src/test/ov_utils_test.cpp +++ b/src/test/ov_utils_test.cpp @@ -168,7 +168,7 @@ TEST(OVUtils, ValidatePluginConfigurationPositive) { std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); ovms::ModelConfig config; config.setTargetDevice("CPU"); - config.setPluginConfig({{"CPU_THROUGHPUT_STREAMS", "10"}}); + config.setPluginConfig({{"NUM_STREAMS", "10"}}); ovms::plugin_config_t supportedPluginConfig = ovms::ModelInstance::prepareDefaultPluginConfig(config); auto status = ovms::validatePluginConfiguration(supportedPluginConfig, "CPU", ieCore); EXPECT_TRUE(status.ok()); From a4e775bbb8681022df85f7f07b1928060066c2ce Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Fri, 18 Nov 2022 09:27:17 +0100 Subject: [PATCH 043/130] Add image name and ovms version postfix for debug builds --- Dockerfile.redhat | 3 +-- Dockerfile.ubuntu | 3 +-- Makefile | 4 ++++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 172e8e2e2d..9ef23aae87 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -129,7 +129,6 @@ RUN ./install_opencv.sh ARG ov_use_binary=1 ARG DLDT_PACKAGE_URL -ARG OPENVINO_NAME=${DLDT_PACKAGE_URL} ################### TAKE OPENVINO FROM A BINARY RELEASE - buildarg ov_use_binary=1 (DEFAULT) ########## WORKDIR / @@ -177,7 +176,7 @@ LABEL description=${PROJECT_NAME} # Set OVMS version strings RUN bash -c "sed -i -e 's|REPLACE_PROJECT_NAME|${PROJECT_NAME}|g' /ovms/src/version.hpp" -RUN bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}|g' /ovms/src/version.hpp" +RUN if [ "$build_type" == "dbg" ] ; then bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}-debug|g' /ovms/src/version.hpp" ; else bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}|g' /ovms/src/version.hpp" ; fi ; RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; sed -i -e "s#REPLACE_OPENVINO_NAME#`find /opt/intel/ -maxdepth 1 -mindepth 1 -type d | grep openvino | grep -Eo '[0-9]{4}.[0-9].[0-9].[0-9]+.[^_]+'`#g" /ovms/src/version.hpp ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 03b2a650a6..a1b184a3c7 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -138,7 +138,6 @@ RUN ./install_opencv.sh ARG ov_use_binary=1 ARG DLDT_PACKAGE_URL -ARG OPENVINO_NAME=${DLDT_PACKAGE_URL} ARG ov_source_branch=master ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ @@ -219,7 +218,7 @@ LABEL description=${PROJECT_NAME} # Set OVMS version strings RUN bash -c "sed -i -e 's|REPLACE_PROJECT_NAME|${PROJECT_NAME}|g' /ovms/src/version.hpp" -RUN bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}|g' /ovms/src/version.hpp" +RUN if [ "$build_type" == "dbg" ] ; then bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}-debug|g' /ovms/src/version.hpp" ; else bash -c "sed -i -e 's|REPLACE_PROJECT_VERSION|${PROJECT_VERSION}|g' /ovms/src/version.hpp" ; fi ; RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; sed -i -e "s#REPLACE_OPENVINO_NAME#`find /opt/intel/ -maxdepth 1 -mindepth 1 -type d | grep openvino | grep -Eo '[0-9]{4}.[0-9].[0-9].[0-9]+.[^_]+'`#g" /ovms/src/version.hpp RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; sed -i -e "s#REPLACE_OPENVINO_NAME#`git --git-dir /openvino/.git log -n 1 | head -n 1 | cut -d' ' -f2 | head -c 12`#g" /ovms/src/version.hpp ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/:/opt/opencv/lib/:/opt/intel/openvino/runtime/3rdparty/tbb/lib/ diff --git a/Makefile b/Makefile index 432f666d27..93f3a228aa 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,10 @@ ifeq ($(BASE_OS),redhat) endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server +ifeq ($(BAZEL_BUILD_TYPE),dbg) + OVMS_CPP_DOCKER_IMAGE:=$(OVMS_CPP_DOCKER_IMAGE)-dbg +endif + OVMS_CPP_IMAGE_TAG ?= latest PRODUCT_NAME = "OpenVINO Model Server" From 01c5d695a6e96da7a2f727db5989c4dde7479d54 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Fri, 18 Nov 2022 13:02:16 +0100 Subject: [PATCH 044/130] update OV backend to 17th Nov version (#1531) * update OV backend to 17th Nov version --- Dockerfile.redhat | 8 ++++++++ Dockerfile.ubuntu | 1 + DockerfileMakePackage | 2 ++ Makefile | 6 +++--- release_files/Dockerfile.ubuntu | 3 +-- src/test/model_cache_test.cpp | 4 ++-- tests/requirements.txt | 3 ++- 7 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 9ef23aae87..437ef17977 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -73,6 +73,14 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n vim \ xz && \ yum clean all + +# Build and install pugixml +RUN git clone -b v1.13 https://github.com/zeux/pugixml && \ + cd pugixml && \ + cmake -DBUILD_SHARED_LIBS=ON && \ + make all && \ + cp libpugixml.so* /usr/lib64/ + # Set up Bazel ENV BAZEL_VERSION 5.3.1 WORKDIR /bazel diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index a1b184a3c7..1f2bd676cf 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -54,6 +54,7 @@ RUN apt update && apt install -y \ gdb \ git \ lcov \ + libpugixml1v5 \ libusb-dev \ libusb-1.0-0-dev \ libcurl4-openssl-dev \ diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 1a60b884e6..8a086f68b7 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -65,6 +65,8 @@ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp /opt/open RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; if [ -f /opt/intel/openvino/docs/licensing/EULA.txt ] ; then true ; else exit 0; fi ; cp /opt/intel/openvino/docs/licensing/EULA.txt /ovms/release_files/thirdparty-licenses/openvino.LICENSE.txt; RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cp /openvino/LICENSE /ovms/release_files/thirdparty-licenses/openvino.LICENSE.txt; +RUN if [ "$BASE_OS" == "redhat" ] ; then true ; else exit 0 ; fi ; cp /usr/lib64/libpugixml.so* /ovms_release/lib/ + RUN find /ovms/bazel-bin/src -name 'ovms' -type f -exec cp -v {} /ovms_release/bin \; WORKDIR /ovms_release/bin RUN patchelf --remove-rpath ./ovms && patchelf --set-rpath '$ORIGIN/../lib/' ./ovms diff --git a/Makefile b/Makefile index 93f3a228aa..eae22dcc8e 100644 --- a/Makefile +++ b/Makefile @@ -73,14 +73,14 @@ ifeq ($(BASE_OS),ubuntu) BASE_OS_TAG=$(BASE_OS_TAG_UBUNTU) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) INSTALL_DRIVER_VERSION ?= "21.48.21782" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_ubuntu20_2022.2.0.7713.af16ea1d79a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.8693.0f647e6b75a_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "21.38.21026" - DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.2/linux/l_openvino_toolkit_rhel8_2022.2.0.7713.af16ea1d79a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.8693.0f647e6b75a_x86_64.tgz endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server @@ -91,7 +91,7 @@ endif OVMS_CPP_IMAGE_TAG ?= latest PRODUCT_NAME = "OpenVINO Model Server" -PRODUCT_VERSION ?= "2022.2" +PRODUCT_VERSION ?= "2022.3" OVMS_CPP_CONTAINTER_NAME ?= server-test$(shell date +%Y-%m-%d-%H.%M.%S) OVMS_CPP_CONTAINTER_PORT ?= 9178 diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 2d42c9387d..1e6c5776df 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -83,7 +83,7 @@ RUN set -e ; \ apt update -y ; \ apt install -y curl ; \ if [ "$GPU" == "1" ] ; then \ - apt install -y libnuma1 ocl-icd-libopencl1; \ + apt install -y libnuma1 ocl-icd-libopencl1 libpugixml1v5 ; \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \ @@ -119,7 +119,6 @@ RUN set -e ; \ cd /drivers/${INSTALL_DRIVER_VERSION} && \ dpkg-scanpackages . > Packages && \ cd - ; \ - echo "deb [trusted=yes arch=amd64] file:/drivers/${INSTALL_DRIVER_VERSION} ./" > /etc/apt/sources.list.d/intel-graphics-${INSTALL_DRIVER_VERSION}.list ; \ apt-get update && \ apt-get install -y \ diff --git a/src/test/model_cache_test.cpp b/src/test/model_cache_test.cpp index 3aefa0c992..313171503d 100644 --- a/src/test/model_cache_test.cpp +++ b/src/test/model_cache_test.cpp @@ -238,7 +238,7 @@ TEST_F(ModelCacheTest, LayoutChangeDoesImpactCache) { ASSERT_EQ(currentCacheFileCount, lastCachedFileCount); } -TEST_F(ModelCacheTest, PluginConfigChangeDoesImpactCache) { +TEST_F(ModelCacheTest, PluginConfigChangeDoesNotImpactCache) { this->prepareImageModelCachedRun(); size_t currentCacheFileCount = this->getCachedFileCount(); @@ -250,7 +250,7 @@ TEST_F(ModelCacheTest, PluginConfigChangeDoesImpactCache) { size_t lastCachedFileCount = currentCacheFileCount; currentCacheFileCount = this->getCachedFileCount(); - ASSERT_GT(currentCacheFileCount, lastCachedFileCount); + ASSERT_EQ(currentCacheFileCount, lastCachedFileCount); manager.reset(); manager = std::make_unique(modelCacheDirectory); diff --git a/tests/requirements.txt b/tests/requirements.txt index 4ffb229bff..5eaf07c2ad 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -8,7 +8,8 @@ paramiko==2.11.0 psutil==5.7.3 pytest==5.3.5 pytest-json==0.4.0 -tensorflow-serving-api==2.7.0 +tensorflow-serving-api==2.10.0 +tensorflow==2.10.0 retry==0.9.2 bandit protobuf>=3.18,<4.0.0 From fca2b71944c52c965f78d89677e5466f549b62b3 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Mon, 21 Nov 2022 10:28:08 +0100 Subject: [PATCH 045/130] add libpugixml1v5 in all versions of the image (#1533) --- release_files/Dockerfile.ubuntu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 1e6c5776df..f635e45de2 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -81,9 +81,9 @@ COPY drivers /drivers RUN set -e ; \ set -x ; \ apt update -y ; \ - apt install -y curl ; \ + apt install -y curl libpugixml1v5 ; \ if [ "$GPU" == "1" ] ; then \ - apt install -y libnuma1 ocl-icd-libopencl1 libpugixml1v5 ; \ + apt install -y libnuma1 ocl-icd-libopencl1 ; \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \ From 1aa5032b97368c3d5f84b5b3edc465d8efb84642 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Mon, 21 Nov 2022 22:08:11 +0100 Subject: [PATCH 046/130] Add support for binary outputs in kfs rest (#1529) --- src/http_rest_api_handler.cpp | 44 +++---- src/http_rest_api_handler.hpp | 16 ++- src/http_server.cpp | 7 +- src/kfs_frontend/kfs_utils.cpp | 5 +- src/rest_utils.cpp | 51 +++++--- src/rest_utils.hpp | 3 +- src/test/kfs_rest_test.cpp | 28 +++-- src/test/metrics_flow_test.cpp | 12 +- src/test/rest_utils_test.cpp | 213 ++++++++++++++++++++++++++++----- 9 files changed, 290 insertions(+), 89 deletions(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index efe0c6e6fa..1a8cd377ca 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -129,12 +129,12 @@ Status HttpRestApiHandler::parseModelVersion(std::string& model_version_str, std return StatusCode::OK; } -void HttpRestApiHandler::registerHandler(RequestType type, std::function f) { +void HttpRestApiHandler::registerHandler(RequestType type, std::function f) { handlers[type] = f; } void HttpRestApiHandler::registerAll() { - registerHandler(Predict, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(Predict, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { if (request_components.processing_method == "predict") { return processPredictRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, request_body, &response); @@ -144,39 +144,39 @@ void HttpRestApiHandler::registerAll() { } }); - registerHandler(GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + registerHandler(GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) { return processModelMetadataRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, &response); }); - registerHandler(GetModelStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + registerHandler(GetModelStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) { return processModelStatusRequest(request_components.model_name, request_components.model_version, request_components.model_version_label, &response); }); - registerHandler(ConfigReload, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(ConfigReload, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processConfigReloadRequest(response, this->modelManager); }); - registerHandler(ConfigStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(ConfigStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processConfigStatusRequest(response, this->modelManager); }); - registerHandler(KFS_GetModelReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(KFS_GetModelReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processModelReadyKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(KFS_GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processModelMetadataKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_Infer, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { - return processInferKFSRequest(request_components, response, request_body); + registerHandler(KFS_Infer, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { + return processInferKFSRequest(request_components, response, request_body, response_components.inferenceHeaderContentLength); }); - registerHandler(KFS_GetServerReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(KFS_GetServerReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processServerReadyKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetServerLive, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(KFS_GetServerLive, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processServerLiveKFSRequest(request_components, response, request_body); }); - registerHandler(KFS_GetServerMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(KFS_GetServerMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processServerMetadataKFSRequest(request_components, response, request_body); }); - registerHandler(Metrics, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + registerHandler(Metrics, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, HttpResponseComponents& response_components) -> Status { return processMetrics(request_components, response, request_body); }); } @@ -497,7 +497,7 @@ Status HttpRestApiHandler::prepareGrpcRequest(const std::string modelName, const return StatusCode::OK; } -Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { +Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::optional& inferenceHeaderContentLength) { Timer timer; timer.start(TOTAL); ServableMetricReporter* reporter = nullptr; @@ -526,11 +526,11 @@ Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& r } std::string output; google::protobuf::util::JsonPrintOptions opts_out; - status = ovms::makeJsonFromPredictResponse(grpc_response, &output); + status = ovms::makeJsonFromPredictResponse(grpc_response, &output, inferenceHeaderContentLength); if (!status.ok()) { return status; } - response = output; + response = std::move(output); timer.stop(TOTAL); double totalTime = timer.elapsed(TOTAL); SPDLOG_DEBUG("Total REST request processing time: {} ms", totalTime / 1000); @@ -541,11 +541,12 @@ Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& r Status HttpRestApiHandler::dispatchToProcessor( const std::string& request_body, std::string* response, - const HttpRequestComponents& request_components) { + const HttpRequestComponents& request_components, + HttpResponseComponents& response_components) { auto handler = handlers.find(request_components.type); if (handler != handlers.end()) { - return handler->second(request_components, *response, request_body); + return handler->second(request_components, *response, request_body, response_components); } else { return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; } @@ -774,7 +775,8 @@ Status HttpRestApiHandler::processRequest( const std::string_view request_path, const std::string& request_body, std::vector>* headers, - std::string* response) { + std::string* response, + HttpResponseComponents& responseComponents) { std::smatch sm; std::string request_path_str(request_path); @@ -792,7 +794,7 @@ Status HttpRestApiHandler::processRequest( if (!status.ok()) return status; - return dispatchToProcessor(request_body, response, requestComponents); + return dispatchToProcessor(request_body, response, requestComponents, responseComponents); } Status HttpRestApiHandler::processPredictRequest( diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 4649026d21..24c10dcb16 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -61,6 +61,10 @@ struct HttpRequestComponents { std::optional inferenceHeaderContentLength; }; +struct HttpResponseComponents { + std::optional inferenceHeaderContentLength; +}; + class HttpRestApiHandler { public: static const std::string predictionRegexExp; @@ -94,13 +98,14 @@ class HttpRestApiHandler { static std::string preprocessInferRequest(std::string request_body); static Status prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::KFSRequest& grpc_request, const std::optional& inferenceHeaderContentLength = {}); - void registerHandler(RequestType type, std::function); + void registerHandler(RequestType type, std::function); void registerAll(); Status dispatchToProcessor( const std::string& request_body, std::string* response, - const HttpRequestComponents& request_components); + const HttpRequestComponents& request_components, + HttpResponseComponents& response_components); /** * @brief Process Request @@ -118,7 +123,8 @@ class HttpRestApiHandler { const std::string_view request_path, const std::string& request_body, std::vector>* headers, - std::string* response); + std::string* response, + HttpResponseComponents& responseComponents); /** * @brief Process predict request @@ -191,7 +197,7 @@ class HttpRestApiHandler { Status processConfigStatusRequest(std::string& response, ModelManager& manager); Status processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); Status processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); - Status processInferKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); + Status processInferKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::optional& inferenceHeaderContentLength); Status processMetrics(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); Status processServerReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); @@ -214,7 +220,7 @@ class HttpRestApiHandler { const std::regex metricsRegex; - std::map> handlers; + std::map> handlers; int timeout_in_ms; ovms::Server& ovmsServer; diff --git a/src/http_server.cpp b/src/http_server.cpp index 9652aee899..c1f838b265 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -194,11 +194,16 @@ class RestApiRequestDispatcher { req->http_method(), req->uri_path(), body.size()); - const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output); + HttpResponseComponents responseComponents; + const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output, responseComponents); if (!status.ok() && output.empty()) { output.append("{\"error\": \"" + status.string() + "\"}"); } const auto http_status = http(status); + if (responseComponents.inferenceHeaderContentLength.has_value()) { + std::pair header{"Inference-Header-Content-Length", std::to_string(responseComponents.inferenceHeaderContentLength.value())}; + headers.emplace_back(header); + } for (const auto& kv : headers) { req->OverwriteResponseHeader(kv.first, kv.second); } diff --git a/src/kfs_frontend/kfs_utils.cpp b/src/kfs_frontend/kfs_utils.cpp index 74a36bc690..043da8d100 100644 --- a/src/kfs_frontend/kfs_utils.cpp +++ b/src/kfs_frontend/kfs_utils.cpp @@ -61,9 +61,8 @@ size_t KFSDataTypeSize(const KFSDataType& datatype) { {"INT64", 8}, {"FP16", 2}, {"FP32", 4}, - {"FP64", 8} - // {"BYTES", }, - }; + {"FP64", 8}, + {"BYTES", 1}}; auto it = datatypeSizeMap.find(datatype); if (it == datatypeSizeMap.end()) { return 0; diff --git a/src/rest_utils.cpp b/src/rest_utils.cpp index 59750f1278..03990c691f 100644 --- a/src/rest_utils.cpp +++ b/src/rest_utils.cpp @@ -230,8 +230,8 @@ static Status parseResponseParameters(const ::KFSResponse& response_proto, rapid return StatusCode::OK; } -static Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer) { - if (output.parameters_size() > 0) { +static Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer, int bytesOutputSize) { + if (output.parameters_size() > 0 || bytesOutputSize > 0) { writer.Key("parameters"); writer.StartObject(); @@ -251,6 +251,10 @@ static Status parseOutputParameters(const inference::ModelInferResponse_InferOut break; // return param error } } + if (bytesOutputSize > 0) { + writer.Key("binary_data_size"); + writer.Int(bytesOutputSize); + } writer.EndObject(); } @@ -275,14 +279,13 @@ static void fillTensorDataWithFloatValuesFromRawContents(const ::KFSResponse& re writer.Double(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); } -static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer) { +static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer, std::string& bytesOutputsBuffer) { writer.Key("outputs"); writer.StartArray(); bool seekDataInValField = false; if (response_proto.raw_output_contents_size() == 0) seekDataInValField = true; - int tensor_it = 0; for (const auto& tensor : response_proto.outputs()) { size_t dataTypeSize = KFSDataTypeSize(tensor.datatype()); @@ -294,7 +297,6 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett if (!seekDataInValField && (response_proto.raw_output_contents(tensor_it).size() != expectedContentSize)) return StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE; - writer.StartObject(); writer.Key("name"); writer.String(tensor.name().c_str()); @@ -306,14 +308,10 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett writer.EndArray(); writer.Key("datatype"); writer.String(tensor.datatype().c_str()); - - auto status = parseOutputParameters(tensor, writer); - if (!status.ok()) { - return status; + if (tensor.datatype() != "BYTES") { + writer.Key("data"); + writer.StartArray(); } - - writer.Key("data"); - writer.StartArray(); if (tensor.datatype() == "FP32") { if (seekDataInValField) { auto status = checkValField(tensor.contents().fp32_contents_size(), expectedElementsNumber); @@ -426,10 +424,27 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett } else { fillTensorDataWithFloatValuesFromRawContents(response_proto, tensor_it, writer); } + } else if (tensor.datatype() == "BYTES") { + if (seekDataInValField) { + auto status = checkValField(tensor.contents().bytes_contents_size(), expectedElementsNumber); + if (!status.ok()) + return status; + for (auto& bytes : tensor.contents().bytes_contents()) { + bytesOutputsBuffer.append(bytes); + } + } else { + bytesOutputsBuffer.append((char*)response_proto.raw_output_contents(tensor_it).data(), response_proto.raw_output_contents(tensor_it).size()); + } } else { return StatusCode::REST_UNSUPPORTED_PRECISION; } - writer.EndArray(); + if (tensor.datatype() != "BYTES") { + writer.EndArray(); + } + auto status = parseOutputParameters(tensor, writer, bytesOutputsBuffer.size()); + if (!status.ok()) { + return status; + } writer.EndObject(); tensor_it++; } @@ -439,7 +454,8 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett Status makeJsonFromPredictResponse( const ::KFSResponse& response_proto, - std::string* response_json) { + std::string* response_json, + std::optional& inferenceHeaderContentLength) { Timer timer; using std::chrono::microseconds; timer.start(CONVERT); @@ -469,13 +485,18 @@ Status makeJsonFromPredictResponse( return StatusCode::REST_PROTO_TO_STRING_ERROR; } - status = parseOutputs(response_proto, writer); + std::string binaryOutputsBuffer; + status = parseOutputs(response_proto, writer, binaryOutputsBuffer); if (!status.ok()) { return status; } writer.EndObject(); response_json->assign(buffer.GetString()); + if (binaryOutputsBuffer.size() > 0) { + inferenceHeaderContentLength = response_json->length(); + } + response_json->append(binaryOutputsBuffer); timer.stop(CONVERT); SPDLOG_DEBUG("GRPC to HTTP response conversion: {:.3f} ms", timer.elapsed(CONVERT) / 1000); diff --git a/src/rest_utils.hpp b/src/rest_utils.hpp index 91d115ec91..1c2932ea27 100644 --- a/src/rest_utils.hpp +++ b/src/rest_utils.hpp @@ -33,7 +33,8 @@ Status makeJsonFromPredictResponse( Status makeJsonFromPredictResponse( const ::KFSResponse& response_proto, - std::string* response_json); + std::string* response_json, + std::optional& inferenceHeaderContentLength); Status decodeBase64(std::string& bytes, std::string& decodedBytes); diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index e3c0dd711a..f6e132a2c4 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -219,13 +219,14 @@ TEST_F(HttpRestApiHandlerTest, dispatchMetadata) { ovms::HttpRequestComponents comp; int c = 0; - handler->registerHandler(KFS_GetModelMetadata, [&](const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + handler->registerHandler(KFS_GetModelMetadata, [&](const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components) { c++; return ovms::StatusCode::OK; }); comp.type = ovms::KFS_GetModelMetadata; std::string discard; - handler->dispatchToProcessor(std::string(), &discard, comp); + ovms::HttpResponseComponents responseComponents; + handler->dispatchToProcessor(std::string(), &discard, comp, responseComponents); ASSERT_EQ(c, 1); } @@ -235,13 +236,14 @@ TEST_F(HttpRestApiHandlerTest, dispatchReady) { ovms::HttpRequestComponents comp; int c = 0; - handler->registerHandler(KFS_GetModelReady, [&](const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + handler->registerHandler(KFS_GetModelReady, [&](const ovms::HttpRequestComponents& request_components, std::string& response, const std::string& request_body, ovms::HttpResponseComponents& response_components) { c++; return ovms::StatusCode::OK; }); comp.type = ovms::KFS_GetModelReady; std::string discard; - handler->dispatchToProcessor(std::string(), &discard, comp); + ovms::HttpResponseComponents responseComponents; + handler->dispatchToProcessor(std::string(), &discard, comp, responseComponents); ASSERT_EQ(c, 1); } @@ -252,7 +254,8 @@ TEST_F(HttpRestApiHandlerTest, modelMetadataRequest) { handler->parseRequestComponents(comp, "GET", request); std::string response; - handler->dispatchToProcessor(std::string(), &response, comp); + ovms::HttpResponseComponents responseComponents; + handler->dispatchToProcessor(std::string(), &response, comp, responseComponents); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -278,7 +281,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequestWithMultidimensionalMatrix) { ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request), ovms::StatusCode::OK); std::string response; - ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp), ovms::StatusCode::OK); + ovms::HttpResponseComponents responseComponents; + ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp, responseComponents), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -296,7 +300,8 @@ TEST_F(HttpRestApiHandlerTest, inferRequest) { ASSERT_EQ(handler->parseRequestComponents(comp, "POST", request), ovms::StatusCode::OK); std::string response; - ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp), ovms::StatusCode::OK); + ovms::HttpResponseComponents responseComponents; + ASSERT_EQ(handler->dispatchToProcessor(request_body, &response, comp, responseComponents), ovms::StatusCode::OK); rapidjson::Document doc; doc.Parse(response.c_str()); @@ -590,7 +595,8 @@ TEST_F(HttpRestApiHandlerTest, serverReady) { comp.type = ovms::KFS_GetServerReady; std::string request; std::string response; - ovms::Status status = handler->dispatchToProcessor(request, &response, comp); + ovms::HttpResponseComponents responseComponents; + ovms::Status status = handler->dispatchToProcessor(request, &response, comp, responseComponents); ASSERT_EQ(status, ovms::StatusCode::OK); } @@ -600,7 +606,8 @@ TEST_F(HttpRestApiHandlerTest, serverLive) { comp.type = ovms::KFS_GetServerLive; std::string request; std::string response; - ovms::Status status = handler->dispatchToProcessor(request, &response, comp); + ovms::HttpResponseComponents responseComponents; + ovms::Status status = handler->dispatchToProcessor(request, &response, comp, responseComponents); ASSERT_EQ(status, ovms::StatusCode::OK); } @@ -610,7 +617,8 @@ TEST_F(HttpRestApiHandlerTest, serverMetadata) { comp.type = ovms::KFS_GetServerMetadata; std::string request; std::string response; - ovms::Status status = handler->dispatchToProcessor(request, &response, comp); + ovms::HttpResponseComponents responseComponents; + ovms::Status status = handler->dispatchToProcessor(request, &response, comp, responseComponents); rapidjson::Document doc; doc.Parse(response.c_str()); diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index 97e048b14c..0e58d474f9 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -458,28 +458,32 @@ TEST_F(MetricFlowTest, RestModelInfer) { components.model_name = modelName; std::string request = R"({"inputs":[{"name":"b","shape":[1,10],"datatype":"FP32","data":[1,2,3,4,5,6,7,8,9,10]}], "parameters":{"binary_data_output":true}})"; std::string response; - ASSERT_EQ(handler.processInferKFSRequest(components, response, request), ovms::StatusCode::OK); + std::optional inferenceHeaderContentLength; + ASSERT_EQ(handler.processInferKFSRequest(components, response, request, inferenceHeaderContentLength), ovms::StatusCode::OK); } for (int i = 0; i < numberOfFailedRequests; i++) { components.model_name = modelName; std::string request = R"({{"inputs":[{"name":"b","shape":[1,10],"datatype":"FP32","data":[1,2,3,4,5,6,7,8,9]}], "parameters":{"binary_data_output":true}})"; std::string response; - ASSERT_EQ(handler.processInferKFSRequest(components, response, request), ovms::StatusCode::JSON_INVALID); + std::optional inferenceHeaderContentLength; + ASSERT_EQ(handler.processInferKFSRequest(components, response, request, inferenceHeaderContentLength), ovms::StatusCode::JSON_INVALID); } for (int i = 0; i < numberOfSuccessRequests; i++) { components.model_name = dagName; std::string request = R"({"inputs":[{"name":"b","shape":[3,1,10],"datatype":"FP32","data":[1,2,3,4,5,6,7,8,9,10,1,2,3,4,5,6,7,8,9,10,1,2,3,4,5,6,7,8,9,10]}], "parameters":{"binary_data_output":true}})"; std::string response; - ASSERT_EQ(handler.processInferKFSRequest(components, response, request), ovms::StatusCode::OK); + std::optional inferenceHeaderContentLength; + ASSERT_EQ(handler.processInferKFSRequest(components, response, request, inferenceHeaderContentLength), ovms::StatusCode::OK); } for (int i = 0; i < numberOfFailedRequests; i++) { components.model_name = dagName; std::string request = R"({{"inputs":[{"name":"b","shape":[3,1,10],"datatype":"FP32","data":[1,2,3,4,5,6,7,8,9,10,1,2,3,4,5,6,7,8,9,10,1,2,3,4,5,6,7,8,9]}], "parameters":{"binary_data_output":true}})"; std::string response; - ASSERT_EQ(handler.processInferKFSRequest(components, response, request), ovms::StatusCode::JSON_INVALID); + std::optional inferenceHeaderContentLength; + ASSERT_EQ(handler.processInferKFSRequest(components, response, request, inferenceHeaderContentLength), ovms::StatusCode::JSON_INVALID); } checkRequestsCounter(server.collect(), METRIC_NAME_REQUESTS_SUCCESS, modelName, 1, "REST", "ModelInfer", "KServe", dynamicBatch * numberOfSuccessRequests + numberOfSuccessRequests); // ran by demultiplexer + real request diff --git a/src/test/rest_utils_test.cpp b/src/test/rest_utils_test.cpp index ce3220362d..cc35b6f474 100644 --- a/src/test/rest_utils_test.cpp +++ b/src/test/rest_utils_test.cpp @@ -648,6 +648,11 @@ class KFSMakeJsonFromPredictResponseRawTest : public ::testing::Test { KFSResponse proto; std::string json; KFSTensorOutputProto *output1, *output2; + std::optional inferenceHeaderContentLength; + const float data1[8] = {5.0f, 10.0f, -3.0f, 2.5f, + 9.0f, 55.5f, -0.5f, -1.5f}; + const int8_t data2[10] = {5, 2, 3, 8, -2, + -100, 0, 125, 4, -1}; void SetUp() override { proto.set_model_name("model"); @@ -662,16 +667,12 @@ class KFSMakeJsonFromPredictResponseRawTest : public ::testing::Test { output1->set_name("output1"); output2->set_name("output2"); - float data1[8] = {5.0f, 10.0f, -3.0f, 2.5f, - 9.0f, 55.5f, -0.5f, -1.5f}; auto* output1_contents = proto.add_raw_output_contents(); output1_contents->assign(reinterpret_cast(data1), 8 * sizeof(float)); output1->mutable_shape()->Add(2); output1->mutable_shape()->Add(1); output1->mutable_shape()->Add(4); - int8_t data2[10] = {5, 2, 3, 8, -2, - -100, 0, 125, 4, -1}; auto* output2_contents = proto.add_raw_output_contents(); output2_contents->assign(reinterpret_cast(data2), 10 * sizeof(int8_t)); output2->mutable_shape()->Add(2); @@ -682,11 +683,13 @@ class KFSMakeJsonFromPredictResponseRawTest : public ::testing::Test { TEST_F(KFSMakeJsonFromPredictResponseRawTest, CannotConvertInvalidPrecision) { output1->set_datatype("INVALID"); proto.mutable_raw_output_contents()->Clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::REST_UNSUPPORTED_PRECISION); + EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::REST_UNSUPPORTED_PRECISION); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); } TEST_F(KFSMakeJsonFromPredictResponseRawTest, Positive) { - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -704,19 +707,56 @@ TEST_F(KFSMakeJsonFromPredictResponseRawTest, Positive) { })"); } +TEST_F(KFSMakeJsonFromPredictResponseRawTest, Positive_binary) { + int output2DataSize = 10 * sizeof(int8_t); + output2->set_datatype("BYTES"); + output2->mutable_shape()->Clear(); + output2->mutable_shape()->Add(1); + output2->mutable_shape()->Add(output2DataSize); + + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output1", + "shape": [2, 1, 4], + "datatype": "FP32", + "data": [5.0, 10.0, -3.0, 2.5, 9.0, 55.5, -0.5, -1.5] + }, { + "name": "output2", + "shape": [1, 10], + "datatype": "BYTES", + "parameters": { + "binary_data_size": 10 + } + }] +})"; + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + output2DataSize); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + for (int i = 0; i < output2DataSize; i++) { + EXPECT_EQ((int)json.substr(inferenceHeaderContentLength.value())[i], data2[i]); + } +} + TEST_F(KFSMakeJsonFromPredictResponseRawTest, EmptyRawOutputContentsError) { proto.mutable_raw_output_contents()->Clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::REST_SERIALIZE_NO_DATA); + EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::REST_SERIALIZE_NO_DATA); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); } TEST_F(KFSMakeJsonFromPredictResponseRawTest, InvalidTensorContentSizeError) { proto.mutable_raw_output_contents(0)->assign("\xFF\xFF\x55\x55", 4); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE); + EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); } TEST_F(KFSMakeJsonFromPredictResponseRawTest, ErrorWhenNoOutputs) { proto.mutable_outputs()->Clear(); - EXPECT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::REST_PROTO_TO_STRING_ERROR); + EXPECT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::REST_PROTO_TO_STRING_ERROR); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); } class KFSMakeJsonFromPredictResponsePrecisionTest : public ::testing::Test { @@ -724,6 +764,7 @@ class KFSMakeJsonFromPredictResponsePrecisionTest : public ::testing::Test { KFSResponse proto; std::string json; KFSTensorOutputProto* output; + std::optional inferenceHeaderContentLength; void SetUp() override { proto.set_model_name("model"); @@ -741,7 +782,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Float) { output->set_datatype("FP32"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(float)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -759,7 +801,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Double) { output->set_datatype("FP64"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(double)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -777,7 +820,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int32) { output->set_datatype("INT32"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(int32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -795,7 +839,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int16) { output->set_datatype("INT16"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(int16_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -813,7 +858,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int8) { output->set_datatype("INT8"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(int8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -831,7 +877,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint8) { output->set_datatype("UINT8"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(uint8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -849,7 +896,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int64) { output->set_datatype("INT64"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(int64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -867,7 +915,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint32) { output->set_datatype("UINT32"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(uint32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -885,7 +934,8 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { output->set_datatype("UINT64"); auto* output_contents = proto.add_raw_output_contents(); output_contents->assign(reinterpret_cast(&data), sizeof(uint64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -898,11 +948,68 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, BYTES_1) { + int8_t data = -53; + output->set_datatype("BYTES"); + auto* output_contents = proto.add_raw_output_contents(); + output_contents->assign(reinterpret_cast(&data), sizeof(int8_t)); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "BYTES", + "parameters": { + "binary_data_size": 1 + } + }] +})"; + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + sizeof(int8_t)); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + EXPECT_EQ((int)json.substr(inferenceHeaderContentLength.value())[0], data); +} + +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, BYTES_2) { + int8_t data[] = {-53, 1, 2, 3}; + int dataSize = 4; + output->set_datatype("BYTES"); + output->mutable_shape()->Clear(); + output->mutable_shape()->Add(1); + output->mutable_shape()->Add(dataSize); + auto* output_contents = proto.add_raw_output_contents(); + output_contents->assign(reinterpret_cast(&data), dataSize); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 4], + "datatype": "BYTES", + "parameters": { + "binary_data_size": 4 + } + }] +})"; + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + dataSize * sizeof(int8_t)); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + for (int i = 0; i < dataSize; i++) { + EXPECT_EQ((int)json.substr(inferenceHeaderContentLength.value())[i], data[i]); + } +} + class KFSMakeJsonFromPredictResponseValTest : public ::testing::Test { protected: KFSResponse proto; std::string json; KFSTensorOutputProto *single_uint64_val, *two_uint32_vals; + std::optional inferenceHeaderContentLength; void SetUp() override { proto.set_model_name("model"); @@ -929,8 +1036,52 @@ class KFSMakeJsonFromPredictResponseValTest : public ::testing::Test { } }; +TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positive_BYTES) { + KFSTensorOutputProto* bytes_val_proto = proto.add_outputs(); + bytes_val_proto->set_name("bytes_val_proto"); + bytes_val_proto->set_datatype("BYTES"); + int dataSize = 10; + bytes_val_proto->mutable_shape()->Add(dataSize); + for (uint8_t i = 0; i < dataSize; i++) { + auto bytes_val = bytes_val_proto->mutable_contents()->mutable_bytes_contents()->Add(); + *bytes_val = i; + } + + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "single_uint64_val", + "shape": [1], + "datatype": "UINT64", + "data": [5000000000] + }, { + "name": "two_uint32_vals", + "shape": [2], + "datatype": "UINT32", + "data": [4000000000, 1] + }, { + "name": "bytes_val_proto", + "shape": [10], + "datatype": "BYTES", + "parameters": { + "binary_data_size": 10 + } + }] +})"; + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + dataSize * sizeof(int8_t)); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + for (uint8_t i = 0; i < dataSize; i++) { + EXPECT_EQ((uint8_t)json.substr(inferenceHeaderContentLength.value())[i], i); + } +} + TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positive) { - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -950,7 +1101,8 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positi TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_OptionalModelVersion) { proto.set_model_version("version"); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -974,7 +1126,8 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option (*protoParameters)["key"].set_string_param("param"); auto outputParameters = single_uint64_val->mutable_parameters(); (*outputParameters)["key"].set_string_param("param"); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -985,10 +1138,10 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option "name": "single_uint64_val", "shape": [1], "datatype": "UINT64", + "data": [5000000000], "parameters": { "key": "param" - }, - "data": [5000000000] + } }, { "name": "two_uint32_vals", "shape": [2], @@ -1003,7 +1156,8 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option (*protoParameters)["key"].set_int64_param(100); auto outputParameters = single_uint64_val->mutable_parameters(); (*outputParameters)["key"].set_int64_param(100); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -1014,10 +1168,10 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option "name": "single_uint64_val", "shape": [1], "datatype": "UINT64", + "data": [5000000000], "parameters": { "key": 100 - }, - "data": [5000000000] + } }, { "name": "two_uint32_vals", "shape": [2], @@ -1032,7 +1186,8 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option (*protoParameters)["key"].set_bool_param(true); auto outputParameters = single_uint64_val->mutable_parameters(); (*outputParameters)["key"].set_bool_param(true); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json), StatusCode::OK); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -1043,10 +1198,10 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Option "name": "single_uint64_val", "shape": [1], "datatype": "UINT64", + "data": [5000000000], "parameters": { "key": true - }, - "data": [5000000000] + } }, { "name": "two_uint32_vals", "shape": [2], From fa59f080e822f89bc162fe23a0b70db45aeed78d Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Tue, 22 Nov 2022 13:30:54 +0100 Subject: [PATCH 047/130] Cvs 95278 example build (#1536) * Tf version fix --- demos/common/cpp/Dockerfile | 7 +- demos/common/cpp/WORKSPACE | 94 ++++++------------- .../bazel_rules_apple.patch | 30 ++++++ 3 files changed, 63 insertions(+), 68 deletions(-) create mode 100644 demos/common/cpp/third_party/build_bazel_rules_apple/bazel_rules_apple.patch diff --git a/demos/common/cpp/Dockerfile b/demos/common/cpp/Dockerfile index a7e770f192..bb0e47805a 100644 --- a/demos/common/cpp/Dockerfile +++ b/demos/common/cpp/Dockerfile @@ -90,11 +90,16 @@ COPY third_party /build/third_party/ RUN cp /build/third_party/opencv/install_opencv.sh /build/third_party/opencv/opencv_cmake_flags.txt . && ./install_opencv.sh +RUN bazel build @org_tensorflow//tensorflow/core:framework + +# Patch apple build scripts +RUN cp -rf /build/third_party/build_bazel_rules_apple/bazel_rules_apple.patch $(bazel info output_base)/external/build_bazel_rules_apple/ +RUN cd $(bazel info output_base)/external/build_bazel_rules_apple/ && patch -p1 < bazel_rules_apple.patch + RUN bazel build \ @tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto \ @com_github_grpc_grpc//:grpc++ \ @com_google_protobuf//:protobuf_lite \ - @org_tensorflow//tensorflow/core:framework \ @org_tensorflow//tensorflow/core:lib \ @opencv//:opencv diff --git a/demos/common/cpp/WORKSPACE b/demos/common/cpp/WORKSPACE index 3e17c763fc..3479907a50 100644 --- a/demos/common/cpp/WORKSPACE +++ b/demos/common/cpp/WORKSPACE @@ -23,80 +23,40 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") git_repository( name = "tensorflow_serving", remote = "https://github.com/tensorflow/serving.git", - tag = "2.2.2" + tag = "2.6.5" ) - -# Tensorflow core -git_repository( +load("@tensorflow_serving//tensorflow_serving:repo.bzl", "tensorflow_http_archive") +tensorflow_http_archive( name = "org_tensorflow", - remote = "https://github.com/tensorflow/tensorflow.git", - tag = "v2.2.2", - patch_args = ["-p1"], - patches = ["tf.patch"] -) - -http_archive( - name = "rules_pkg", - sha256 = "5bdc04987af79bd27bc5b00fe30f59a858f77ffa0bd2d8143d5b31ad8b1bd71c", - url = "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.0/rules_pkg-0.2.0.tar.gz", + sha256 = "fd687f8e26833cb917ae0bd8e434c9bd30c92042361c8ae69679983d3c66a440", + git_commit = "15198b1818bd2bf1b5b55bf5b02bf42398d222fc", + patch = "tf.patch", + repo_mapping = {"@curl" : "@curl"} ) -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") -rules_pkg_dependencies() - -load( - "@org_tensorflow//third_party/toolchains/preconfig/generate:archives.bzl", - "bazel_toolchains_archive", -) -bazel_toolchains_archive() - -load( - "@bazel_toolchains//repositories:repositories.bzl", - bazel_toolchains_repositories = "repositories", -) -bazel_toolchains_repositories() - - -# START: Upstream TensorFlow dependencies -# TensorFlow build depends on these dependencies. -# Needs to be in-sync with TensorFlow sources. -http_archive( - name = "io_bazel_rules_closure", - sha256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9", - strip_prefix = "rules_closure-308b05b2419edb5c8ee0471b67a40403df940149", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/308b05b2419edb5c8ee0471b67a40403df940149.tar.gz", # 2019-06-13 - ], -) -http_archive( - name = "bazel_skylib", - sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz", - ], -) # https://github.com/bazelbuild/bazel-skylib/releases - - -# END: Upstream TensorFlow dependencies - load("@tensorflow_serving//tensorflow_serving:workspace.bzl", "tf_serving_workspace") tf_serving_workspace() -# Specify the minimum required bazel version. -load("@org_tensorflow//tensorflow:version_check.bzl", "check_bazel_version_at_least") - -check_bazel_version_at_least("2.0.0") - -# GPRC deps, required to match TF's. Only after calling tf_serving_workspace() -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") - -grpc_deps() - -load("@upb//bazel:repository_defs.bzl", "bazel_version_repository") - -bazel_version_repository(name = "bazel_version") +# Check bazel version requirement, which is stricter than TensorFlow's. +load( + "@org_tensorflow//tensorflow:version_check.bzl", + "check_bazel_version_at_least" +) +check_bazel_version_at_least("5.3.1") + +# Initialize TensorFlow's external dependencies. +load("@org_tensorflow//tensorflow:workspace3.bzl", "workspace") +workspace() +load("@org_tensorflow//tensorflow:workspace2.bzl", "workspace") +workspace() +load("@org_tensorflow//tensorflow:workspace1.bzl", "workspace") +workspace() +load("@org_tensorflow//tensorflow:workspace0.bzl", "workspace") +workspace() + +# Initialize bazel package rules' external dependencies. +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +rules_pkg_dependencies() load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") grpc_deps() diff --git a/demos/common/cpp/third_party/build_bazel_rules_apple/bazel_rules_apple.patch b/demos/common/cpp/third_party/build_bazel_rules_apple/bazel_rules_apple.patch new file mode 100644 index 0000000000..fa2e07aaee --- /dev/null +++ b/demos/common/cpp/third_party/build_bazel_rules_apple/bazel_rules_apple.patch @@ -0,0 +1,30 @@ +diff -uraN a/apple/internal/rule_factory.bzl b/apple/internal/rule_factory.bzl +--- a/apple/internal/rule_factory.bzl 2020-04-03 18:27:03.000000000 +0000 ++++ b/apple/internal/rule_factory2.bzl 2022-10-19 15:31:40.437342517 +0000 +@@ -230,7 +230,7 @@ + + def _common_binary_linking_attrs(rule_descriptor): + deps_aspects = [ +- apple_common.objc_proto_aspect, ++# apple_common.objc_proto_aspect, + apple_resource_aspect, + framework_import_aspect, + swift_usage_aspect, +@@ -250,7 +250,7 @@ + """, + ), + "bundle_loader": attr.label( +- aspects = [apple_common.objc_proto_aspect], ++# aspects = [apple_common.objc_proto_aspect], + providers = [[apple_common.AppleExecutableBinary]], + doc = """ + This attribute is public as an implementation detail while we migrate the architecture of the rules. +@@ -258,7 +258,7 @@ + """, + ), + "dylibs": attr.label_list( +- aspects = [apple_common.objc_proto_aspect], ++# aspects = [apple_common.objc_proto_aspect], + doc = """ + This attribute is public as an implementation detail while we migrate the architecture of the rules. + Do not change its value. \ No newline at end of file From 81f7f20fbdc533c4253f32e3443cc76d4e926b41 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Tue, 22 Nov 2022 14:10:21 +0100 Subject: [PATCH 048/130] update opencl version (#1534) --- Makefile | 4 ++-- docs/docker_container.md | 10 +++++----- release_files/Dockerfile.redhat | 11 ++++++++++- release_files/Dockerfile.ubuntu | 15 ++++++++++++++- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index eae22dcc8e..4ae70c0c04 100644 --- a/Makefile +++ b/Makefile @@ -72,14 +72,14 @@ DIST_OS_TAG ?= $(BASE_OS_TAG) ifeq ($(BASE_OS),ubuntu) BASE_OS_TAG=$(BASE_OS_TAG_UBUNTU) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) - INSTALL_DRIVER_VERSION ?= "21.48.21782" + INSTALL_DRIVER_VERSION ?= "22.35.24055" DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.8693.0f647e6b75a_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat - INSTALL_DRIVER_VERSION ?= "21.38.21026" + INSTALL_DRIVER_VERSION ?= "22.28.23726" DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.8693.0f647e6b75a_x86_64.tgz endif diff --git a/docs/docker_container.md b/docs/docker_container.md index 185e7defbb..b2ff5e04f0 100644 --- a/docs/docker_container.md +++ b/docs/docker_container.md @@ -76,7 +76,7 @@ You can build your own Docker image executing the `make docker_build` command in In the `./dist` directory it will generate: - image tagged as openvino/model_server:latest - with CPU, NCS, and HDDL support -- image tagged as openvino/model_server:latest-gpu - with CPU, NCS, HDDL, and iGPU support +- image tagged as openvino/model_server:latest-gpu - with CPU, NCS, HDDL, and iGPU, dGPU support - image tagged as openvino/model_server:latest-nginx-mtls - with CPU, NCS, and HDDL support and a reference nginx setup of mTLS integration - release package (.tar.gz, with ovms binary and necessary libraries) @@ -98,13 +98,13 @@ docker run --rm -it --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/rende --model_name resnet --model_path gs://ovms-public-eu/resnet50-binary --port 9001 --target_device GPU ``` -*Note:* The public docker image includes the OpenCL drivers for GPU in version 21.38.21026. +*Note:* The public docker image includes the OpenCL drivers for GPU in version 22.35.24055 for ubuntu20.04 based image and 22.28.23726 for UBI8. -### Model Server image with DG2 support (Ubuntu 20.04) +### Model Server image using preproduction gpu drivers (Ubuntu 20.04) -Image with DG2 GPU support has not been published. To build the image yourself you need to have DG2 drivers installed on the host and NEO Runtime packages available. +To build the image, you need to have NEO Runtime packages available. Contact Intel representative to get the access to the preproduction drivers. -Put NEO Runtime packages in the catalog `/release_files/drivers/dg2` and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. +Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2` and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. Example: ``` diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index 894fb2fcd0..650a043fb2 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -38,7 +38,7 @@ LABEL "description"="OpenVINO(TM) Model Server is a solution for serving AI mode ARG INSTALL_RPMS_FROM_URL= ENV INSTALL_RPMS_FROM_URL=$INSTALL_RPMS_FROM_URL -ARG INSTALL_DRIVER_VERSION="21.38.21026" +ARG INSTALL_DRIVER_VERSION="22.28.23726" ENV INSTALL_DRIVER_VERSION=$INSTALL_DRIVER_VERSION ARG GPU=1 ENV GPU=$GPU @@ -78,6 +78,15 @@ RUN set -e ; \ rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-igc-opencl-1.0.10409-i699.3.el8.x86_64.rpm ; \ rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-ocloc-22.10.22597-i699.3.el8.x86_64.rpm ; \ rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-opencl-22.10.22597-i699.3.el8.x86_64.rpm ; \ + ;; \ + "22.28.23726") \ + microdnf install -y libedit ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-gmmlib-22.1.7-i419.el8.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-igc-core-1.0.11485-i419.el8.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-igc-opencl-1.0.11485-i419.el8.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5/intel-opencl-22.28.23726.1-i419.el8.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5-devel/intel-level-zero-gpu-1.3.23453-i392.el8.x86_64.rpm ; \ + rpm -ivh https://repositories.intel.com/graphics/rhel/8.5-devel/level-zero-1.8.1-i392.el8.x86_64.rpm ; \ ;; \ *) \ echo "ERROR: Unrecognized driver ${INSTALL_DRIVER_VERSION}." ; \ diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index f635e45de2..5f0319e22d 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -68,7 +68,7 @@ FROM $BASE_IMAGE as release ARG INSTALL_RPMS_FROM_URL= ENV INSTALL_RPMS_FROM_URL=$INSTALL_RPMS_FROM_URL ENV HDDL_INSTALL_DIR=/ovms/lib/hddl -ARG INSTALL_DRIVER_VERSION="21.48.21782" +ARG INSTALL_DRIVER_VERSION="22.35.24055" ENV INSTALL_DRIVER_VERSION=$INSTALL_DRIVER_VERSION ARG GPU=1 ENV GPU=$GPU @@ -113,6 +113,19 @@ RUN set -e ; \ curl -L -O https://github.com/intel/compute-runtime/releases/download/22.10.22597/intel-level-zero-gpu_1.3.22597_amd64.deb ; \ dpkg -i intel*.deb && rm -Rf /tmp/gpu_deps ; \ ;; \ + "22.35.24055") \ + apt-get update && apt-get install -y gpg-agent && \ + curl https://repositories.intel.com/graphics/intel-graphics.key |gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \ + echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu focal-legacy main' | tee /etc/apt/sources.list.d/intel.gpu.focal.list && \ + apt-get update && \ + apt-get install -y \ + intel-opencl-icd=22.35.24055+i815~u20.04 \ + intel-level-zero-gpu=1.3.24055+i815~u20.04 \ + level-zero=1.8.5+i815~u20.04 && \ + apt-get purge gpg-agent --yes && apt-get --yes autoremove && \ + apt clean ; \ + rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* ; \ + ;; \ *) \ dpkg -P intel-gmmlib intel-igc-core intel-igc-opencl intel-level-zero-gpu intel-ocloc intel-opencl intel-opencl-icd ; \ apt-get update && apt-get -y install dpkg-dev ; \ From 11046f52c62ce4e2c5d019ad3e0e40f41103fa48 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Tue, 22 Nov 2022 14:43:10 +0100 Subject: [PATCH 049/130] add plugin_config legacy parsing options (#1535) --- src/modelconfig.cpp | 17 ++++++++++++++++- src/test/modelconfig_test.cpp | 16 +++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index 47b354b75e..6cae53fd07 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -295,11 +295,26 @@ Status ModelConfig::parsePluginConfig(const rapidjson::Value& node) { } else { if ((it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS")) || (it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS"))) { pluginConfig["NUM_STREAMS"] = it->value.GetString(); - SPDLOG_WARN("{} plugin config key is deprecated. Use NUM_STREAMS instead", it->name.GetString()); + SPDLOG_WARN("{} plugin config key is deprecated. Use NUM_STREAMS instead", it->name.GetString()); + } else if (it->name.GetString() == std::string("CPU_BIND_THREAD")) { + if (it->value.GetString() == std::string("YES")) { + pluginConfig["AFFINITY"] = "CORE"; + SPDLOG_WARN("{} plugin config key is deprecated. Use AFFINITY instead", it->name.GetString()); + } else if (it->value.GetString() == std::string("NO")) { + pluginConfig["AFFINITY"] = "NONE"; + SPDLOG_WARN("{} plugin config key is deprecated. Use AFFINITY instead", it->name.GetString()); + } else { + SPDLOG_ERROR("{} plugin config key has invalid value and is deprecated. Use AFFINITY key instead", it->name.GetString()); + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + } else if (it->name.GetString() == std::string("CPU_THREADS_NUM")) { + pluginConfig["INFERENCE_NUM_THREADS"] = it->value.GetString(); + SPDLOG_WARN("{} plugin config key is deprecated. Use INFERENCE_NUM_THREADS instead", it->name.GetString()); } else { pluginConfig[it->name.GetString()] = it->value.GetString(); } } + } else if (it->value.IsInt64()) { if (it->name.GetString() == std::string("CPU_THROUGHPUT_STREAMS") || it->name.GetString() == std::string("GPU_THROUGHPUT_STREAMS")) { pluginConfig["NUM_STREAMS"] = std::to_string(it->value.GetInt64()); diff --git a/src/test/modelconfig_test.cpp b/src/test/modelconfig_test.cpp index f185a12158..b784b90a65 100644 --- a/src/test/modelconfig_test.cpp +++ b/src/test/modelconfig_test.cpp @@ -498,11 +498,16 @@ TEST(ModelConfig, plugin_config_legacy_cpu_num) { TEST(ModelConfig, plugin_config_legacy_cpu_str) { ovms::ModelConfig config; - std::string pluginConfig_str = "{\"CPU_THROUGHPUT_STREAMS\":\"5\"}"; + std::string pluginConfig_str = "{\"CPU_THROUGHPUT_STREAMS\":\"5\", \"CPU_BIND_THREAD\":\"NO\", \"CPU_THREADS_NUM\": \"2\"}"; auto status = config.parsePluginConfig(pluginConfig_str); auto actualPluginConfig = config.getPluginConfig(); EXPECT_EQ(status, ovms::StatusCode::OK); EXPECT_EQ(actualPluginConfig["NUM_STREAMS"], "5"); + EXPECT_EQ(actualPluginConfig["AFFINITY"], "NONE"); + EXPECT_EQ(actualPluginConfig["INFERENCE_NUM_THREADS"], "2"); + EXPECT_EQ(actualPluginConfig.count("CPU_THREADS_NUM"), 0); + EXPECT_EQ(actualPluginConfig.count("CPU_THROUGHPUT_STREAMS"), 0); + EXPECT_EQ(actualPluginConfig.count("CPU_BIND_THREAD"), 0); } TEST(ModelConfig, plugin_config_legacy_gpu) { @@ -514,6 +519,15 @@ TEST(ModelConfig, plugin_config_legacy_gpu) { EXPECT_EQ(actualPluginConfig["PERFORMANCE_HINT"], "THROUGHPUT"); } +TEST(ModelConfig, plugin_config_cpu_bind_thread) { + ovms::ModelConfig config; + std::string pluginConfig_str = "{\"CPU_BIND_THREAD\":\"YES\"}"; + auto status = config.parsePluginConfig(pluginConfig_str); + auto actualPluginConfig = config.getPluginConfig(); + EXPECT_EQ(status, ovms::StatusCode::OK); + EXPECT_EQ(actualPluginConfig["AFFINITY"], "CORE"); +} + TEST(ModelConfig, plugin_config_legacy_gpu_num) { ovms::ModelConfig config; std::string pluginConfig_str = "{\"GPU_THROUGHPUT_STREAMS\":5}"; From 7ffe704191c26ead0e468fee74da747eaba25c60 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Wed, 23 Nov 2022 15:46:34 +0100 Subject: [PATCH 050/130] patch zlib in opencv component (#1540) --- third_party/opencv/install_opencv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/opencv/install_opencv.sh b/third_party/opencv/install_opencv.sh index 80159a3052..65140ba40e 100755 --- a/third_party/opencv/install_opencv.sh +++ b/third_party/opencv/install_opencv.sh @@ -55,7 +55,7 @@ fi current_working_dir=$(pwd) cd $work_dir -git clone https://github.com/opencv/opencv.git --depth 1 -b $opencv_branch $work_dir/opencv_repo +git clone https://github.com/opencv/opencv.git --depth 1 -b $opencv_branch $work_dir/opencv_repo && git fetch 4.x && git cherry-pick 1b1bbe426277715a876878890a3dc88231b871bc mkdir -p $work_dir/opencv_repo/build cd $work_dir/opencv_repo/build cmake $(cat $current_working_dir/opencv_cmake_flags.txt) $work_dir/opencv_repo && \ From cabf350133d9e19b0c5ddc191656d9c2bbeb3571 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Thu, 24 Nov 2022 14:20:05 +0100 Subject: [PATCH 051/130] Fix grpc_infer_binary_resnet.py (#1537) --- .../samples/grpc_infer_binary_resnet.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/client/python/kserve-api/samples/grpc_infer_binary_resnet.py b/client/python/kserve-api/samples/grpc_infer_binary_resnet.py index e75d23e6af..85a3948119 100644 --- a/client/python/kserve-api/samples/grpc_infer_binary_resnet.py +++ b/client/python/kserve-api/samples/grpc_infer_binary_resnet.py @@ -17,6 +17,7 @@ import sys sys.path.append("../../../../demos/common/python") +import ast import grpc import numpy as np import classes @@ -26,6 +27,18 @@ from tritonclient.grpc import service_pb2, service_pb2_grpc from tritonclient.utils import * +DataTypeToContentsFieldName = { + 'BOOL' : 'bool_contents', + 'BYTES' : 'bytes_contents', + 'FP32' : 'fp32_contents', + 'FP64' : 'fp64_contents', + 'INT64' : 'int64_contents', + 'INT32' : 'int_contents', + 'UINT64' : 'uint64_contents', + 'UINT32' : 'uint_contents', + 'INT64' : 'int64_contents', + 'INT32' : 'int_contents', +} def as_numpy(response, name): index = 0 @@ -35,11 +48,14 @@ def as_numpy(response, name): for value in output.shape: shape.append(value) datatype = output.datatype + field_name = DataTypeToContentsFieldName[datatype] + contents = getattr(output, "contents") + contents = getattr(contents, f"{field_name}") if index < len(response.raw_output_contents): np_array = np.frombuffer( response.raw_output_contents[index], dtype=triton_to_np_dtype(output.datatype)) - elif len(output.contents.bytes_contents) != 0: - np_array = np.array(output.contents.bytes_contents, + elif len(contents) != 0: + np_array = np.array(contents, copy=False) else: np_array = np.empty(0) From 40dbbf8dd2dcc8afde40d75de55f80fd4d746520 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Fri, 25 Nov 2022 13:19:15 +0100 Subject: [PATCH 052/130] Fix EnsemleFlowTest sporadic fail (#1542) --- src/test/ensemble_tests.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp index e310002d32..2d1cbf635c 100644 --- a/src/test/ensemble_tests.cpp +++ b/src/test/ensemble_tests.cpp @@ -1673,12 +1673,23 @@ TEST_F(EnsembleFlowTest, OrderOfScheduling) { pipeline.push(std::move(node_3)); ASSERT_EQ(pipeline.execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); - EXPECT_THAT(order, ElementsAre( - 1, // try to schedule node_1 with success - 2, // try to schedule node_2, defer (with order ticket #1) - 3, // after node_1 ends, try to run next node (node_3), defer with order ticket #2 - 2, // also try to schedule previously deferred nodes, node_2 gets scheduled with success - 3)); // node_2 ends, try to schedule previously deferred node_3 with success + std::vector expectedOrder = { + 1, // try to schedule node_1 with success + 2, // try to schedule node_2, defer (with order ticket #1) + 3, // after node_1 ends, try to run next node (node_3), defer with order ticket #2 + 2, // also try to schedule previously deferred nodes, node_2 gets scheduled with success + 3}; // node_2 ends, try to schedule previously deferred node_3 with success + int expectedOrderIt = 0; + int lastValue = 0; + for (int orderElement : order) { + if (orderElement != lastValue) { + EXPECT_EQ(orderElement, expectedOrder[expectedOrderIt]); + expectedOrderIt++; + } + lastValue = orderElement; + } + // This fragment above is implemented that way because amount of scheduling retries may differ between different machines + // depending on the inference time of the dummy model /* -----O1-----O3---- O---< >----O From 5ff3f66ed68df938e058f918ee0616042bef7f89 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:25:23 +0100 Subject: [PATCH 053/130] CVS-93692_release_files_drivers_readme_doc_fix (#1522) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * updates for automatic tests * Apply suggestions from code review Co-authored-by: Miłosz Żeglarski Co-authored-by: ngrozae Co-authored-by: Miłosz Żeglarski --- release_files/drivers/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/release_files/drivers/README.md b/release_files/drivers/README.md index ac9500442b..b945e52e90 100644 --- a/release_files/drivers/README.md +++ b/release_files/drivers/README.md @@ -17,8 +17,13 @@ drivers With such structure you can build OpenVINO Model Server GPU image with custom NEO Runtime by specifying `INSTALL_DRIVER_VERSION` parameter in `make docker_build` command. The value of `INSTALL_DRIVER_VERSION` should be the name of the subfolder i.e. the `` in above schema. -**Example:** For packages placed in `drivers/dg2` location, the build command could look like this: +**Example:** For packages placed in `drivers/neo` location, the build command could look like this: +```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server ``` -make docker_build BASE_OS=ubuntu OVMS_CPP_DOCKER_IMAGE=ovms_dg2 INSTALL_DRIVER_VERSION=dg2 + +```bash +make docker_build BASE_OS=ubuntu OVMS_CPP_DOCKER_IMAGE=ovms_custom_neo INSTALL_DRIVER_VERSION=neo ``` From 220165f8242cde1733da3bd45b9559a7b802829e Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Mon, 28 Nov 2022 08:55:35 +0100 Subject: [PATCH 054/130] OV package 2022.3.0.8844 (#1548) --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4ae70c0c04..b0eb00c73c 100644 --- a/Makefile +++ b/Makefile @@ -73,14 +73,14 @@ ifeq ($(BASE_OS),ubuntu) BASE_OS_TAG=$(BASE_OS_TAG_UBUNTU) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) INSTALL_DRIVER_VERSION ?= "22.35.24055" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.8693.0f647e6b75a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_ubuntu20_2022.3.0.8844.12f74f066a4_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0f647e6b75a87e334f44e0451f0f7ef9c66141c3/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.8693.0f647e6b75a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_rhel8_2022.3.0.8844.12f74f066a4_x86_64.tgz endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server From a7bd266f340e92dae0ae87a91efb8354cf613710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Mon, 28 Nov 2022 09:06:59 +0100 Subject: [PATCH 055/130] OpenCV remove TIFF codec installation (#1544) --- third_party/opencv/opencv_cmake_flags.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/third_party/opencv/opencv_cmake_flags.txt b/third_party/opencv/opencv_cmake_flags.txt index f794800474..2668f35545 100644 --- a/third_party/opencv/opencv_cmake_flags.txt +++ b/third_party/opencv/opencv_cmake_flags.txt @@ -5,4 +5,5 @@ -D WITH_OPENJPEG=OFF -D WITH_JASPER=OFF -D WITH_OPENEXR=OFF +-D WITH_TIFF=OFF From a8634dd05b22513f330583f5b089038e60ae6115 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Mon, 28 Nov 2022 11:21:56 +0100 Subject: [PATCH 056/130] correct zlib patch changes (#1549) --- third_party/opencv/install_opencv.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/third_party/opencv/install_opencv.sh b/third_party/opencv/install_opencv.sh index 65140ba40e..40ef3dccff 100755 --- a/third_party/opencv/install_opencv.sh +++ b/third_party/opencv/install_opencv.sh @@ -14,8 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set -e - +set -exo pipefail #=================================================================================================== # Option parsing @@ -55,7 +54,10 @@ fi current_working_dir=$(pwd) cd $work_dir -git clone https://github.com/opencv/opencv.git --depth 1 -b $opencv_branch $work_dir/opencv_repo && git fetch 4.x && git cherry-pick 1b1bbe426277715a876878890a3dc88231b871bc +git clone https://github.com/opencv/opencv.git --depth 1 -b $opencv_branch $work_dir/opencv_repo +cd $work_dir/opencv_repo +git fetch origin 4.x:4.4 +git cherry-pick -n 1b1bbe426277715a876878890a3dc88231b871bc mkdir -p $work_dir/opencv_repo/build cd $work_dir/opencv_repo/build cmake $(cat $current_working_dir/opencv_cmake_flags.txt) $work_dir/opencv_repo && \ @@ -65,4 +67,3 @@ cmake $(cat $current_working_dir/opencv_cmake_flags.txt) $work_dir/opencv_repo & #=================================================================================================== # end -exit 0 From a7bef6c296489c414eb3683bc6bda051304f2499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Mon, 28 Nov 2022 16:26:42 +0100 Subject: [PATCH 057/130] Fix main README invalid link (#1550) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c118f8670..97e5272e07 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ For more information on using Model Server in various scenarios you can check th * [Serving stateful models](https://docs.openvino.ai/2022.2/ovms_docs_stateful_models.html) -* [Deploy using a Kubernetes Helm Chart](https://docs.openvino.ai/2022.2/ovms_deploy_helm_chart.html) +* [Deploy using a Kubernetes Helm Chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) * [Deployment using Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) From 44a39df46af2b39f8d9286d5ecc01116682c6a40 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Mon, 28 Nov 2022 17:47:35 +0100 Subject: [PATCH 058/130] Nvidia plugin automation (#1539) * Nvidia plugin support Co-authored-by: michalkulakowski --- Dockerfile.ubuntu | 80 +++++++++++++++++++++---- DockerfileMakePackage | 12 ++-- Makefile | 38 ++++++++---- docs/accelerators.md | 13 ++++ release_files/Dockerfile.ubuntu | 21 +++++-- src/example/SampleCpuExtension/Makefile | 8 ++- 6 files changed, 139 insertions(+), 33 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 1f2bd676cf..0351ae9b2d 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -27,16 +27,19 @@ ARG DL_DIR=/tmp ARG JOBS ARG APT_OV_PACKAGE=openvino-2022.1.0 ARG CHECK_COVERAGE=1 +ARG NVIDIA=0 +ARG http_proxy +ARG CMAKE_BUILD_TYPE=Release # build_type=[ opt, dbg ] -ARG build_type=dbg +ARG build_type=opt ARG debug_bazel_flags=--strip=never\ --copt="-g"\ -c\ dbg ARG minitrace_flags ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/external/hddl ENV DEBIAN_FRONTEND=noninteractive ENV TF_SYSTEM_LIBS="curl" SHELL ["/bin/bash", "-c"] -RUN apt update && apt install -y \ +RUN apt-get update && apt-get install -y \ libboost-atomic1.71.0 \ libboost-chrono1.71.0 \ libboost-filesystem1.71.0 \ @@ -79,7 +82,42 @@ RUN apt update && apt install -y \ unzip \ vim \ xz-utils && \ - apt clean + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + + +# Add Nvidia dev tool if needed +RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin -O /etc/apt/preferences.d/cuda-repository-pin-600 ; \ + set -exuo pipefail ; \ + rm -f /etc/apt/apt.conf.d/docker-clean ; \ + apt-get update && \ + apt-get install -y \ + gnupg2 \ + software-properties-common; \ + if [[ ${enable_tensorrt-} == "1" ]] ; then dpkg -i /nv-tensorrt-repo-*.deb ; fi; \ + apt-key adv --keyserver-options http-proxy=$http_proxy --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub ; \ + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"; \ + apt-get update && apt install -y \ + libzstd-dev \ + cuda \ + libcudnn8 \ + libcudnn8-dev \ + libcutensor1 \ + libcutensor-dev; \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + pip3 install --no-cache-dir cython && \ + curl -L https://github.com/Kitware/ninja/releases/download/v1.10.0.gfb670.kitware.jobserver-1/ninja-1.10.0.gfb670.kitware.jobserver-1_x86_64-linux-gnu.tar.gz | tar xzv --strip-components=1 -C /usr/local/bin ; \ + curl https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz -L | tar xvzC /usr/local/bin --strip-components=1 --wildcards '*/sccache' ; \ + chmod a+x /usr/local/bin/sccache ; \ + curl https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-x86_64.tar.gz -L | tar xzvC /usr/local --exclude={doc,man} --strip-components=1 ; \ + curl -L https://github.com/ccache/ccache/releases/download/v4.3/ccache-4.3.tar.xz | tar xJv ; \ + cd ccache-4.3 ; \ + mkdir build ; \ + cd build ; \ + cmake -DCMAKE_BUILD_TYPE=Release -G Ninja .. ; \ + ninja -v install # Set up Bazel ENV BAZEL_VERSION 5.3.1 @@ -107,7 +145,7 @@ COPY third_party /ovms/third_party/ ####### Azure SDK WORKDIR /azure -RUN apt update && apt-get install -y uuid uuid-dev +RUN apt-get update && apt-get install -y uuid uuid-dev && rm -rf /var/lib/apt/lists/* RUN git clone https://github.com/Microsoft/cpprestsdk.git && cd cpprestsdk && git checkout tags/v2.10.18 -b v2.10.18 && git submodule update --init RUN git clone https://github.com/Azure/azure-storage-cpp.git && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage && git checkout tags/v7.5.0 && mkdir build.release @@ -140,12 +178,13 @@ RUN ./install_opencv.sh ARG ov_use_binary=1 ARG DLDT_PACKAGE_URL ARG ov_source_branch=master +ARG ov_contrib_branch=master ################### BUILD OPENVINO FROM SOURCE - buildarg ov_use_binary=0 ############################ # Build OpenVINO and nGraph (OV dependency) with D_GLIBCXX_USE_CXX11_ABI=0 or 1 -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/openvinotoolkit/openvino /openvino ; cd /openvino ; git checkout $ov_source_branch; git submodule update --init --recursive +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; git clone https://github.com/openvinotoolkit/openvino /openvino ; cd /openvino ; git checkout $ov_source_branch; git submodule update --init --recursive WORKDIR /openvino/build -RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " .. +RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DENABLE_SAMPLES=0 -DNGRAPH_USE_CXX_ABI=1 -DCMAKE_CXX_FLAGS=" -D_GLIBCXX_USE_CXX11_ABI=1 -Wno-error=parentheses " .. RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; make --jobs=$JOBS RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; make install RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; \ @@ -181,20 +220,41 @@ RUN if [ "$ov_use_binary" = "1" ] && [ "$DLDT_PACKAGE_URL" = "" ] ; then true ; wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \ apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \ echo "deb https://apt.repos.intel.com/openvino/2022 focal main" | tee /etc/apt/sources.list.d/intel-openvino-2022.list && \ - apt update && \ - apt install -y $APT_OV_PACKAGE && \ + apt-get update && \ + apt-get install -y $APT_OV_PACKAGE && \ + rm -rf /var/lib/apt/lists/* && \ ln -s /opt/intel/openvino_2022 /opt/intel/openvino # install sample apps including benchmark_app RUN if [ -f /opt/intel/openvino/samples/cpp/build_samples.sh ]; then /opt/intel/openvino/samples/cpp/build_samples.sh ; fi #################### END OF OPENVINO BINARY INSTALL +# NVIDIA +ENV OPENVINO_BUILD_PATH=/cuda_plugin_build +ENV OPENVINO_HOME=/openvino +ENV OPENVINO_CONTRIB=/openvino_contrib + +RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ + git clone --recurse-submodules --branch=$ov_contrib_branch https://github.com/openvinotoolkit/openvino_contrib.git /openvino_contrib && cd /openvino_contrib && \ + mkdir ${OPENVINO_BUILD_PATH} && \ + cd "${OPENVINO_BUILD_PATH}" && \ + cmake "${OPENVINO_HOME}" \ + -DENABLE_NVIDIA=ON \ + -DENABLE_TESTS=ON \ + -DBUILD_arm_plugin=OFF \ + -DBUILD_java_api=OFF \ + -DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB}/modules" \ + -DWHEEL_VERSION=2022.1.0 \ + -DVERBOSE_BUILD=ON \ + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE && \ + cmake --build "${OPENVINO_BUILD_PATH}" --target openvino_nvidia_gpu_plugin -j $JOBS + # Build OpenVINO Model Server WORKDIR /ovms COPY .bazelrc WORKSPACE /ovms/ COPY external /ovms/external/ -RUN apt install -y python-is-python3 +RUN apt-get update && apt-get install -y python-is-python3 && rm -rf /var/lib/apt/lists/* RUN bazel build --jobs=$JOBS ${debug_bazel_flags} @org_tensorflow//tensorflow/core:framework @@ -234,7 +294,7 @@ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else bazel test ${debug_bazel_ RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so -RUN cd /ovms/src/example/SampleCpuExtension/ && make +RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 8a086f68b7..6879fac178 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -20,6 +20,7 @@ FROM $BUILD_IMAGE ARG BASE_OS=ubuntu ARG ov_use_binary=1 +ARG NVIDIA=0 SHELL ["/bin/bash", "-c"] RUN mkdir /patchelf && cd /patchelf && \ @@ -35,6 +36,7 @@ RUN mkdir -vp /ovms_release/lib/custom_nodes RUN if [ -d /ovms/src/custom_nodes/lib/${BASE_OS} ] ; then true ; else exit 0 ; fi ; cp /ovms/src/custom_nodes/lib/${BASE_OS}/*.so /ovms_release/lib/custom_nodes/ RUN cp /ovms/metadata.json /ovms_release/ +RUN if [ "$NVIDIA" == "1" ]; then true ; else exit 0 ; fi ; cp -v /openvino/bin/intel64/Release/libopenvino_nvidia_gpu_plugin.so /ovms_release/lib/ RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cp -v /openvino/bin/intel64/Release/plugins.xml /ovms_release/lib/ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp -v /opt/intel/openvino/runtime/3rdparty/hddl/config/* /ovms_release/lib/hddl/config/ || true RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp -vr /opt/intel/openvino/runtime/3rdparty/hddl/etc/* /ovms_release/lib/hddl/etc/ || true @@ -57,11 +59,11 @@ libjavajpeg.so libjava.so libjawt.so libjdwp.so libjimage.so libjli.so libjsig.s liblcms.so libmanagement_agent.so libmanagement_ext.so libmanagement.so libmlib_image.so libnet.so libnio.so \ libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libsystemconf.so libunpack.so libverify.so libzip.so -RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; -RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; -RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/intel/openvino/runtime/3rdparty/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; -RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; find /opt/opencv/lib/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; -RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; cp /opt/opencv/share/licenses/opencv4/* /ovms/release_files/thirdparty-licenses/ +RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; +RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; +RUN find /opt/intel/openvino/runtime/3rdparty/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; +RUN find /opt/opencv/lib/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; +RUN cp /opt/opencv/share/licenses/opencv4/* /ovms/release_files/thirdparty-licenses/ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; if [ -f /opt/intel/openvino/docs/licensing/EULA.txt ] ; then true ; else exit 0; fi ; cp /opt/intel/openvino/docs/licensing/EULA.txt /ovms/release_files/thirdparty-licenses/openvino.LICENSE.txt; RUN if [ "$ov_use_binary" == "0" ] ; then true ; else exit 0 ; fi ; cp /openvino/LICENSE /ovms/release_files/thirdparty-licenses/openvino.LICENSE.txt; diff --git a/Makefile b/Makefile index b0eb00c73c..6b8bea8f3a 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ BASE_OS_TAG_REDHAT ?= 8.6 INSTALL_RPMS_FROM_URL ?= CHECK_COVERAGE ?=1 +NVIDIA ?=0 # NOTE: when changing any value below, you'll need to adjust WORKSPACE file by hand: # - uncomment source build section, comment binary section @@ -50,6 +51,7 @@ OV_USE_BINARY ?= 1 APT_OV_PACKAGE ?= openvino-2022.1.0 # opt, dbg: BAZEL_BUILD_TYPE ?= opt +CMAKE_BUILD_TYPE ?= Release MINITRACE ?= OFF ifeq ($(BAZEL_BUILD_TYPE),dbg) @@ -71,13 +73,16 @@ DIST_OS_TAG ?= $(BASE_OS_TAG) ifeq ($(BASE_OS),ubuntu) BASE_OS_TAG=$(BASE_OS_TAG_UBUNTU) - BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) + ifeq ($(NVIDIA),1) + BASE_IMAGE=docker.io/nvidia/cuda:11.8.0-runtime-ubuntu20.04 + else + BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) + endif INSTALL_DRIVER_VERSION ?= "22.35.24055" DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_ubuntu20_2022.3.0.8844.12f74f066a4_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) - BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_rhel8_2022.3.0.8844.12f74f066a4_x86_64.tgz @@ -89,6 +94,9 @@ ifeq ($(BAZEL_BUILD_TYPE),dbg) endif OVMS_CPP_IMAGE_TAG ?= latest +ifeq ($(NVIDIA),1) + IMAGE_TAG_SUFFIX = -cuda +endif PRODUCT_NAME = "OpenVINO Model Server" PRODUCT_VERSION ?= "2022.3" @@ -162,6 +170,11 @@ clang-format: venv .PHONY: docker_build docker_build: +ifeq ($(NVIDIA),1) + ifeq ($(OV_USE_BINARY),1) + @echo "Building NVIDIA plugin requires OV built from source. To build NVIDIA plugin and OV from source make command should look like this 'NVIDIA=1 OV_USE_BINARY=0 make docker_build'"; exit 1 ; + endif +endif ifeq ($(BUILD_CUSTOM_NODES),true) @echo "Building custom nodes" @cd src/custom_nodes && make BASE_OS=$(BASE_OS) @@ -189,21 +202,24 @@ endif --build-arg ov_use_binary=$(OV_USE_BINARY) --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL) \ --build-arg APT_OV_PACKAGE=$(APT_OV_PACKAGE) --build-arg CHECK_COVERAGE=$(CHECK_COVERAGE) \ --build-arg build_type=$(BAZEL_BUILD_TYPE) --build-arg debug_bazel_flags=$(BAZEL_DEBUG_FLAGS) \ + --build-arg CMAKE_BUILD_TYPE=$(CMAKE_BUILD_TYPE) \ --build-arg minitrace_flags=$(MINITRACE_FLAGS) \ --build-arg PROJECT_NAME=${PROJECT_NAME} \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ - -t $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) \ + --build-arg NVIDIA=$(NVIDIA) \ + -t $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \ --build-arg JOBS=$(JOBS) docker build $(NO_CACHE_OPTION) -f DockerfileMakePackage . \ --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" \ --build-arg ov_use_binary=$(OV_USE_BINARY) --build-arg DLDT_PACKAGE_URL=$(DLDT_PACKAGE_URL) --build-arg BASE_OS=$(BASE_OS) \ + --build-arg NVIDIA=$(NVIDIA) \ -t $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) \ - --build-arg BUILD_IMAGE=$(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) + --build-arg BUILD_IMAGE=$(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) rm -vrf dist/$(DIST_OS) && mkdir -vp dist/$(DIST_OS) && cd dist/$(DIST_OS) && \ docker run $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) bash -c \ "tar -c -C / ovms.tar* ; sleep 2" | tar -x - -docker rm -v $$(docker ps -a -q -f status=exited -f ancestor=$(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) ) + -docker rm -v $$(docker ps -a -q -f status=exited -f ancestor=$(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX)) cd dist/$(DIST_OS) && sha256sum --check ovms.tar.gz.sha256 cd dist/$(DIST_OS) && sha256sum --check ovms.tar.xz.sha256 cp -vR release_files/* dist/$(DIST_OS)/ @@ -213,7 +229,8 @@ endif --build-arg INSTALL_RPMS_FROM_URL="$(INSTALL_RPMS_FROM_URL)" \ --build-arg GPU=0 \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ - -t $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG) + --build-arg NVIDIA=$(NVIDIA) \ + -t $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) cd dist/$(DIST_OS)/ && docker build $(NO_CACHE_OPTION) -f Dockerfile.$(BASE_OS) . \ --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" \ --build-arg no_proxy=$(NO_PROXY) \ @@ -221,11 +238,12 @@ endif --build-arg INSTALL_DRIVER_VERSION="$(INSTALL_DRIVER_VERSION)" \ --build-arg GPU=1 \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ - -t $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG) && \ - docker tag $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG) $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)-gpu + --build-arg NVIDIA=$(NVIDIA) \ + -t $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) && \ + docker tag $(OVMS_CPP_DOCKER_IMAGE)-gpu:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)-gpu$(IMAGE_TAG_SUFFIX) cd extras/nginx-mtls-auth && \ - http_proxy=$(HTTP_PROXY) https_proxy=$(HTTPS_PROXY) no_proxy=$(NO_PROXY) ./build.sh "$(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)" "$(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG)" "$(BASE_OS)" && \ - docker tag $(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG) $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)-nginx-mtls + http_proxy=$(HTTP_PROXY) https_proxy=$(HTTPS_PROXY) no_proxy=$(NO_PROXY) ./build.sh "$(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX)" "$(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX)" "$(BASE_OS)" && \ + docker tag $(OVMS_CPP_DOCKER_IMAGE)-nginx-mtls:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) $(OVMS_CPP_DOCKER_IMAGE):$(OVMS_CPP_IMAGE_TAG)-nginx-mtls$(IMAGE_TAG_SUFFIX) # Ci build expects index.html in genhtml directory get_coverage: diff --git a/docs/accelerators.md b/docs/accelerators.md index 2aa9e187ad..e84a3514b7 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -214,3 +214,16 @@ THROUGHPUT ``` > **NOTE**: currently, AUTO plugin cannot be used with `--shape auto` parameter while GPU device is enabled. + +## Using NVIDIA Plugin + +*Note:* To build container with NVIDIA plugin use command: +```bash + make docker_build NVIDIA=1 OV_USE_BINARY=0 +``` + +Example command to run container with NVIDIA support: + +```bash + docker run -it --gpus all -p 9178:9178 -v /model:/model openvino/model_server-cuda:latest --model_path /model --model_name resnet --target_device NVIDIA +``` diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 5f0319e22d..9ad40a2b61 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -25,6 +25,8 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends curl tzdata ca-certificates && \ rm -rf /var/lib/apt/lists/* +ARG NVIDIA=0 + # for VPU ARG BUILD_DEPENDENCIES="autoconf \ automake \ @@ -71,6 +73,7 @@ ENV HDDL_INSTALL_DIR=/ovms/lib/hddl ARG INSTALL_DRIVER_VERSION="22.35.24055" ENV INSTALL_DRIVER_VERSION=$INSTALL_DRIVER_VERSION ARG GPU=1 +ARG NVIDIA=0 ENV GPU=$GPU ENV DEBIAN_FRONTEND=noninteractive SHELL ["/bin/bash", "-c"] @@ -83,7 +86,7 @@ RUN set -e ; \ apt update -y ; \ apt install -y curl libpugixml1v5 ; \ if [ "$GPU" == "1" ] ; then \ - apt install -y libnuma1 ocl-icd-libopencl1 ; \ + apt-get update && apt-get install -y libnuma1 ocl-icd-libopencl1 && rm -rf /var/lib/apt/lists/* && \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \ @@ -127,8 +130,8 @@ RUN set -e ; \ rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* ; \ ;; \ *) \ - dpkg -P intel-gmmlib intel-igc-core intel-igc-opencl intel-level-zero-gpu intel-ocloc intel-opencl intel-opencl-icd ; \ - apt-get update && apt-get -y install dpkg-dev ; \ + dpkg -P intel-gmmlib intel-igc-core intel-igc-opencl intel-level-zero-gpu intel-ocloc intel-opencl intel-opencl-icd && \ + apt-get update && apt-get -y install dpkg-dev && rm -rf /var/lib/apt/lists/* && \ cd /drivers/${INSTALL_DRIVER_VERSION} && \ dpkg-scanpackages . > Packages && \ cd - ; \ @@ -137,10 +140,11 @@ RUN set -e ; \ apt-get install -y \ intel-opencl-icd \ intel-level-zero-gpu level-zero \ - intel-media-va-driver-non-free libmfx1 ; \ + intel-media-va-driver-non-free libmfx1 && \ + rm -rf /var/lib/apt/lists/* ; \ esac ; \ fi ; \ - apt clean ; \ + apt-get clean ; \ rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* ; \ groupadd --gid 5000 ovms ; \ useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --groups 39,44 --shell /bin/bash --skel /dev/null ovms @@ -163,6 +167,13 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* # for NCS RUN if [ -f /ovms/lib/hddl/etc/rules.d/97-myriad-usbboot.rules ]; then mkdir -p /etc/udev/rules.d/ && cp /ovms/lib/hddl/etc/rules.d/97-myriad-usbboot.rules /etc/udev/rules.d/ && ldconfig ; fi +# for NVIDIA +RUN if [ "$NVIDIA" == "1" ]; then true ; else exit 0 ; fi ; echo "installing cuda apt package"; \ + apt update -y && \ + apt install -y curl && \ + cd etc/apt/preferences.d && curl -o cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin ; \ + apt update && apt install -y --no-install-recommends libcudnn8 libcutensor1 libpng16-16 && \ + apt clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* RUN echo "The source code of added GPL components is stored in https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/container_gpl_sources/ubuntu20/" > /ovms/thirdparty-licenses/GPL.txt USER ovms diff --git a/src/example/SampleCpuExtension/Makefile b/src/example/SampleCpuExtension/Makefile index 9c955a6c93..2fc9c7afb5 100644 --- a/src/example/SampleCpuExtension/Makefile +++ b/src/example/SampleCpuExtension/Makefile @@ -14,9 +14,11 @@ # limitations under the License. # +OPENVINO_PATH ?= /opt/intel/openvino_2022 + all: $(eval SHELL:=/bin/bash) /usr/bin/g++ --version - . /opt/intel/openvino_2022/setupvars.sh && /usr/bin/g++ -std=gnu++11 -fPIC -shared ov_extension.cpp CustomReluOp.cpp \ - -I /opt/intel/openvino_2022/runtime/include/ie \ - -I /opt/intel/openvino_2022/runtime/include \ + . $(OPENVINO_PATH)/setupvars.sh && /usr/bin/g++ -std=gnu++11 -fPIC -shared ov_extension.cpp CustomReluOp.cpp \ + -I $(OPENVINO_PATH)/runtime/include/ie \ + -I $(OPENVINO_PATH)/runtime/include \ -o libcustom_relu_cpu_extension.so From ce0570a6b72d78d558618e7a60e8c7739e879a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Tue, 29 Nov 2022 15:22:58 +0100 Subject: [PATCH 059/130] Pull latest ubi-minimal image on no cache build option (#1552) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 6b8bea8f3a..0e2a8e20fd 100644 --- a/Makefile +++ b/Makefile @@ -189,6 +189,9 @@ ifeq ($(NO_DOCKER_CACHE),true) $(eval NO_CACHE_OPTION:=--no-cache) @echo "Docker image will be rebuilt from scratch" @docker pull $(BASE_IMAGE) + ifeq ($(BASE_OS),redhat) + @docker pull registry.access.redhat.com/ubi8/ubi-minimal:8.6 + endif endif ifneq ($(OVMS_METADATA_FILE),) @cp $(OVMS_METADATA_FILE) .workspace/metadata.json From fbdba4581fc46edfff51cce07cc4eea7ed8c0bbd Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Thu, 1 Dec 2022 08:33:36 +0100 Subject: [PATCH 060/130] CVS-93701_doc_test_fixes (#1554) * git clone + pip install added * review updates Co-authored-by: ngrozae --- tests/models/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/models/README.md b/tests/models/README.md index 7750d0de2e..2f8ea4319f 100644 --- a/tests/models/README.md +++ b/tests/models/README.md @@ -1,5 +1,13 @@ # Generating models with simple arithmetical operations +## Prepare Python environment + +```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server/tests/models +pip3 install -r ../requirements.txt +``` + ## Model incrementing an input tensor ```bash From e0e1d0d94e85f8fed88b9aa35ad37924d7169d68 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Thu, 1 Dec 2022 17:24:01 +0100 Subject: [PATCH 061/130] Remove coverage logs (#1547) * Remove coverage logs --- Dockerfile.redhat | 13 ++++++------- Dockerfile.ubuntu | 11 +++++------ Makefile | 3 ++- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 437ef17977..75449f0dcb 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -27,13 +27,14 @@ ARG TEMP_DIR=/tmp/openvino_installer ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools ARG DL_DIR=/tmp ARG JOBS -ARG CHECK_COVERAGE=1 +ARG CHECK_COVERAGE=0 # build_type=[ opt, dbg ] ARG build_type=dbg ARG debug_bazel_flags=--strip=never\ --copt="-g"\ -c\ dbg ARG minitrace_flags ENV TF_SYSTEM_LIBS="curl" +ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/k8-opt/testlogs/test.log" RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum update -d6 -y && yum install -d6 -y \ boost169-atomic \ @@ -194,12 +195,10 @@ RUN wget https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov COPY check_coverage.bat /ovms/ -# Test coverage and unit tests in one layer -RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test && exit 0; fi ;\ - bazel coverage --test_output=streamed --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ - bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test - +# Test Coverage +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ + bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} + RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 0351ae9b2d..0a8cdc3109 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -26,7 +26,7 @@ ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools ARG DL_DIR=/tmp ARG JOBS ARG APT_OV_PACKAGE=openvino-2022.1.0 -ARG CHECK_COVERAGE=1 +ARG CHECK_COVERAGE=0 ARG NVIDIA=0 ARG http_proxy ARG CMAKE_BUILD_TYPE=Release @@ -38,6 +38,7 @@ ARG minitrace_flags ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/external/hddl ENV DEBIAN_FRONTEND=noninteractive ENV TF_SYSTEM_LIBS="curl" +ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/k8-opt/testlogs/test.log" SHELL ["/bin/bash", "-c"] RUN apt-get update && apt-get install -y \ libboost-atomic1.71.0 \ @@ -286,11 +287,9 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: COPY check_coverage.bat /ovms/ # Test Coverage -RUN if [ "$CHECK_COVERAGE" == "1" ] ; then true ; else bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test && exit 0; fi ;\ - bazel coverage --test_output=streamed --combined_report=lcov //src:ovms_test &&\ - genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" &&\ - bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test - +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ + bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} + RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so diff --git a/Makefile b/Makefile index 0e2a8e20fd..66683e6f9e 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,8 @@ BASE_OS_TAG_UBUNTU ?= 20.04 BASE_OS_TAG_REDHAT ?= 8.6 INSTALL_RPMS_FROM_URL ?= -CHECK_COVERAGE ?=1 + +CHECK_COVERAGE ?=0 NVIDIA ?=0 # NOTE: when changing any value below, you'll need to adjust WORKSPACE file by hand: From b23ce6bb9318d54623bdcb5bcf0f09c2de4d9531 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Fri, 2 Dec 2022 10:56:36 +0100 Subject: [PATCH 062/130] drop redundant documentation (#1558) --- docs/docker_container.md | 18 ++++++++++++++++-- release_files/drivers/.gitkeep | 0 release_files/drivers/README.md | 29 ----------------------------- 3 files changed, 16 insertions(+), 31 deletions(-) create mode 100644 release_files/drivers/.gitkeep delete mode 100644 release_files/drivers/README.md diff --git a/docs/docker_container.md b/docs/docker_container.md index b2ff5e04f0..df2718903a 100644 --- a/docs/docker_container.md +++ b/docs/docker_container.md @@ -104,11 +104,25 @@ docker run --rm -it --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/rende To build the image, you need to have NEO Runtime packages available. Contact Intel representative to get the access to the preproduction drivers. -Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2` and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. +Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2`. Expected structure is like below: +``` +drivers +└── dg2 + ├── intel-igc-core__amd64.deb + ├── intel-igc-opencl__amd64.deb + ├── intel-level-zero-gpu-dbgsym__amd64.ddeb + ├── intel-level-zero-gpu__amd64.deb + ├── intel-opencl-icd-dbgsym__amd64.ddeb + ├── intel-opencl-icd__amd64.deb + ├── libigdgmm12__amd64.deb + └── libigdgmm12__amd64.deb +``` + +and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. Example: ``` -make docker_build BASE_OS=ubuntu OVMS_CPP_DOCKER_IMAGE=ovms_dg2 INSTALL_DRIVER_VERSION=dg2 +make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 ``` ## Using Multi-Device Plugin diff --git a/release_files/drivers/.gitkeep b/release_files/drivers/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/release_files/drivers/README.md b/release_files/drivers/README.md deleted file mode 100644 index b945e52e90..0000000000 --- a/release_files/drivers/README.md +++ /dev/null @@ -1,29 +0,0 @@ -## Custom NEO Runtime packages - -This directory is supposed to contain NEO Runtime packages placed in a separate directory like: - -``` -drivers -└── - ├── intel-igc-core__amd64.deb - ├── intel-igc-opencl__amd64.deb - ├── intel-level-zero-gpu-dbgsym__amd64.ddeb - ├── intel-level-zero-gpu__amd64.deb - ├── intel-opencl-icd-dbgsym__amd64.ddeb - ├── intel-opencl-icd__amd64.deb - ├── libigdgmm12__amd64.deb - └── libigdgmm12__amd64.deb -``` - -With such structure you can build OpenVINO Model Server GPU image with custom NEO Runtime by specifying `INSTALL_DRIVER_VERSION` parameter in `make docker_build` command. The value of `INSTALL_DRIVER_VERSION` should be the name of the subfolder i.e. the `` in above schema. - -**Example:** For packages placed in `drivers/neo` location, the build command could look like this: - -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -``` - -```bash -make docker_build BASE_OS=ubuntu OVMS_CPP_DOCKER_IMAGE=ovms_custom_neo INSTALL_DRIVER_VERSION=neo -``` From de133110be7918d8e7c43c06c6ac822e86bc593d Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Fri, 2 Dec 2022 11:45:48 +0100 Subject: [PATCH 063/130] set tfs-api import on version 2.11.0 (#1559) --- .../python/tensorflow-serving-api/samples/requirements.txt | 2 +- demos/bert_question_answering/python/requirements.txt | 3 +-- demos/common/python/requirements.txt | 5 ++--- .../python/asr_demo/README.md | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/client/python/tensorflow-serving-api/samples/requirements.txt b/client/python/tensorflow-serving-api/samples/requirements.txt index fa1be3398e..3ccf3afac4 100644 --- a/client/python/tensorflow-serving-api/samples/requirements.txt +++ b/client/python/tensorflow-serving-api/samples/requirements.txt @@ -1 +1 @@ -tensorflow-serving-api==2.* +tensorflow-serving-api==2.11.0 diff --git a/demos/bert_question_answering/python/requirements.txt b/demos/bert_question_answering/python/requirements.txt index 9ae8c0e84b..a553919019 100644 --- a/demos/bert_question_answering/python/requirements.txt +++ b/demos/bert_question_answering/python/requirements.txt @@ -1,3 +1,2 @@ -tensorflow-serving-api==2.7.0 +tensorflow-serving-api==2.11.0 validators -protobuf>=3.18,<4.0.0 \ No newline at end of file diff --git a/demos/common/python/requirements.txt b/demos/common/python/requirements.txt index 8cc727a593..116852469f 100644 --- a/demos/common/python/requirements.txt +++ b/demos/common/python/requirements.txt @@ -1,4 +1,3 @@ futures==3.1.1 -opencv-python==4.4.0.46 -tensorflow-serving-api==2.7.0 -protobuf>=3.18,<4.0.0 +opencv-python>=4.6.0 +tensorflow-serving-api==2.11.0 diff --git a/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md b/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md index abf1de1b43..6cffa21938 100644 --- a/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md +++ b/demos/speech_recognition_with_kaldi_model/python/asr_demo/README.md @@ -49,7 +49,7 @@ RUN git clone https://github.com/openvinotoolkit/model_server.git /opt/model_ser cd /opt/model_server && \ virtualenv -p python3 .venv && \ . .venv/bin/activate && \ - pip install tensorflow-serving-api==2.* kaldi-python-io==1.2.1 && \ + pip install tensorflow-serving-api==2.11.0 kaldi-python-io==1.2.1 && \ echo "source /opt/model_server/.venv/bin/activate" | tee -a /root/.bashrc && \ mkdir /opt/workspace From a585b6afdeb4438f5bffebc73a9321a00c0a5bf3 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Fri, 2 Dec 2022 12:12:13 +0100 Subject: [PATCH 064/130] Fix binary outputs (#1553) --- src/http_rest_api_handler.cpp | 32 +++- src/rest_utils.cpp | 189 ++++++-------------- src/rest_utils.hpp | 4 +- src/test/rest_utils_test.cpp | 320 +++++++++++++++++++++++++++++----- 4 files changed, 361 insertions(+), 184 deletions(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 1a8cd377ca..f726ab75e1 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -497,6 +498,33 @@ Status HttpRestApiHandler::prepareGrpcRequest(const std::string modelName, const return StatusCode::OK; } +static std::set getRequestedBinaryOutputsNames(::KFSRequest& grpc_request) { + std::set binaryOutputs; + bool byDefaultBinaryOutpuRequested = false; + for (auto& parameter : grpc_request.parameters()) { + if (parameter.second.parameter_choice_case(), inference::InferParameter::ParameterChoiceCase::kBoolParam) { + if (parameter.first == "binary_data_output") { + byDefaultBinaryOutpuRequested = parameter.second.bool_param(); + break; + } + } + } + for (const inference::ModelInferRequest_InferRequestedOutputTensor& output : grpc_request.outputs()) { + bool specificBinaryOutputRequested = byDefaultBinaryOutpuRequested; + for (auto& parameter : output.parameters()) { + if ((parameter.second.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kBoolParam) && + (parameter.first == "binary_data")) { + specificBinaryOutputRequested = parameter.second.bool_param(); + break; + } + } + if (specificBinaryOutputRequested) { + binaryOutputs.insert(output.name()); + } + } + return binaryOutputs; +} + Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body, std::optional& inferenceHeaderContentLength) { Timer timer; timer.start(TOTAL); @@ -524,9 +552,9 @@ Status HttpRestApiHandler::processInferKFSRequest(const HttpRequestComponents& r if (!gstatus.ok()) { return gstatus; } + std::set requestedBinaryOutputsNames = getRequestedBinaryOutputsNames(grpc_request); std::string output; - google::protobuf::util::JsonPrintOptions opts_out; - status = ovms::makeJsonFromPredictResponse(grpc_response, &output, inferenceHeaderContentLength); + status = ovms::makeJsonFromPredictResponse(grpc_response, &output, inferenceHeaderContentLength, requestedBinaryOutputsNames); if (!status.ok()) { return status; } diff --git a/src/rest_utils.cpp b/src/rest_utils.cpp index 03990c691f..a3cd3653c0 100644 --- a/src/rest_utils.cpp +++ b/src/rest_utils.cpp @@ -15,6 +15,8 @@ //***************************************************************************** #include "rest_utils.hpp" +#include + #include #include #include @@ -230,8 +232,8 @@ static Status parseResponseParameters(const ::KFSResponse& response_proto, rapid return StatusCode::OK; } -static Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer, int bytesOutputSize) { - if (output.parameters_size() > 0 || bytesOutputSize > 0) { +static Status parseOutputParameters(const inference::ModelInferResponse_InferOutputTensor& output, rapidjson::PrettyWriter& writer, int binaryOutputSize) { + if (output.parameters_size() > 0 || binaryOutputSize > 0) { writer.Key("parameters"); writer.StartObject(); @@ -251,9 +253,9 @@ static Status parseOutputParameters(const inference::ModelInferResponse_InferOut break; // return param error } } - if (bytesOutputSize > 0) { + if (binaryOutputSize > 0) { writer.Key("binary_data_size"); - writer.Int(bytesOutputSize); + writer.Int(binaryOutputSize); } writer.EndObject(); } @@ -261,25 +263,32 @@ static Status parseOutputParameters(const inference::ModelInferResponse_InferOut return StatusCode::OK; } -template -static void fillTensorDataWithIntValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { - for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) - writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); -} - -template -static void fillTensorDataWithUintValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { - for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) - writer.Int(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); +static void appendBinaryOutput(std::string& bytesOutputsBuffer, char* output, size_t outputSize) { + bytesOutputsBuffer.append(output, outputSize); } -template -static void fillTensorDataWithFloatValuesFromRawContents(const ::KFSResponse& response_proto, int tensor_it, rapidjson::PrettyWriter& writer) { - for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(ValueType)) - writer.Double(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); -} +#define PARSE_OUTPUT_DATA(CONTENTS_FIELD, DATATYPE, WRITER_TYPE) \ + if (seekDataInValField) { \ + auto status = checkValField(tensor.contents().CONTENTS_FIELD##_size(), expectedElementsNumber); \ + if (!status.ok()) \ + return status; \ + if (binaryOutput) { \ + appendBinaryOutput(bytesOutputsBuffer, (char*)tensor.contents().CONTENTS_FIELD().data(), expectedContentSize); \ + } else { \ + for (auto& number : tensor.contents().CONTENTS_FIELD()) { \ + writer.WRITER_TYPE(number); \ + } \ + } \ + } else { \ + if (binaryOutput) { \ + appendBinaryOutput(bytesOutputsBuffer, (char*)response_proto.raw_output_contents(tensor_it).data(), expectedContentSize); \ + } else { \ + for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(DATATYPE)) \ + writer.WRITER_TYPE(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); \ + } \ + } -static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer, std::string& bytesOutputsBuffer) { +static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::PrettyWriter& writer, std::string& bytesOutputsBuffer, const std::set& binaryOutputsNames) { writer.Key("outputs"); writer.StartArray(); @@ -308,140 +317,53 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett writer.EndArray(); writer.Key("datatype"); writer.String(tensor.datatype().c_str()); - if (tensor.datatype() != "BYTES") { + bool binaryOutput = ((binaryOutputsNames.find(tensor.name().c_str()) != binaryOutputsNames.end()) || (tensor.datatype() == "BYTES")); + if (!binaryOutput) { writer.Key("data"); writer.StartArray(); } if (tensor.datatype() == "FP32") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().fp32_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().fp32_contents()) { - writer.Double(number); - } - } else { - fillTensorDataWithFloatValuesFromRawContents(response_proto, tensor_it, writer); - } - } else if (tensor.datatype() == "INT64") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().int64_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().int64_contents()) { - writer.Int64(number); - } - } else { - for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(int64_t)) - writer.Int64(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); - } + PARSE_OUTPUT_DATA(fp32_contents, float, Double) } else if (tensor.datatype() == "INT32") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().int_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().int_contents()) { - writer.Int(number); - } - } else { - fillTensorDataWithIntValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(int_contents, int32_t, Int) } else if (tensor.datatype() == "INT16") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().int_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().int_contents()) { - writer.Int(number); - } - } else { - fillTensorDataWithIntValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(int_contents, int16_t, Int) } else if (tensor.datatype() == "INT8") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().int_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().int_contents()) { - writer.Int(number); - } - } else { - fillTensorDataWithIntValuesFromRawContents(response_proto, tensor_it, writer); - } - } else if (tensor.datatype() == "UINT64") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().uint64_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().uint64_contents()) { - writer.Uint64(number); - } - } else { - for (size_t i = 0; i < response_proto.raw_output_contents(tensor_it).size(); i += sizeof(uint64_t)) - writer.Uint64(*(reinterpret_cast(response_proto.raw_output_contents(tensor_it).data() + i))); - } + PARSE_OUTPUT_DATA(int_contents, int8_t, Int) } else if (tensor.datatype() == "UINT32") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().uint_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().uint_contents()) { - writer.Uint(number); - } - } else { - fillTensorDataWithUintValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(uint_contents, uint32_t, Uint) } else if (tensor.datatype() == "UINT16") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().uint_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().uint_contents()) { - writer.Uint(number); - } - } else { - fillTensorDataWithUintValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(uint_contents, uint16_t, Uint) } else if (tensor.datatype() == "UINT8") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().uint_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().uint_contents()) { - writer.Uint(number); - } - } else { - fillTensorDataWithUintValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(uint_contents, uint8_t, Uint) } else if (tensor.datatype() == "FP64") { - if (seekDataInValField) { - auto status = checkValField(tensor.contents().fp64_contents_size(), expectedElementsNumber); - if (!status.ok()) - return status; - for (auto& number : tensor.contents().fp64_contents()) { - writer.Double(number); - } - } else { - fillTensorDataWithFloatValuesFromRawContents(response_proto, tensor_it, writer); - } + PARSE_OUTPUT_DATA(fp64_contents, double, Double) + } else if (tensor.datatype() == "INT64") { + PARSE_OUTPUT_DATA(int64_contents, int64_t, Int64) + } else if (tensor.datatype() == "UINT64") { + PARSE_OUTPUT_DATA(uint64_contents, uint64_t, Uint64) } else if (tensor.datatype() == "BYTES") { if (seekDataInValField) { - auto status = checkValField(tensor.contents().bytes_contents_size(), expectedElementsNumber); + size_t bytesContentsSize = 0; + for (auto bytesStr : tensor.contents().bytes_contents()) { + bytesContentsSize += bytesStr.size(); + } + auto status = checkValField(bytesContentsSize, expectedElementsNumber); if (!status.ok()) return status; - for (auto& bytes : tensor.contents().bytes_contents()) { - bytesOutputsBuffer.append(bytes); + for (auto bytesStr : tensor.contents().bytes_contents()) { + bytesOutputsBuffer.append(bytesStr); } } else { - bytesOutputsBuffer.append((char*)response_proto.raw_output_contents(tensor_it).data(), response_proto.raw_output_contents(tensor_it).size()); + appendBinaryOutput(bytesOutputsBuffer, (char*)response_proto.raw_output_contents(tensor_it).data(), expectedContentSize); } } else { return StatusCode::REST_UNSUPPORTED_PRECISION; } - if (tensor.datatype() != "BYTES") { + if (!binaryOutput) { writer.EndArray(); } - auto status = parseOutputParameters(tensor, writer, bytesOutputsBuffer.size()); + auto status = parseOutputParameters(tensor, writer, binaryOutput ? expectedContentSize : 0); if (!status.ok()) { return status; } @@ -455,7 +377,8 @@ static Status parseOutputs(const ::KFSResponse& response_proto, rapidjson::Prett Status makeJsonFromPredictResponse( const ::KFSResponse& response_proto, std::string* response_json, - std::optional& inferenceHeaderContentLength) { + std::optional& inferenceHeaderContentLength, + const std::set& requestedBinaryOutputsNames) { Timer timer; using std::chrono::microseconds; timer.start(CONVERT); @@ -486,7 +409,7 @@ Status makeJsonFromPredictResponse( } std::string binaryOutputsBuffer; - status = parseOutputs(response_proto, writer, binaryOutputsBuffer); + status = parseOutputs(response_proto, writer, binaryOutputsBuffer, requestedBinaryOutputsNames); if (!status.ok()) { return status; } diff --git a/src/rest_utils.hpp b/src/rest_utils.hpp index 1c2932ea27..79f6d8b60b 100644 --- a/src/rest_utils.hpp +++ b/src/rest_utils.hpp @@ -15,6 +15,7 @@ //***************************************************************************** #pragma once +#include #include #pragma GCC diagnostic push @@ -34,7 +35,8 @@ Status makeJsonFromPredictResponse( Status makeJsonFromPredictResponse( const ::KFSResponse& response_proto, std::string* response_json, - std::optional& inferenceHeaderContentLength); + std::optional& inferenceHeaderContentLength, + const std::set& requestedBinaryOutputsNames = {}); Status decodeBase64(std::string& bytes, std::string& decodedBytes); diff --git a/src/test/rest_utils_test.cpp b/src/test/rest_utils_test.cpp index cc35b6f474..608a95f371 100644 --- a/src/test/rest_utils_test.cpp +++ b/src/test/rest_utils_test.cpp @@ -759,31 +759,62 @@ TEST_F(KFSMakeJsonFromPredictResponseRawTest, ErrorWhenNoOutputs) { ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); } +template +static void assertBinaryOutput(T data, std::string json, std::string expectedJson, std::optional inferenceHeaderContentLength) { + ASSERT_TRUE(inferenceHeaderContentLength.has_value()); + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + sizeof(T)); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + EXPECT_EQ(*(T*)json.substr(inferenceHeaderContentLength.value()).data(), data); +} + class KFSMakeJsonFromPredictResponsePrecisionTest : public ::testing::Test { protected: KFSResponse proto; std::string json; KFSTensorOutputProto* output; std::optional inferenceHeaderContentLength; + std::string outputName = "output"; void SetUp() override { proto.set_model_name("model"); proto.set_id("id"); output = proto.add_outputs(); - output->set_name("output"); + output->set_name(outputName); output->mutable_shape()->Add(1); output->mutable_shape()->Add(1); } + + template + void prepareData(T data, std::string datatype) { + output->set_datatype(datatype); + auto* output_contents = proto.add_raw_output_contents(); + output_contents->assign(reinterpret_cast(&data), sizeof(T)); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + } + + template + void prepareDataBinary(T data, std::string datatype) { + output->set_datatype(datatype); + auto* output_contents = proto.add_raw_output_contents(); + output_contents->assign(reinterpret_cast(&data), sizeof(T)); + std::set binaryOutputs; + binaryOutputs.insert(outputName); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength, binaryOutputs), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + } + + template + void assertDataBinary(T data, std::string expectedJson) { + assertBinaryOutput(data, json, expectedJson, inferenceHeaderContentLength); + } }; TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Float) { float data = 92.5f; - output->set_datatype("FP32"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(float)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "FP32"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -796,13 +827,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Float) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Float_binary) { + float data = 50000000000.99; + prepareDataBinary(data, "FP32"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "FP32", + "parameters": { + "binary_data_size": 4 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Double) { double data = 50000000000.99; - output->set_datatype("FP64"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(double)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "FP64"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -815,13 +860,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Double) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Double_binary) { + double data = 50000000000.99; + prepareDataBinary(data, "FP64"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "FP64", + "parameters": { + "binary_data_size": 8 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int32) { int32_t data = -82; - output->set_datatype("INT32"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(int32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "INT32"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -834,13 +893,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int32) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int32_binary) { + int32_t data = -82; + prepareDataBinary(data, "INT32"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "INT32", + "parameters": { + "binary_data_size": 4 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int16) { int16_t data = -945; - output->set_datatype("INT16"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(int16_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "INT16"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -853,13 +926,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int16) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int16_binary) { + int16_t data = -945; + prepareDataBinary(data, "INT16"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "INT16", + "parameters": { + "binary_data_size": 2 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int8) { int8_t data = -53; - output->set_datatype("INT8"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(int8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "INT8"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -872,13 +959,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int8) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int8_binary) { + int8_t data = -53; + prepareDataBinary(data, "INT8"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "INT8", + "parameters": { + "binary_data_size": 1 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint8) { uint8_t data = 250; - output->set_datatype("UINT8"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(uint8_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "UINT8"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -891,13 +992,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint8) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint8_binary) { + uint8_t data = 250; + prepareDataBinary(data, "UINT8"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "UINT8", + "parameters": { + "binary_data_size": 1 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int64) { int64_t data = -658324; - output->set_datatype("INT64"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(int64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "INT64"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -910,13 +1025,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int64) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Int64_binary) { + int64_t data = -658324; + prepareDataBinary(data, "INT64"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "INT64", + "parameters": { + "binary_data_size": 8 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint32) { uint32_t data = 1245353; - output->set_datatype("UINT32"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(uint32_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "UINT32"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -929,13 +1058,27 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint32) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint32_binary) { + uint32_t data = 1245353; + prepareDataBinary(data, "UINT32"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "UINT32", + "parameters": { + "binary_data_size": 4 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { uint64_t data = 63456412; - output->set_datatype("UINT64"); - auto* output_contents = proto.add_raw_output_contents(); - output_contents->assign(reinterpret_cast(&data), sizeof(uint64_t)); - ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); - ASSERT_EQ(inferenceHeaderContentLength.has_value(), false); + prepareData(data, "UINT64"); EXPECT_EQ(json, R"({ "model_name": "model", "id": "id", @@ -948,6 +1091,24 @@ TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64) { })"); } +TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, Uint64_binary) { + uint64_t data = 63456412; + prepareDataBinary(data, "UINT64"); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "output", + "shape": [1, 1], + "datatype": "UINT64", + "parameters": { + "binary_data_size": 8 + } + }] +})"; + assertDataBinary(data, expectedJson); +} + TEST_F(KFSMakeJsonFromPredictResponsePrecisionTest, BYTES_1) { int8_t data = -53; output->set_datatype("BYTES"); @@ -1042,9 +1203,10 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positi bytes_val_proto->set_datatype("BYTES"); int dataSize = 10; bytes_val_proto->mutable_shape()->Add(dataSize); + auto bytes_val = bytes_val_proto->mutable_contents()->mutable_bytes_contents()->Add(); + // *bytes_val = i; for (uint8_t i = 0; i < dataSize; i++) { - auto bytes_val = bytes_val_proto->mutable_contents()->mutable_bytes_contents()->Add(); - *bytes_val = i; + bytes_val->append((char*)&i, 1); } ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); @@ -1099,6 +1261,68 @@ TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positi })"); } +TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positive_oneOutputsBinary) { + std::set binaryOutputs; + binaryOutputs.insert("single_uint64_val"); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength, binaryOutputs), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "single_uint64_val", + "shape": [1], + "datatype": "UINT64", + "parameters": { + "binary_data_size": 8 + } + }, { + "name": "two_uint32_vals", + "shape": [2], + "datatype": "UINT32", + "data": [4000000000, 1] + }] +})"; + uint64_t expectedData = 5000000000; + assertBinaryOutput(expectedData, json, expectedJson, inferenceHeaderContentLength); +} + +TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_Positive_bothOutputsBinary) { + std::set binaryOutputs; + binaryOutputs.insert("single_uint64_val"); + binaryOutputs.insert("two_uint32_vals"); + ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength, binaryOutputs), StatusCode::OK); + ASSERT_EQ(inferenceHeaderContentLength.has_value(), true); + std::string expectedJson = R"({ + "model_name": "model", + "id": "id", + "outputs": [{ + "name": "single_uint64_val", + "shape": [1], + "datatype": "UINT64", + "parameters": { + "binary_data_size": 8 + } + }, { + "name": "two_uint32_vals", + "shape": [2], + "datatype": "UINT32", + "parameters": { + "binary_data_size": 8 + } + }] +})"; + ASSERT_EQ(inferenceHeaderContentLength.value(), expectedJson.size()); + ASSERT_EQ(json.size(), expectedJson.size() + sizeof(uint64_t) + 2 * sizeof(uint32_t)); + EXPECT_EQ(json.substr(0, inferenceHeaderContentLength.value()), expectedJson); + uint64_t firstOutputExpectedData = 5000000000; + EXPECT_EQ(*(uint64_t*)json.substr(inferenceHeaderContentLength.value()).data(), firstOutputExpectedData); + uint32_t secondOutputExpectedData_1 = 4000000000; + uint32_t secondOutputExpectedData_2 = 1; + EXPECT_EQ(*(uint32_t*)json.substr(inferenceHeaderContentLength.value() + sizeof(uint64_t)).data(), secondOutputExpectedData_1); + EXPECT_EQ(*(uint32_t*)json.substr(inferenceHeaderContentLength.value() + sizeof(uint64_t) + sizeof(uint32_t)).data(), secondOutputExpectedData_2); +} + TEST_F(KFSMakeJsonFromPredictResponseValTest, MakeJsonFromPredictResponse_OptionalModelVersion) { proto.set_model_version("version"); ASSERT_EQ(makeJsonFromPredictResponse(proto, &json, inferenceHeaderContentLength), StatusCode::OK); From b15a0ea735d605299ee0f59b0be1999b4dee8f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Fri, 2 Dec 2022 12:24:04 +0100 Subject: [PATCH 065/130] Update packages in redhat release stage dockerfile (#1556) --- Dockerfile.redhat | 6 +++--- Dockerfile.ubuntu | 6 +++--- release_files/Dockerfile.redhat | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 75449f0dcb..468c16c898 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -197,10 +197,10 @@ COPY check_coverage.bat /ovms/ # Test Coverage RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ - bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} + bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} -RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so +RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms +RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && make diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 0a8cdc3109..a84953b3da 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -288,10 +288,10 @@ ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/openvino/runtime/lib/intel64/: COPY check_coverage.bat /ovms/ # Test Coverage RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ - bazel test ${debug_bazel_flags} --jobs $JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} + bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} -RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs $JOBS //src:ovms -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libsampleloader.so +RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms +RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index 650a043fb2..5d14c2d007 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -45,6 +45,7 @@ ENV GPU=$GPU WORKDIR / RUN set -e ; \ set -x ; \ + microdnf upgrade -y ; \ microdnf install -y pkg-config ; \ if [ "$GPU" == "1" ] ; then \ case $INSTALL_DRIVER_VERSION in \ From 316da7c9b8af8cb091737816a7efd42f56d673e1 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Mon, 5 Dec 2022 08:17:59 +0100 Subject: [PATCH 066/130] Kfs cpp samples (#1557) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Kserve c++ samples with vector input format (#1434) * added cmake lists * added basic README * added infer example with dummy * args fix * removed redundant defaults * added documentation for grpc infer dummy * Apply suggestions from code review Co-authored-by: Miłosz Żeglarski * converted ns to ms * removed redundant output * added server live to readme * added mentions of dummy model * minor fix * changed file extensions from .cc to .cpp * Update client/cpp/kserve-api/README.md Co-authored-by: Miłosz Żeglarski * review fix * fixed rps * fixed docker start * Apply suggestions from code review Co-authored-by: Miłosz Żeglarski Co-authored-by: Miłosz Żeglarski * Kfs cpp grpc samples (#1546) * added utility clients * added documentation for grpc clients * added http samples * review fix * doc fix * doc fix v2 * documented http_infer_dummy * doc fix Co-authored-by: Miłosz Żeglarski --- client/cpp/kserve-api/CMakeLists.txt | 43 ++ client/cpp/kserve-api/README.md | 437 ++++++++++++++++++ client/cpp/kserve-api/samples/CMakeLists.txt | 34 ++ .../kserve-api/samples/grpc_infer_dummy.cpp | 173 +++++++ .../samples/grpc_model_metadata.cpp | 75 +++ .../kserve-api/samples/grpc_model_ready.cpp | 75 +++ .../kserve-api/samples/grpc_server_live.cpp | 73 +++ .../samples/grpc_server_metadata.cpp | 74 +++ .../kserve-api/samples/grpc_server_ready.cpp | 73 +++ .../kserve-api/samples/http_infer_dummy.cpp | 181 ++++++++ .../samples/http_model_metadata.cpp | 75 +++ .../kserve-api/samples/http_model_ready.cpp | 75 +++ .../kserve-api/samples/http_server_live.cpp | 73 +++ .../samples/http_server_metadata.cpp | 73 +++ .../kserve-api/samples/http_server_ready.cpp | 73 +++ 15 files changed, 1607 insertions(+) create mode 100644 client/cpp/kserve-api/CMakeLists.txt create mode 100644 client/cpp/kserve-api/README.md create mode 100644 client/cpp/kserve-api/samples/CMakeLists.txt create mode 100644 client/cpp/kserve-api/samples/grpc_infer_dummy.cpp create mode 100644 client/cpp/kserve-api/samples/grpc_model_metadata.cpp create mode 100644 client/cpp/kserve-api/samples/grpc_model_ready.cpp create mode 100644 client/cpp/kserve-api/samples/grpc_server_live.cpp create mode 100644 client/cpp/kserve-api/samples/grpc_server_metadata.cpp create mode 100644 client/cpp/kserve-api/samples/grpc_server_ready.cpp create mode 100644 client/cpp/kserve-api/samples/http_infer_dummy.cpp create mode 100644 client/cpp/kserve-api/samples/http_model_metadata.cpp create mode 100644 client/cpp/kserve-api/samples/http_model_ready.cpp create mode 100644 client/cpp/kserve-api/samples/http_server_live.cpp create mode 100644 client/cpp/kserve-api/samples/http_server_metadata.cpp create mode 100644 client/cpp/kserve-api/samples/http_server_ready.cpp diff --git a/client/cpp/kserve-api/CMakeLists.txt b/client/cpp/kserve-api/CMakeLists.txt new file mode 100644 index 0000000000..8c9b097485 --- /dev/null +++ b/client/cpp/kserve-api/CMakeLists.txt @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 3.17) + +project(kserve-samples LANGUAGES C CXX) + +set(CMAKE_INSTALL_PREFIX install) + +set(TRITON_ENABLE_CC_HTTP ON) +set(TRITON_ENABLE_CC_GRPC ON) + +include(FetchContent) +FetchContent_Declare( + triton-client + GIT_REPOSITORY https://github.com/triton-inference-server/client.git + GIT_TAG main +) +FetchContent_MakeAvailable(triton-client) +FetchContent_Declare( + cxxopts + GIT_REPOSITORY https://github.com/jarro2783/cxxopts.git + GIT_TAG v3.0.0 +) +FetchContent_MakeAvailable(cxxopts) +link_directories("${triton-client_BINARY_DIR}/cc-clients/install/lib") +include_directories("${triton-client_BINARY_DIR}/cc-clients/install/include") +include_directories("${cxxopts_SOURCE_DIR}/include") + +add_subdirectory(samples) diff --git a/client/cpp/kserve-api/README.md b/client/cpp/kserve-api/README.md new file mode 100644 index 0000000000..d85a2d3e02 --- /dev/null +++ b/client/cpp/kserve-api/README.md @@ -0,0 +1,437 @@ +# KServe API usage samples + +OpenVINO Model Server introduced support for [KServe API](https://github.com/kserve/kserve/tree/master/docs/predict-api/v2), including [Triton](https://github.com/triton-inference-server)'s raw format externsion. + +This guide shows how to interact with KServe API endpoints on both gRPC and HTTP interfaces using [Triton](https://github.com/triton-inference-server)'s client library. It covers following topics: +- GRPC API Examples + - grpc_server_live.py + - grpc_server_ready.py + - grpc_server_metadata.py + - grpc_model_ready.py + - grpc_model_metadata.py + - grpc_infer_resnet.py + - grpc_infer_binary_resnet.py +- HTTP API Example + - http_server_live.py + - http_server_ready.py + - http_server_metadata.py + - http_model_ready.py + - http_model_metadata.py + - http_infer_resnet.py + - http_infer_binary_resnet.py + +## Before you run the samples + +### Clone OpenVINO™ Model Server GitHub repository and go to the top directory. +```Bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server +``` + +### Start the Model Server Container with Dummy Model +```Bash +docker run --rm -d -v $(pwd)/src/test/dummy:/models -p 9000:9000 openvino/model_server:latest --model_name dummy --model_path /models --port 9000 +``` + +### Build client library and samples +```Bash +cd client/cpp/kserve-api +cmake . && make +cd samples +``` + +## GRPC Examples + + +## GRPC Examples with Dummy Model + +This section demonstrates inference on a simple model, which increments each provided value. + + +Once you finish above steps, you are ready to run the samples. + +### Run the Client to get server liveness + +- Command + +```Bash +./grpc_server_live --help +Sends requests via KServe gRPC API to check if server is alive. +Usage: + grpc_server_live [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_server_live --grpc_port 9000 --grpc_address localhost +Server Live: True +``` + +### Run the Client to get server readiness + +- Command + +```Bash +./grpc_server_ready --help +Sends requests via KServe gRPC API to check if server is ready. +Usage: + grpc_server_ready [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_server_ready --grpc_port 9000 --grpc_address localhost +Server Ready: True +``` + +### Run the Client to get server metadata + +- Command + +```Bash +./grpc_server_metadata --help +Sends requests via KServe gRPC API to get server metadata. +Usage: + grpc_server_metadata [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_server_metadata --grpc_port 9000 --grpc_address localhost +Name: "OpenVINO Model Server" +Version: "2022.2.c290da85" +``` + +### Run the Client to get model readiness + +- Command + +```Bash +./grpc_model_ready --help +Sends requests via KServe gRPC API to check if model is ready for inference. +Usage: + grpc_model_ready [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. (default: "") + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_model_ready --grpc_port 9000 --grpc_address localhost --model_name resnet +Model Ready: True +``` + +### Run the Client to get metadata + +- Command + +```Bash +./grpc_model_metadata --help +Sends requests via KServe gRPC API to get model metadata. +Usage: + grpc_ready [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. (default: "") + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_model_metadata --grpc_port 9000 --grpc_address localhost --model_name resnet +model metadata: +name: "resnet" +versions: "1" +platform: "OpenVINO" +inputs { + name: "0" + datatype: "FP32" + shape: 1 + shape: 224 + shape: 224 + shape: 3 +} +outputs { + name: "1463" + datatype: "FP32" + shape: 1 + shape: 1000 +} +``` +### Run the Client to perform inference + +```Bash +./grpc_infer_dummy --help +Sends requests via KServe gRPC API. +Usage: + grpc_infer_dummy [OPTION...] + + -h, --help Show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --input_name INPUT_NAME Specify input tensor name. (default: b) + --output_name OUTPUT_NAME + Specify input tensor name. (default: a) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_infer_dummy --grpc_port 9000 --model_name dummy +0 => 1 +1 => 2 +2 => 3 +3 => 4 +4 => 5 +5 => 6 +6 => 7 +7 => 8 +8 => 9 +9 => 10 +======Client Statistics====== +Number of requests: 1 +Total processing time: 5.28986 ms +Latency: 5.28986 ms +Requests per second: 189.041 +``` + +## HTTP Examples + +### Run the Client to get server liveness + +- Command + +```Bash +./http_server_live --help +Sends requests via KServe rest API to check if server is alive. +Usage: + http_server_live [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_server_live --http_port 8000 --http_address localhost +Server Live: True +``` + +### Run the Client to get server readiness + +- Command + +```Bash +./http_server_ready --help +Sends requests via KServe rest API to check if server is ready. +Usage: + http_server_ready [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_server_ready --http_port 8000 --http_address localhost +Server Ready: True +``` + +### Run the Client to get server metadata + +- Command + +```Bash +./http_server_metadata --help +Sends requests via KServe rest API to get server metadata. +Usage: + http_server_metadata [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_server_metadata --http_port 8000 --http_address localhost +{'name': 'OpenVINO Model Server', 'version': '2022.2.c290da85'} +``` + +### Run the Client to get model readiness + +- Command + +```Bash +./http_model_ready --help +Sends requests via KServe rest API to check if model is ready for inference. +Usage: + http_model_ready [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. (default: "") + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_model_ready --http_port 8000 --http_address localhost +Model Ready: True +``` + +### Run the Client to get model metadata + +- Command + +```Bash +./http_model_metadata --help +Sends requests via KServe rest API to get model metadata. +Usage: + http_ready [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. (default: "") + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_model_metadata --http_port 8000 --http_address localhost +{"name":"dummy","versions":["1"],"platform":"OpenVINO","inputs":[{"name":"b","datatype":"FP32","shape":[1,10]}],"outputs":[{"name":"a","datatype":"FP32","shape":[1,10]}]} +``` +### Run the Client to perform inference + +- Command + +```Bash +./http_infer_resnet --help +Sends requests via KServe rest API. +Usage: + http_infer_dummy [OPTION...] + + -h, --help Show this help message and exit + --http_address HTTP_ADDRESS + Specify url to grpc service. (default: + localhost) + --http_port PORT Specify port to grpc service. (default: + 8000) + --input_name INPUT_NAME Specify input tensor name. (default: b) + --output_name OUTPUT_NAME + Specify input tensor name. (default: a) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: dummy) + --model_version MODEL_VERSION + Define model version. + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./http_infer_resnety --http_port 8000 +0 => 1 +1 => 2 +2 => 3 +3 => 4 +4 => 5 +5 => 6 +6 => 7 +7 => 8 +8 => 9 +9 => 10 +======Client Statistics====== +Number of requests: 1 +Total processing time: 2.18683 ms +Latency: 2.18683 ms +Requests per second: 457.283 +``` \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/CMakeLists.txt b/client/cpp/kserve-api/samples/CMakeLists.txt new file mode 100644 index 0000000000..bc7f15e3c8 --- /dev/null +++ b/client/cpp/kserve-api/samples/CMakeLists.txt @@ -0,0 +1,34 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 3.17) + +function(sample sample_name) + add_executable(grpc_${sample_name} grpc_${sample_name}.cpp) + add_dependencies(grpc_${sample_name} cc-clients) + target_link_libraries(grpc_${sample_name} grpcclient libprotobuf.a) + + add_executable(http_${sample_name} http_${sample_name}.cpp) + add_dependencies(http_${sample_name} cc-clients) + target_link_libraries(http_${sample_name} httpclient) +endfunction() + +sample(infer_dummy) +sample(server_live) +sample(server_ready) +sample(server_metadata) +sample(model_ready) +sample(model_metadata) \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/grpc_infer_dummy.cpp b/client/cpp/kserve-api/samples/grpc_infer_dummy.cpp new file mode 100644 index 0000000000..5f9e5f2bd7 --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_infer_dummy.cpp @@ -0,0 +1,173 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + +namespace { + +void ValidateShapeAndDatatype( + const std::string& name, std::shared_ptr result) { + std::vector shape; + FAIL_IF_ERR( + result->Shape(name, &shape), "unable to get shape for '" + name + "'"); + // Validate shape + if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 10)) { + std::cerr << "error: received incorrect shapes for '" << name << "'" + << std::endl; + exit(1); + } + std::string datatype; + FAIL_IF_ERR( + result->Datatype(name, &datatype), + "unable to get datatype for '" + name + "'"); + // Validate datatype + if (datatype.compare("FP32") != 0) { + std::cerr << "error: received incorrect datatype for '" << name + << "': " << datatype << std::endl; + exit(1); + } +} + +} // namespace + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_infer_dummy", "Sends requests via KServe gRPC API."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("input_name", "Specify input tensor name. ", cxxopts::value()->default_value("b"), "INPUT_NAME") + ("output_name", "Specify input tensor name. ", cxxopts::value()->default_value("a"), "OUTPUT_NAME") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value(), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + std::string input_name(args["input_name"].as()); + std::string output_name(args["output_name"].as()); + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + std::string model_name = args["model_name"].as(); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + std::vector input_data(10); + for (size_t i = 0; i < 10; ++i) { + input_data[i] = i; + } + + std::vector shape{1, 10}; + + // Initialize the inputs with the data. + tc::InferInput* input; + + FAIL_IF_ERR( + tc::InferInput::Create(&input, input_name, shape, "FP32"), + "unable to get input"); + std::shared_ptr input_ptr; + input_ptr.reset(input); + + FAIL_IF_ERR( + input_ptr->AppendRaw( + reinterpret_cast(&input_data[0]), + input_data.size() * sizeof(float)), + "unable to set data for input"); + + + tc::InferOptions options(model_name); + if (args.count("model_version")) + options.model_version_ = args["model_version"].as(); + options.client_timeout_ = args["timeout"].as(); + + std::vector inputs = {input_ptr.get()}; + + tc::InferResult* results; + FAIL_IF_ERR( + client->Infer(&results, options, inputs), + "unable to run model"); + std::shared_ptr results_ptr; + results_ptr.reset(results); + + // Validate the results... + ValidateShapeAndDatatype(output_name, results_ptr); + + // Get pointer to the result returned... + float* output_data; + size_t output_byte_size; + FAIL_IF_ERR( + results_ptr->RawData( + output_name, (const uint8_t**)&output_data, &output_byte_size), + "unable to get result data for output"); + if (output_byte_size != 40) { + std::cerr << "error: received incorrect byte size for output: " + << output_byte_size << std::endl; + exit(1); + } + + for (size_t i = 0; i < 10; ++i) { + std::cout << input_data[i] << " => " + << *(output_data + i) << std::endl; + + if ((input_data[i] + 1) != *(output_data + i)) { + std::cerr << "error: Incorrect sum" << std::endl; + } + } + + tc::InferStat infer_stat; + client->ClientInferStat(&infer_stat); + std::cout << "======Client Statistics======" << std::endl; + std::cout << "Number of requests: " + << infer_stat.completed_request_count << std::endl; + std::cout << "Total processing time: " + << double(infer_stat.cumulative_total_request_time_ns)/1.0e+6 << " ms" << std::endl; + std::cout << "Latency: " + << double(infer_stat.cumulative_total_request_time_ns/infer_stat.completed_request_count)/1.0e+6 << " ms" << std::endl; + std::cout << "Requests per second: " + << double(1.0e+9/infer_stat.cumulative_total_request_time_ns/infer_stat.completed_request_count) << std::endl; + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/grpc_model_metadata.cpp b/client/cpp/kserve-api/samples/grpc_model_metadata.cpp new file mode 100644 index 0000000000..de9c4ae66c --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_model_metadata.cpp @@ -0,0 +1,75 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_ready", "Sends requests via KServe gRPC API to get model metadata."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + inference::ModelMetadataResponse model_metadata; + FAIL_IF_ERR( + client->ModelMetadata(&model_metadata, args["model_name"].as(), args["model_version"].as()), + "unable to get model metadata"); + std::cout< +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_model_ready", "Sends requests via KServe gRPC API to check if model is ready for inference."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + bool ready; + FAIL_IF_ERR( + client->IsModelReady(&ready, args["model_name"].as(), args["model_version"].as()), + "unable to get server readiness"); + std::cout<<"Model Ready: "<<(ready?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/grpc_server_live.cpp b/client/cpp/kserve-api/samples/grpc_server_live.cpp new file mode 100644 index 0000000000..d5a6cda352 --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_server_live.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_server_live", "Sends requests via KServe gRPC API to check if server is alive."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + bool live; + FAIL_IF_ERR( + client->IsServerLive(&live), + "unable to get server liveness"); + std::cout<<"Server Live: "<<(live?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/grpc_server_metadata.cpp b/client/cpp/kserve-api/samples/grpc_server_metadata.cpp new file mode 100644 index 0000000000..4b5a5a5b2f --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_server_metadata.cpp @@ -0,0 +1,74 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_server_metadata", "Sends requests via KServe gRPC API to get server metadata."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + inference::ServerMetadataResponse server_metadata; + FAIL_IF_ERR( + client->ServerMetadata(&server_metadata), + "unable to get server metadata"); + std::cout<<"Name: "<<(server_metadata.name()) << std::endl; + std::cout<<"Version: "<<(server_metadata.version()) << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/grpc_server_ready.cpp b/client/cpp/kserve-api/samples/grpc_server_ready.cpp new file mode 100644 index 0000000000..af225a7342 --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_server_ready.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_server_ready", "Sends requests via KServe gRPC API to check if server is ready."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + bool ready; + FAIL_IF_ERR( + client->IsServerReady(&ready), + "unable to get server readyness"); + std::cout<<"Server Ready: "<<(ready?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_infer_dummy.cpp b/client/cpp/kserve-api/samples/http_infer_dummy.cpp new file mode 100644 index 0000000000..7f4d1b310a --- /dev/null +++ b/client/cpp/kserve-api/samples/http_infer_dummy.cpp @@ -0,0 +1,181 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + +namespace { + +void ValidateShapeAndDatatype( + const std::string& name, std::shared_ptr result) { + std::vector shape; + FAIL_IF_ERR( + result->Shape(name, &shape), "unable to get shape for '" + name + "'"); + // Validate shape + if ((shape.size() != 2) || (shape[0] != 1) || (shape[1] != 10)) { + std::cerr << "error: received incorrect shapes for '" << name << "'" + << std::endl; + exit(1); + } + std::string datatype; + FAIL_IF_ERR( + result->Datatype(name, &datatype), + "unable to get datatype for '" + name + "'"); + // Validate datatype + if (datatype.compare("FP32") != 0) { + std::cerr << "error: received incorrect datatype for '" << name + << "': " << datatype << std::endl; + exit(1); + } +} + +} // namespace + +int main(int argc, char** argv) { + cxxopts::Options opt("http_infer_dummy", "Sends requests via KServe rest API."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("input_name", "Specify input tensor name. ", cxxopts::value()->default_value("b"), "INPUT_NAME") + ("output_name", "Specify input tensor name. ", cxxopts::value()->default_value("a"), "OUTPUT_NAME") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value(), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + std::string input_name(args["input_name"].as()); + std::string output_name(args["output_name"].as()); + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + std::string model_name = args["model_name"].as(); + + // Create a InferenceServerhttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + std::vector input_data(10); + for (size_t i = 0; i < 10; ++i) { + input_data[i] = i; + } + + std::vector shape{1, 10}; + + // Initialize the inputs with the data. + tc::InferInput* input; + + FAIL_IF_ERR( + tc::InferInput::Create(&input, input_name, shape, "FP32"), + "unable to get input"); + std::shared_ptr input_ptr; + input_ptr.reset(input); + + FAIL_IF_ERR( + input_ptr->AppendRaw( + reinterpret_cast(&input_data[0]), + input_data.size() * sizeof(float)), + "unable to set data for input"); + + tc::InferRequestedOutput* output; + + FAIL_IF_ERR( + tc::InferRequestedOutput::Create(&output, "a"), + "unable to get output"); + std::shared_ptr output_ptr; + output_ptr.reset(output); + + tc::InferOptions options(model_name); + if (args.count("model_version")) + options.model_version_ = args["model_version"].as(); + options.client_timeout_ = args["timeout"].as(); + + std::vector inputs = {input_ptr.get()}; + std::vector outputs = {output_ptr.get()}; + + tc::InferResult* results; + FAIL_IF_ERR( + client->Infer(&results, options, inputs, outputs), + "unable to run model"); + std::shared_ptr results_ptr; + results_ptr.reset(results); + + // Validate the results... + ValidateShapeAndDatatype(output_name, results_ptr); + + // Get pointer to the result returned... + float* output_data; + size_t output_byte_size; + FAIL_IF_ERR( + results_ptr->RawData( + output_name, (const uint8_t**)&output_data, &output_byte_size), + "unable to get result data for output"); + if (output_byte_size != 40) { + std::cerr << "error: received incorrect byte size for output: " + << output_byte_size << std::endl; + exit(1); + } + + for (size_t i = 0; i < 10; ++i) { + std::cout << input_data[i] << " => " + << *(output_data + i) << std::endl; + + if ((input_data[i] + 1) != *(output_data + i)) { + std::cerr << "error: Incorrect sum" << std::endl; + } + } + + tc::InferStat infer_stat; + client->ClientInferStat(&infer_stat); + std::cout << "======Client Statistics======" << std::endl; + std::cout << "Number of requests: " + << infer_stat.completed_request_count << std::endl; + std::cout << "Total processing time: " + << double(infer_stat.cumulative_total_request_time_ns)/1.0e+6 << " ms" << std::endl; + std::cout << "Latency: " + << double(infer_stat.cumulative_total_request_time_ns/infer_stat.completed_request_count)/1.0e+6 << " ms" << std::endl; + std::cout << "Requests per second: " + << double(1.0e+9/infer_stat.cumulative_total_request_time_ns/infer_stat.completed_request_count) << std::endl; + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_model_metadata.cpp b/client/cpp/kserve-api/samples/http_model_metadata.cpp new file mode 100644 index 0000000000..13fee6e2d7 --- /dev/null +++ b/client/cpp/kserve-api/samples/http_model_metadata.cpp @@ -0,0 +1,75 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("http_ready", "Sends requests via KServe rest API to get model metadata."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + + // Create a InferenceServerHttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + std::string model_metadata; + FAIL_IF_ERR( + client->ModelMetadata(&model_metadata, args["model_name"].as(), args["model_version"].as()), + "unable to get model metadata"); + std::cout< +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("http_model_ready", "Sends requests via KServe rest API to check if model is ready for inference."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + + // Create a InferenceServerhttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + bool ready; + FAIL_IF_ERR( + client->IsModelReady(&ready, args["model_name"].as(), args["model_version"].as()), + "unable to get server readiness"); + std::cout<<"Model Ready: "<<(ready?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_server_live.cpp b/client/cpp/kserve-api/samples/http_server_live.cpp new file mode 100644 index 0000000000..dd514ef9c3 --- /dev/null +++ b/client/cpp/kserve-api/samples/http_server_live.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("http_server_live", "Sends requests via KServe rest API to check if server is alive."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + + // Create a InferenceServerhttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + bool live; + FAIL_IF_ERR( + client->IsServerLive(&live), + "unable to get server liveness"); + std::cout<<"Server Live: "<<(live?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_server_metadata.cpp b/client/cpp/kserve-api/samples/http_server_metadata.cpp new file mode 100644 index 0000000000..c909842d86 --- /dev/null +++ b/client/cpp/kserve-api/samples/http_server_metadata.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("http_server_metadata", "Sends requests via KServe rest API to get server metadata."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + + // Create a InferenceServerHttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + std::string server_metadata; + FAIL_IF_ERR( + client->ServerMetadata(&server_metadata), + "unable to get server metadata"); + std::cout< +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + exit(1); \ + } \ + } + + +int main(int argc, char** argv) { + cxxopts::Options opt("http_server_ready", "Sends requests via KServe rest API to check if server is ready."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost")) + ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000")) + ("timeout", "Request timeout.", cxxopts::value()->default_value("0")) + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + exit(0); + } + + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + + // Create a InferenceServerHttpClient instance to communicate with the + // server using http protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + bool ready; + FAIL_IF_ERR( + client->IsServerReady(&ready), + "unable to get server readyness"); + std::cout<<"Server Ready: "<<(ready?"True":"False") << std::endl; + + + return 0; +} \ No newline at end of file From 9474be854bb1df777d823611d99a214a9bb5a5be Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 5 Dec 2022 15:51:12 +0100 Subject: [PATCH 067/130] Kw fix (#1565) * Kw fix --- ci/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/Dockerfile b/ci/Dockerfile index e24d8de4df..ac8b0267ac 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -26,6 +26,11 @@ RUN http_proxy=${HTTP_PROXY} yum install -y glibc.i686 libgcc.x86_64 libgcc.i686 ADD ./kwbuildtools /tmp/kwbuildtools WORKDIR /example_cpp_client/cpp +RUN bazel build @org_tensorflow//tensorflow/core:framework +# Patch apple build scripts +RUN cp -rf /ovms/third_party/build_bazel_rules_apple/bazel_rules_apple.patch $(bazel info output_base)/external/build_bazel_rules_apple/ +RUN cd $(bazel info output_base)/external/build_bazel_rules_apple/ && patch -p1 < bazel_rules_apple.patch + RUN /tmp/kwbuildtools/bin/kwinject --output /tmp/out.out bazel build //src:all WORKDIR /ovms/src From 5e38ce4510b8ea7417ca52ae048f4bc9db962621 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Tue, 6 Dec 2022 09:27:57 +0100 Subject: [PATCH 068/130] fix default base image for redhat os (#1563) --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 66683e6f9e..75fde3b55a 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,7 @@ ifeq ($(BASE_OS),ubuntu) endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) + BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_rhel8_2022.3.0.8844.12f74f066a4_x86_64.tgz From ae6de815f8ebda80eeb11342871cb4272be1b2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20=C5=BBeglarski?= Date: Tue, 6 Dec 2022 16:09:59 +0100 Subject: [PATCH 069/130] Upgrade UBI base image to 8.7 (#1567) --- Dockerfile.redhat | 2 +- Makefile | 4 ++-- client/python/ovmsclient/README.md | 2 +- release_files/Dockerfile.redhat | 4 ++-- src/custom_nodes/Dockerfile.redhat | 2 +- src/example/SampleCpuExtension/Dockerfile.redhat | 2 +- src/example/SampleCustomLoader/Dockerfile.redhat | 2 +- tools/deps/redhat/Dockerfile | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 468c16c898..8293be66de 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.6 +ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.7 FROM $BASE_IMAGE as base_build LABEL version="1.0.0" diff --git a/Makefile b/Makefile index 75fde3b55a..887d0db3a9 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ BASE_OS ?= ubuntu BASE_OS_TAG ?= latest BASE_OS_TAG_UBUNTU ?= 20.04 -BASE_OS_TAG_REDHAT ?= 8.6 +BASE_OS_TAG_REDHAT ?= 8.7 INSTALL_RPMS_FROM_URL ?= @@ -192,7 +192,7 @@ ifeq ($(NO_DOCKER_CACHE),true) @echo "Docker image will be rebuilt from scratch" @docker pull $(BASE_IMAGE) ifeq ($(BASE_OS),redhat) - @docker pull registry.access.redhat.com/ubi8/ubi-minimal:8.6 + @docker pull registry.access.redhat.com/ubi8/ubi-minimal:$(BASE_OS_TAG_REDHAT) endif endif ifneq ($(OVMS_METADATA_FILE),) diff --git a/client/python/ovmsclient/README.md b/client/python/ovmsclient/README.md index 75f8bd0ecd..1ed1a93ba0 100644 --- a/client/python/ovmsclient/README.md +++ b/client/python/ovmsclient/README.md @@ -20,7 +20,7 @@ If you're looking for a quick way to try `ovmsclient` out or want to look up how There are also Dockerfiles available that prepare Docker image with `ovmsclient` installed and ready to use. Simply run `docker build` with the Dockerfile of your choice to get the minimal image: - [Ubuntu 20.04 based image](Dockerfile.ubuntu) -- [UBI 8.4 based image](Dockerfile.redhat) +- [UBI 8.7 based image](Dockerfile.redhat) ```bash git clone https://github.com/openvinotoolkit/model_server.git diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index 5d14c2d007..ca59e05084 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.6 +ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.7 FROM $BASE_IMAGE as base_build RUN yum install -y xz WORKDIR / @@ -27,7 +27,7 @@ RUN groupadd --gid 5000 ovms && \ RUN mkdir /licenses && ln -s /ovms/LICENSE /licenses && ln -s /ovms/thirdparty-licenses /licenses/thirdparty-licenses # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6 as release +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 as release LABEL "name"="OVMS" LABEL "vendor"="Intel Corporation" LABEL "version"="2022.1" diff --git a/src/custom_nodes/Dockerfile.redhat b/src/custom_nodes/Dockerfile.redhat index 84ad481791..cd1d212ae1 100644 --- a/src/custom_nodes/Dockerfile.redhat +++ b/src/custom_nodes/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -FROM registry.access.redhat.com/ubi8/ubi:8.6 +FROM registry.access.redhat.com/ubi8/ubi:8.7 RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum update -d6 -y && yum install -d6 -y gcc-c++ curl xz # OpenCV diff --git a/src/example/SampleCpuExtension/Dockerfile.redhat b/src/example/SampleCpuExtension/Dockerfile.redhat index 10582bb8ad..94dd8d4822 100644 --- a/src/example/SampleCpuExtension/Dockerfile.redhat +++ b/src/example/SampleCpuExtension/Dockerfile.redhat @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 +ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.7 FROM $BASE_IMAGE as base_build ARG TEMP_DIR=/tmp/openvino_installer diff --git a/src/example/SampleCustomLoader/Dockerfile.redhat b/src/example/SampleCustomLoader/Dockerfile.redhat index 737aadb352..ad37f22640 100644 --- a/src/example/SampleCustomLoader/Dockerfile.redhat +++ b/src/example/SampleCustomLoader/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -FROM registry.access.redhat.com/ubi8/ubi:8.6 +FROM registry.access.redhat.com/ubi8/ubi:8.7 RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum update -d6 -y && yum install -d6 -y gcc-c++ wget make unzip COPY Makefile /src/example/SampleCustomLoader/ COPY sampleCustLoader.cpp /src/example/SampleCustomLoader/ diff --git a/tools/deps/redhat/Dockerfile b/tools/deps/redhat/Dockerfile index 70a5ceba1e..b6244096a7 100644 --- a/tools/deps/redhat/Dockerfile +++ b/tools/deps/redhat/Dockerfile @@ -14,13 +14,13 @@ # limitations under the License. # -FROM registry.access.redhat.com/ubi8/ubi:8.4 as rpmbuild +FROM registry.access.redhat.com/ubi8/ubi:8.7 as rpmbuild WORKDIR /root COPY rpmbuild.sh /root/ RUN /root/rpmbuild.sh -FROM registry.access.redhat.com/ubi8/ubi:8.4 +FROM registry.access.redhat.com/ubi8/ubi:8.7 RUN mkdir /rpmbuild/ COPY --from=rpmbuild /ovms-rpmbuild-deps.tar.xz /rpmbuild/ From 1dc2ca7756444649e7e13fb045deecc8fd85a9af Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Wed, 7 Dec 2022 08:12:49 +0100 Subject: [PATCH 070/130] Direct support for TF models in OVMS (#1541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added support for tf model loading * added demo * added decompresion step * code cleanup * minor fix * Update demos/image_classification_using_tf_model/python/README.md Co-authored-by: Miłosz Żeglarski * minor fix but better * actual fix * style fix Co-authored-by: Miłosz Żeglarski --- .../python/README.md | 64 +++++++++++++++++++ .../image_classification_using_tf_model.py | 55 ++++++++++++++++ .../python/requirements.txt | 2 + src/localfilesystem.cpp | 2 +- src/modelinstance.cpp | 51 ++++++--------- src/modelinstance.hpp | 8 +++ 6 files changed, 151 insertions(+), 31 deletions(-) create mode 100644 demos/image_classification_using_tf_model/python/README.md create mode 100644 demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py create mode 100644 demos/image_classification_using_tf_model/python/requirements.txt diff --git a/demos/image_classification_using_tf_model/python/README.md b/demos/image_classification_using_tf_model/python/README.md new file mode 100644 index 0000000000..b2e555cd13 --- /dev/null +++ b/demos/image_classification_using_tf_model/python/README.md @@ -0,0 +1,64 @@ +## Overview + +This guide demonstrates how to run inference requests for TensorFlow model with OpenVINO Model Server. +As an example, we will use [InceptionResNetV2](https://storage.googleapis.com/download.tensorflow.org/models/tflite/model_zoo/upload_20180427/inception_resnet_v2_2018_04_27.tgz) to perform classification of an image. + +## Prerequisites + +- [Docker](https://docs.docker.com/engine/install/) installed + +- Python 3.6 or newer installed + +## Preparing to Run + +Clone the repository and enter image_classification_using_tf_model directory + +```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd model_server/demos/image_classification_using_tf_model/python +``` + +## Download the InceptionResNetV2 model + +```bash +mkdir -p model/1 +wget -P model/1 https://storage.googleapis.com/download.tensorflow.org/models/tflite/model_zoo/upload_20180427/inception_resnet_v2_2018_04_27.tgz +tar xzf model/1/inception_resnet_v2_2018_04_27.tgz -C model/1 +ls model/1 +``` + +## Run Openvino Model Server + +```bash +docker run -d -v $PWD/model:/models -p 9000:9000 openvino/model_server:latest --model_path /models --model_name resnet --port 9000 +``` + +## Run the client + +Install python dependencies: +```bash +pip3 install -r requirements.txt +``` + +Now you can run the client: +```bash +python3 image_classification_using_tf_model.py --help +usage: image_classification_using_tf_model.py [-h] [--grpc_address GRPC_ADDRESS] [--grpc_port GRPC_PORT] --image_input_path IMAGE_INPUT_PATH + +Client for OCR pipeline + +optional arguments: + -h, --help show this help message and exit + --grpc_address GRPC_ADDRESS + Specify url to grpc service. default:localhost + --grpc_port GRPC_PORT + Specify port to grpc service. default: 9000 + --image_input_path IMAGE_INPUT_PATH + Image input path +``` + +Exemplary result of running the demo: +```bash +python3 image_classification_using_tf_model.py --image_input_path ../../common/static/images/zebra.jpeg +Image classified as zebra +``` \ No newline at end of file diff --git a/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py b/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py new file mode 100644 index 0000000000..5bdb35ad69 --- /dev/null +++ b/demos/image_classification_using_tf_model/python/image_classification_using_tf_model.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from ovmsclient import make_grpc_client +import cv2 +import numpy as np +import argparse +import sys +sys.path.append("../../common/python") +import classes + + + +def load_img(path): + img = cv2.imread(path) + img_f = img.astype(np.float32) + img_f = cv2.cvtColor(img_f, cv2.COLOR_BGR2RGB) + img_f = cv2.resize(img_f, (299,299)) + mean = [127.5,127.5,127.5] + scale = [127.5,127.5,127.5] + img_f = (img_f - np.array(mean, dtype=np.float32))/np.array(scale, dtype=np.float32) + img_f = img_f.reshape(1, img_f.shape[0], img_f.shape[1], 3) + return {"input": img_f} + +def build_parser(): + parser = argparse.ArgumentParser(description='Client for OCR pipeline') + parser.add_argument('--grpc_address', required=False, default='localhost', help='Specify url to grpc service. default:localhost') + parser.add_argument('--grpc_port', required=False, default=9000, help='Specify port to grpc service. default: 9000') + parser.add_argument('--image_input_path', required=True, help='Image input path') + return parser + +if __name__ == "__main__": + args = vars(build_parser().parse_args()) + + img_path = args['image_input_path'] + address = "{}:{}".format(args['grpc_address'],args['grpc_port']) + input = load_img(img_path) + + client = make_grpc_client(address) + classification_output = client.predict(input, "resnet") + + print("Image classified as " + classes.imagenet_classes[classification_output.argmax() - 1]) diff --git a/demos/image_classification_using_tf_model/python/requirements.txt b/demos/image_classification_using_tf_model/python/requirements.txt new file mode 100644 index 0000000000..cefea1ee3d --- /dev/null +++ b/demos/image_classification_using_tf_model/python/requirements.txt @@ -0,0 +1,2 @@ +ovmsclient +opencv-python \ No newline at end of file diff --git a/src/localfilesystem.cpp b/src/localfilesystem.cpp index 99fd1e7536..29ad2259c5 100644 --- a/src/localfilesystem.cpp +++ b/src/localfilesystem.cpp @@ -31,7 +31,7 @@ namespace ovms { namespace fs = std::filesystem; constexpr uint64_t NANOS_PER_SECOND = 1000000000; -const std::vector FileSystem::acceptedFiles = {".bin", ".onnx", ".xml", "mapping_config.json", ".pdiparams", ".pdmodel"}; +const std::vector FileSystem::acceptedFiles = {".bin", ".onnx", ".xml", "mapping_config.json", ".pdiparams", ".pdmodel", ".pb"}; StatusCode LocalFileSystem::fileExists(const std::string& path, bool* exists) { try { diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 849299aed6..5b54189ff5 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -739,6 +739,21 @@ Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { return StatusCode::OK; } +template +void ModelInstance::fetchModelFiles(bool& found, ArrayType ext) { + if (!found) { + found = true; + modelFiles.clear(); + for (auto extension : ext) { + auto file = findModelFilePathWithExtension(extension); + if (file.empty()) { + found = false; + } + modelFiles.push_back(file); + } + } +} + Status ModelInstance::fetchModelFilepaths() { if (this->config.isCustomLoaderRequiredToLoadModel()) { // not required if the model is loaded using a custom loader and can be returned from here @@ -751,36 +766,12 @@ Status ModelInstance::fetchModelFilepaths() { return StatusCode::PATH_INVALID; } - bool found = true; - for (auto extension : OV_MODEL_FILES_EXTENSIONS) { - auto file = findModelFilePathWithExtension(extension); - if (file.empty()) { - found = false; - } - modelFiles.push_back(file); - } - if (!found) { - found = true; - modelFiles.clear(); - for (auto extension : ONNX_MODEL_FILES_EXTENSIONS) { - auto file = findModelFilePathWithExtension(extension); - if (file.empty()) { - found = false; - } - modelFiles.push_back(file); - } - } - if (!found) { - found = true; - modelFiles.clear(); - for (auto extension : PADDLE_MODEL_FILES_EXTENSIONS) { - auto file = findModelFilePathWithExtension(extension); - if (file.empty()) { - found = false; - } - modelFiles.push_back(file); - } - } + bool found = false; + fetchModelFiles(found, OV_MODEL_FILES_EXTENSIONS); + fetchModelFiles(found, ONNX_MODEL_FILES_EXTENSIONS); + fetchModelFiles(found, PADDLE_MODEL_FILES_EXTENSIONS); + fetchModelFiles(found, TF_MODEL_FILES_EXTENSIONS); + if (!found) { SPDLOG_ERROR("Could not find file for model: {} version: {} in path: {}", getName(), getVersion(), path); return StatusCode::FILE_INVALID; diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index b3d33667c5..edc590264d 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -148,6 +148,11 @@ class ModelInstance { */ static constexpr std::array PADDLE_MODEL_FILES_EXTENSIONS{".pdmodel", ".pdiparams"}; + /** + * @brief Stores required tensorflow model files extensions to be able to load model + */ + static constexpr std::array TF_MODEL_FILES_EXTENSIONS{".pb"}; + /** * @brief Notifies model instance users who wait for loading */ @@ -554,5 +559,8 @@ class ModelInstance { ModelMetricReporter& getMetricReporter() const { return *this->reporter; } uint32_t getNumOfStreams() const; + + template + void fetchModelFiles(bool& found, ArrayType ext); }; } // namespace ovms From 539177b216ef5a21a28ed667817974d05bd3205e Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Wed, 7 Dec 2022 14:37:38 +0100 Subject: [PATCH 071/130] update ov backend package (#1569) --- Dockerfile.ubuntu | 5 ++++- Makefile | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index a84953b3da..515b4827b2 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -48,6 +48,7 @@ RUN apt-get update && apt-get install -y \ libboost-thread1.71.0 \ libboost-system1.71.0 \ libboost-date-time1.71.0 \ + libgflags-dev \ bc \ build-essential \ cmake \ @@ -66,6 +67,7 @@ RUN apt-get update && apt-get install -y \ libxml2-dev \ libnuma-dev \ libssl-dev \ + nlohmann-json3-dev \ patch \ pkg-config \ python2 \ @@ -82,7 +84,8 @@ RUN apt-get update && apt-get install -y \ wget \ unzip \ vim \ - xz-utils && \ + xz-utils \ + zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/Makefile b/Makefile index 887d0db3a9..d459a3f17f 100644 --- a/Makefile +++ b/Makefile @@ -80,14 +80,14 @@ ifeq ($(BASE_OS),ubuntu) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) endif INSTALL_DRIVER_VERSION ?= "22.35.24055" - DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_ubuntu20_2022.3.0.8844.12f74f066a4_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0357a97c40a41be0f362a3d0907e378b236b3c71/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.8963.0357a97c40a_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" - DLDT_PACKAGE_URL ?= http://s3.toolbox.iotg.sclab.intel.com/ov-packages/l_openvino_toolkit_rhel8_2022.3.0.8844.12f74f066a4_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0357a97c40a41be0f362a3d0907e378b236b3c71/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.8963.0357a97c40a_x86_64.tgz endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server From bec10dfe748d02f46b64084997ca9ca3cb07b6de Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Wed, 7 Dec 2022 15:10:56 +0100 Subject: [PATCH 072/130] CVS-98238_doc_test_fixes (#1574) * update NVIDIA part * docker image updated --- docs/accelerators.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/accelerators.md b/docs/accelerators.md index e84a3514b7..a619be0ac6 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -217,13 +217,15 @@ THROUGHPUT ## Using NVIDIA Plugin -*Note:* To build container with NVIDIA plugin use command: +*Note:* To build container with NVIDIA plugin use commands: ```bash - make docker_build NVIDIA=1 OV_USE_BINARY=0 + git clone https://github.com/openvinotoolkit/model_server.git + cd model_server + make docker_build NVIDIA=1 OV_USE_BINARY=0 ``` Example command to run container with NVIDIA support: ```bash - docker run -it --gpus all -p 9178:9178 -v /model:/model openvino/model_server-cuda:latest --model_path /model --model_name resnet --target_device NVIDIA + docker run -it --gpus all -p 9178:9178 -v ${PWD}/models/public/resnet-50-tf:/opt/model openvino/model_server:latest-cuda --model_path /opt/model --model_name resnet --target_device NVIDIA ``` From 0e561733fc21896e8998c1eb98df6bc5cd919013 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 7 Dec 2022 16:24:53 +0100 Subject: [PATCH 073/130] test log for debug build (#1571) --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 8293be66de..ef2e82b1d1 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -34,7 +34,7 @@ ARG build_type=dbg ARG debug_bazel_flags=--strip=never\ --copt="-g"\ -c\ dbg ARG minitrace_flags ENV TF_SYSTEM_LIBS="curl" -ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/k8-opt/testlogs/test.log" +ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum update -d6 -y && yum install -d6 -y \ boost169-atomic \ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 515b4827b2..9d2866a520 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -38,7 +38,7 @@ ARG minitrace_flags ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/external/hddl ENV DEBIAN_FRONTEND=noninteractive ENV TF_SYSTEM_LIBS="curl" -ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/k8-opt/testlogs/test.log" +ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" SHELL ["/bin/bash", "-c"] RUN apt-get update && apt-get install -y \ libboost-atomic1.71.0 \ From d384c9e98e1dfa7d2ecac5c51aa83a6d0fd5c3b4 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 7 Dec 2022 16:54:17 +0100 Subject: [PATCH 074/130] CVS-95536 Make all unsupported http methods defined in RFC2616 return 406 (#1572) Adding similar handling for CONNECT & TRACE as for other unsupported methods: HEAD/PUT/DELETE/OPTIONS/TRACE - HTTP status 406. Other methods not specified in RFC2616 are still falling back to TFS handling, returning 501. http://www.wangafu.net/~nickm/libevent-2.0/doxygen/html/http_8h.html#ac858319d667267f9fc848c2bb6931aa3 --- WORKSPACE | 2 ++ external/net_http.patch | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 7189ba9af6..ce28c3b851 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -66,6 +66,8 @@ git_repository( patches = ["net_http.patch", "listen.patch"] # ^^^^^^^^^^^^ # make bind address configurable + # ^^^^^^^^^^^^ + # allow all http methods ) # minitrace diff --git a/external/net_http.patch b/external/net_http.patch index ebf46d5f09..0b4411b49f 100644 --- a/external/net_http.patch +++ b/external/net_http.patch @@ -1,9 +1,9 @@ diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD -index 5beb2fb..e7b166c 100644 +index 5beb2fb1..e7b166c1 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -4,7 +4,7 @@ load("//tensorflow_serving:serving.bzl", "serving_proto_library") - + package( default_visibility = [ - "//tensorflow_serving:internal", @@ -12,13 +12,13 @@ index 5beb2fb..e7b166c 100644 features = ["-layering_check"], ) diff --git a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -index 36c925a..86a5ba9 100644 +index 237443dd..1b7d0563 100644 --- a/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc +++ b/tensorflow_serving/util/net_http/server/internal/evhttp_server.cc -@@ -105,6 +105,11 @@ bool EvHTTPServer::Initialize() { +@@ -105,13 +105,18 @@ bool EvHTTPServer::Initialize() { return false; } - + + std::size_t maxBodySize = 1024 * 1024 * 1024; + evhttp_set_max_body_size(ev_http_, maxBodySize); + std::size_t maxHeadersSize = 8 * 1024; @@ -27,12 +27,20 @@ index 36c925a..86a5ba9 100644 // By default libevents only allow GET, POST, HEAD, PUT, DELETE request // we have to manually turn OPTIONS and PATCH flag on documentation: // (http://www.wangafu.net/~nickm/libevent-2.0/doxygen/html/http_8h.html) + evhttp_set_allowed_methods( + ev_http_, EVHTTP_REQ_GET | EVHTTP_REQ_POST | EVHTTP_REQ_HEAD | + EVHTTP_REQ_PUT | EVHTTP_REQ_DELETE | EVHTTP_REQ_OPTIONS | +- EVHTTP_REQ_PATCH); ++ EVHTTP_REQ_CONNECT | EVHTTP_REQ_TRACE | EVHTTP_REQ_PATCH); + evhttp_set_gencb(ev_http_, &DispatchEvRequestFn, this); + + return true; diff --git a/tensorflow_serving/util/net_http/server/public/BUILD b/tensorflow_serving/util/net_http/server/public/BUILD -index e7f96d9..89d1a9c 100644 +index e7f96d98..89d1a9c0 100644 --- a/tensorflow_serving/util/net_http/server/public/BUILD +++ b/tensorflow_serving/util/net_http/server/public/BUILD @@ -2,9 +2,7 @@ - + package( default_visibility = [ - ":http_server_clients", @@ -41,4 +49,4 @@ index e7f96d9..89d1a9c 100644 + "//visibility:public", ], ) - + From f6c097adcfc23881feb14e9cf1e2c28edece6ece Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 8 Dec 2022 11:46:54 +0100 Subject: [PATCH 075/130] CVS-97927 Remove workaround in DAG for models with GPU (#1577) Previously it was required to clone tensor before setting the input to DAG node on GPU due to bug in the opencl driver. Removing the workaround since it is tested that issue has been fixed. --- src/dlnodesession.cpp | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/dlnodesession.cpp b/src/dlnodesession.cpp index bd3fd205f5..7b9fe5587f 100644 --- a/src/dlnodesession.cpp +++ b/src/dlnodesession.cpp @@ -229,22 +229,8 @@ Status DLNodeSession::setInputsForInference(ov::InferRequest& inferRequest) { __FUNCTION__, getName(), name); return StatusCode::INTERNAL_ERROR; } - // Workaround for GPU. - if (this->model->getModelConfig().isDeviceUsed("GPU")) { - ov::Tensor clonedTensor; - status = ovms::tensorClone(clonedTensor, tensor); - if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor clone error: {}", getName(), status.string()); - return status; - } - OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::set_tensor"); - inferRequest.set_tensor(realModelInputName, clonedTensor); - OVMS_PROFILE_SYNC_END("ov::InferRequest::set_tensor"); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor name: {} cloned before GPU inference", getName(), name); - } else { - OVMS_PROFILE_SCOPE("ov::InferRequest::set_tensor"); - inferRequest.set_tensor(realModelInputName, tensor); - } + OVMS_PROFILE_SCOPE("ov::InferRequest::set_tensor"); + inferRequest.set_tensor(realModelInputName, tensor); } // OV implementation the ov::Exception is not // a base class for all other exceptions thrown from OV. From 7ae5fb83e8bad23da64f48aa02b715153109da08 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Fri, 9 Dec 2022 08:22:02 +0100 Subject: [PATCH 076/130] updates for test (#1581) --- demos/image_classification_using_tf_model/python/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/demos/image_classification_using_tf_model/python/README.md b/demos/image_classification_using_tf_model/python/README.md index b2e555cd13..076c042723 100644 --- a/demos/image_classification_using_tf_model/python/README.md +++ b/demos/image_classification_using_tf_model/python/README.md @@ -24,7 +24,6 @@ cd model_server/demos/image_classification_using_tf_model/python mkdir -p model/1 wget -P model/1 https://storage.googleapis.com/download.tensorflow.org/models/tflite/model_zoo/upload_20180427/inception_resnet_v2_2018_04_27.tgz tar xzf model/1/inception_resnet_v2_2018_04_27.tgz -C model/1 -ls model/1 ``` ## Run Openvino Model Server @@ -59,6 +58,6 @@ optional arguments: Exemplary result of running the demo: ```bash -python3 image_classification_using_tf_model.py --image_input_path ../../common/static/images/zebra.jpeg +python3 image_classification_using_tf_model.py --grpc_port 9000 --image_input_path ../../common/static/images/zebra.jpeg Image classified as zebra -``` \ No newline at end of file +``` From b634dc99eb7f84ff5f8db0003487341261059175 Mon Sep 17 00:00:00 2001 From: Tatiana Savina Date: Mon, 12 Dec 2022 09:29:06 +0100 Subject: [PATCH 077/130] Documentation restructure (#1551) --- docs/accelerators.md | 27 ++++- docs/advanced_topics.md | 28 +++++ docs/architecture.md | 30 ----- docs/dag_scheduler.md | 6 +- docs/deploying_server.md | 108 +++++++++++++++++ docs/docker_container.md | 164 -------------------------- docs/features.md | 67 +++++++++++ docs/home.md | 25 ++-- docs/host.md | 53 --------- docs/installations_kubernetes.md | 9 -- docs/models_repository.md | 4 +- docs/multiple_models_mode.md | 70 ----------- docs/ovms_quickstart.md | 4 +- docs/single_model_mode.md | 58 ---------- docs/starting_server.md | 191 +++++++++++++++++++------------ docs/writing_app.md | 23 ++++ 16 files changed, 384 insertions(+), 483 deletions(-) create mode 100644 docs/advanced_topics.md delete mode 100644 docs/architecture.md create mode 100644 docs/deploying_server.md delete mode 100644 docs/docker_container.md create mode 100644 docs/features.md delete mode 100644 docs/host.md delete mode 100644 docs/installations_kubernetes.md delete mode 100644 docs/multiple_models_mode.md delete mode 100644 docs/single_model_mode.md create mode 100644 docs/writing_app.md diff --git a/docs/accelerators.md b/docs/accelerators.md index a619be0ac6..18a534c6ea 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -56,6 +56,31 @@ Check out our recommendations for [throughput optimization on HDDL](performance_ > It requires RW permissions in the docker container security context. > It is recommended to start the docker container in the same context as the account starting _hddldaemon_. For example, if you start the _hddldaemon_ as root, add `--user root` to the `docker run` command. +## Model Server image with Intel® Data Center GPU Flex Series and Intel® Arc™ GPU support (Ubuntu 20.04) + +To build the image, you need to have NEO Runtime packages available. Contact Intel representative to get the access to the preproduction drivers. + +Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2`. Expected structure is like below: +``` +drivers +└── dg2 + ├── intel-igc-core__amd64.deb + ├── intel-igc-opencl__amd64.deb + ├── intel-level-zero-gpu-dbgsym__amd64.ddeb + ├── intel-level-zero-gpu__amd64.deb + ├── intel-opencl-icd-dbgsym__amd64.ddeb + ├── intel-opencl-icd__amd64.deb + ├── libigdgmm12__amd64.deb + └── libigdgmm12__amd64.deb +``` + +and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. + +Example: +``` +make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 +``` + ## Starting a Docker Container with Intel GPU The [GPU plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html) uses the Intel Compute Library for @@ -105,7 +130,6 @@ git clone https://github.com/openvinotoolkit/model_server.git cd model_server make docker_build INSTALL_DRIVER_VERSION=22.10.22597 ``` - ## Using Multi-Device Plugin If you have multiple inference devices available (e.g. Myriad VPUs and CPU) you can increase inference throughput by enabling the Multi-Device Plugin. @@ -147,7 +171,6 @@ docker run -d --net=host -u root --privileged --name ie-serving --rm -v ${PWD}/m The deployed model will perform inference on both Intel Movidius Neural Compute Stick and CPU. The total throughput will be roughly equal to the sum of CPU and Intel Movidius Neural Compute Stick throughputs. - ## Using Heterogeneous Plugin The [HETERO plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_Hetero_execution.html) makes it possible to distribute inference load of one model diff --git a/docs/advanced_topics.md b/docs/advanced_topics.md new file mode 100644 index 0000000000..8632b50998 --- /dev/null +++ b/docs/advanced_topics.md @@ -0,0 +1,28 @@ +# Advanced Features {#ovms_docs_advanced} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + ovms_sample_cpu_extension + ovms_docs_model_cache + ovms_docs_custom_loader + +@endsphinxdirective + +## CPU Extensions +Implement any CPU layer, that is not support by OpenVINO yet, as a shared library. + +[Learn more](../src/example/SampleCpuExtension/README.md) + +## Model Cache +Leverage the OpenVINO [model caching](https://docs.openvino.ai/latest/openvino_docs_OV_UG_Model_caching_overview.html) feature to speed up subsequent model loading on a target device. + +[Learn more](model_cache.md) + +## Custom Model Loader +Write your own custom model loader based on a predefined interface and load it similar to a dynamic library. + +[Learn more](custom_model_loader.md) \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 0bb5c6761b..0000000000 --- a/docs/architecture.md +++ /dev/null @@ -1,30 +0,0 @@ -# Architecture {#ovms_docs_architecture} - -- OpenVINO™ Model Server provides a C++ implementation of the gRPC and RESTful API interfaces compatible with [Tensorflow Serving](https://www.tensorflow.org/tfx/guide/serving). - -- In the backend, OpenVINO™ Model Server uses [OpenVINO™ Runtime](https://docs.openvino.ai/2022.2/index.html) libraries from OpenVINO™ toolkit. This speeds up execution on the CPU and enables it on AI accelerators, like [Neural Compute Stick 2](https://software.intel.com/content/www/us/en/main/hardware/neural-compute-stick.html), iGPU(Integrated Graphics Processing Unit), and [HDDL](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_ivad_vpu.html). - - -- API requests in gRPC code backbone are created based on [TensorFlow Serving Core Framework](https://www.tensorflow.org/tfx/guide/serving) with tuned implementation of request handling. - -- Services are designed via a set of C++ classes managing AI models in the Intermediate Representation format. [OpenVINO™ Runtime](https://docs.openvino.ai/2022.2/index.html) executes the model's operations. - -![serving](serving-c.png) - -

Figure 1: Docker Container (VM or Bare Metal Host)
- -- The models used by OpenVINO Model Server need to be stored locally or hosted remotely by object storage services. Storage compatible with -Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage is supported. For more details, refer to [Preparing the Model Repository](./models_repository.md). -- OpenVINO™ Model Server is suitable for landing in the Kubernetes environment. It can be also hosted on a bare metal server, virtual machine, or inside a docker container. - -- The only two exposed network interfaces are **gRPC API**: - - * TensorFlow Serving compatible API(./model_server_grpc_api_tfs.md) - * KServe compatible API(./model_server_grpc_api_kfs.md) - - ... and **RESTful API**: - - * TensorFlow Serving compatible API(./model_server_rest_api_tfs.md) - * KServe compatible API(./model_server_rest_api_kfs.md) - - They _do not_ include authorization, authentication, or data encryption. There is, however, a [documented method](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/extras/nginx-mtls-auth) for including NGINX reverse proxy with mTLS traffic termination. diff --git a/docs/dag_scheduler.md b/docs/dag_scheduler.md index d21f9ff67e..60be9f7b03 100644 --- a/docs/dag_scheduler.md +++ b/docs/dag_scheduler.md @@ -72,9 +72,9 @@ It assumes the batches are combined on the first dimension which is dropped afte ## Configuration file Pipelines configuration is to be placed in the same json file like the -[models config file](multiple_models_mode.md). -While models are defined in section `model_config_list`, pipelines are to be configured in -section `pipeline_config_list`. +[models config file](starting_server.md). +While models are defined in section `model_config_list`, pipelines are configured in +the `pipeline_config_list` section. Nodes in the pipelines can reference only the models configured in model_config_list section. Basic pipeline section template is depicted below: diff --git a/docs/deploying_server.md b/docs/deploying_server.md new file mode 100644 index 0000000000..c9a7b52de6 --- /dev/null +++ b/docs/deploying_server.md @@ -0,0 +1,108 @@ +# Deploying Model Server {#ovms_docs_deploying_server} + +1. Docker is the recommended way to deploy OpenVINO Model Server. Pre-built container images are available on Docker Hub and Red Hat Ecosystem Catalog. +2. Host Model Server on baremetal. +3. Deploy OpenVINO Model Server in Kubernetes via helm chart, Kubernetes Operator or OpenShift Operator. + +## Deploying Model Server in Docker Container + +This is a step-by-step guide on how to deploy OpenVINO™ Model Server on Linux, using a pre-build Docker Container. + +**Before you start, make sure you have:** + +- [Docker Engine](https://docs.docker.com/engine/) installed +- Intel® Core™ processor (6-12th gen.) or Intel® Xeon® processor +- Linux, macOS or Windows via [WSL](https://docs.microsoft.com/en-us/windows/wsl/) +- (optional) AI accelerators [supported by OpenVINO](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html). Accelerators are tested only on bare-metal Linux hosts. + +### Launch Model Server Container + +This example shows how to launch the model server with a ResNet50 image classification model from a cloud storage: + +#### Step 1. Pull Model Server Image + +Pull an image from Docker: + +```bash +docker pull openvino/model_server:latest +``` + +or [RedHat Ecosystem Catalog](https://catalog.redhat.com/software/containers/intel/openvino-model-server/607833052937385fc98515de): + +``` +docker pull registry.connect.redhat.com/intel/openvino-model-server:latest +``` + +#### Step 2. Prepare Data for Serving + +```bash +# start the container with the model +docker run -p 9000:9000 openvino/model_server:latest \ +--model_name resnet --model_path gs://ovms-public-eu/resnet50-binary \ +--layout NHWC:NCHW --port 9000 + +# download input files: an image and a label mapping file +wget https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/static/images/zebra.jpeg +wget https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/python/classes.py + +# install the Python-based ovmsclient package +pip3 install ovmsclient +``` + +#### Step 3. Run Prediction + + +```bash +echo 'import numpy as np +from classes import imagenet_classes +from ovmsclient import make_grpc_client + +client = make_grpc_client("localhost:9000") + +with open("zebra.jpeg", "rb") as f: + img = f.read() + +output = client.predict({"0": img}, "resnet") +result_index = np.argmax(output[0]) +print(imagenet_classes[result_index])' >> predict.py + +python predict.py +zebra +``` +If everything is set up correctly, you will see 'zebra' prediction in the output. + +## Deploying Model Server on Baremetal + +There are two ways to start the server: + +- using the ```./ovms/bin/ovms --help``` command from inside the directory where OVMS is installed +- in interactive mode as a background process or a daemon initiated by ```systemctl/initd``` depending on the Linux distribution and specific hosting requirements + +Learn more about model server [starting parameters](parameters.md). + +> **NOTE**: +> When serving models on [AI accelerators](accelerators.md), some additional steps may be required to install device drivers and dependencies. +> Learn more in the [Additional Configurations for Hardware](https://docs.openvino.ai/latest/openvino_docs_install_guides_configurations_header.html) documentation. + + +## Deploying Model Server in Kubernetes + +There are three recommended methods for deploying OpenVINO Model Server in Kubernetes: +1. [helm chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) - deploys Model Server instances using the [helm](https://helm.sh) package manager for Kubernetes +2. [Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) - manages Model Server using a Kubernetes Operator +3. [OpenShift Operator](https://github.com/openvinotoolkit/operator/blob/main/docs/operator_installation.md#openshift) - manages Model Server instances in Red Hat OpenShift + +For operators mentioned in 2. and 3. see the [description of the deployment process](https://github.com/openvinotoolkit/operator/blob/main/docs/modelserver.md) + +## Next Steps + +- [Start the server](starting_server.md) +- Try the model server [features](features.md) +- Explore the model server [demos](../demos/README.md) + +## Additional Resources + +- [Preparing Model Repository](models_repository.md) +- [Using Cloud Storage](using_cloud_storage.md) +- [Troubleshooting](troubleshooting.md) +- [Model server parameters](parameters.md) diff --git a/docs/docker_container.md b/docs/docker_container.md deleted file mode 100644 index df2718903a..0000000000 --- a/docs/docker_container.md +++ /dev/null @@ -1,164 +0,0 @@ -# Model Server in Docker Containers {#ovms_docs_docker_container} - -This is a step-by-step guide on how to deploy OpenVINO™ Model Server on Linux, using a Docker Container. Links are provided for different compatible hardware. - -**Before you start, make sure you have:** - -- [Docker Engine](https://docs.docker.com/engine/) installed ([How to Install Docker Engine](https://docs.docker.com/engine/install/)) -- Intel® Core™ processor (6-12th gen.) or Intel® Xeon® processor -- (optional) AI accelerators [supported by OpenVINO](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) -- Linux, macOS or Windows via [WSL](https://docs.microsoft.com/en-us/windows/wsl/) - -**NOTE:** accelerators are only tested on bare-metal Linux hosts. - - -## Starting with a container - -- Pull OpenVINO™ Model Server Image. -- Start a Docker Container with OVMS and your chosen model from cloud storage. -- Provide the input files, (arrange an input Dataset). -- Prepare a client package. -- Run the prediction using ovmsclient. - -Here is an example of this process using a ResNet50 model for image classification: - -Pull an image from Docker or [RedHat Ecosystem Catalog](https://catalog.redhat.com/software/containers/intel/openvino-model-server/607833052937385fc98515de) - -```bash -docker pull openvino/model_server:latest -``` -or, alternatively -``` -docker pull registry.connect.redhat.com/intel/openvino-model-server:latest -``` - -Start the container -```bash -# start the container -docker run -p 9000:9000 openvino/model_server:latest \ ---model_name resnet --model_path gs://ovms-public-eu/resnet50-binary \ ---layout NHWC:NCHW --port 9000 - -# download input files, an image, and a label mapping file -wget https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/static/images/zebra.jpeg -wget https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/python/classes.py - -# Install the Python-based ovmsclient package -pip3 install ovmsclient -``` - -Run prediction -```bash -echo 'import numpy as np -from classes import imagenet_classes -from ovmsclient import make_grpc_client - -client = make_grpc_client("localhost:9000") - -with open("zebra.jpeg", "rb") as f: - img = f.read() - -output = client.predict({"0": img}, "resnet") -result_index = np.argmax(output[0]) -print(imagenet_classes[result_index])' >> predict.py - -python predict.py -zebra -``` - -To learn how to set up OpenVINO Model Server, refer to the [Quick Start guide](./ovms_quickstart.md). - - - -## Building an OpenVINO™ Model Server Docker Image - -You can build your own Docker image executing the `make docker_build` command in the [git repository root folder](https://github.com/openvinotoolkit/model_server). -In the `./dist` directory it will generate: - -- image tagged as openvino/model_server:latest - with CPU, NCS, and HDDL support -- image tagged as openvino/model_server:latest-gpu - with CPU, NCS, HDDL, and iGPU, dGPU support -- image tagged as openvino/model_server:latest-nginx-mtls - with CPU, NCS, and HDDL support and a reference nginx setup of mTLS integration -- release package (.tar.gz, with ovms binary and necessary libraries) - -**Note:** OVMS docker image can be created with ubi8-minimal base image or the default ubuntu20. -Note that OVMS with the ubi base image doesn’t support NCS and HDDL accelerators. - -To do so, use either of these commands: - -Running the inference operation on GPU requires the ovms process security context account to have correct permissions. -It has to belong to the render group identified by the command: -```bash -stat -c "group_name=%G group_id=%g" /dev/dri/render* -``` -The default account in the docker image is already preconfigured. In case you change the security context, use the following command -to start the ovms container: -```bash -docker run --rm -it --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ --p 9001:9001 openvino/model_server:latest-gpu \ ---model_name resnet --model_path gs://ovms-public-eu/resnet50-binary --port 9001 --target_device GPU -``` - -*Note:* The public docker image includes the OpenCL drivers for GPU in version 22.35.24055 for ubuntu20.04 based image and 22.28.23726 for UBI8. - -### Model Server image using preproduction gpu drivers (Ubuntu 20.04) - -To build the image, you need to have NEO Runtime packages available. Contact Intel representative to get the access to the preproduction drivers. - -Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2`. Expected structure is like below: -``` -drivers -└── dg2 - ├── intel-igc-core__amd64.deb - ├── intel-igc-opencl__amd64.deb - ├── intel-level-zero-gpu-dbgsym__amd64.ddeb - ├── intel-level-zero-gpu__amd64.deb - ├── intel-opencl-icd-dbgsym__amd64.ddeb - ├── intel-opencl-icd__amd64.deb - ├── libigdgmm12__amd64.deb - └── libigdgmm12__amd64.deb -``` - -and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. - -Example: -``` -make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 -``` - -## Using Multi-Device Plugin - -If you have multiple inference devices available (e.g. Myriad VPUs and CPU) you can increase inference throughput by enabling the Multi-Device Plugin. -With Multi-Device Plugin enabled, inference requests will be load balanced between multiple devices. -For more detailed information read [OpenVino's Multi-Device plugin documentation](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_Running_on_multiple_devices.html). - -In order to use this feature in OpenVino™ Model Server, following steps are required: - -Set target_device for the model in configuration json file to MULTI:DEVICE_1,DEVICE_2 (e.g. MULTI:MYRIAD,CPU, order of the devices defines their priority, so MYRIAD devices will be used first in this example) - -Below is exemplary config.json setting up Multi-Device Plugin for resnet model, using Intel® Movidius™ Neural Compute Stick and CPU devices: -``` -{ - "model_config_list": [ - {"config": { - "name": "resnet", - "base_path": "/opt/model", - "batch_size": "1", - "target_device": "MULTI:MYRIAD,CPU"} - }] -} -``` - -Additionally, you can use the `INSTALL_DRIVER_VERSION` argument command to choose which GPU driver version is used by the produced image. -If not provided, most recent version is used. - -Currently, the following versions are available: -- 21.38.21026 - Redhat -- 21.48.21782 - Ubuntu - -Example: -```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -make docker_build INSTALL_DRIVER_VERSION=21.38.21026 -``` -If not provided, version 21.38.21026 is used for Redhat and 21.48.21782 is used for Ubuntu. diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 0000000000..bab8d50904 --- /dev/null +++ b/docs/features.md @@ -0,0 +1,67 @@ +# Model Server Features {#ovms_docs_features} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + ovms_docs_dag + ovms_docs_binary_input + ovms_docs_model_version_policy + ovms_docs_shape_batch_layout + ovms_docs_online_config_changes + ovms_docs_stateful_models + ovms_docs_metrics + ovms_docs_dynamic_input + ovms_docs_advanced + +@endsphinxdirective + +## Serving Pipelines of Models +Connect multiple models in a pipeline and reduce data transfer overhead with Directed Acyclic Graph (DAG) Scheduler. +Implement model inference and data transformations using a custom node C/C++ dynamic library. + +[Learn more](dag_scheduler.md) + +## Processing Raw Data +Send data in JPEG or PNG formats to reduce traffic and offload data pre-processing to the server. + +[Learn more](binary_input.md) + +## Model Versioning Policies +The model repository structure enables adding or deleting numerical version directories and the server will automatically adjust which models are served. +Control which model versions are served by setting the model version policy to serve all models, a specific model or set of models or just the latest version of the model (default setting). + +[Learn more](model_version_policy.md) + +## Model Reshaping +Change the batch size, shape and layout of the model at runtime to achieve high throughput and low latency. + +[Learn more](shape_batch_size_and_layout.md) + +## Modify Model Configuration at Runtime +OpenVINO Model Server regularly checks for changes to the configuration file and applies them during runtime. This means that you can change model configurations +(for example, change the device where a model is served), add a new model or completely remove one that is no longer needed. These changes will be applied without any disruption to the service. + +[Learn more](online_config_changes.md) + +## Working with Stateful Models +Serve models that operate on sequences of data and maintain their state between inference requests. + +[Learn more](stateful_models.md) + +## Metrics +Use the metrics endpoint compatible with the Prometheus to access performance and utilization statistics. + +[Learn more](metrics.md) + +## Enabling Dynamic Inputs +Configure served models to accept data with variable batch sizes and input shapes. + +[Learn more](dynamic_input.md) + +## Advanced Features +Use CPU Extensions, model cache feature or a custom model loader. + +[Learn more](advanced_topics.md) diff --git a/docs/home.md b/docs/home.md index 1f0b98960e..f998ffbd43 100644 --- a/docs/home.md +++ b/docs/home.md @@ -7,21 +7,12 @@ :hidden: ovms_docs_quick_start_guide - ovms_docs_architecture ovms_docs_models_repository - ovms_docs_starting_server - ovms_docs_server_api - ovms_docs_clients - ovms_docs_dag - ovms_docs_binary_input - ovms_docs_model_cache - ovms_docs_metrics - ovms_sample_cpu_extension - ovms_docs_dynamic_input - ovms_docs_stateful_models - ovms_docs_custom_loader + ovms_docs_deploying_server + ovms_docs_serving_model + ovms_docs_features + ovms_docs_server_app ovms_docs_performance_tuning - ovms_docs_kubernetes ovms_docs_demos ovms_docs_troubleshooting @@ -40,8 +31,6 @@ Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage. Read [release notes](https://github.com/openvinotoolkit/model_server/releases) to find out what’s new. -Review the [Architecture concept](architecture.md) document for more details. - Key features: - support for multiple frameworks, such as Caffe, TensorFlow, MXNet, PaddlePaddle and ONNX - online deployment of new [model versions](model_version_policy.md) @@ -50,7 +39,7 @@ Key features: [Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), [GPU](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html), and [HDDL](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_HDDL.html) -- works with [Bare Metal Hosts](host.md) as well as [Docker containers](docker_container.md) +- works with [Bare Metal Hosts](deploying_server.md) as well as [Docker containers](deploying_server.md) - [model reshaping](shape_batch_size_and_layout.md) in runtime - [directed Acyclic Graph Scheduler](dag_scheduler.md) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead - [custom nodes in DAG pipelines](custom_node_development.md) - allowing model inference and data transformations to be implemented with a custom node C/C++ dynamic library @@ -72,9 +61,9 @@ For more information on using Model Server in various scenarios you can check th * [Model repository configuration](models_repository.md) -* [Using a docker container](docker_container.md) +* [Using a docker container](deploying_server.md) -* [Landing on bare metal or virtual machine](host.md) +* [Landing on bare metal or virtual machine](deploying_server.md) * [Performance tuning](performance_tuning.md) diff --git a/docs/host.md b/docs/host.md deleted file mode 100644 index f396bc9034..0000000000 --- a/docs/host.md +++ /dev/null @@ -1,53 +0,0 @@ -# Bare Metal and Virtual Hosts {#ovms_docs_baremetal} - -OpenVINO™ Model Server includes a C++ implementation of gRPC and RESTful API interfaces defined by TensorFlow Serving. -In the backend, it uses OpenVINO™ Runtime libraries from OpenVINO™ toolkit, which speeds up the execution on CPU, and enables it on iGPU and Movidius devices. - -OpenVINO Model Server can be hosted on a bare metal server, virtual machine, or inside a docker container. It is also suitable for landing in the Kubernetes environment. - -**Before you start:** - -OpenVINO Model Server execution on baremetal is tested on Ubuntu 20.04.x. For other operating systems we recommend using [OVMS docker containers](./docker_container.md). - -For supported hardware, refer to [supported configurations](https://docs.openvino.ai/2022.2/_docs_IE_DG_supported_plugins_Supported_Devices.html). -Always verify if your model is supported by the VPU Plugins and convert it to the OpenVINO format, using [OpenVINO Model Optimizer](https://software.intel.com/en-us/articles/OpenVINO-ModelOptimizer). - -## Installing Model Server - -- Clone model server git repository. -- Navigate to the model server directory. -- To install Model Server, you can either use a precompiled binary or build it on your own, in a Docker container. -- Navigate to the folder containing the binary package and unpack the included `tar.gz` file. - -Here is an example of this process: - -```bash - -git clone https://github.com/openvinotoolkit/model_server.git - -cd model_server - -# replace to 'redhat` if using UBI base image -export BASE_OS=ubuntu - -# automatically build a container from source -# it will also place a copy of the binary package in the `dist` subfolder in the Model Server root directory -make docker_build BASE_OS=${BASE_OS} - -# unpack the `tar.gz` file -cd dist/${BASE_OS} && tar -xzvf ovms.tar.gz - -``` - -## Running the Server - -The server can be started in two ways: - -- using the ```./ovms/bin/ovms --help``` command in the folder, where OVMS was is installed -- in the interactive mode - as a background process or a daemon initiated by ```systemctl/initd``` depending on the Linux distribution and specific hosting requirements - -Refer to [Running Model Server using Docker Container](./docker_container.md) to get more details on the OpenVINO Model Server parameters and configuration. - -> **NOTE**: -> When AI accelerators are used for inference execution, additional steps may be required to install their drivers and dependencies. Learn more about it -> Learn more about it on [OpenVINO installation guide](https://docs.openvino.ai/2022.2/openvino_docs_install_guides_installing_openvino_from_archive_linux.html). diff --git a/docs/installations_kubernetes.md b/docs/installations_kubernetes.md deleted file mode 100644 index 1271a4d86d..0000000000 --- a/docs/installations_kubernetes.md +++ /dev/null @@ -1,9 +0,0 @@ -# Deploy Model Server in Kubernetes {#ovms_docs_kubernetes} - -There are three recommended methods for deploying OpenVINO Model Server in Kubernetes: -1. [helm chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) - deploys Model Server instances using the [helm](https://helm.sh) package manager for Kubernetes - -2. [Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) - manages Model Server using a Kubernetes Operator -3. [OpenShift Operator](https://github.com/openvinotoolkit/operator/blob/main/docs/operator_installation.md#openshift) - manages Model Server instances in Red Hat OpenShift - -For operators mentioned in 2. and 3. see the [description of the deployment process](https://github.com/openvinotoolkit/operator/blob/main/docs/modelserver.md) \ No newline at end of file diff --git a/docs/models_repository.md b/docs/models_repository.md index c8d8aca01c..c3d2545b8d 100644 --- a/docs/models_repository.md +++ b/docs/models_repository.md @@ -1,6 +1,6 @@ -# Model Repository {#ovms_docs_models_repository} +# Preparing a Model Repository {#ovms_docs_models_repository} -The AI models served by OpenVINO™ Model Server must be in either of the three formats: +Served models must be in one of the following formats: - [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets), where the graph is represented in .bin and .xml files - [ONNX](https://onnx.ai/), using the .onnx file - [PaddlePaddle](https://www.paddlepaddle.org.cn/en), using .pdiparams and .pdmodel files diff --git a/docs/multiple_models_mode.md b/docs/multiple_models_mode.md deleted file mode 100644 index e03351c98e..0000000000 --- a/docs/multiple_models_mode.md +++ /dev/null @@ -1,70 +0,0 @@ -# Multiple-Model mode with a Config File {#ovms_docs_multiple_models} - -To use a container that has several models, you need an additional JSON configuration file defining each of them. In the file, provide an array, -`model_config_list`, that includes a collection of config objects for each served model. Every such config object needs the minimum of -the name and the base_path attribute values provided. - -An example of the configuration file: -```json -{ - "model_config_list":[ - { - "config":{ - "name":"model_name1", - "base_path":"/opt/ml/models/model1", - "batch_size": "16" - } - }, - { - "config":{ - "name":"model_name2", - "base_path":"/opt/ml/models/model2", - "batch_size": "auto", - "model_version_policy": {"all": {}} - } - }, - { - "config":{ - "name":"model_name3", - "base_path":"gs://bucket/models/model3", - "model_version_policy": {"specific": { "versions":[1, 3] }}, - "shape": "auto" - } - }, - { - "config":{ - "name":"model_name4", - "base_path":"s3://bucket/models/model4", - "shape": { - "input1": "(1,3,200,200)", - "input2": "(1,3,50,50)" - }, - "plugin_config": {"CPU_THROUGHPUT_STREAMS": "CPU_THROUGHPUT_AUTO"} - } - }, - { - "config":{ - "name":"model_name5", - "base_path":"s3://bucket/models/model5", - "shape": "auto", - "nireq": 32, - "target_device": "HDDL" - } - } - ] -} -``` - -When the Docker container has the config file mounted, it can be started - the command is minimalistic, as arguments are read from the config file. -Note that models with a cloud storage path require setting specific environmental variables. - -``` - -docker run --rm -d -v /models/:/opt/ml:ro -p 9001:9001 -p 8001:8001 -v :/opt/ml/config.json openvino/model_server:latest \ ---config_path /opt/ml/config.json --port 9001 --rest_port 8001 - -``` - -For additional details, refer to: -- [Model Repository](models_repository.md) -- [Using Cloud Storage](using_cloud_storage.md) diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index 8f4c4e8969..718e4d0860 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -69,9 +69,9 @@ model/ └── face-detection-retail-0004.xml ``` -For more information on the folder structure and how to deploy more than one model at a time, check these links: +For more information about the directory structure and how to deploy multiple models at a time, check out the following sections: - [Prepare models](models_repository.md) -- [Deploy multiple models at once and to start a Docker container with a configuration file](multiple_models_mode.md) +- [Deploy multiple models at once and to start a Docker container with a configuration file](starting_server.md) ### Step 5: Prepare the Example Client Components diff --git a/docs/single_model_mode.md b/docs/single_model_mode.md deleted file mode 100644 index fc4091c131..0000000000 --- a/docs/single_model_mode.md +++ /dev/null @@ -1,58 +0,0 @@ -# Single-Model Mode {#ovms_docs_single_model} - -Learn about the structure of a [Model Repository](models_repository.md) before running the Docker image. - -Launch Model Server by running the following command: - -``` -docker run -d --rm -v :/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ ---model_path --model_name --port 9000 --rest_port 9001 --log_level DEBUG -``` - -Example: - -```bash -mkdir -p models/resnet/1 -wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin -wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml - -docker run -d --rm -v ${PWD}/models:/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ ---model_path /models/resnet/ --model_name resnet --port 9000 --rest_port 9001 --log_level DEBUG -``` - - -**Configuration Arguments for Running Model Server:** - -@sphinxdirective -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `--rm` | | remove the container when exiting the Docker container | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `-d` | | runs the container in the background | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `-v` | | defines how to mount the model folder in the Docker container | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `-p` | | exposes the model serving port outside the Docker container | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `openvino/model_server:latest` | | represents the image name; the ovms binary is the Docker entry point | -| | | varies by tag and build process - see tags: https://hub.docker.com/r/openvino/model_server/tags/ for a full tag list. | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `--model_path` | | model location, which can be: | -| | | a Docker container path that is mounted during start-up | -| | | a Google Cloud Storage path `gs:///` | -| | | an AWS S3 path `s3:///` | -| | | an Azure blob path `az:///` | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `--model_name` | | the name of the model in the model_path | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `--port` | | the gRPC server port | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -| `--rest_port` | | the REST server port | -+--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -@endsphinxdirective - - -### Notes -- Publish the container's port to your host's **open ports**. -- In the command above, port 9000 is exposed for gRPC and port 9001 is exposed for REST API calls. -- For preparing and saving models to serve with OpenVINO™ Model Server refer to the [Model Repository](models_repository.md) article. -- Add model_name for the client gRPC/REST API calls. diff --git a/docs/starting_server.md b/docs/starting_server.md index 036374c44e..782fcbf60c 100644 --- a/docs/starting_server.md +++ b/docs/starting_server.md @@ -1,4 +1,4 @@ -# Starting the Server {#ovms_docs_starting_server} +# Starting Model Server {#ovms_docs_serving_model} @sphinxdirective @@ -6,92 +6,139 @@ :maxdepth: 1 :hidden: - ovms_docs_single_model - ovms_docs_multiple_models - ovms_docs_docker_container - ovms_docs_baremetal ovms_docs_parameters - ovms_docs_cloud_storage ovms_docs_target_devices - ovms_docs_model_version_policy - ovms_docs_shape_batch_layout - ovms_docs_online_config_changes + ovms_docs_cloud_storage ovms_docs_security - - @endsphinxdirective -## Serving a Single Model - -The simplest way to deploy OpenVINO™ Model Server is in the single-model mode - only one model is served and the whole configuration is passed via CLI parameters. - -> **NOTE**: In the single-model mode, changing configuration in runtime is not possible. - -[Learn more](single_model_mode.md) - -## Serving Multiple Models - -To serve multiple models, use the multi-model mode. This requires a configuration file that stores settings for all served models. -In this mode you can add and delete models, as well as update their configurations in runtime, without restarting Model Server. - -[Learn more](multiple_models_mode.md) - -## Running in a Docker Container - -Using Docker is the recommended way of running OpenVINO Model Server. Its images are available via -[DockerHub](https://hub.docker.com/r/openvino/model_server) and [RedHat Ecosystem Catalog](https://catalog.redhat.com/software/containers/intel/openvino-model-server/607833052937385fc98515de). -They are minimal and contain only the necessary dependencies. +Serving a single model is the simplest way to deploy OpenVINO™ Model Server. Only one model is served and the whole configuration is passed via CLI parameters. +Note that changing configuration in runtime while serving a single model is not possible. Serving multiple models requires a configuration file that stores settings for all served models. +When deploying model(s) with a configuration file, you can add or delete models, as well as update their configurations in runtime, without needing to restart the server. -[Learn more](docker_container.md) - -## Running on Bare Metal and Virtual Machine (VM) Hosts - -OpenVINO Model Server is an open-source project written in C++. Therefore you can download and compile the code to obtain the binary and run it on bare metal. -The `make` targets are provided to simplify the process. - -[Learn more](host.md) - -## Configuring Deployments - -Depending on performance requirements, traffic expectations, and what models OVMS serves, you may want to make certain adjustments to: -configuration of server options, like: -- ports used -- enable/disable REST API -- set configuration monitoring - -configuration for each of the served models, like: -- the device to load the model onto -- the model version policy -- inference related options - -[Learn more](parameters.md) - -## Keeping Models in a Remote Storage +## Serving a Single Model -Leverage remote storages, compatible with Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage, to create more flexible model repositories -that are easy to use and manage, for example, in Kubernetes deployments. +Before starting the container, make sure you have [prepared the model for serving](models_repository.md). -[Learn more](using_cloud_storage.md) +Start the model server by running the following command with your parameters: -## Setting Model Versioning Policies for served models +``` +docker run -d --rm -v :/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ +--model_path --model_name --port 9000 --rest_port 9001 --log_level DEBUG +``` -Take advantage of the model repository structure. Add or delete version directories and Model Server will automatically adjust. -Take full control over the served model versions by setting a model version policy and serving all, the chosen, or just the latest version of the model. +Example using a ResNet model: -[Learn more](model_version_policy.md) +```bash +mkdir -p models/resnet/1 +wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin +wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -## Modifying Model Configuration in Runtime +docker run -d --rm -v ${PWD}/models:/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ +--model_path /models/resnet/ --model_name resnet --port 9000 --rest_port 9001 --log_level DEBUG +``` -OpenVINO Model Server tracks changes to the configuration file and applies them in runtime. It means that you can change model configurations -(for example serve the model on a different device), add a new model or completely remove one that is no longer needed. All changes will be applied with no -disruption to the service and no restart will berequired. +The required Model Server parameters are listed below. For additional configuration options, see the [Model Server Parameters](parameters.md) section. -[Learn more](online_config_changes.md) +@sphinxdirective -## Keeping Deployments Secure ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `--rm` | | remove the container when exiting the Docker container | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `-d` | | runs the container in the background | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `-v` | | defines how to mount the model folder in the Docker container | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `-p` | | exposes the model serving port outside the Docker container | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `openvino/model_server:latest` | | represents the image name; the ovms binary is the Docker entry point | +| | | varies by tag and build process - see tags: https://hub.docker.com/r/openvino/model_server/tags/ for a full tag list. | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `--model_path` | | model location, which can be: | +| | | a Docker container path that is mounted during start-up | +| | | a Google Cloud Storage path `gs:///` | +| | | an AWS S3 path `s3:///` | +| | | an Azure blob path `az:///` | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `--model_name` | | the name of the model in the model_path | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `--port` | | the gRPC server port | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| `--rest_port` | | the REST server port | ++--------------------------------+---------------------------------------------------------------------------------------------------------------------------------+ -While deploying model server, think about security of your deployment. Take care of appropriate permissions and keeping your models in a safe place. -Consider configuring access restrictions and traffic encryption to secure communication with the model server. +@endsphinxdirective -[Learn more](security_considerations.md) +- Expose the container ports to **open ports** on your host or virtual machine. +- In the command above, port 9000 is exposed for gRPC and port 9001 is exposed for REST API calls. +- Add model_name for the client gRPC/REST API calls. + +## Serving Multiple Models + +To serve multiple models from the same container you will need an additional JSON configuration file that defines each model. To use a container with several models, you need an additional JSON configuration file defining each model. `model_config_list` array that includes a collection of config objects for each served model. The `name` and the `base_path` values of the model are required for each config object. + + +```json +{ + "model_config_list":[ + { + "config":{ + "name":"model_name1", + "base_path":"/opt/ml/models/model1", + "batch_size": "16" + } + }, + { + "config":{ + "name":"model_name2", + "base_path":"/opt/ml/models/model2", + "batch_size": "auto", + "model_version_policy": {"all": {}} + } + }, + { + "config":{ + "name":"model_name3", + "base_path":"gs://bucket/models/model3", + "model_version_policy": {"specific": { "versions":[1, 3] }}, + "shape": "auto" + } + }, + { + "config":{ + "name":"model_name4", + "base_path":"s3://bucket/models/model4", + "shape": { + "input1": "(1,3,200,200)", + "input2": "(1,3,50,50)" + }, + "plugin_config": {"CPU_THROUGHPUT_STREAMS": "CPU_THROUGHPUT_AUTO"} + } + }, + { + "config":{ + "name":"model_name5", + "base_path":"s3://bucket/models/model5", + "shape": "auto", + "nireq": 32, + "target_device": "HDDL" + } + } + ] +} +``` + +Once the Docker container has the path to your config file mounted, it can be started. This simplifies the `docker run` command, as arguments are now read from the config file. + +## Next Steps + +- Explore all model serving [features](features.md) +- Try model serving [demos](../demos/README.md) + +## Additional Resources + +- [Preparing a Model Repository](models_repository.md) +- [Using Cloud Storage](using_cloud_storage.md) +- [Troubleshooting](troubleshooting.md) +- [Model Server Parameters](parameters.md) diff --git a/docs/writing_app.md b/docs/writing_app.md new file mode 100644 index 0000000000..f4ae7b5c77 --- /dev/null +++ b/docs/writing_app.md @@ -0,0 +1,23 @@ +# Writing a Client Application {#ovms_docs_server_app} + +@sphinxdirective + +.. toctree:: + :maxdepth: 1 + :hidden: + + ovms_docs_clients + ovms_docs_server_api + +@endsphinxdirective + +OpenVINO™ Model Server exposes two sets of APIs: one compatible with TensorFlow Serving and another one, with KServe API, for inference. Both APIs work on gRPC and REST interfaces. Supporting two sets of APIs makes OpenVINO Model Server easier to plug into existing systems the already leverage one of these APIs for inference. Learn more about supported APIs: + +- [TensorFlow Serving gRPC API](./model_server_grpc_api_tfs.md) +- [KServe gRPC API](./model_server_grpc_api_kfs.md) +- [TensorFlow Serving REST API](./model_server_rest_api_tfs.md) +- [KServe REST API](./model_server_rest_api_kfs.md) + +In this section you can find short code samples to interact with OpenVINO Model Server endpoints via: +- [TensorFlow Serving API](./clients_tfs.md) +- [KServe API](./clients_kfs.md) From 6247e0f0ed29967d521e5cf336018ec6f8588d11 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Mon, 12 Dec 2022 09:38:51 +0100 Subject: [PATCH 078/130] Remove TODOs (#1579) --- src/config.hpp | 1 - src/http_rest_api_handler.cpp | 42 ----------------------------------- src/http_rest_api_handler.hpp | 1 - src/modelmanager.hpp | 2 -- 4 files changed, 46 deletions(-) diff --git a/src/config.hpp b/src/config.hpp index 6b507c116e..152ec1818b 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -57,7 +57,6 @@ class Config { * @brief Gets the instance of the config */ static Config& instance() { - // TODO to remove singleton static Config instance; return instance; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index f726ab75e1..c6850d3fac 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -239,48 +239,6 @@ void HttpRestApiHandler::parseParams(Value& scope, Document& doc) { } } -std::string HttpRestApiHandler::preprocessInferRequest(std::string request_body) { - static const std::unordered_map types = { - {"BOOL", "bool_contents"}, - {"INT8", "int_contents"}, - {"INT16", "int_contents"}, - {"INT32", "int_contents"}, - {"INT64", "int64_contents"}, - {"UINT8", "uint_contents"}, - {"UINT16", "uint_contents"}, - {"UINT32", "uint_contents"}, - {"UINT64", "uint64_contents"}, - {"FP32", "fp32_contents"}, - {"FP64", "fp64_contents"}, - {"BYTES", "bytes_contents"}}; - - Document doc; - doc.Parse(request_body.c_str()); - Value& inputs = doc["inputs"]; - for (SizeType i = 0; i < inputs.Size(); i++) { - Value data = inputs[i].GetObject()["data"].GetArray(); - Value contents(rapidjson::kObjectType); - auto it = types.find(inputs[i].GetObject()["datatype"].GetString()); - if (it == types.end()) - return ""; - // TODO confirm its not an issue - Value datatype(it->second.c_str(), doc.GetAllocator()); - contents.AddMember(datatype, data, doc.GetAllocator()); - inputs[i].AddMember("contents", contents, doc.GetAllocator()); - parseParams(inputs[i], doc); - } - Value& outputs = doc["outputs"]; - for (SizeType i = 0; i < outputs.Size(); i++) { - parseParams(outputs[i], doc); - } - parseParams(doc, doc); - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - doc.Accept(writer); - - return buffer.GetString(); -} - static Status convertStringToVectorOfSizes(const std::string& comma_separated_numbers, std::vector& sizes) { std::stringstream streamData(comma_separated_numbers); std::vector sizes_; diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index 24c10dcb16..c404e94a0a 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -95,7 +95,6 @@ class HttpRestApiHandler { Status parseModelVersion(std::string& model_version_str, std::optional& model_version); static void parseParams(rapidjson::Value&, rapidjson::Document&); - static std::string preprocessInferRequest(std::string request_body); static Status prepareGrpcRequest(const std::string modelName, const std::optional& modelVersion, const std::string& request_body, ::KFSRequest& grpc_request, const std::optional& inferenceHeaderContentLength = {}); void registerHandler(RequestType type, std::function); diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index 75eaa82c83..c84c3b3754 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -165,8 +165,6 @@ class ModelManager { * */ std::unordered_map servedModelConfigs; - // TODO we should either dispose this member or at least keep it as a pointer - // so we do not disclose details about ModelConfig to all modelmanager users /** * @brief Retires models non existing in config file From 31a0a3ca6c6e522440f9ca79637fd87b6db61999 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Tue, 13 Dec 2022 00:26:52 +0100 Subject: [PATCH 079/130] develop branch links --- docs/clients_kfs.md | 2 +- docs/model_server_rest_api_kfs.md | 12 ++++++------ src/example/SampleCpuExtension/README.md | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/clients_kfs.md b/docs/clients_kfs.md index 42ce3fcc52..48b75cbb87 100644 --- a/docs/clients_kfs.md +++ b/docs/clients_kfs.md @@ -84,4 +84,4 @@ When creating a Python-based client application, you can use Triton client libra @endsphinxdirective -For complete usage examples see [Kserve samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples). +For complete usage examples see [Kserve samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples). diff --git a/docs/model_server_rest_api_kfs.md b/docs/model_server_rest_api_kfs.md index 178f7e08d1..0ffc04f99a 100644 --- a/docs/model_server_rest_api_kfs.md +++ b/docs/model_server_rest_api_kfs.md @@ -36,7 +36,7 @@ Date: Tue, 09 Aug 2022 09:20:24 GMT Content-Length: 2 ``` -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for getting server liveness with KServe API on HTTP Server Live endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for getting server liveness with KServe API on HTTP Server Live endpoint. ## Server Ready API **Description** @@ -63,7 +63,7 @@ Date: Tue, 09 Aug 2022 09:22:14 GMT Content-Length: 2 ``` -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for getting server readiness with KServe API on HTTP Server Ready endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for getting server readiness with KServe API on HTTP Server Ready endpoint. ## Server Metadata API **Description** @@ -103,7 +103,7 @@ $ curl http://localhost:5000/v2 For detailed description of the response contents see [KServe API docs](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md#server-metadata). -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for getting server metadata with KServe API on HTTP Server Metadata endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for getting server metadata with KServe API on HTTP Server Metadata endpoint. ## Model Ready API **Description** @@ -130,7 +130,7 @@ Date: Tue, 09 Aug 2022 09:25:31 GMT Content-Length: 2 ``` -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for getting model readiness with KServe API on HTTP Model Ready endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for getting model readiness with KServe API on HTTP Model Ready endpoint. @@ -185,7 +185,7 @@ $ curl http://localhost:8000/v2/models/resnet For detailed description of the response contents see [KServe API docs](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md#model-metadata). -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for running getting model metadata with KServe API on HTTP Model Metadata endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for running getting model metadata with KServe API on HTTP Model Metadata endpoint. ## Inference API **Description** @@ -263,4 +263,4 @@ For detailed description of request and response contents see [KServe API docs]( > Note: More efficient way of running inference via REST is sending data in a binary format outside of the JSON object, by using [binary data extension](./binary_input_kfs.md). -See also [code samples](https://github.com/openvinotoolkit/model_server/tree/v2022.2/client/python/kserve-api/samples) for running inference with KServe API on HTTP Inference endpoint. +See also [code samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples) for running inference with KServe API on HTTP Inference endpoint. diff --git a/src/example/SampleCpuExtension/README.md b/src/example/SampleCpuExtension/README.md index 068f79ee10..941048e477 100644 --- a/src/example/SampleCpuExtension/README.md +++ b/src/example/SampleCpuExtension/README.md @@ -8,7 +8,7 @@ custom extension execution. ## Creating cpu_extension library -Compile the library by running `make cpu_extension BASE_OS=ubuntu` in root directory of [Model Server repository](https://github.com/openvinotoolkit/model_server/tree/v2022.2). The implementation of this library slightly differs from the template in OpenVINO™ repository and can be found in [SampleCpuExtension directory](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/src/example/SampleCpuExtension). +Compile the library by running `make cpu_extension BASE_OS=ubuntu` in root directory of [Model Server repository](https://github.com/openvinotoolkit/model_server/tree/develop). The implementation of this library slightly differs from the template in OpenVINO™ repository and can be found in [SampleCpuExtension directory](https://github.com/openvinotoolkit/model_server/tree/develop/src/example/SampleCpuExtension). Shared library will be generated in the `lib` folder. Such library can be used to run Model Server, using `--cpu_extension` argument. From 848cdf7e5e0c127b2a8f43d2e386ce41e7eeb153 Mon Sep 17 00:00:00 2001 From: ngrozae <104074686+ngrozae@users.noreply.github.com> Date: Tue, 13 Dec 2022 10:24:45 +0100 Subject: [PATCH 080/130] rest port number update (#1590) --- docs/starting_server.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/starting_server.md b/docs/starting_server.md index 782fcbf60c..0e2abddb77 100644 --- a/docs/starting_server.md +++ b/docs/starting_server.md @@ -24,8 +24,8 @@ Before starting the container, make sure you have [prepared the model for servin Start the model server by running the following command with your parameters: ``` -docker run -d --rm -v :/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ ---model_path --model_name --port 9000 --rest_port 9001 --log_level DEBUG +docker run -d --rm -v :/models -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ +--model_path --model_name --port 9000 --rest_port 8000 --log_level DEBUG ``` Example using a ResNet model: @@ -35,8 +35,8 @@ mkdir -p models/resnet/1 wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin wget -P models/resnet/1 https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -docker run -d --rm -v ${PWD}/models:/models -p 9000:9000 -p 9001:9001 openvino/model_server:latest \ ---model_path /models/resnet/ --model_name resnet --port 9000 --rest_port 9001 --log_level DEBUG +docker run -d --rm -v ${PWD}/models:/models -p 9000:9000 -p 8000:8000 openvino/model_server:latest \ +--model_path /models/resnet/ --model_name resnet --port 9000 --rest_port 8000 --log_level DEBUG ``` The required Model Server parameters are listed below. For additional configuration options, see the [Model Server Parameters](parameters.md) section. @@ -71,7 +71,7 @@ The required Model Server parameters are listed below. For additional configurat @endsphinxdirective - Expose the container ports to **open ports** on your host or virtual machine. -- In the command above, port 9000 is exposed for gRPC and port 9001 is exposed for REST API calls. +- In the command above, port 9000 is exposed for gRPC and port 8000 is exposed for REST API calls. - Add model_name for the client gRPC/REST API calls. ## Serving Multiple Models From c81ef745fb182365928de9458e175dd913ded79a Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Tue, 13 Dec 2022 13:32:04 +0100 Subject: [PATCH 081/130] coverity scan automation (#1506) --- Dockerfile.redhat | 3 +++ Dockerfile.ubuntu | 2 ++ ci/Dockerfile.coverity | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 ci/Dockerfile.coverity diff --git a/Dockerfile.redhat b/Dockerfile.redhat index ef2e82b1d1..1bf59fdac9 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -209,3 +209,6 @@ COPY ${ovms_metadata_file} metadata.json RUN ./bazel-bin/src/./ovms COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE +COPY client /client +COPY demos /demos + diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 9d2866a520..db51698bc7 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -305,4 +305,6 @@ RUN ./bazel-bin/src/./ovms --version RUN ./bazel-bin/src/./ovms COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE +COPY client /client +COPY demos /demos diff --git a/ci/Dockerfile.coverity b/ci/Dockerfile.coverity new file mode 100644 index 0000000000..5098b7bbb4 --- /dev/null +++ b/ci/Dockerfile.coverity @@ -0,0 +1,41 @@ +# +# Copyright (c) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BUILD_IMAGE=openvino/model_server-build:latest +FROM $BUILD_IMAGE + +COPY ./cov-analysis-linux64-2022.6.0 /cov + +RUN apt-get remove --yes cmake && apt-get update && apt-get install -y rapidjson-dev && pip3 install cmake + +RUN /cov/bin/cov-configure --gcc --config coverity_config.xml && \ + /cov/bin/cov-configure --comptype gcc --compiler /usr/bin/gcc && \ + /cov/bin/cov-build --dir cov-int bash -c 'bazel shutdown; bazel clean; bazel build --spawn_strategy=standalone //src:static_analysis && cmake /client/cpp/kserve-api && make --jobs=$(nproc)' + +WORKDIR /ovms/ +RUN tar czvf /model_server.tgz cov-int + +ARG ENV_COVERITY_PROJECT +ARG ENV_COVERITY_TOKEN +ARG ENV_BUILD_VERSION=3 +WORKDIR / +RUN curl --form token=$ENV_COVERITY_TOKEN \ + --form email=dariusz.trawinski@intel.com \ + --form file=@model_server.tgz \ + --form version=$ENV_BUILD_VERSION \ + --form description="SDL build" \ + https://scan.coverity.com/builds?project=$ENV_COVERITY_PROJECT + From d61cf7255874103238646a3c15040a23e07ab77c Mon Sep 17 00:00:00 2001 From: atobisze Date: Mon, 22 Aug 2022 11:34:04 +0200 Subject: [PATCH 082/130] Mini POC CAPI -> target in bazel to build app that starts the server and then uses ovms to perform inference -> no one shared library to link to -> inference only in model not in dag -> this is just working snapshot - we shouldn't have a copy of sources in src/ovms_lib -> rule in bazel has to be rewritten from scratch - my_cc_combine --- MakefileCapi | 42 + src/BUILD | 65 +- src/deserialization.cpp | 2 +- src/main2.cpp | 48 + src/modelinstance.cpp | 39 + src/modelinstance.hpp | 1 + src/my_cc_combine.bzl | 70 + src/ovms_lib/BUILD | 630 ++++++++ src/ovms_lib/aliases.hpp | 25 + src/ovms_lib/azurefilesystem.cpp | 320 ++++ src/ovms_lib/azurefilesystem.hpp | 166 ++ src/ovms_lib/azurestorage.cpp | 1217 +++++++++++++++ src/ovms_lib/azurestorage.hpp | 195 +++ src/ovms_lib/binaryutils.cpp | 447 ++++++ src/ovms_lib/binaryutils.hpp | 26 + src/ovms_lib/cleaner_utils.hpp | 43 + src/ovms_lib/config.cpp | 303 ++++ src/ovms_lib/config.hpp | 394 +++++ src/ovms_lib/custom_node.cpp | 103 ++ src/ovms_lib/custom_node.hpp | 65 + src/ovms_lib/custom_node_interface.h | 79 + ..._node_library_internal_manager_wrapper.cpp | 27 + ..._node_library_internal_manager_wrapper.hpp | 40 + src/ovms_lib/custom_node_library_manager.cpp | 135 ++ src/ovms_lib/custom_node_library_manager.hpp | 36 + src/ovms_lib/custom_node_output_allocator.cpp | 54 + src/ovms_lib/custom_node_output_allocator.hpp | 39 + src/ovms_lib/customloaderconfig.hpp | 145 ++ src/ovms_lib/customloaderinterface.hpp | 117 ++ src/ovms_lib/customloaders.cpp | 90 ++ src/ovms_lib/customloaders.hpp | 92 ++ src/ovms_lib/customnodesession.cpp | 250 +++ src/ovms_lib/customnodesession.hpp | 59 + src/ovms_lib/deserialization.cpp | 73 + src/ovms_lib/deserialization.hpp | 405 +++++ src/ovms_lib/dl_node.cpp | 134 ++ src/ovms_lib/dl_node.hpp | 98 ++ src/ovms_lib/dlnodesession.cpp | 345 +++++ src/ovms_lib/dlnodesession.hpp | 73 + src/ovms_lib/entry_node.cpp | 177 +++ src/ovms_lib/entry_node.hpp | 63 + src/ovms_lib/executingstreamidguard.hpp | 39 + src/ovms_lib/exit_node.cpp | 77 + src/ovms_lib/exit_node.hpp | 63 + src/ovms_lib/exitnodesession.cpp | 46 + src/ovms_lib/exitnodesession.hpp | 37 + src/ovms_lib/filesystem.hpp | 196 +++ src/ovms_lib/gatherexitnodeinputhandler.cpp | 53 + src/ovms_lib/gatherexitnodeinputhandler.hpp | 68 + src/ovms_lib/gathernodeinputhandler.cpp | 116 ++ src/ovms_lib/gathernodeinputhandler.hpp | 48 + src/ovms_lib/gcsfilesystem.cpp | 407 +++++ src/ovms_lib/gcsfilesystem.hpp | 165 ++ src/ovms_lib/get_model_metadata_impl.cpp | 202 +++ src/ovms_lib/get_model_metadata_impl.hpp | 71 + src/ovms_lib/global_sequences_viewer.cpp | 73 + src/ovms_lib/global_sequences_viewer.hpp | 46 + src/ovms_lib/grpcservermodule.cpp | 183 +++ src/ovms_lib/grpcservermodule.hpp | 46 + src/ovms_lib/http_rest_api_handler.cpp | 606 ++++++++ src/ovms_lib/http_rest_api_handler.hpp | 190 +++ src/ovms_lib/http_server.cpp | 131 ++ src/ovms_lib/http_server.hpp | 41 + src/ovms_lib/kfs_grpc_inference_service.cpp | 343 +++++ src/ovms_lib/kfs_grpc_inference_service.hpp | 68 + src/ovms_lib/layout.cpp | 169 ++ src/ovms_lib/layout.hpp | 59 + src/ovms_lib/layout_configuration.cpp | 98 ++ src/ovms_lib/layout_configuration.hpp | 52 + src/ovms_lib/localfilesystem.cpp | 173 +++ src/ovms_lib/localfilesystem.hpp | 116 ++ src/ovms_lib/log | 0 src/ovms_lib/logging.cpp | 87 ++ src/ovms_lib/logging.hpp | 36 + src/ovms_lib/main.cpp | 24 + src/ovms_lib/main2.cpp | 15 + src/ovms_lib/model.cpp | 305 ++++ src/ovms_lib/model.hpp | 246 +++ src/ovms_lib/model_service.cpp | 220 +++ src/ovms_lib/model_service.hpp | 65 + src/ovms_lib/model_version_policy.cpp | 67 + src/ovms_lib/model_version_policy.hpp | 135 ++ src/ovms_lib/modelchangesubscription.cpp | 57 + src/ovms_lib/modelchangesubscription.hpp | 42 + src/ovms_lib/modelconfig.cpp | 669 ++++++++ src/ovms_lib/modelconfig.hpp | 992 ++++++++++++ src/ovms_lib/modelinstance.cpp | 1225 +++++++++++++++ src/ovms_lib/modelinstance.hpp | 556 +++++++ src/ovms_lib/modelinstanceunloadguard.cpp | 29 + src/ovms_lib/modelinstanceunloadguard.hpp | 30 + src/ovms_lib/modelmanager.cpp | 1316 ++++++++++++++++ src/ovms_lib/modelmanager.hpp | 427 +++++ src/ovms_lib/modelversion.hpp | 24 + src/ovms_lib/modelversionstatus.cpp | 122 ++ src/ovms_lib/modelversionstatus.hpp | 105 ++ src/ovms_lib/my_cc_combine.bzl | 70 + src/ovms_lib/node.cpp | 272 ++++ src/ovms_lib/node.hpp | 102 ++ src/ovms_lib/node_library.cpp | 29 + src/ovms_lib/node_library.hpp | 52 + src/ovms_lib/node_library_utils.cpp | 143 ++ src/ovms_lib/node_library_utils.hpp | 40 + src/ovms_lib/nodeinfo.hpp | 92 ++ src/ovms_lib/nodeinputhandler.cpp | 60 + src/ovms_lib/nodeinputhandler.hpp | 54 + src/ovms_lib/nodeoutputhandler.cpp | 20 + src/ovms_lib/nodeoutputhandler.hpp | 28 + src/ovms_lib/nodesession.cpp | 72 + src/ovms_lib/nodesession.hpp | 62 + src/ovms_lib/nodesessionmetadata.cpp | 197 +++ src/ovms_lib/nodesessionmetadata.hpp | 48 + src/ovms_lib/nodesessionresult.hpp | 29 + src/ovms_lib/nodestreamidguard.hpp | 70 + src/ovms_lib/ov_utils.cpp | 131 ++ src/ovms_lib/ov_utils.hpp | 47 + src/ovms_lib/ovinferrequestsqueue.hpp | 45 + src/ovms_lib/pipeline.cpp | 270 ++++ src/ovms_lib/pipeline.hpp | 63 + src/ovms_lib/pipeline_factory.cpp | 146 ++ src/ovms_lib/pipeline_factory.hpp | 84 + src/ovms_lib/pipelinedefinition.cpp | 1367 +++++++++++++++++ src/ovms_lib/pipelinedefinition.hpp | 189 +++ src/ovms_lib/pipelinedefinitionstatus.hpp | 407 +++++ .../pipelinedefinitionunloadguard.cpp | 29 + .../pipelinedefinitionunloadguard.hpp | 30 + src/ovms_lib/pipelineeventqueue.hpp | 28 + src/ovms_lib/precision.cpp | 249 +++ src/ovms_lib/precision.hpp | 70 + .../predict_request_validation_utils.cpp | 696 +++++++++ .../predict_request_validation_utils.hpp | 42 + src/ovms_lib/prediction_service.cpp | 137 ++ src/ovms_lib/prediction_service.hpp | 64 + src/ovms_lib/prediction_service_utils.cpp | 90 ++ src/ovms_lib/prediction_service_utils.hpp | 38 + src/ovms_lib/profiler.cpp | 56 + src/ovms_lib/profiler.hpp | 49 + src/ovms_lib/queue.hpp | 136 ++ src/ovms_lib/rest_parser.cpp | 468 ++++++ src/ovms_lib/rest_parser.hpp | 218 +++ src/ovms_lib/rest_utils.cpp | 198 +++ src/ovms_lib/rest_utils.hpp | 36 + src/ovms_lib/s3filesystem.cpp | 552 +++++++ src/ovms_lib/s3filesystem.hpp | 155 ++ src/ovms_lib/schema.cpp | 351 +++++ src/ovms_lib/schema.hpp | 28 + src/ovms_lib/sequence.cpp | 67 + src/ovms_lib/sequence.hpp | 60 + src/ovms_lib/sequence_manager.cpp | 157 ++ src/ovms_lib/sequence_manager.hpp | 83 + src/ovms_lib/sequence_processing_spec.hpp | 36 + src/ovms_lib/serialization.cpp | 245 +++ src/ovms_lib/serialization.hpp | 126 ++ src/ovms_lib/servablemanagermodule.cpp | 56 + src/ovms_lib/servablemanagermodule.hpp | 35 + src/ovms_lib/server.cpp | 348 +++++ src/ovms_lib/server.hpp | 75 + src/ovms_lib/session_id.hpp | 24 + src/ovms_lib/shape.cpp | 505 ++++++ src/ovms_lib/shape.hpp | 128 ++ src/ovms_lib/statefulmodelinstance.cpp | 324 ++++ src/ovms_lib/statefulmodelinstance.hpp | 101 ++ src/ovms_lib/status.cpp | 426 +++++ src/ovms_lib/status.hpp | 381 +++++ src/ovms_lib/stringutils.cpp | 139 ++ src/ovms_lib/stringutils.hpp | 91 ++ src/ovms_lib/tensor_utils.hpp | 46 + src/ovms_lib/tensorinfo.cpp | 289 ++++ src/ovms_lib/tensorinfo.hpp | 238 +++ src/ovms_lib/tensormap.hpp | 33 + src/ovms_lib/tfs_frontend/tfs_utils.cpp | 98 ++ src/ovms_lib/tfs_frontend/tfs_utils.hpp | 33 + src/ovms_lib/threadsafequeue.hpp | 69 + src/ovms_lib/timer.hpp | 50 + src/ovms_lib/version.hpp | 21 + src/server.cpp | 5 + src/server.hpp | 5 +- 176 files changed, 29704 insertions(+), 7 deletions(-) create mode 100644 MakefileCapi create mode 100644 src/main2.cpp create mode 100644 src/my_cc_combine.bzl create mode 100644 src/ovms_lib/BUILD create mode 100644 src/ovms_lib/aliases.hpp create mode 100644 src/ovms_lib/azurefilesystem.cpp create mode 100644 src/ovms_lib/azurefilesystem.hpp create mode 100644 src/ovms_lib/azurestorage.cpp create mode 100644 src/ovms_lib/azurestorage.hpp create mode 100644 src/ovms_lib/binaryutils.cpp create mode 100644 src/ovms_lib/binaryutils.hpp create mode 100644 src/ovms_lib/cleaner_utils.hpp create mode 100644 src/ovms_lib/config.cpp create mode 100644 src/ovms_lib/config.hpp create mode 100644 src/ovms_lib/custom_node.cpp create mode 100644 src/ovms_lib/custom_node.hpp create mode 100644 src/ovms_lib/custom_node_interface.h create mode 100644 src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp create mode 100644 src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp create mode 100644 src/ovms_lib/custom_node_library_manager.cpp create mode 100644 src/ovms_lib/custom_node_library_manager.hpp create mode 100644 src/ovms_lib/custom_node_output_allocator.cpp create mode 100644 src/ovms_lib/custom_node_output_allocator.hpp create mode 100644 src/ovms_lib/customloaderconfig.hpp create mode 100644 src/ovms_lib/customloaderinterface.hpp create mode 100644 src/ovms_lib/customloaders.cpp create mode 100644 src/ovms_lib/customloaders.hpp create mode 100644 src/ovms_lib/customnodesession.cpp create mode 100644 src/ovms_lib/customnodesession.hpp create mode 100644 src/ovms_lib/deserialization.cpp create mode 100644 src/ovms_lib/deserialization.hpp create mode 100644 src/ovms_lib/dl_node.cpp create mode 100644 src/ovms_lib/dl_node.hpp create mode 100644 src/ovms_lib/dlnodesession.cpp create mode 100644 src/ovms_lib/dlnodesession.hpp create mode 100644 src/ovms_lib/entry_node.cpp create mode 100644 src/ovms_lib/entry_node.hpp create mode 100644 src/ovms_lib/executingstreamidguard.hpp create mode 100644 src/ovms_lib/exit_node.cpp create mode 100644 src/ovms_lib/exit_node.hpp create mode 100644 src/ovms_lib/exitnodesession.cpp create mode 100644 src/ovms_lib/exitnodesession.hpp create mode 100644 src/ovms_lib/filesystem.hpp create mode 100644 src/ovms_lib/gatherexitnodeinputhandler.cpp create mode 100644 src/ovms_lib/gatherexitnodeinputhandler.hpp create mode 100644 src/ovms_lib/gathernodeinputhandler.cpp create mode 100644 src/ovms_lib/gathernodeinputhandler.hpp create mode 100644 src/ovms_lib/gcsfilesystem.cpp create mode 100644 src/ovms_lib/gcsfilesystem.hpp create mode 100644 src/ovms_lib/get_model_metadata_impl.cpp create mode 100644 src/ovms_lib/get_model_metadata_impl.hpp create mode 100644 src/ovms_lib/global_sequences_viewer.cpp create mode 100644 src/ovms_lib/global_sequences_viewer.hpp create mode 100644 src/ovms_lib/grpcservermodule.cpp create mode 100644 src/ovms_lib/grpcservermodule.hpp create mode 100644 src/ovms_lib/http_rest_api_handler.cpp create mode 100644 src/ovms_lib/http_rest_api_handler.hpp create mode 100644 src/ovms_lib/http_server.cpp create mode 100644 src/ovms_lib/http_server.hpp create mode 100644 src/ovms_lib/kfs_grpc_inference_service.cpp create mode 100644 src/ovms_lib/kfs_grpc_inference_service.hpp create mode 100644 src/ovms_lib/layout.cpp create mode 100644 src/ovms_lib/layout.hpp create mode 100644 src/ovms_lib/layout_configuration.cpp create mode 100644 src/ovms_lib/layout_configuration.hpp create mode 100644 src/ovms_lib/localfilesystem.cpp create mode 100644 src/ovms_lib/localfilesystem.hpp create mode 100644 src/ovms_lib/log create mode 100644 src/ovms_lib/logging.cpp create mode 100644 src/ovms_lib/logging.hpp create mode 100644 src/ovms_lib/main.cpp create mode 100644 src/ovms_lib/main2.cpp create mode 100644 src/ovms_lib/model.cpp create mode 100644 src/ovms_lib/model.hpp create mode 100644 src/ovms_lib/model_service.cpp create mode 100644 src/ovms_lib/model_service.hpp create mode 100644 src/ovms_lib/model_version_policy.cpp create mode 100644 src/ovms_lib/model_version_policy.hpp create mode 100644 src/ovms_lib/modelchangesubscription.cpp create mode 100644 src/ovms_lib/modelchangesubscription.hpp create mode 100644 src/ovms_lib/modelconfig.cpp create mode 100644 src/ovms_lib/modelconfig.hpp create mode 100644 src/ovms_lib/modelinstance.cpp create mode 100644 src/ovms_lib/modelinstance.hpp create mode 100644 src/ovms_lib/modelinstanceunloadguard.cpp create mode 100644 src/ovms_lib/modelinstanceunloadguard.hpp create mode 100644 src/ovms_lib/modelmanager.cpp create mode 100644 src/ovms_lib/modelmanager.hpp create mode 100644 src/ovms_lib/modelversion.hpp create mode 100644 src/ovms_lib/modelversionstatus.cpp create mode 100644 src/ovms_lib/modelversionstatus.hpp create mode 100644 src/ovms_lib/my_cc_combine.bzl create mode 100644 src/ovms_lib/node.cpp create mode 100644 src/ovms_lib/node.hpp create mode 100644 src/ovms_lib/node_library.cpp create mode 100644 src/ovms_lib/node_library.hpp create mode 100644 src/ovms_lib/node_library_utils.cpp create mode 100644 src/ovms_lib/node_library_utils.hpp create mode 100644 src/ovms_lib/nodeinfo.hpp create mode 100644 src/ovms_lib/nodeinputhandler.cpp create mode 100644 src/ovms_lib/nodeinputhandler.hpp create mode 100644 src/ovms_lib/nodeoutputhandler.cpp create mode 100644 src/ovms_lib/nodeoutputhandler.hpp create mode 100644 src/ovms_lib/nodesession.cpp create mode 100644 src/ovms_lib/nodesession.hpp create mode 100644 src/ovms_lib/nodesessionmetadata.cpp create mode 100644 src/ovms_lib/nodesessionmetadata.hpp create mode 100644 src/ovms_lib/nodesessionresult.hpp create mode 100644 src/ovms_lib/nodestreamidguard.hpp create mode 100644 src/ovms_lib/ov_utils.cpp create mode 100644 src/ovms_lib/ov_utils.hpp create mode 100644 src/ovms_lib/ovinferrequestsqueue.hpp create mode 100644 src/ovms_lib/pipeline.cpp create mode 100644 src/ovms_lib/pipeline.hpp create mode 100644 src/ovms_lib/pipeline_factory.cpp create mode 100644 src/ovms_lib/pipeline_factory.hpp create mode 100644 src/ovms_lib/pipelinedefinition.cpp create mode 100644 src/ovms_lib/pipelinedefinition.hpp create mode 100644 src/ovms_lib/pipelinedefinitionstatus.hpp create mode 100644 src/ovms_lib/pipelinedefinitionunloadguard.cpp create mode 100644 src/ovms_lib/pipelinedefinitionunloadguard.hpp create mode 100644 src/ovms_lib/pipelineeventqueue.hpp create mode 100644 src/ovms_lib/precision.cpp create mode 100644 src/ovms_lib/precision.hpp create mode 100644 src/ovms_lib/predict_request_validation_utils.cpp create mode 100644 src/ovms_lib/predict_request_validation_utils.hpp create mode 100644 src/ovms_lib/prediction_service.cpp create mode 100644 src/ovms_lib/prediction_service.hpp create mode 100644 src/ovms_lib/prediction_service_utils.cpp create mode 100644 src/ovms_lib/prediction_service_utils.hpp create mode 100644 src/ovms_lib/profiler.cpp create mode 100644 src/ovms_lib/profiler.hpp create mode 100644 src/ovms_lib/queue.hpp create mode 100644 src/ovms_lib/rest_parser.cpp create mode 100644 src/ovms_lib/rest_parser.hpp create mode 100644 src/ovms_lib/rest_utils.cpp create mode 100644 src/ovms_lib/rest_utils.hpp create mode 100644 src/ovms_lib/s3filesystem.cpp create mode 100644 src/ovms_lib/s3filesystem.hpp create mode 100644 src/ovms_lib/schema.cpp create mode 100644 src/ovms_lib/schema.hpp create mode 100644 src/ovms_lib/sequence.cpp create mode 100644 src/ovms_lib/sequence.hpp create mode 100644 src/ovms_lib/sequence_manager.cpp create mode 100644 src/ovms_lib/sequence_manager.hpp create mode 100644 src/ovms_lib/sequence_processing_spec.hpp create mode 100644 src/ovms_lib/serialization.cpp create mode 100644 src/ovms_lib/serialization.hpp create mode 100644 src/ovms_lib/servablemanagermodule.cpp create mode 100644 src/ovms_lib/servablemanagermodule.hpp create mode 100644 src/ovms_lib/server.cpp create mode 100644 src/ovms_lib/server.hpp create mode 100644 src/ovms_lib/session_id.hpp create mode 100644 src/ovms_lib/shape.cpp create mode 100644 src/ovms_lib/shape.hpp create mode 100644 src/ovms_lib/statefulmodelinstance.cpp create mode 100644 src/ovms_lib/statefulmodelinstance.hpp create mode 100644 src/ovms_lib/status.cpp create mode 100644 src/ovms_lib/status.hpp create mode 100644 src/ovms_lib/stringutils.cpp create mode 100644 src/ovms_lib/stringutils.hpp create mode 100644 src/ovms_lib/tensor_utils.hpp create mode 100644 src/ovms_lib/tensorinfo.cpp create mode 100644 src/ovms_lib/tensorinfo.hpp create mode 100644 src/ovms_lib/tensormap.hpp create mode 100644 src/ovms_lib/tfs_frontend/tfs_utils.cpp create mode 100644 src/ovms_lib/tfs_frontend/tfs_utils.hpp create mode 100644 src/ovms_lib/threadsafequeue.hpp create mode 100644 src/ovms_lib/timer.hpp create mode 100644 src/ovms_lib/version.hpp diff --git a/MakefileCapi b/MakefileCapi new file mode 100644 index 0000000000..9913f156b0 --- /dev/null +++ b/MakefileCapi @@ -0,0 +1,42 @@ +all: + g++ -pthread --std=c++17 -I./src -L./bazel-bin/src/ -L/usr/local/lib -L/opt/intel/openvino/runtime/lib/intel64/ -fPIC src/main2.cpp -L/opt/opencv/lib/ \ + -lovms_lib \ + -lovms_shared \ + -lopenvino \ + -lopenvino_c \ + -lopenvino_gapi_preproc \ + -lopenvino \ + -lazurestorage \ + -lcpprest \ + -lboost_thread \ + -lboost_regex \ + -lboost_system \ + -lboost_log \ + -lboost_atomic \ + -lboost_chrono \ + -lboost_date_time \ + -lopenvino \ + -lopenvino_intel_hddl_plugin \ + -lopenvino_c \ + -lopenvino_ov_c \ + -lopenvino_intel_gpu_plugin \ + -lopenvino_intel_gna_plugin \ + -lopenvino_intel_cpu_plugin \ + -lopenvino_hetero_plugin \ + -lopenvino_tensorflow_fe \ + -lopenvino_paddle_frontend \ + -lopenvino_onnx_frontend \ + -lopenvino_ir_frontend \ + -lopenvino_intel_myriad_plugin \ + -lopenvino_gapi_preproc \ + -lopenvino_auto_plugin \ + -lopenvino_auto_batch_plugin \ + -lgna \ + -lstdc++fs \ + -lopencv_core \ + -lopencv_imgcodecs \ + -lopencv_imgproc \ + -lssl \ + -lcrypto \ + -ldl \ + 2>&1 | tee log; wc -l log diff --git a/src/BUILD b/src/BUILD index e8abbdca8c..24672ff7a9 100644 --- a/src/BUILD +++ b/src/BUILD @@ -43,10 +43,24 @@ cc_library( ) load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") +load(":my_cc_combine.bzl", "my_cc_combine") + +my_cc_combine( + name = "hello_combined", + # Here merge all the static libraries into one static library + genstatic = True, + output = "libcombined.so", + deps = [ + "//src:libovms_shared.so", + # "//src:ovms_lib", + ], +) cc_library( name = "ovms_lib", - linkstatic = 1, + linkstatic = True, + # linkshared = True, + hdrs = ["server.hpp"], srcs = [ "aliases.hpp", "azurestorage.hpp", @@ -271,6 +285,9 @@ cc_library( "-Wno-unknown-pragmas", "-Werror", ], + visibility = [ + "//visibility:public", + ], ) cc_binary( @@ -469,6 +486,27 @@ cc_binary( ] ) +cc_binary( + name = "poc", + srcs = [ + "main2.cpp", + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + ], + deps = [ + "//src:ovms_lib", + # "//src:libovms_shared.so", + ], + linkstatic = True, +) + + cc_binary( name = "ovms", srcs = [ @@ -486,7 +524,30 @@ cc_binary( ], deps = [ "//src:ovms_lib", - ] + # "//src:libovms_shared.so", + ], + linkstatic = False, +) + +cc_binary( + name = "libovms_shared.so", + srcs = [ + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + "-Wconversion", + "-Werror", + ], + deps = [ + "//src/ovms_lib:ovms_lib", + ], + linkstatic = True, + linkshared = True, ) cc_test( diff --git a/src/deserialization.cpp b/src/deserialization.cpp index 536ee99354..5f8ae1c682 100644 --- a/src/deserialization.cpp +++ b/src/deserialization.cpp @@ -46,7 +46,7 @@ ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, for (int i = 0; i < requestInput.tensor_shape().dim_size(); i++) { shape.push_back(requestInput.tensor_shape().dim(i).size()); } - ov::element::Type precision = tensorInfo->getOvPrecision(); + ov::element::Type_t precision = tensorInfo->getOvPrecision(); return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.tensor_content().data()))); } diff --git a/src/main2.cpp b/src/main2.cpp new file mode 100644 index 0000000000..f075c5c18b --- /dev/null +++ b/src/main2.cpp @@ -0,0 +1,48 @@ +#include +#include +#include +#include + +#include "server.hpp" + +#include "modelmanager.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "servablemanagermodule.hpp" + +using ovms::Server; + +int main(int argc, char** argv) { + Server& server = Server::instance(); + std::thread t([&server, &argv, &argc](){ + std::cout << server.start(argc, argv) << std::endl; + }); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + std::cout << __LINE__ << "AERO" << std::endl; + // get model instance and have a lock on reload + std::shared_ptr instance; + std::unique_ptr modelInstanceUnloadGuardPtr; + auto module = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + std::cout << __LINE__ << "AERO" << std::endl; + if (nullptr == module) { + return 0; + } + std::cout << __LINE__ << "AERO" << std::endl; + auto servableManagerModule = dynamic_cast(module); + auto& manager = servableManagerModule->getServableManager(); + manager.getModelInstance("dummy", 0, instance, modelInstanceUnloadGuardPtr); // status code + float a[10] = {1,2,3,4,5,6,7,8,9,10}; + float b[10] = {1,2,3,4,5,6,7,8,9,10}; + std::cout << __LINE__ << "AERO" << std::endl; + instance->infer(a,b); + std::cout << __LINE__ << "AERO" << std::endl; + for (int i = 0; i < 10; ++i) { + std::cout << b[i] << " "; + } + std::cout << std::endl; + std::cout << __LINE__ << "AERO" << std::endl; + t.join(); + std::cout << __LINE__ << "AERO" << std::endl; + std::cout << __LINE__ << "AERO" << std::endl; + return 0; +} diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 5b54189ff5..7744c3952f 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1144,6 +1144,45 @@ Status ModelInstance::performInference(ov::InferRequest& inferRequest) { return StatusCode::OK; } +#include +Status ModelInstance::infer(float* data, float* output) { + OVMS_PROFILE_FUNCTION(); + Timer timer; + using std::chrono::microseconds; + timer.start("get infer request"); + ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); + int executingInferId = executingStreamIdGuard.getId(); + ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); + timer.stop("get infer request"); + SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); + timer.start("deserialize"); + static ov::Shape shape{1,10}; + ov::element::Type precision = ov::element::Type_t::f32; + ov::Tensor tensor(precision, + shape, + (void*)data); + inferRequest.set_tensor("b", tensor); + timer.stop("deserialize"); + SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); + timer.start("prediction"); + auto status = performInference(inferRequest); + timer.stop("prediction"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); + timer.start("serialize"); + auto otensor = inferRequest.get_tensor("a"); + std::memcpy((void*)output, otensor.data(), otensor.get_byte_size()); + timer.stop("serialize"); + SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); + SPDLOG_ERROR("ER"); + return StatusCode::OK; +} + Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, tensorflow::serving::PredictResponse* responseProto, std::unique_ptr& modelUnloadGuardPtr) { diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index edc590264d..3c0c56f9e4 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -562,5 +562,6 @@ class ModelInstance { template void fetchModelFiles(bool& found, ArrayType ext); + Status infer(float* data, float* output); }; } // namespace ovms diff --git a/src/my_cc_combine.bzl b/src/my_cc_combine.bzl new file mode 100644 index 0000000000..27113911f2 --- /dev/null +++ b/src/my_cc_combine.bzl @@ -0,0 +1,70 @@ +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +def _combine_impl(ctx): + cc_toolchain = find_cpp_toolchain(ctx) + + target_list = [] + for dep_target in ctx.attr.deps: + # CcInfo, InstrumentedFilesInfo, OutputGroupInfo + cc_info_linker_inputs = dep_target[CcInfo].linking_context.linker_inputs + + target_dirname_list = [] + for linker_in in cc_info_linker_inputs.to_list(): + for linker_in_lib in linker_in.libraries: + if linker_in_lib.pic_static_library != None: + target_list += [linker_in_lib.pic_static_library] + if linker_in_lib.static_library != None: + target_list += [linker_in_lib.static_library] + + output = ctx.outputs.output + if ctx.attr.genstatic: + cp_command = "" + processed_list = [] + processed_path_list = [] + for dep in target_list: + cp_command += "cp -a "+ dep.path +" "+ output.dirname + "/&&" + processed = ctx.actions.declare_file(dep.basename) + processed_list += [processed] + processed_path_list += [dep.path] + cp_command += "echo'starting to run shell'" + processed_path_list += [output.path] + + ctx.actions.run_shell( + outputs = processed_list, + inputs = target_list, + command = cp_command, + ) + + command = "cd {} && ar -x {} {}".format( + output.dirname, + "&& ar -x ".join([dep.basename for dep in target_list]), + "&& ar -rc libauto.a *.o" + ) + print("command = ", command) + ctx.actions.run_shell( + outputs = [output], + inputs = processed_list, + command = command, + ) + else: + command = "export PATH=$PATH:{} && {} -shared -fPIC -Wl,--whole-archive {} -Wl,--no-whole-archive -Wl,-soname -o {}".format ( + cc_toolchain.ld_executable, + cc_toolchain.compiler_executable, + "".join([dep.path for dep in target_list]), + output.path) + print("command = ", command) + ctx.actions.run_shell( + outputs = [output], + inputs = target_list, + command = command, + ) + +my_cc_combine = rule( + implementation = _combine_impl, + attrs = { + "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), + "genstatic": attr.bool(default = False), + "deps": attr.label_list(), + "output": attr.output() + }, +) diff --git a/src/ovms_lib/BUILD b/src/ovms_lib/BUILD new file mode 100644 index 0000000000..624258cc33 --- /dev/null +++ b/src/ovms_lib/BUILD @@ -0,0 +1,630 @@ +# +# Copyright (c) 2020,2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#config_setting( +# name = "linux_distribution_family", +# constraint_values = [ +# ":debian", # like Ubuntu +# ":fedora", # like RHEL/CentOS +# ], +#) + +constraint_setting(name = "linux_distribution_family") +constraint_value(constraint_setting = "linux_distribution_family", name = "fedora") # like RHEL/CentOS +constraint_value(constraint_setting = "linux_distribution_family", name = "debian") # like Ubuntu + +cc_library( + name = "custom_nodes_common_lib", + linkstatic = 1, + hdrs = ["custom_nodes/common/buffersqueue.hpp"], + srcs = [ + "queue.hpp", + "custom_nodes/common/buffersqueue.hpp", + "custom_nodes/common/buffersqueue.cpp", + ], + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror", + "-Wno-sign-compare", + ], +) + +load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") +load(":my_cc_combine.bzl", "my_cc_combine") + +my_cc_combine( + name = "hello_combined", + # Here merge all the static libraries into one static library + genstatic = True, + output = "libcombined.so", + deps = [ + "//src:libovms_shared.so", + # "//src:ovms_lib", + ], +) + +cc_library( + name = "ovms_lib", + linkstatic = True, + # linkshared = True, + hdrs = ["server.hpp"], + srcs = [ + "aliases.hpp", + "azurestorage.hpp", + "azurestorage.cpp", + "azurefilesystem.cpp", + "azurefilesystem.hpp", + "cleaner_utils.hpp", + "config.cpp", + "config.hpp", + "custom_node.cpp", + "custom_node.hpp", + "custom_node_interface.h", + "custom_node_library_manager.cpp", + "custom_node_library_manager.hpp", + "custom_node_output_allocator.cpp", + "custom_node_output_allocator.hpp", + "custom_node_library_internal_manager_wrapper.hpp", + "custom_node_library_internal_manager_wrapper.cpp", + "customnodesession.cpp", + "customnodesession.hpp", + "customloaderconfig.hpp", + "customloaders.hpp", + "customloaders.cpp", + "customloaderinterface.hpp", + "deserialization.cpp", + "deserialization.hpp", + "dl_node.cpp", + "dl_node.hpp", + "dlnodesession.cpp", + "dlnodesession.hpp", + "entry_node.cpp", + "entry_node.hpp", + "executingstreamidguard.hpp", + "exit_node.cpp", + "exit_node.hpp", + "exitnodesession.cpp", + "exitnodesession.hpp", + "filesystem.hpp", + "get_model_metadata_impl.cpp", + "get_model_metadata_impl.hpp", + "global_sequences_viewer.hpp", + "global_sequences_viewer.cpp", + "http_rest_api_handler.cpp", + "http_rest_api_handler.hpp", + "http_server.cpp", + "http_server.hpp", + "layout.cpp", + "layout.hpp", + "layout_configuration.cpp", + "layout_configuration.hpp", + "localfilesystem.cpp", + "localfilesystem.hpp", + "gathernodeinputhandler.cpp", + "gathernodeinputhandler.hpp", + "gatherexitnodeinputhandler.cpp", + "gatherexitnodeinputhandler.hpp", + "gcsfilesystem.cpp", + "gcsfilesystem.hpp", + "grpcservermodule.cpp", + "grpcservermodule.hpp", + "kfs_grpc_inference_service.cpp", + "kfs_grpc_inference_service.hpp", + "model.cpp", + "model.hpp", + "model_version_policy.cpp", + "model_version_policy.hpp", + "modelchangesubscription.cpp", + "modelchangesubscription.hpp", + "modelconfig.cpp", + "modelconfig.hpp", + "modelmanager.cpp", + "modelmanager.hpp", + "modelinstance.cpp", + "modelinstance.hpp", + "modelinstanceunloadguard.cpp", + "modelinstanceunloadguard.hpp", + "modelversion.hpp", + "modelversionstatus.cpp", + "modelversionstatus.hpp", + "model_service.hpp", + "model_service.cpp", + "node.cpp", + "node.hpp", + "nodeinfo.hpp", + "node_library.cpp", + "node_library.hpp", + "node_library_utils.cpp", + "node_library_utils.hpp", + "nodesession.cpp", + "nodesession.hpp", + "nodesessionresult.hpp", + "nodeinputhandler.cpp", + "nodeinputhandler.hpp", + "nodeoutputhandler.cpp", + "nodeoutputhandler.hpp", + "nodesessionmetadata.hpp", + "nodesessionmetadata.cpp", + "nodestreamidguard.hpp", + "ovinferrequestsqueue.hpp", + "ov_utils.cpp", + "ov_utils.hpp", + "pipeline.cpp", + "pipeline.hpp", + "pipelinedefinition.cpp", + "pipelinedefinition.hpp", + "pipelinedefinitionstatus.hpp", + "pipelinedefinitionunloadguard.cpp", + "pipelinedefinitionunloadguard.hpp", + "pipelineeventqueue.hpp", + "pipeline_factory.cpp", + "pipeline_factory.hpp", + "precision.cpp", + "precision.hpp", + "prediction_service.cpp", + "prediction_service.hpp", + "prediction_service_utils.hpp", + "prediction_service_utils.cpp", + "predict_request_validation_utils.hpp", + "predict_request_validation_utils.cpp", + "profiler.cpp", + "profiler.hpp", + "rest_parser.cpp", + "rest_parser.hpp", + "rest_utils.cpp", + "rest_utils.hpp", + "s3filesystem.cpp", + "s3filesystem.hpp", + "schema.hpp", + "schema.cpp", + "serialization.cpp", + "serialization.hpp", + "servablemanagermodule.cpp", + "servablemanagermodule.hpp", + "server.cpp", + "server.hpp", + "session_id.hpp", + "sequence.cpp", + "sequence.hpp", + "sequence_manager.cpp", + "sequence_manager.hpp", + "sequence_processing_spec.hpp", + "shape.cpp", + "shape.hpp", + "statefulmodelinstance.cpp", + "statefulmodelinstance.hpp", + "status.cpp", + "status.hpp", + "stringutils.cpp", + "stringutils.hpp", + "queue.hpp", + "tensorinfo.cpp", + "tensorinfo.hpp", + "tfs_frontend/tfs_utils.cpp", + "tfs_frontend/tfs_utils.hpp", + "tensormap.hpp", + "tensor_utils.hpp", + "threadsafequeue.hpp", + "timer.hpp", + "version.hpp", + "logging.hpp", + "logging.cpp", + "binaryutils.hpp", + "binaryutils.cpp", + ], + deps = [ + "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", + "@tensorflow_serving//tensorflow_serving/apis:model_service_cc_proto", + "@minitrace//:trace", + "@com_github_grpc_grpc//:grpc++", + "@org_tensorflow//tensorflow/core:framework", + "@rapidjson//:rapidjson", + "@spdlog//:spdlog", + "@cxxopts//:cxxopts", + "@awssdk//:s3", + "@awssdk//:core", + "@awssdk//:deps", + "@azure//:storage", + "@cpprest//:sdk", + "@boost//:lib", + "@com_github_googleapis_google_cloud_cpp//google/cloud/storage:storage_client", + "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server", + "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server_api", + "@tensorflow_serving//tensorflow_serving/util:threadpool_executor", + "@tensorflow_serving//tensorflow_serving/util:json_tensor", + "@openvino//:openvino", + "@opencv//:opencv", + #":kfserving_api_cpp", + "//src/kfserving_api:kfserving_api_cpp", + ], + local_defines = [ + "SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE", + ], + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror", + "-Wno-sign-compare", + ], + visibility = [ + "//visibility:public", + ], +) + +cc_binary( + name = "libsampleloader.so", + srcs = [ + "example/SampleCustomLoader/sampleCustLoader.cpp", + "customloaderinterface.hpp", + ], + linkshared = 1, + deps = [ + "@rapidjson//:rapidjson", + ], +) + +cc_binary( + name = "lib_node_mock.so", + srcs = [ + "test/custom_nodes/node_mock.c", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "lib_node_missing_implementation.so", + srcs = [ + "test/custom_nodes/node_missing_implementation.c", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "lib_node_add_sub.so", + srcs = [ + "test/custom_nodes/node_add_sub.c", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "lib_node_dynamic_image.so", + srcs = [ + "test/custom_nodes/node_dynamic_image.c", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "lib_node_choose_maximum.so", + srcs = [ + "test/custom_nodes/node_choose_maximum.cpp", + "custom_node_interface.h", + ], + linkshared = 1, +) +cc_binary( + name = "lib_node_perform_different_operations.so", + srcs = [ + "test/custom_nodes/node_perform_different_operations.cpp", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "lib_node_dynamic_demultiplex.so", + srcs = [ + "test/custom_nodes/node_dynamic_demultiplex.cpp", + "custom_node_interface.h", + ], + linkshared = 1, +) + +cc_binary( + name = "libcustom_node_east_ocr.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/opencv_utils.hpp", + "custom_nodes/east_ocr/east_ocr.cpp", + "custom_nodes/east_ocr/nms.hpp", + "custom_node_interface.h", + ], + deps = [ + "@opencv//:opencv" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "libcustom_node_face_blur.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/opencv_utils.hpp", + "custom_nodes/face_blur/face_blur.cpp", + "custom_node_interface.h", + ], + deps = [ + "@opencv//:opencv" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "libcustom_node_add_one.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/buffersqueue.hpp", + "custom_nodes/common/buffersqueue.cpp", + "custom_nodes/common/custom_node_library_internal_manager.hpp", + "custom_nodes/common/custom_node_library_internal_manager.cpp", + "queue.hpp", + "custom_nodes/add_one/add_one.cpp", + "custom_node_interface.h", + "custom_nodes/add_one/add_one_internal_manager.hpp" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "libcustom_node_model_zoo_intel_object_detection.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/opencv_utils.hpp", + "custom_nodes/common/buffersqueue.hpp", + "custom_nodes/common/buffersqueue.cpp", + "custom_nodes/common/custom_node_library_internal_manager.hpp", + "custom_nodes/common/custom_node_library_internal_manager.cpp", + "queue.hpp", + "custom_nodes/model_zoo_intel_object_detection/model_zoo_intel_object_detection.cpp", + "custom_node_interface.h", + ], + deps = [ + "@opencv//:opencv" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "libcustom_node_image_transformation.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/opencv_utils.hpp", + "custom_nodes/image_transformation/image_transformation.cpp", + "custom_node_interface.h", + ], + deps = [ + "@opencv//:opencv" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "libcustom_node_horizontal_ocr.so", + srcs = [ + "custom_nodes/common/utils.hpp", + "custom_nodes/common/opencv_utils.hpp", + "custom_nodes/horizontal_ocr/horizontal_ocr.cpp", + "custom_node_interface.h", + ], + deps = [ + "@opencv//:opencv" + ], + linkshared = 1, + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror" + ] +) + +cc_binary( + name = "ovms", + srcs = [ + "main.cpp", + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + "-Wconversion", + "-Werror", + ], + deps = [ + "//src:ovms_lib", + # "//src:libovms_shared.so", + ], + linkstatic = False, +) + +cc_binary( + name = "libovms_shared.so", + srcs = [ + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + "-Wconversion", + "-Werror", + ], + deps = [ + "//src/ovms_lib:ovms_lib", + ], + linkstatic = True, + linkshared = True, +) + +cc_test( + name = "ovms_test", + linkstatic = 1, + srcs = [ + "test/azurefilesystem_test.cpp", + "test/binaryutils_test.cpp", + "test/custom_loader_test.cpp", + "test/custom_node_output_allocator_test.cpp", + "test/custom_node_buffersqueue_test.cpp", + "test/demultiplexer_node_test.cpp", + "test/deserialization_tests.cpp", + "test/ensemble_tests.cpp", + "test/ensemble_flow_custom_node_tests.cpp", + "test/ensemble_mapping_config_tests.cpp", + "test/ensemble_metadata_test.cpp", + "test/ensemble_config_change_stress.cpp", + "test/environment.hpp", + "test/gather_node_test.cpp", + "test/gcsfilesystem_test.cpp", + "test/get_model_metadata_response_test.cpp", + "test/get_pipeline_metadata_response_test.cpp", + "test/get_model_metadata_signature_test.cpp", + "test/get_model_metadata_validation_test.cpp", + "test/http_rest_api_handler_test.cpp", + "test/kfs_metadata_test.cpp", + "test/kfs_rest_test.cpp", + "test/layout_test.cpp", + "test/localfilesystem_test.cpp", + "test/mockmodelinstancechangingstates.hpp", + "test/model_cache_test.cpp", + "test/model_service_test.cpp", + "test/model_version_policy_test.cpp", + "test/model_test.cpp", + "test/modelinstance_test.cpp", + "test/modelconfig_test.cpp", + "test/node_library_manager_test.cpp", + "test/modelmanager_test.cpp", + "test/modelversionstatus_test.cpp", + "test/nodesessionmetadata_test.cpp", + "test/ovmsconfig_test.cpp", + "test/ovinferrequestqueue_test.cpp", + "test/ov_utils_test.cpp", + "test/pipelinedefinitionstatus_test.cpp", + "test/predict_validation_test.cpp", + "test/prediction_service_test.cpp", + "test/rest_parser_row_test.cpp", + "test/rest_parser_column_test.cpp", + "test/rest_parser_binary_inputs_test.cpp", + "test/rest_parser_nonamed_test.cpp", + "test/rest_utils_test.cpp", + "test/schema_test.cpp", + "test/sequence_test.cpp", + "test/serialization_tests.cpp", + "test/server_test.cpp", + "test/sequence_manager_test.cpp", + "test/shape_test.cpp", + "test/stateful_config_test.cpp", + "test/stateful_modelinstance_test.cpp", + "test/stateful_test_utils.hpp", + "test/status_test.cpp", + "test/stringutils_test.cpp", + "test/tensorinfo_test.cpp", + "test/tensorutils_test.cpp", + "test/test_utils.cpp", + "test/test_utils.hpp", + "test/threadsafequeue_test.cpp", + "test/unit_tests.cpp", + ], + data = [ + "test/dummy/1/dummy.xml", + "test/dummy/1/dummy.bin", + "test/dummy_fp64/1/saved_model.xml", + "test/dummy_fp64/1/saved_model.bin", + "test/add_two_inputs_model/1/add.xml", + "test/add_two_inputs_model/1/add.bin", + "test/increment_1x3x4x5/1/increment_1x3x4x5.xml", + "test/increment_1x3x4x5/1/increment_1x3x4x5.bin", + "test/summator/1/summator.xml", + "test/summator/1/summator.bin", + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + "-lssl", + ], + deps = [ + "//src:ovms_lib", + "//src:custom_nodes_common_lib", + "//src:libsampleloader.so", + "//src:lib_node_mock.so", + "//src:lib_node_missing_implementation.so", + "//src:lib_node_add_sub.so", + "//src:lib_node_dynamic_image.so", + "//src:lib_node_dynamic_demultiplex.so", + "//src:lib_node_perform_different_operations.so", + "//src:lib_node_choose_maximum.so", + "//src:libcustom_node_east_ocr.so", + "//src:libcustom_node_face_blur.so", + "//src:libcustom_node_model_zoo_intel_object_detection.so", + "//src:libcustom_node_image_transformation.so", + "//src:libcustom_node_add_one.so", + "//src:libcustom_node_horizontal_ocr.so", + "@com_google_googletest//:gtest", + ], + copts = [ + "-Wall", + "-Wno-unknown-pragmas", + "-Werror", + ], +) + +filegroup( + name = "static_analysis", + srcs = [ + "//src:ovms", + "//src:libcustom_node_add_one.so", + "//src:libcustom_node_east_ocr.so", + "//src:libcustom_node_face_blur.so", + "//src:libcustom_node_horizontal_ocr.so", + "//src:libcustom_node_model_zoo_intel_object_detection.so", + "//src:libcustom_node_image_transformation.so", + ] +) diff --git a/src/ovms_lib/aliases.hpp b/src/ovms_lib/aliases.hpp new file mode 100644 index 0000000000..b5a11ee16c --- /dev/null +++ b/src/ovms_lib/aliases.hpp @@ -0,0 +1,25 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +namespace ovms { + +using Aliases = std::vector>; +} // namespace ovms diff --git a/src/ovms_lib/azurefilesystem.cpp b/src/ovms_lib/azurefilesystem.cpp new file mode 100644 index 0000000000..6b92031f45 --- /dev/null +++ b/src/ovms_lib/azurefilesystem.cpp @@ -0,0 +1,320 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "azurefilesystem.hpp" + +#include +#include +#include + +#include "logging.hpp" +#include "stringutils.hpp" + +namespace ovms { + +const std::string AzureFileSystem::AZURE_URL_FILE_PREFIX = "azfs://"; +const std::string AzureFileSystem::AZURE_URL_BLOB_PREFIX = "az://"; + +as::cloud_storage_account createDefaultOrAnonymousAccount() { + try { + const char* env_cred = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); + + std::string credentials = std::string(_XPLATSTR("DefaultEndpointsProtocol = https;")); + + if (!env_cred) { + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Creating AzureFileSystem anonymous connection string."); + } else { + credentials = std::string(_XPLATSTR(env_cred)); + } + + as::cloud_storage_account storage_account = as::cloud_storage_account::parse(credentials); + if (!storage_account.is_initialized()) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account"); + throw std::runtime_error("Unable to create default azure storage account"); + } + + const char* use_http = std::getenv("AZURE_STORAGE_USE_HTTP_PROXY"); + + const char* proxy_env; + + std::string https_proxy = std::string(""); + if (!use_http) { + proxy_env = std::getenv("https_proxy"); + } else { + proxy_env = std::getenv("http_proxy"); + } + + if (!proxy_env) { + SPDLOG_LOGGER_DEBUG(azurestorage_logger, "No proxy detected."); + } else { + https_proxy = std::string(proxy_env); + web::web_proxy wproxy(https_proxy); + as::operation_context::set_default_proxy(wproxy); + + SPDLOG_LOGGER_DEBUG(azurestorage_logger, "Proxy detected: {}" + https_proxy); + } + + return storage_account; + } catch (const as::storage_exception& e) { + as::request_result result = e.result(); + as::storage_extended_error extended_error = result.extended_error(); + if (!extended_error.message().empty()) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", extended_error.message()); + } else { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); + } + throw e; + } catch (const std::exception& e) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); + throw e; + } +} + +AzureFileSystem::AzureFileSystem() : + account_{createDefaultOrAnonymousAccount()} { + SPDLOG_LOGGER_TRACE(azurestorage_logger, "AzureFileSystem default ctor"); +} + +AzureFileSystem::~AzureFileSystem() { SPDLOG_LOGGER_TRACE(azurestorage_logger, "AzureFileSystem dtor"); } + +StatusCode AzureFileSystem::fileExists(const std::string& path, bool* exists) { + *exists = false; + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->fileExists(exists); + + return status; +} + +StatusCode AzureFileSystem::isDirectory(const std::string& path, + bool* is_directory) { + *is_directory = false; + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->isDirectory(is_directory); + + return status; +} + +StatusCode AzureFileSystem::fileModificationTime(const std::string& path, + int64_t* mtime_ns) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->fileModificationTime(mtime_ns); + + return status; +} + +StatusCode +AzureFileSystem::getDirectoryContents(const std::string& path, + std::set* contents) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->getDirectoryContents(contents); + + return status; +} + +StatusCode AzureFileSystem::getDirectorySubdirs(const std::string& path, + std::set* subdirs) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->getDirectorySubdirs(subdirs); + + return status; +} + +StatusCode AzureFileSystem::getDirectoryFiles(const std::string& path, + std::set* files) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->getDirectoryFiles(files); + + return status; +} + +StatusCode AzureFileSystem::readTextFile(const std::string& path, + std::string* contents) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->readTextFile(contents); + + return status; +} + +StatusCode AzureFileSystem::downloadModelVersions(const std::string& path, + std::string* local_path, + const std::vector& versions) { + + auto sc = createTempPath(local_path); + if (sc != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to create a temporary path {}", sc); + return sc; + } + + for (auto& ver : versions) { + std::string versionpath = path; + if (!endsWith(versionpath, "/")) { + versionpath.append("/"); + } + versionpath.append(std::to_string(ver)); + std::string lpath = *local_path; + if (!endsWith(lpath, "/")) { + lpath.append("/"); + } + lpath.append(std::to_string(ver)); + fs::create_directory(lpath); + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(versionpath, account_); + auto status = azureStorageObj->checkPath(versionpath); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", versionpath, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->downloadFileFolderTo(lpath); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to download model version {}", versionpath); + return status; + } + } + + return StatusCode::OK; +} + +StatusCode AzureFileSystem::downloadFile(const std::string& remote_path, + const std::string& local_path) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(remote_path, account_); + auto status = azureStorageObj->checkPath(remote_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->downloadFile(local_path); + + return status; +} + +StatusCode AzureFileSystem::downloadFileFolder(const std::string& path, + const std::string& local_path) { + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->downloadFileFolder(local_path); + + return status; +} + +StatusCode AzureFileSystem::downloadFileFolderTo(const std::string& path, + const std::string& local_path) { + + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->downloadFileFolderTo(local_path); + + return status; +} + +StatusCode AzureFileSystem::deleteFileFolder(const std::string& path) { + auto factory = std::make_shared(); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto status = azureStorageObj->checkPath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + status = azureStorageObj->deleteFileFolder(); + + return status; +} + +} // namespace ovms diff --git a/src/ovms_lib/azurefilesystem.hpp b/src/ovms_lib/azurefilesystem.hpp new file mode 100644 index 0000000000..ed7dd1096e --- /dev/null +++ b/src/ovms_lib/azurefilesystem.hpp @@ -0,0 +1,166 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#define _TURN_OFF_PLATFORM_STRING + +#include +#include +#include + +#include "azurestorage.hpp" +#include "filesystem.hpp" +#include "status.hpp" + +namespace ovms { + +namespace as = azure::storage; + +class AzureFileSystem : public FileSystem { +public: + /** + * @brief Construct a new AzureFileSystem object + * + */ + AzureFileSystem(); + + /** + * @brief Destroy the AzureFileSystem object + * + */ + virtual ~AzureFileSystem(); + + /** + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ + StatusCode fileExists(const std::string& path, bool* exists) override; + + /** + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ + StatusCode isDirectory(const std::string& path, bool* is_dir) override; + + /** + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode getDirectoryContents(const std::string& path, + files_list_t* contents) override; + + /** + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ + StatusCode getDirectorySubdirs(const std::string& path, + files_list_t* subdirs) override; + + /** + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ + StatusCode getDirectoryFiles(const std::string& path, + files_list_t* files) override; + + /** + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode readTextFile(const std::string& path, + std::string* contents) override; + + /** + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ + StatusCode downloadFileFolder(const std::string& path, + const std::string& local_path) override; + + /** + * @brief Download selected model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ + StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; + + StatusCode fileModificationTime(const std::string& path, int64_t* mtime_ns); + + /** + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ + StatusCode deleteFileFolder(const std::string& path) override; + + static const std::string AZURE_URL_FILE_PREFIX; + + static const std::string AZURE_URL_BLOB_PREFIX; + +private: + /** + * + * @brief + * + * @param remote_path + * @param local_path + */ + StatusCode downloadFile(const std::string& remote_path, + const std::string& local_path); + + /** + * @brief Download a remote directory to a callee-provided local_path. + * + * @param path + * @param local_path + * @return StatusCode + */ + StatusCode downloadFileFolderTo(const std::string& path, + const std::string& local_path); + + /** + * @brief + * + */ + as::cloud_storage_account account_; +}; + +} // namespace ovms diff --git a/src/ovms_lib/azurestorage.cpp b/src/ovms_lib/azurestorage.cpp new file mode 100644 index 0000000000..5abd65ee80 --- /dev/null +++ b/src/ovms_lib/azurestorage.cpp @@ -0,0 +1,1217 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "azurestorage.hpp" + +#include + +#include "azurefilesystem.hpp" +#include "logging.hpp" + +namespace ovms { + +using namespace utility; + +const std::string UNAVAILABLE_PATH_ERROR = "Unable to access path: {}"; + +const std::string AzureStorageAdapter::extractAzureStorageExceptionMessage(const as::storage_exception& e) { + as::request_result result = e.result(); + as::storage_extended_error extended_error = result.extended_error(); + if (!extended_error.message().empty()) { + return extended_error.message(); + } else { + return e.what(); + } +} + +std::string AzureStorageAdapter::joinPath(std::initializer_list segments) { + std::string joined; + + for (const auto& seg : segments) { + if (joined.empty()) { + joined = seg; + } else if (isAbsolutePath(seg)) { + if (joined[joined.size() - 1] == '/') { + joined.append(seg.substr(1)); + } else { + joined.append(seg); + } + } else { + if (joined[joined.size() - 1] != '/') { + joined.append("/"); + } + joined.append(seg); + } + } + + return joined; +} + +StatusCode AzureStorageAdapter::CreateLocalDir(const std::string& path) { + int status = + mkdir(const_cast(path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); + if (status == -1) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to create local folder: {} {} ", path, + strerror(errno)); + return StatusCode::PATH_INVALID; + } + return StatusCode::OK; +} + +bool AzureStorageAdapter::isAbsolutePath(const std::string& path) { + return !path.empty() && (path[0] == '/'); +} + +AzureStorageBlob::AzureStorageBlob(const std::string& path, as::cloud_storage_account account) { + account_ = account; + as_blob_client_ = account_.create_cloud_blob_client(); + isPathValidationOk_ = false; +} + +StatusCode AzureStorageBlob::checkPath(const std::string& path) { + try { + if (FileSystem::isPathEscaped(path)) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + + auto status = this->parseFilePath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to parse path: {} -> {}", fullPath_, + ovms::Status(status).string()); + return status; + } + + as_container_ = as_blob_client_.get_container_reference(container_); + + if (!as_container_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Container does not exist: {} -> {}", fullPath_, container_); + return StatusCode::AS_CONTAINER_NOT_FOUND; + } + + isPathValidationOk_ = true; + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::fileExists(bool* exists) { + try { + *exists = false; + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_blob_ = as_container_.get_blob_reference(blockpath_); + if (!as_blob_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + *exists = true; + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::isDirectory(bool* is_directory) { + try { + *is_directory = false; + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_blob_directory temp_directory = as_container_.get_directory_reference(blockpath_); + as::cloud_blob_directory parent_directory = temp_directory.get_parent_reference(); + + // List blobs in the blob container + as::continuation_token token; + do { + as::list_blob_item_segment result; + // Check if we are at container root + if (parent_directory.prefix() == "") + result = as_container_.list_blobs_segmented(token); + else + result = parent_directory.list_blobs_segmented(token); + + for (auto& item : result.results()) { + if (!item.is_blob()) { + std::string prefix = item.as_directory().prefix(); + if (prefix.back() == '/') + prefix.pop_back(); + if (prefix == blockpath_) { + *is_directory = true; + } + } + } + + token = result.continuation_token(); + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::fileModificationTime(int64_t* mtime_ns) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_blob_ = as_container_.get_blob_reference(blockpath_); + if (!as_blob_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as::cloud_blob_properties& prop = as_blob_.properties(); + utility::datetime time = prop.last_modified(); + std::string date = time.to_string(); + + auto nanoseconds = time.to_interval(); + + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); + *mtime_ns = nanoseconds; + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::getDirectoryContents(files_list_t* contents) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); + + // List blobs in the blob container + as::continuation_token token; + do { + as::list_blob_item_segment result; + // Check if we are at container root + if (parent_directory.prefix() == "") + result = as_container_.list_blobs_segmented(token); + else + result = parent_directory.list_blobs_segmented(token); + + for (auto& item : result.results()) { + if (item.is_blob()) { + contents->insert(getLastPathPart(item.as_blob().name())); + } else { + contents->insert(getLastPathPart(item.as_directory().prefix())); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::getDirectorySubdirs(files_list_t* subdirs) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); + + // List blobs in the blob container + as::continuation_token token; + do { + as::list_blob_item_segment result; + // Check if we are at container root + if (parent_directory.prefix() == "") + result = as_container_.list_blobs_segmented(token); + else + result = parent_directory.list_blobs_segmented(token); + + for (auto& item : result.results()) { + if (!item.is_blob()) { + subdirs->insert(getLastPathPart(item.as_directory().prefix())); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::getDirectoryFiles(files_list_t* files) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); + + // List blobs in the blob container + as::continuation_token token; + do { + as::list_blob_item_segment result; + // Check if we are at container root + if (parent_directory.prefix() == "") + result = as_container_.list_blobs_segmented(token); + else + result = parent_directory.list_blobs_segmented(token); + + for (auto& item : result.results()) { + if (item.is_blob()) { + files->insert(getLastPathPart(item.as_blob().name())); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::readTextFile(std::string* contents) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_blob_ = as_container_.get_blob_reference(blockpath_); + if (!as_blob_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Nlock blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_block_blob_ = as_container_.get_block_blob_reference(blockpath_); + + concurrency::streams::container_buffer> buffer; + concurrency::streams::ostream output_stream(buffer); + as_block_blob_.download_to_stream(output_stream); + *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::downloadFileFolder(const std::string& local_path) { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + SPDLOG_LOGGER_DEBUG(azurestorage_logger, + "Downloading dir {} (recursive) and saving a new local path: {}", + fullUri_, local_path); + return downloadFileFolderTo(local_path); +} + +StatusCode AzureStorageBlob::deleteFileFolder() { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_blob_ = as_container_.get_blob_reference(blockpath_); + if (!as_blob_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_blob_.delete_blob(); + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +std::string AzureStorageBlob::getLastPathPart(const std::string& path) { + std::string proper_path = path; + if (path.back() == '/') + proper_path = path.substr(0, path.size() - 1); + + int part_start = proper_path.find_last_of("/"); + int part_end = proper_path.length(); + + return proper_path.substr(part_start + 1, part_end - part_start - 1); +} + +StatusCode AzureStorageBlob::downloadFile(const std::string& local_path) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_blob_ = as_container_.get_blob_reference(blockpath_); + if (!as_blob_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_blob_.download_to_file(local_path); + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Downloading dir {} and saving to {}", fullPath_, local_path); + bool is_dir; + auto status = this->isDirectory(&is_dir); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File/folder does not exist at {}", fullPath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + if (!is_dir) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Path is not a directory: {}", fullPath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + std::set dirs; + status = getDirectorySubdirs(&dirs); + if (status != StatusCode::OK) { + return status; + } + + std::set files; + status = getDirectoryFiles(&files); + if (status != StatusCode::OK) { + return status; + } + + for (auto&& d : dirs) { + std::string remote_dir_path = joinPath({fullUri_, d}); + std::string local_dir_path = joinPath({local_path, d}); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, + local_dir_path); + + auto factory = std::make_shared(); + auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); + status = azureSubdirStorageObj->checkPath(remote_dir_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, + ovms::Status(status).string()); + return status; + } + + auto mkdir_status = CreateLocalDir(local_dir_path); + if (mkdir_status != StatusCode::OK) { + return status; + } + auto download_dir_status = + azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); + if (download_dir_status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", + remote_dir_path, local_dir_path); + return download_dir_status; + } + } + + for (auto&& f : files) { + std::string remote_file_path = joinPath({fullUri_, f}); + std::string local_file_path = joinPath({local_path, f}); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing file {} from {} -> {}", f, remote_file_path, + local_file_path); + + auto factory = std::make_shared(); + auto azureFiledirStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); + status = azureFiledirStorageObj->checkPath(remote_file_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", + remote_file_path, local_file_path); + return status; + } + + auto download_status = + azureFiledirStorageObj->downloadFile(local_file_path); + if (download_status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, + local_file_path); + return download_status; + } + } + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +std::string AzureStorageBlob::getNameFromPath(std::string& path) { + int name_start = path.find_last_of("/"); + int name_end = path.length(); + + if (name_start == name_end) + path = path.substr(0, path.size() - 1); + + name_start = path.find_last_of("/"); + name_end = path.length(); + + return path.substr(name_start, name_end - name_start); +} + +StatusCode AzureStorageBlob::parseFilePath(const std::string& path) { + // az://share/blockpath/file + // az://share/blockpath + // az://share/ + if (path.back() == '/') { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Path can not end with '/'", path); + return StatusCode::AS_INVALID_PATH; + } + + fullUri_ = path; + int share_start = 0; + // Blob path + if (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos) { + share_start = path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) + AzureFileSystem::AZURE_URL_BLOB_PREFIX.size(); + } else if (path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) != std::string::npos) { + // File path + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Wrong object type - az:// prefix in path required, azure:// found:", path); + return StatusCode::AS_INVALID_PATH; + } else { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Missing az:// prefix in path:", path); + return StatusCode::AS_INVALID_PATH; + } + + int share_end = path.find_first_of("/", share_start); + int file_end = path.length(); + + fullPath_ = path.substr(share_end + 1, file_end - share_end - 1); + + subdirs_ = FindSubdirectories(fullPath_); + + std::set subdirs; + + if (share_end > share_start) { + container_ = path.substr(share_start, share_end - share_start); + + blockpath_ = path.substr(share_end + 1, file_end - share_end - 1); + + } else { + // No directory and no file + container_ = path.substr(share_start); + blockpath_ = ""; + } + + // No container + if (container_.empty()) { + return StatusCode::AS_INVALID_PATH; + } + + return StatusCode::OK; +} + +AzureStorageFile::AzureStorageFile(const std::string& path, as::cloud_storage_account account) { + account_ = account; + as_file_client_ = account_.create_cloud_file_client(); + isPathValidationOk_ = false; +} + +StatusCode AzureStorageFile::checkPath(const std::string& path) { + try { + if (FileSystem::isPathEscaped(path)) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + + auto status = this->parseFilePath(path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to parse path: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + as_file_client_ = account_.create_cloud_file_client(); + as_share_ = as_file_client_.get_share_reference(share_); + + if (!as_share_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Share does not exist: {} -> {}", path, share_); + return StatusCode::AS_SHARE_NOT_FOUND; + } + + if (directory_.empty()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory required in path: {} -> {}", path, directory_); + return StatusCode::AS_INVALID_PATH; + } + + isPathValidationOk_ = true; + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::fileExists(bool* exists) { + try { + *exists = false; + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + as_file1_ = as_last_working_subdir.get_file_reference(file_); + if (!as_file1_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + *exists = true; + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::isDirectory(bool* is_directory) { + try { + *is_directory = false; + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + return StatusCode::OK; + } + } + + *is_directory = true; + return StatusCode::OK; + } catch (const as::storage_exception& e) { + } + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::fileModificationTime(int64_t* mtime_ns) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as_directory_ = as_share_.get_directory_reference(_XPLATSTR(directory_)); + if (!as_directory_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory does not exist: {} -> {}", fullPath_, directory_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_file1_ = as_directory_.get_file_reference(_XPLATSTR(file_)); + if (!as_file1_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as::cloud_file_properties& prop = as_file1_.properties(); + utility::datetime time = prop.last_modified(); + std::string date = time.to_string(); + + auto nanoseconds = time.to_interval(); + + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); + *mtime_ns = nanoseconds; + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::getDirectoryContents(files_list_t* contents) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + // List files and directories in the directory + as::continuation_token token; + do { + as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); + for (auto& item : result.results()) { + if (item.is_file()) { + contents->insert(item.as_file().name()); + } + if (item.is_directory()) { + contents->insert(item.as_directory().name()); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::getDirectorySubdirs(files_list_t* subdirs) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + // List files and directories in the directory + as::continuation_token token; + do { + as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); + for (auto& item : result.results()) { + if (item.is_directory()) { + subdirs->insert(item.as_directory().name()); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::getDirectoryFiles(files_list_t* files) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + // List files and directories in the directory + as::continuation_token token; + do { + as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); + for (auto& item : result.results()) { + if (item.is_file()) { + files->insert(item.as_file().name()); + } + } + } while (!token.empty()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::readTextFile(std::string* contents) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); + if (!as_file1_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + concurrency::streams::container_buffer> buffer; + concurrency::streams::ostream output_stream(buffer); + as_file1_.download_to_stream(output_stream); + *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); + + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::downloadFileFolder(const std::string& local_path) { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + SPDLOG_LOGGER_DEBUG(azurestorage_logger, + "Downloading dir {} (recursive) and saving a new local path: {}", + fullPath_, local_path); + return downloadFileFolderTo(local_path); +} + +StatusCode AzureStorageFile::deleteFileFolder() { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); + if (!as_file1_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_file1_.delete_file(); + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::downloadFile(const std::string& local_path) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + as::cloud_file_directory as_last_working_subdir; + std::string tmp_dir = ""; + + try { + for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { + tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; + as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); + if (!as_tmp_subdir.exists()) { + break; + } + + as_last_working_subdir = as_tmp_subdir; + } + } catch (const as::storage_exception& e) { + } + + as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); + if (!as_file1_.exists()) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + as_file1_.download_to_file(local_path); + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) { + try { + if (!isPathValidationOk_) { + auto status = checkPath(fullUri_); + if (status != StatusCode::OK) + return status; + } + + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Downloading dir {} and saving to {}", fullPath_, local_path); + bool is_dir; + auto status = this->isDirectory(&is_dir); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Folder does not exist at {}", fullPath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + if (!is_dir) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Path is not a directory: {}", fullPath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + + std::set dirs; + status = getDirectorySubdirs(&dirs); + if (status != StatusCode::OK) { + return status; + } + + std::set files; + status = getDirectoryFiles(&files); + if (status != StatusCode::OK) { + return status; + } + + for (auto&& d : dirs) { + std::string remote_dir_path = joinPath({fullUri_, d}); + std::string local_dir_path = joinPath({local_path, d}); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, + local_dir_path); + + auto factory = std::make_shared(); + auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); + auto status = azureSubdirStorageObj->checkPath(remote_dir_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, + ovms::Status(status).string()); + return status; + } + + auto mkdir_status = CreateLocalDir(local_dir_path); + if (mkdir_status != StatusCode::OK) { + return status; + } + auto download_dir_status = + azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); + if (download_dir_status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", + remote_dir_path, local_dir_path); + return download_dir_status; + } + } + + for (auto&& f : files) { + std::string remote_file_path = joinPath({fullUri_, f}); + std::string local_file_path = joinPath({local_path, f}); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing file {} from {} -> {}", f, remote_file_path, + local_file_path); + + auto factory = std::make_shared(); + auto azureFileStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); + auto status = azureFileStorageObj->checkPath(remote_file_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_file_path, + ovms::Status(status).string()); + return status; + } + + auto download_status = + azureFileStorageObj->downloadFile(local_file_path); + if (download_status != StatusCode::OK) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, + local_file_path); + return download_status; + } + } + return StatusCode::OK; + } catch (const as::storage_exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); + } + + return StatusCode::AS_FILE_NOT_FOUND; +} + +std::vector AzureStorageAdapter::FindSubdirectories(std::string path) { + std::vector output; + + std::string::size_type prev_pos = 0, pos = 0; + + while ((pos = path.find('/', pos)) != std::string::npos) { + std::string substring(path.substr(prev_pos, pos - prev_pos)); + + output.push_back(substring); + + prev_pos = ++pos; + } + + output.push_back(path.substr(prev_pos, pos - prev_pos)); + + return output; +} + +StatusCode AzureStorageFile::parseFilePath(const std::string& path) { + // azure://share/directory/file + // azure://share/directory + // azure://share/ + if (path.back() == '/') { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Path can not end with '/'", path); + return StatusCode::AS_INVALID_PATH; + } + + fullUri_ = path; + int share_start = 0; + // File or directory path + if (path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) != std::string::npos) { + share_start = path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) + AzureFileSystem::AZURE_URL_FILE_PREFIX.size(); + } else if (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos) { + // Blob path + SPDLOG_LOGGER_ERROR(azurestorage_logger, "Wrong object type. azfs:// prefix in path required, found az://:", path); + return StatusCode::AS_INVALID_PATH; + } else { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Missing azfs:// prefix in path:", path); + return StatusCode::AS_INVALID_PATH; + } + + int share_end = path.find_first_of("/", share_start); + int file_start = path.find_last_of("/"); + int file_end = path.length(); + + fullPath_ = path.substr(share_end + 1, file_end - share_end - 1); + + subdirs_ = FindSubdirectories(fullPath_); + + if (share_end > share_start) { + share_ = path.substr(share_start, share_end - share_start); + + directory_ = path.substr(share_end + 1, file_start - share_end - 1); + + // No file or no directory + if (share_end == file_start) { + file_ = ""; + } else { + // No file + if (file_start == file_end) { + file_ = ""; + } else { + file_ = path.substr(file_start + 1, file_end - file_start); + } + } + } else { + // No directory and no file + share_ = path.substr(share_start); + directory_ = ""; + file_ = ""; + } + + // No share + if (share_.empty()) { + return StatusCode::AS_INVALID_PATH; + } + + return StatusCode::OK; +} + +std::shared_ptr AzureStorageFactory::getNewAzureStorageObject(const std::string& path, as::cloud_storage_account account) { + if (isBlobStoragePath(path)) + return std::make_shared(path, account); + + return std::make_shared(path, account); +} + +bool AzureStorageFactory::isBlobStoragePath(std::string path) { + return (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos); +} + +} // namespace ovms diff --git a/src/ovms_lib/azurestorage.hpp b/src/ovms_lib/azurestorage.hpp new file mode 100644 index 0000000000..3a8b6bd79b --- /dev/null +++ b/src/ovms_lib/azurestorage.hpp @@ -0,0 +1,195 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "status.hpp" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#pragma GCC diagnostic ignored "-Wreorder" +#pragma GCC diagnostic ignored "-Wunused-value" +#include + +#include "was/blob.h" +#include "was/common.h" +#include "was/storage_account.h" +#pragma GCC diagnostic pop + +namespace ovms { + +namespace as = azure::storage; +using files_list_t = std::set; + +class AzureStorageAdapter { +public: + AzureStorageAdapter() {} + + virtual StatusCode fileExists(bool* exists) = 0; + virtual StatusCode isDirectory(bool* is_directory) = 0; + virtual StatusCode fileModificationTime(int64_t* mtime_ns) = 0; + virtual StatusCode getDirectoryContents(files_list_t* contents) = 0; + virtual StatusCode getDirectorySubdirs(files_list_t* subdirs) = 0; + virtual StatusCode getDirectoryFiles(files_list_t* files) = 0; + virtual StatusCode readTextFile(std::string* contents) = 0; + virtual StatusCode downloadFileFolder(const std::string& local_path) = 0; + virtual StatusCode deleteFileFolder() = 0; + virtual StatusCode downloadFile(const std::string& local_path) = 0; + virtual StatusCode downloadFileFolderTo(const std::string& local_path) = 0; + virtual StatusCode checkPath(const std::string& path) = 0; + + std::string joinPath(std::initializer_list segments); + StatusCode CreateLocalDir(const std::string& path); + bool isAbsolutePath(const std::string& path); + std::vector FindSubdirectories(std::string path); + +protected: + const std::string extractAzureStorageExceptionMessage(const as::storage_exception& e); + +private: + virtual StatusCode parseFilePath(const std::string& path) = 0; +}; + +class AzureStorageBlob : public AzureStorageAdapter { +public: + AzureStorageBlob(const std::string& path, as::cloud_storage_account account); + + StatusCode checkPath(const std::string& path) override; + + StatusCode fileExists(bool* exists) override; + + StatusCode isDirectory(bool* is_directory) override; + + StatusCode fileModificationTime(int64_t* mtime_ns) override; + + StatusCode getDirectoryContents(files_list_t* contents) override; + + StatusCode getDirectorySubdirs(files_list_t* subdirs) override; + + StatusCode getDirectoryFiles(files_list_t* files) override; + + StatusCode readTextFile(std::string* contents) override; + + StatusCode downloadFileFolder(const std::string& local_path) override; + + StatusCode deleteFileFolder() override; + + StatusCode downloadFile(const std::string& local_path) override; + + StatusCode downloadFileFolderTo(const std::string& local_path) override; + +private: + std::string getLastPathPart(const std::string& path); + + StatusCode parseFilePath(const std::string& path) override; + + std::string getNameFromPath(std::string& path); + + bool isPathValidationOk_; + + std::string fullPath_; + + std::string fullUri_; + + std::string blockpath_; + + std::vector subdirs_; + + std::string container_; + + as::cloud_blob_container as_container_; + + as::cloud_block_blob as_block_blob_; + + as::cloud_blob as_blob_; + + as::cloud_storage_account account_; + + as::cloud_blob_client as_blob_client_; +}; + +class AzureStorageFile : public AzureStorageAdapter { +public: + AzureStorageFile(const std::string& path, as::cloud_storage_account account); + + StatusCode checkPath(const std::string& path) override; + + StatusCode fileExists(bool* exists) override; + + StatusCode isDirectory(bool* is_directory) override; + + StatusCode fileModificationTime(int64_t* mtime_ns) override; + + StatusCode getDirectoryContents(files_list_t* contents) override; + + StatusCode getDirectorySubdirs(files_list_t* subdirs) override; + + StatusCode getDirectoryFiles(files_list_t* files) override; + + StatusCode readTextFile(std::string* contents) override; + + StatusCode downloadFileFolder(const std::string& local_path) override; + + StatusCode deleteFileFolder() override; + + StatusCode downloadFile(const std::string& local_path) override; + + StatusCode downloadFileFolderTo(const std::string& local_path) override; + +private: + StatusCode parseFilePath(const std::string& path) override; + + bool isPathValidationOk_; + + std::string fullPath_; + + std::string fullUri_; + + std::string file_; + + std::string directory_; + + std::vector subdirs_; + + std::string share_; + + as::cloud_storage_account account_; + + as::cloud_file_client as_file_client_; + + as::cloud_file_share as_share_; + + as::cloud_file_directory as_directory_; + + as::cloud_file as_file1_; +}; + +class AzureStorageFactory { +public: + std::shared_ptr getNewAzureStorageObject(const std::string& path, as::cloud_storage_account account); + +private: + bool isBlobStoragePath(std::string path); +}; + +} // namespace ovms diff --git a/src/ovms_lib/binaryutils.cpp b/src/ovms_lib/binaryutils.cpp new file mode 100644 index 0000000000..97130add96 --- /dev/null +++ b/src/ovms_lib/binaryutils.cpp @@ -0,0 +1,447 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "binaryutils.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include +#include +#include +#include + +#include + +#include "logging.hpp" +#include "opencv2/opencv.hpp" +#include "status.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "kfs_grpc_inference_service.hpp" +#pragma GCC diagnostic pop + +namespace ovms { + +int getMatTypeFromTensorPrecision(ovms::Precision tensorPrecision) { + switch (tensorPrecision) { + case ovms::Precision::FP32: + return CV_32F; + case ovms::Precision::FP64: + return CV_64F; + case ovms::Precision::FP16: + return CV_16F; + case ovms::Precision::I16: + return CV_16S; + case ovms::Precision::U8: + return CV_8U; + case ovms::Precision::I8: + return CV_8S; + case ovms::Precision::U16: + return CV_16U; + case ovms::Precision::I32: + return CV_32S; + default: + return -1; + } +} + +bool isPrecisionEqual(int matPrecision, ovms::Precision tensorPrecision) { + int convertedTensorPrecision = getMatTypeFromTensorPrecision(tensorPrecision); + if (convertedTensorPrecision == matPrecision) { + return true; + } + return false; +} + +cv::Mat convertStringToMat(const std::string& image) { + std::vector data(image.begin(), image.end()); + cv::Mat dataMat(data, true); + + try { + return cv::imdecode(dataMat, cv::IMREAD_UNCHANGED); + } catch (const cv::Exception& e) { + SPDLOG_DEBUG("Error during string_val to mat conversion: {}", e.what()); + return cv::Mat{}; + } +} + +Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision requestedPrecision) { + int type = getMatTypeFromTensorPrecision(requestedPrecision); + if (type == -1) { + SPDLOG_DEBUG("Error during binary input conversion: not supported precision: {}", toString(requestedPrecision)); + return StatusCode::INVALID_PRECISION; + } + + src.convertTo(dst, type); + return StatusCode::OK; +} + +Status validateLayout(const std::shared_ptr& tensorInfo) { + static const std::string binarySupportedLayout = "N...HWC"; + if (!tensorInfo->getLayout().createIntersection(Layout(binarySupportedLayout), tensorInfo->getShape().size()).has_value()) { + SPDLOG_DEBUG("Endpoint needs to be compatible with {} to support binary image inputs, actual: {}", + binarySupportedLayout, + tensorInfo->getLayout()); + return StatusCode::UNSUPPORTED_LAYOUT; + } + return StatusCode::OK; +} + +bool resizeNeeded(const cv::Mat& image, const dimension_value_t height, const dimension_value_t width) { + if (height != image.rows || width != image.cols) { + return true; + } + return false; +} + +Status resizeMat(const cv::Mat& src, cv::Mat& dst, const dimension_value_t height, const dimension_value_t width) { + cv::resize(src, dst, cv::Size(width, height)); + return StatusCode::OK; +} + +Status validateNumberOfChannels(const std::shared_ptr& tensorInfo, + const cv::Mat input, + cv::Mat* firstBatchImage) { + + // At this point we can either have nhwc format or pretendant to be nhwc but with ANY layout in pipeline info + Dimension numberOfChannels; + if (tensorInfo->getShape().size() == 4) { + numberOfChannels = tensorInfo->getShape()[3]; + } else if (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5) { + numberOfChannels = tensorInfo->getShape()[4]; + } else { + return StatusCode::INVALID_NO_OF_CHANNELS; + } + if (numberOfChannels.isAny() && firstBatchImage) { + numberOfChannels = firstBatchImage->channels(); + } + if (numberOfChannels.isAny()) { + return StatusCode::OK; + } + if (!numberOfChannels.match(input.channels())) { + SPDLOG_DEBUG("Binary data sent to input: {} has invalid number of channels. Expected: {} Actual: {}", + tensorInfo->getMappedName(), + numberOfChannels.toString(), + input.channels()); + return StatusCode::INVALID_NO_OF_CHANNELS; + } + + return StatusCode::OK; +} + +Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* firstBatchImage) { + if (input.cols == firstBatchImage->cols && input.rows == firstBatchImage->rows) { + return StatusCode::OK; + } + SPDLOG_DEBUG("Each binary image in request needs to have resolution matched. First cols: {}, rows: {}, current cols: {}, rows: {}", + firstBatchImage->cols, firstBatchImage->rows, input.cols, input.rows); + return StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH; +} + +bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, + const int batchSize) { + if (!tensorInfo->getBatchSize().has_value()) { + return true; + } + return !tensorInfo->getBatchSize().value().match(batchSize); +} + +Status validateInput(const std::shared_ptr& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage, bool enforceResolutionAlignment) { + // Binary inputs are supported for any endpoint that is compatible with N...HWC layout. + // With unknown layout, there is no way to deduce expected endpoint input resolution. + // This forces binary utility to create tensors with resolution inherited from first batch of binary input image (request). + // In case of any dimension in endpoint shape is dynamic, we need to validate images against first image resolution. + // Otherwise we can omit that, and proceed to image resize. + if (firstBatchImage && enforceResolutionAlignment) { + auto status = validateResolutionAgainstFirstBatchImage(input, firstBatchImage); + if (!status.ok()) { + return status; + } + } + return validateNumberOfChannels(tensorInfo, input, firstBatchImage); +} + +Status validateTensor(const std::shared_ptr& tensorInfo, + const tensorflow::TensorProto& src) { + auto status = validateLayout(tensorInfo); + if (!status.ok()) { + return status; + } + // 4 for default pipelines, 5 for pipelines with demultiplication at entry + bool isShapeLengthValid = tensorInfo->getShape().size() == 4 || + (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5); + if (!isShapeLengthValid) { + return StatusCode::INVALID_SHAPE; + } + + if (checkBatchSizeMismatch(tensorInfo, src.string_val_size())) { + SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", + tensorInfo->getMappedName(), + tensorInfo->getBatchSize().has_value() ? tensorInfo->getBatchSize().value().toString() : std::string{"none"}, + src.string_val_size()); + return StatusCode::INVALID_BATCH_SIZE; + } + + for (size_t i = 0; i < src.string_val_size(); i++) { + if (src.string_val(i).size() <= 0) { + return StatusCode::STRING_VAL_EMPTY; + } + } + + return StatusCode::OK; +} + +Status validateTensor(const std::shared_ptr& tensorInfo, + const ::inference::ModelInferRequest::InferInputTensor& src) { + auto status = validateLayout(tensorInfo); + if (!status.ok()) { + return status; + } + // 4 for default pipelines, 5 for pipelines with demultiplication at entry + bool isShapeLengthValid = tensorInfo->getShape().size() == 4 || + (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5); + if (!isShapeLengthValid) { + return StatusCode::INVALID_SHAPE; + } + + if (checkBatchSizeMismatch(tensorInfo, src.contents().bytes_contents_size())) { + SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", + tensorInfo->getMappedName(), + tensorInfo->getBatchSize().has_value() ? tensorInfo->getBatchSize().value().toString() : std::string{"none"}, + src.contents().bytes_contents_size()); + return StatusCode::INVALID_BATCH_SIZE; + } + + for (size_t i = 0; i < src.contents().bytes_contents_size(); i++) { + if (src.contents().bytes_contents(i).size() <= 0) { + return StatusCode::BYTES_CONTENTS_EMPTY; + } + } + + if (src.contents().bytes_contents_size() <= 0) { + return StatusCode::BYTES_CONTENTS_EMPTY; + } + + return StatusCode::OK; +} + +Dimension getTensorInfoHeightDim(const std::shared_ptr& tensorInfo) { + size_t numberOfShapeDimensions = tensorInfo->getShape().size(); + if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { + throw std::logic_error("wrong number of shape dimensions"); + } + size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 1 : /*N?HWC*/ 2; + return tensorInfo->getShape()[position]; +} + +Dimension getTensorInfoWidthDim(const std::shared_ptr& tensorInfo) { + size_t numberOfShapeDimensions = tensorInfo->getShape().size(); + if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { + throw std::logic_error("wrong number of shape dimensions"); + } + size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 2 : /*N?HWC*/ 3; + return tensorInfo->getShape()[position]; +} + +void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& image) { + if (height.isAny()) { + height = image.rows; + } else if (height.isDynamic()) { + if (height.match(image.rows)) { + height = image.rows; + } else { + if (image.rows > height.getMaxValue()) { + height = height.getMaxValue(); + } else { + height = height.getMinValue(); + } + } + } + if (width.isAny()) { + width = image.cols; + } else if (width.isDynamic()) { + if (width.match(image.cols)) { + width = image.cols; + } else { + if (image.cols > width.getMaxValue()) { + width = width.getMaxValue(); + } else { + width = width.getMinValue(); + } + } + } +} + +bool isResizeSupported(const std::shared_ptr& tensorInfo) { + for (const auto& dim : tensorInfo->getShape()) { + if (dim.isAny()) { + return false; + } + } + if (tensorInfo->getLayout() != "NHWC" && + tensorInfo->getLayout() != "N?HWC" && + tensorInfo->getLayout() != Layout::getUnspecifiedLayout()) { + return false; + } + return true; +} + +const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i) { + return tensor.string_val(i); +} + +const std::string& getBinaryInput(const ::inference::ModelInferRequest::InferInputTensor& tensor, size_t i) { + return tensor.contents().bytes_contents(i); +} + +size_t getBinaryInputsSize(const tensorflow::TensorProto& tensor) { + return tensor.string_val_size(); +} + +size_t getBinaryInputsSize(const ::inference::ModelInferRequest::InferInputTensor& tensor) { + return tensor.contents().bytes_contents_size(); +} + +template +Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo) { + Dimension targetHeight = getTensorInfoHeightDim(tensorInfo); + Dimension targetWidth = getTensorInfoWidthDim(tensorInfo); + + // Enforce resolution alignment against first image in the batch if resize is not supported. + bool resizeSupported = isResizeSupported(tensorInfo); + bool enforceResolutionAlignment = !resizeSupported; + + for (int i = 0; i < getBinaryInputsSize(src); i++) { + cv::Mat image = convertStringToMat(getBinaryInput(src, i)); + if (image.data == nullptr) + return StatusCode::IMAGE_PARSING_FAILED; + + cv::Mat* firstImage = images.size() == 0 ? nullptr : &images.at(0); + auto status = validateInput(tensorInfo, image, firstImage, enforceResolutionAlignment); + if (status != StatusCode::OK) { + return status; + } + + if (i == 0) { + updateTargetResolution(targetHeight, targetWidth, image); + } + + if (!isPrecisionEqual(image.depth(), tensorInfo->getPrecision())) { + cv::Mat imageCorrectPrecision; + status = convertPrecision(image, imageCorrectPrecision, tensorInfo->getPrecision()); + + if (status != StatusCode::OK) { + return status; + } + image = std::move(imageCorrectPrecision); + } + if (!targetHeight.isStatic() || !targetWidth.isStatic()) { + return StatusCode::INTERNAL_ERROR; + } + if (resizeNeeded(image, targetHeight.getStaticValue(), targetWidth.getStaticValue())) { + if (!resizeSupported) { + return StatusCode::INVALID_SHAPE; + } + cv::Mat imageResized; + status = resizeMat(image, imageResized, targetHeight.getStaticValue(), targetWidth.getStaticValue()); + if (!status.ok()) { + return status; + } + image = std::move(imageResized); + } + + // if (i == 0 && src.contents().bytes_contents_size() > 1) { + // // Multiply src.string_val_size() * image resolution * precision size + // } + + images.push_back(image); + } + + return StatusCode::OK; +} +shape_t getShapeFromImages(const std::vector& images, const std::shared_ptr& tensorInfo) { + shape_t dims; + dims.push_back(images.size()); + if (tensorInfo->isInfluencedByDemultiplexer()) { + dims.push_back(1); + } + dims.push_back(images[0].rows); + dims.push_back(images[0].cols); + dims.push_back(images[0].channels()); + return dims; +} + +ov::Tensor createTensorFromMats(const std::vector& images, const std::shared_ptr& tensorInfo) { + ov::Shape shape = getShapeFromImages(images, tensorInfo); + ov::element::Type precision = tensorInfo->getOvPrecision(); + ov::Tensor tensor(precision, shape); + char* ptr = (char*)tensor.data(); + for (cv::Mat image : images) { + memcpy(ptr, (char*)image.data, image.total() * image.elemSize()); + ptr += (image.total() * image.elemSize()); + } + return tensor; +} + +ov::Tensor convertMatsToTensor(std::vector& images, const std::shared_ptr& tensorInfo) { + switch (tensorInfo->getPrecision()) { + case ovms::Precision::FP32: + case ovms::Precision::I32: + case ovms::Precision::FP64: + case ovms::Precision::I8: + case ovms::Precision::U8: + case ovms::Precision::FP16: + case ovms::Precision::U16: + case ovms::Precision::I16: + return createTensorFromMats(images, tensorInfo); + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + case ovms::Precision::BOOL: + case ovms::Precision::CUSTOM: + default: + return ov::Tensor(); + } +} + +template +Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo) { + auto status = validateTensor(tensorInfo, src); + if (status != StatusCode::OK) { + return status; + } + + std::vector images; + + status = convertTensorToMatsMatchingTensorInfo(src, images, tensorInfo); + if (!status.ok()) { + return status; + } + + tensor = convertMatsToTensor(images, tensorInfo); + if (!tensor) { + return StatusCode::IMAGE_PARSING_FAILED; + } + return StatusCode::OK; +} + +template Status convertBinaryRequestTensorToOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +template Status convertBinaryRequestTensorToOVTensor<::inference::ModelInferRequest::InferInputTensor>(const ::inference::ModelInferRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +} // namespace ovms diff --git a/src/ovms_lib/binaryutils.hpp b/src/ovms_lib/binaryutils.hpp new file mode 100644 index 0000000000..b41806bb61 --- /dev/null +++ b/src/ovms_lib/binaryutils.hpp @@ -0,0 +1,26 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { +template +Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +} // namespace ovms diff --git a/src/ovms_lib/cleaner_utils.hpp b/src/ovms_lib/cleaner_utils.hpp new file mode 100644 index 0000000000..3ac0cea675 --- /dev/null +++ b/src/ovms_lib/cleaner_utils.hpp @@ -0,0 +1,43 @@ +//***************************************************************************** +// Copyright 2020-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "global_sequences_viewer.hpp" +#include "modelmanager.hpp" + +namespace ovms { +struct FunctorSequenceCleaner { + GlobalSequencesViewer& globalSequencesViewer; + + FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer) : + globalSequencesViewer(globalSequencesViewer) {} + + virtual void cleanup() { + globalSequencesViewer.removeIdleSequences(); + } +}; + +struct FunctorResourcesCleaner { + ModelManager& modelManager; + + FunctorResourcesCleaner(ModelManager& modelManager) : + modelManager(modelManager) {} + + virtual void cleanup() { + modelManager.cleanupResources(); + } +}; +} // namespace ovms diff --git a/src/ovms_lib/config.cpp b/src/ovms_lib/config.cpp new file mode 100644 index 0000000000..39c3b05a41 --- /dev/null +++ b/src/ovms_lib/config.cpp @@ -0,0 +1,303 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "config.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include "logging.hpp" +#include "version.hpp" + +namespace ovms { + +const uint AVAILABLE_CORES = std::thread::hardware_concurrency(); +const uint MAX_PORT_NUMBER = std::numeric_limits::max(); + +const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; +const std::string DEFAULT_REST_WORKERS_STRING{std::to_string(DEFAULT_REST_WORKERS)}; +const uint64_t MAX_REST_WORKERS = 10'000; + +Config& Config::parse(int argc, char** argv) { + try { + options = std::make_unique(argv[0], "OpenVINO Model Server"); + + // clang-format off + options->add_options() + ("h, help", + "Show this help message and exit") + ("version", + "Show binary version") + ("port", + "gRPC server port", + cxxopts::value()->default_value("9178"), + "PORT") + ("grpc_bind_address", + "Network interface address to bind to for the gRPC API", + cxxopts::value()->default_value("0.0.0.0"), + "GRPC_BIND_ADDRESS") + ("rest_port", + "REST server port, the REST server will not be started if rest_port is blank or set to 0", + cxxopts::value()->default_value("0"), + "REST_PORT") + ("rest_bind_address", + "Network interface address to bind to for the REST API", + cxxopts::value()->default_value("0.0.0.0"), + "REST_BIND_ADDRESS") + ("grpc_workers", + "Number of gRPC servers. Default 1. Increase for multi client, high throughput scenarios", + cxxopts::value()->default_value("1"), + "GRPC_WORKERS") + ("rest_workers", + "Number of worker threads in REST server - has no effect if rest_port is not set. Default value depends on number of CPUs. ", + cxxopts::value()->default_value(DEFAULT_REST_WORKERS_STRING.c_str()), + "REST_WORKERS") + ("log_level", + "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", + cxxopts::value()->default_value("INFO"), "LOG_LEVEL") + ("log_path", + "Optional path to the log file", + cxxopts::value(), "LOG_PATH") +#ifdef MTR_ENABLED + ("trace_path", + "Path to the trace file", + cxxopts::value(), "TRACE_PATH") +#endif + ("grpc_channel_arguments", + "A comma separated list of arguments to be passed to the grpc server. (e.g. grpc.max_connection_age_ms=2000)", + cxxopts::value(), "GRPC_CHANNEL_ARGUMENTS") + ("file_system_poll_wait_seconds", + "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", + cxxopts::value()->default_value("1"), + "FILE_SYSTEM_POLL_WAIT_SECONDS") + ("sequence_cleaner_poll_wait_minutes", + "Time interval between two consecutive sequence cleanup scans. Default is 5. Zero value disables sequence cleaner.", + cxxopts::value()->default_value("5"), + "SEQUENCE_CLEANER_POLL_WAIT_MINUTES") + ("custom_node_resources_cleaner_interval", + "Time interval between two consecutive resources cleanup scans. Default is 1. Must be greater than 0.", + cxxopts::value()->default_value("1"), + "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL") + ("cache_dir", + "Overrides model cache directory. By default cache files are saved into /opt/cache if the directory is present. When enabled, first model load will produce cache files.", + cxxopts::value(), + "CACHE_DIR") + ("cpu_extension", + "A path to shared library containing custom CPU layer implementation. Default: empty.", + cxxopts::value()->default_value(""), + "CPU_EXTENSION"); + options->add_options("multi model") + ("config_path", + "Absolute path to json configuration file", + cxxopts::value(), "CONFIG_PATH"); + + options->add_options("single model") + ("model_name", + "Name of the model", + cxxopts::value(), + "MODEL_NAME") + ("model_path", + "Absolute path to model, as in tf serving", + cxxopts::value(), + "MODEL_PATH") + ("batch_size", + "Resets models batchsize, int value or auto. This parameter will be ignored if shape is set", + cxxopts::value(), + "BATCH_SIZE") + ("shape", + "Resets models shape (model must support reshaping). If set, batch_size parameter is ignored", + cxxopts::value(), + "SHAPE") + ("layout", + "Resets model layout.", + cxxopts::value(), + "LAYOUT") + ("model_version_policy", + "Model version policy", + cxxopts::value(), + "MODEL_VERSION_POLICY") + ("nireq", + "Size of inference request queue for model executions. Recommended to be >= parallel executions. Default value calculated by OpenVINO based on available resources. Request for 0 is treated as request for default value", + cxxopts::value(), + "NIREQ") + ("target_device", + "Target device to run the inference", + cxxopts::value()->default_value("CPU"), + "TARGET_DEVICE") + ("plugin_config", + "A dictionary of plugin configuration keys and their values, eg \"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"1\\\"}\". Default throughput streams for CPU and GPU are calculated by OpenVINO", + cxxopts::value(), + "PLUGIN_CONFIG") + ("stateful", + "Flag indicating model is stateful", + cxxopts::value()->default_value("false"), + "STATEFUL") + ("idle_sequence_cleanup", + "Flag indicating if model is subject to sequence cleaner scans", + cxxopts::value()->default_value("true"), + "IDLE_SEQUENCE_CLEANUP") + ("low_latency_transformation", + "Flag indicating that Model Server should perform low latency transformation on that model", + cxxopts::value()->default_value("false"), + "LOW_LATENCY_TRANSFORMATION") + ("max_sequence_number", + "Determines how many sequences can be processed concurrently by one model instance. When that value is reached, attempt to start a new sequence will result in error.", + cxxopts::value(), + "MAX_SEQUENCE_NUMBER"); + + // clang-format on + + result = std::make_unique(options->parse(argc, argv)); + + if (result->count("version")) { + std::string project_name(PROJECT_NAME); + std::string project_version(PROJECT_VERSION); + std::cout << project_name + " " + project_version << std::endl; + std::cout << "OpenVINO backend " << OPENVINO_NAME << std::endl; + exit(EX_OK); + } + + if (result->count("help") || result->arguments().size() == 0) { + std::cout << options->help({"", "multi model", "single model"}) << std::endl; + exit(EX_OK); + } + + validate(); + } catch (const cxxopts::OptionException& e) { + std::cerr << "error parsing options: " << e.what() << std::endl; + exit(EX_USAGE); + } + + return instance(); +} + +bool Config::check_hostname_or_ip(const std::string& input) { + if (input.size() > 255) { + return false; + } + bool all_numeric = true; + for (char c : input) { + if (c == '.') { + continue; + } + if (!::isdigit(c)) { + all_numeric = false; + } + } + if (all_numeric) { + std::regex valid_ip_regex("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"); + return std::regex_match(input, valid_ip_regex); + } else { + std::regex valid_hostname_regex("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$"); + return std::regex_match(input, valid_hostname_regex); + } +} + +void Config::validate() { + // cannot set both config path & model_name/model_path + if (result->count("config_path") && (result->count("model_name") || result->count("model_path"))) { + std::cerr << "Use either config_path or model_path with model_name" << std::endl; + exit(EX_USAGE); + } + + if (!result->count("config_path") && !(result->count("model_name") && result->count("model_path"))) { + std::cerr << "Use config_path or model_path with model_name" << std::endl; + exit(EX_USAGE); + } + + if (result->count("config_path") && (result->count("batch_size") || result->count("shape") || + result->count("nireq") || result->count("model_version_policy") || result->count("target_device") || + result->count("plugin_config"))) { + std::cerr << "Model parameters in CLI are exclusive with the config file" << std::endl; + exit(EX_USAGE); + } + + // check grpc_workers value + if (result->count("grpc_workers") && ((this->grpcWorkers() > AVAILABLE_CORES) || (this->grpcWorkers() < 1))) { + std::cerr << "grpc_workers count should be from 1 to CPU core count : " << AVAILABLE_CORES << std::endl; + exit(EX_USAGE); + } + + // check rest_workers value + if (result->count("rest_workers") && ((this->restWorkers() > MAX_REST_WORKERS) || (this->restWorkers() < 2))) { + std::cerr << "rest_workers count should be from 2 to " << MAX_REST_WORKERS << std::endl; + exit(EX_USAGE); + } + + if (result->count("rest_workers") && (this->restWorkers() != DEFAULT_REST_WORKERS) && this->restPort() == 0) { + std::cerr << "rest_workers is set but rest_port is not set. rest_port is required to start rest servers" << std::endl; + exit(EX_USAGE); + } + + // check docker ports + if (result->count("port") && ((this->port() > MAX_PORT_NUMBER) || (this->port() < 0))) { + std::cerr << "port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; + exit(EX_USAGE); + } + if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { + std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; + exit(EX_USAGE); + } + + // check bind addresses: + if (result->count("rest_bind_address") && check_hostname_or_ip(this->restBindAddress()) == false) { + std::cerr << "rest_bind_address has invalid format: proper hostname or IP address expected." << std::endl; + exit(EX_USAGE); + } + if (result->count("grpc_bind_address") && check_hostname_or_ip(this->grpcBindAddress()) == false) { + std::cerr << "grpc_bind_address has invalid format: proper hostname or IP address expected." << std::endl; + exit(EX_USAGE); + } + if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { + std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; + exit(EX_USAGE); + } + + // port and rest_port cannot be the same + if (this->port() == this->restPort()) { + std::cerr << "port and rest_port cannot have the same values" << std::endl; + exit(EX_USAGE); + } + + // check cpu_extension path: + if (result->count("cpu_extension") && !std::filesystem::exists(this->cpuExtensionLibraryPath())) { + std::cerr << "File path provided as an --cpu_extension parameter does not exists in the filesystem: " << this->cpuExtensionLibraryPath() << std::endl; + exit(EX_USAGE); + } + + // check log_level values + if (result->count("log_level")) { + std::vector v({"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"}); + if (std::find(v.begin(), v.end(), this->logLevel()) == v.end()) { + std::cerr << "log_level should be one of: TRACE, DEBUG, INFO, WARNING, ERROR" << std::endl; + exit(EX_USAGE); + } + } + + // check stateful flags: + if ((result->count("low_latency_transformation") || result->count("max_sequence_number") || result->count("idle_sequence_cleanup")) && !result->count("stateful")) { + std::cerr << "Setting low_latency_transformation, max_sequence_number and idle_sequence_cleanup require setting stateful flag for the model." << std::endl; + exit(EX_USAGE); + } + return; +} + +} // namespace ovms diff --git a/src/ovms_lib/config.hpp b/src/ovms_lib/config.hpp new file mode 100644 index 0000000000..ac52e39d9e --- /dev/null +++ b/src/ovms_lib/config.hpp @@ -0,0 +1,394 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "modelconfig.hpp" + +namespace ovms { +/** + * @brief Provides all the configuration options from command line + */ +class Config { +private: + /** + * @brief A default constructor is private + */ + Config() = default; + + /** + * @brief Private copying constructor + */ + Config(const Config&) = delete; + + /** + * @brief cxxopts options dictionary definition + */ + std::unique_ptr options; + + /** + * @brief cxxopts contains parsed parameters + */ + std::unique_ptr result; + + /** + * @brief + */ + const std::string empty; + +public: + /** + * @brief Gets the instance of the config + */ + static Config& instance() { + static Config instance; + + return instance; + } + + /** + * @brief Parse the commandline parameters + * + * @param argc + * @param argv + * @return Config& + */ + Config& parse(int argc, char** argv); + + /** + * @brief Validate passed arguments + * + * @return void + */ + void validate(); + + /** + * @brief checks if input is a proper hostname or IP address value + * + * @return bool + */ + static bool check_hostname_or_ip(const std::string& input); + + /** + * @brief Get the config path + * + * @return std::string + */ + const std::string& configPath() const { + if (result->count("config_path")) + return result->operator[]("config_path").as(); + return empty; + } + + /** + * @brief Gets the grpc port + * + * @return uint64_t + */ + uint64_t port() const { + return result->operator[]("port").as(); + } + + /** + * @brief Get the gRPC network interface address to bind to + * + * @return const std::string + */ + const std::string cpuExtensionLibraryPath() const { + if (result != nullptr && result->count("cpu_extension")) { + return result->operator[]("cpu_extension").as(); + } + return ""; + } + + /** + * @brief Get the gRPC network interface address to bind to + * + * @return const std::string& + */ + const std::string grpcBindAddress() const { + if (result->count("grpc_bind_address")) + return result->operator[]("grpc_bind_address").as(); + return "0.0.0.0"; + } + /** + * @brief Gets the REST port + * + * @return uint64_t + */ + uint64_t restPort() const { + return result->operator[]("rest_port").as(); + } + + /** + * @brief Get the rest network interface address to bind to + * + * @return const std::string& + */ + const std::string restBindAddress() const { + if (result->count("rest_bind_address")) + return result->operator[]("rest_bind_address").as(); + return "0.0.0.0"; + } + + /** + * @brief Gets the gRPC workers count + * + * @return uint + */ + uint grpcWorkers() const { + return result->operator[]("grpc_workers").as(); + } + + /** + * @brief Gets the rest workers count + * + * @return uint + */ + uint restWorkers() const { + return result->operator[]("rest_workers").as(); + } + + /** + * @brief Get the model name + * + * @return const std::string& + */ + const std::string& modelName() const { + if (result->count("model_name")) + return result->operator[]("model_name").as(); + return empty; + } + + /** + * @brief Gets the model path + * + * @return const std::string& + */ + const std::string& modelPath() const { + if (result->count("model_path")) + return result->operator[]("model_path").as(); + return empty; + } + + /** + * @brief Gets the batch size + * + * @return const std::string& + */ + const std::string& batchSize() const { + if (!result->count("batch_size")) { + static const std::string d = "0"; + return d; + } + return result->operator[]("batch_size").as(); + } + + /** + * @brief Get the shape + * + * @return const std::string& + */ + const std::string& shape() const { + if (result->count("shape")) + return result->operator[]("shape").as(); + return empty; + } + + /** + * @brief Get the layout + * + * @return const std::string& + */ + const std::string& layout() const { + if (result->count("layout")) + return result->operator[]("layout").as(); + return empty; + } + + /** + * @brief Get the shape + * + * @return const std::string& + */ + const std::string& modelVersionPolicy() const { + if (result->count("model_version_policy")) + return result->operator[]("model_version_policy").as(); + return empty; + } + + /** + * @brief Get the nireq + * + * @return uint + */ + uint32_t nireq() const { + if (!result->count("nireq")) { + return 0; + } + return result->operator[]("nireq").as(); + } + + /** + * @brief Get the target device + * + * @return const std::string& + */ + const std::string& targetDevice() const { + return result->operator[]("target_device").as(); + } + + /** + * @brief Get the plugin config + * + * @return const std::string& + */ + const std::string& pluginConfig() const { + if (result->count("plugin_config")) + return result->operator[]("plugin_config").as(); + return empty; + } + + /** + * @brief Get stateful flag + * + * @return bool + */ + bool stateful() const { + return result->operator[]("stateful").as(); + } + + /** + * @brief Get idle sequence cleanup flag + * + * @return uint + */ + bool idleSequenceCleanup() const { + return result->operator[]("idle_sequence_cleanup").as(); + } + + /** + * @brief Get low latency transformation flag + * + * @return bool + */ + bool lowLatencyTransformation() const { + return result->operator[]("low_latency_transformation").as(); + } + + /** + * @brief Get max number of sequences that can be processed concurrently + * + * @return uint + */ + uint32_t maxSequenceNumber() const { + if (!result->count("max_sequence_number")) { + return DEFAULT_MAX_SEQUENCE_NUMBER; + } + return result->operator[]("max_sequence_number").as(); + } + + /** + * @brief Get the log level + * + * @return const std::string& + */ + const std::string& logLevel() const { + if (result->count("log_level")) + return result->operator[]("log_level").as(); + return empty; + } + + /** + * @brief Get the log path + * + * @return const std::string& + */ + const std::string& logPath() const { + if (result->count("log_path")) + return result->operator[]("log_path").as(); + return empty; + } + +#ifdef MTR_ENABLED + /** + * @brief Get the log path + * + * @return const std::string& + */ + const std::string& tracePath() const { + if (result->count("trace_path")) + return result->operator[]("trace_path").as(); + return empty; + } +#endif + + /** + * @brief Get the plugin config + * + * @return const std::string& + */ + const std::string& grpcChannelArguments() const { + if (result->count("grpc_channel_arguments")) + return result->operator[]("grpc_channel_arguments").as(); + return empty; + } + + /** + * @brief Get the filesystem poll wait time in seconds + * + * @return uint + */ + uint filesystemPollWaitSeconds() const { + return result->operator[]("file_system_poll_wait_seconds").as(); + } + + /** + * @brief Get the sequence cleanup poll wait time in minutes + * + * @return uint32_t + */ + uint32_t sequenceCleanerPollWaitMinutes() const { + return result->operator[]("sequence_cleaner_poll_wait_minutes").as(); + } + + /** + * @brief Get the resources cleanup poll wait time in seconds + * + * @return uint32_t + */ + uint32_t resourcesCleanerPollWaitSeconds() const { + return result->operator[]("custom_node_resources_cleaner_interval").as(); + } + + /** + * @brief Model cache directory + * + * @return const std::string& + */ + const std::string cacheDir() const { + if (result != nullptr && result->count("cache_dir")) { + return result->operator[]("cache_dir").as(); + } + return ""; + } +}; +} // namespace ovms diff --git a/src/ovms_lib/custom_node.cpp b/src/ovms_lib/custom_node.cpp new file mode 100644 index 0000000000..3feb6c3c6e --- /dev/null +++ b/src/ovms_lib/custom_node.cpp @@ -0,0 +1,103 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "custom_node.hpp" + +#include + +#include "custom_node_output_allocator.hpp" +#include "customnodesession.hpp" +#include "logging.hpp" +#include "node_library.hpp" +#include "node_library_utils.hpp" + +namespace ovms { + +CustomNode::CustomNode( + const std::string& nodeName, + const NodeLibrary& library, + const parameters_t& parameters, + const std::unordered_map& nodeOutputNameAlias, + std::optional demultiplyCount, + std::set gatherFromNode, + std::shared_ptr customNodeLibraryInternalManager) : + Node(nodeName, demultiplyCount, gatherFromNode), + library(library), + parameters(parameters), + nodeOutputNameAlias(nodeOutputNameAlias), + libraryParameters(createCustomNodeParamArray(this->parameters)), + customNodeLibraryInternalManager(customNodeLibraryInternalManager) { +} + +Status CustomNode::execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) { + auto& nodeSession = getNodeSession(sessionKey); + auto& customNodeSession = static_cast(nodeSession); + return customNodeSession.execute(notifyEndQueue, *this, this->library, this->libraryParameters, this->parameters.size(), getCNLIMWrapperPtr(customNodeLibraryInternalManager)); +} + +Status CustomNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { + auto& customNodeSession = static_cast(nodeSession); + const auto& sessionMetadata = nodeSession.getNodeSessionMetadata(); + SessionResult sessionResults{sessionMetadata, {}}; + auto it = nodeSessionOutputs.emplace(sessionMetadata.getSessionKey(), std::move(sessionResults)); + if (!it.second) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to put node: {} session: {} results in node session outputs", + getName(), nodeSession.getSessionKey()); + customNodeSession.release(); + return StatusCode::INTERNAL_ERROR; + } + auto& metadataTensorResultsPair = it.first->second; + auto& tensorResults = metadataTensorResultsPair.second; + return this->fetchResults( + tensorResults, + nodeSession.getSessionKey()); +} + +Status CustomNode::fetchResults(TensorWithSourceMap& outputs, session_key_t sessionKey) { + auto& session = static_cast(this->getNodeSession(sessionKey)); + session.clearInputs(); + + for (const auto& node : this->next) { + for (const auto& pair : node.get().getMappingByDependency(*this)) { + const auto& output_name = pair.first; + if (outputs.find(output_name) != outputs.end()) { + continue; + } + const auto& realOutputName = this->getRealOutputName(output_name); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Getting custom node output tensor with name: {}", + getName(), sessionKey, realOutputName); + + ov::Tensor resultTensor; + auto status = session.fetchResult(realOutputName, resultTensor); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} Custom node output with name {} is missing", + getName(), sessionKey, realOutputName); + return StatusCode::NODE_LIBRARY_MISSING_OUTPUT; + } + + outputs.emplace(std::make_pair(output_name, TensorWithSource(std::move(resultTensor)))); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Tensor with name {} has been prepared under alias {}", + getName(), sessionKey, realOutputName, output_name); + } + } + + return StatusCode::OK; +} + +std::unique_ptr CustomNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { + return std::make_unique(metadata, getName(), previous.size(), collapsingDetails); +} + +} // namespace ovms diff --git a/src/ovms_lib/custom_node.hpp b/src/ovms_lib/custom_node.hpp new file mode 100644 index 0000000000..358dbc4af1 --- /dev/null +++ b/src/ovms_lib/custom_node.hpp @@ -0,0 +1,65 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include "custom_node_interface.h" // NOLINT +#include "custom_node_library_internal_manager_wrapper.hpp" +#include "node.hpp" +#include "nodeinfo.hpp" +#include "pipelineeventqueue.hpp" + +namespace ovms { + +class NodeLibrary; + +class CustomNode : public Node { + NodeLibrary library; + parameters_t parameters; + std::unordered_map nodeOutputNameAlias; + + std::unique_ptr libraryParameters = nullptr; + + std::shared_ptr customNodeLibraryInternalManager; + +public: + CustomNode( + const std::string& nodeName, + const NodeLibrary& library, + const parameters_t& parameters, + const std::unordered_map& nodeOutputNameAlias = {}, + std::optional demultiplyCount = std::nullopt, + std::set gatherFromNode = {}, + std::shared_ptr customNodeLibraryInternalManager = nullptr); + + Status execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) override; + + Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; + Status fetchResults(TensorWithSourceMap& outputs, session_key_t sessionKey); + + const std::string& getRealOutputName(const std::string& alias) const { + auto it = nodeOutputNameAlias.find(alias); + return it != nodeOutputNameAlias.end() ? it->second : alias; + } + + std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; +}; + +} // namespace ovms diff --git a/src/ovms_lib/custom_node_interface.h b/src/ovms_lib/custom_node_interface.h new file mode 100644 index 0000000000..96ab2ecb58 --- /dev/null +++ b/src/ovms_lib/custom_node_interface.h @@ -0,0 +1,79 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +typedef enum { + UNSPECIFIED, + FP32, + FP16, + U8, + I8, + I16, + U16, + I32, + FP64, + I64 +} CustomNodeTensorPrecision; + +struct CustomNodeTensor { + const char* name; + uint8_t* data; + uint64_t dataBytes; + uint64_t* dims; + uint64_t dimsCount; + CustomNodeTensorPrecision precision; +}; + +struct CustomNodeTensorInfo { + const char* name; + uint64_t* dims; + uint64_t dimsCount; + CustomNodeTensorPrecision precision; +}; + +struct CustomNodeParam { + const char *key, *value; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Custom node library initialize enables creation of resources to be reused between predictions. + * Potential use cases include optimized temporary buffers allocation. + * Using initialize is optional and not required for custom node to work. + * CustomNodeLibraryInternalManager should be created here if initialize is used. + * On initialize failure status not equal to zero is returned and error log is printed. + */ +int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount); +/** + * @brief Custom node library deinitialize enables destruction of resources that were used between predictions. + * Using deinitialize is optional and not required for custom node to work. + * CustomNodeLibraryInternalManager should be destroyed here if deinitialize is used. + * On deinitialize failure only error log is printed. + */ +int deinitialize(void* customNodeLibraryInternalManager); +int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); +int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); +int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); +int release(void* ptr, void* customNodeLibraryInternalManager); + +#ifdef __cplusplus +} +#endif diff --git a/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp b/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp new file mode 100644 index 0000000000..849d1a271c --- /dev/null +++ b/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp @@ -0,0 +1,27 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "custom_node_library_internal_manager_wrapper.hpp" + +namespace ovms { + +void* getCNLIMWrapperPtr(const std::shared_ptr& wrapper) { + if (wrapper == nullptr) { + return nullptr; + } + return wrapper->ptr; +} + +} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp b/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp new file mode 100644 index 0000000000..8b68bc33f7 --- /dev/null +++ b/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp @@ -0,0 +1,40 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include "node_library.hpp" + +namespace ovms { + +struct CNLIMWrapper { + void* ptr; + deinitialize_fn deinitialize = nullptr; + + CNLIMWrapper(void* CNLIM, deinitialize_fn deinitialize) : + ptr(CNLIM), + deinitialize(deinitialize) {} + + ~CNLIMWrapper() { + deinitialize(ptr); + } +}; + +void* getCNLIMWrapperPtr(const std::shared_ptr& wrapper); + +} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_manager.cpp b/src/ovms_lib/custom_node_library_manager.cpp new file mode 100644 index 0000000000..2eeb5eb8d1 --- /dev/null +++ b/src/ovms_lib/custom_node_library_manager.cpp @@ -0,0 +1,135 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "custom_node_library_manager.hpp" + +#include + +#include + +#include "filesystem.hpp" +#include "logging.hpp" +#include "status.hpp" + +namespace ovms { + +Status CustomNodeLibraryManager::loadLibrary(const std::string& name, const std::string& basePath) { + if (FileSystem::isPathEscaped(basePath)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", basePath); + return StatusCode::PATH_INVALID; + } + + auto it = libraries.find(name); + if (it != libraries.end() && it->second.basePath == basePath) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom node library name: {} is already loaded", name); + return StatusCode::NODE_LIBRARY_ALREADY_LOADED; + } + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Loading custom node library name: {}; base_path: {}", name, basePath); + + void* handle = dlopen(basePath.c_str(), RTLD_LAZY | RTLD_LOCAL); + char* error = dlerror(); + if (handle == NULL) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Library name: {} failed to open base_path: {} with error: {}", name, basePath, error); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_OPEN; + } + + initialize_fn initialize = reinterpret_cast(dlsym(handle, "initialize")); + error = dlerror(); + if (error || initialize == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + deinitialize_fn deinitialize = reinterpret_cast(dlsym(handle, "deinitialize")); + error = dlerror(); + if (error || deinitialize == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + execute_fn execute = reinterpret_cast(dlsym(handle, "execute")); + error = dlerror(); + if (error || execute == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + metadata_fn getInputsInfo = reinterpret_cast(dlsym(handle, "getInputsInfo")); + error = dlerror(); + if (error || getInputsInfo == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + metadata_fn getOutputsInfo = reinterpret_cast(dlsym(handle, "getOutputsInfo")); + error = dlerror(); + if (error || getOutputsInfo == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + release_fn release = reinterpret_cast(dlsym(handle, "release")); + error = dlerror(); + if (error || release == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); + dlclose(handle); + return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; + } + + libraries[name] = NodeLibrary{ + initialize, + deinitialize, + execute, + getInputsInfo, + getOutputsInfo, + release, + basePath}; + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Successfully loaded custom node library name: {}; base_path: {}", name, basePath); + return StatusCode::OK; +} + +Status CustomNodeLibraryManager::getLibrary(const std::string& name, NodeLibrary& library) const { + auto it = libraries.find(name); + if (it == libraries.end()) { + return StatusCode::NODE_LIBRARY_MISSING; + } else { + library = it->second; + return StatusCode::OK; + } +} + +void CustomNodeLibraryManager::unloadLibrariesRemovedFromConfig(const std::set& librariesInConfig) { + std::set librariesCurrentlyLoaded; + for (auto& library : libraries) { + librariesCurrentlyLoaded.emplace(library.first); + } + std::set librariesToUnload; + std::set_difference( + librariesCurrentlyLoaded.begin(), librariesCurrentlyLoaded.end(), + librariesInConfig.begin(), librariesInConfig.end(), + std::inserter(librariesToUnload, librariesToUnload.end())); + for (auto& library : librariesToUnload) { + libraries.erase(library); + } +} + +} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_manager.hpp b/src/ovms_lib/custom_node_library_manager.hpp new file mode 100644 index 0000000000..0a899f0e57 --- /dev/null +++ b/src/ovms_lib/custom_node_library_manager.hpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "node_library.hpp" +#include "status.hpp" + +namespace ovms { + +class CustomNodeLibraryManager { + std::unordered_map libraries; + +public: + Status loadLibrary(const std::string& name, const std::string& basePath); + Status getLibrary(const std::string& name, NodeLibrary& library) const; + void unloadLibrariesRemovedFromConfig(const std::set& librariesInConfig); +}; + +} // namespace ovms diff --git a/src/ovms_lib/custom_node_output_allocator.cpp b/src/ovms_lib/custom_node_output_allocator.cpp new file mode 100644 index 0000000000..a4a4b6e536 --- /dev/null +++ b/src/ovms_lib/custom_node_output_allocator.cpp @@ -0,0 +1,54 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "custom_node_output_allocator.hpp" + +#include "logging.hpp" + +namespace ovms { +bool operator==(const CustomNodeTensor& t1, const CustomNodeTensor& t2) { + return (t1.name == t2.name) && + (t1.data == t2.data) && + (t1.dataBytes == t2.dataBytes) && + (t1.dims == t2.dims) && + (t1.dimsCount == t2.dimsCount) && + (t1.precision == t2.precision); +} +CustomNodeOutputAllocator::CustomNodeOutputAllocator(struct CustomNodeTensor tensor, NodeLibrary nodeLibrary, void* customNodeLibraryInternalManager) : + tensor(tensor), + nodeLibrary(nodeLibrary), + customNodeLibraryInternalManager(customNodeLibraryInternalManager) {} +void* CustomNodeOutputAllocator::allocate(const size_t bytes, const size_t alignment) { + return (void*)tensor.data; +} +void CustomNodeOutputAllocator::deallocate(void* handle, const size_t bytes, size_t alignment) { + bool succeeded = nodeLibrary.release(tensor.data, customNodeLibraryInternalManager) == 0; + if (false == succeeded) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to release custom node tensor:{} buffer using library:{}", tensor.name, nodeLibrary.basePath); + } +} +bool CustomNodeOutputAllocator::is_equal(const CustomNodeOutputAllocator& other) const { + return (customNodeLibraryInternalManager == other.customNodeLibraryInternalManager) && + (nodeLibrary == other.nodeLibrary) && + (tensor == other.tensor); +} +bool CustomNodeOutputAllocator::is_equal(const AllocatorImpl& other) const { + const CustomNodeOutputAllocator* otherPtr = dynamic_cast(&other); + if (otherPtr == nullptr) { + return false; + } + return this->is_equal(*otherPtr); +} +} // namespace ovms diff --git a/src/ovms_lib/custom_node_output_allocator.hpp b/src/ovms_lib/custom_node_output_allocator.hpp new file mode 100644 index 0000000000..b2b0a84e43 --- /dev/null +++ b/src/ovms_lib/custom_node_output_allocator.hpp @@ -0,0 +1,39 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "custom_node_interface.h" // NOLINT +#include "node_library.hpp" + +namespace ovms { + +bool operator==(const CustomNodeTensor& t1, const CustomNodeTensor& t2); + +class CustomNodeOutputAllocator : public ov::AllocatorImpl { + struct CustomNodeTensor tensor; + NodeLibrary nodeLibrary; + void* customNodeLibraryInternalManager; + +public: + CustomNodeOutputAllocator(struct CustomNodeTensor tensor, NodeLibrary nodeLibrary, void* customNodeLibraryInternalManager); + void* allocate(const size_t bytes, const size_t alignment = alignof(max_align_t)) override; + void deallocate(void* handle, const size_t bytes, size_t alignment = alignof(max_align_t)) override; + bool is_equal(const CustomNodeOutputAllocator& other) const; + bool is_equal(const AllocatorImpl& other) const override; +}; +} // namespace ovms diff --git a/src/ovms_lib/customloaderconfig.hpp b/src/ovms_lib/customloaderconfig.hpp new file mode 100644 index 0000000000..8251acc2fb --- /dev/null +++ b/src/ovms_lib/customloaderconfig.hpp @@ -0,0 +1,145 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "status.hpp" +#include "stringutils.hpp" + +namespace ovms { + +/** + * @brief This class represents Custom Loader configuration + */ +class CustomLoaderConfig { +private: + /** + * @brief Custom Loader Name + */ + std::string loaderName; + + /** + * @brief Custom Loader Library Path + */ + std::string libraryPath; + + /** + * @brief Custom Loader Config Path + */ + std::string loaderConfigFile; + +public: + /** + * @brief Construct a new Custom Loader Config object + * + * @param name + * @param libraryPath + * @param configPath + */ + CustomLoaderConfig(const std::string& loaderName = "", + const std::string& libraryPath = "", + const std::string& loaderConfigFile = "") : + loaderName(loaderName), + libraryPath(libraryPath), + loaderConfigFile(loaderConfigFile) { + } + + void clear() { + loaderName.clear(); + libraryPath.clear(); + loaderConfigFile.clear(); + } + + /** + * @brief Get the name + * + * @return const std::string& + */ + const std::string& getLoaderName() const { + return this->loaderName; + } + + /** + * @brief Set the name + * + * @param name + */ + void setLoaderName(const std::string& loaderName) { + this->loaderName = loaderName; + } + + /** + * @brief Get the Library Path + * + * @return const std::string& + */ + const std::string& getLibraryPath() const { + return this->libraryPath; + } + + /** + * @brief Set the Library Path + * + * @param libraryPath + */ + void setLibraryPath(const std::string& libraryPath) { + this->libraryPath = libraryPath; + } + + /** + * @brief Get the Config Path + * + * @return const std::string& + */ + const std::string& getLoaderConfigFile() const { + return this->loaderConfigFile; + } + + /** + * @brief Set the Config Path + * + * @param configPath + */ + void setLoaderConfigFile(const std::string& loaderConfigFile) { + this->loaderConfigFile = loaderConfigFile; + } + + /** + * @brief Parses all settings from a JSON node + * + * @return Status + */ + Status parseNode(const rapidjson::Value& v) { + try { + this->setLoaderName(v["loader_name"].GetString()); + this->setLibraryPath(v["library_path"].GetString()); + if (v.HasMember("loader_config_file")) + this->setLoaderConfigFile(v["loader_config_file"].GetString()); + } catch (...) { + SPDLOG_ERROR("There was an error parsing the custom loader config"); + return StatusCode::JSON_INVALID; + } + return StatusCode::OK; + } +}; +} // namespace ovms diff --git a/src/ovms_lib/customloaderinterface.hpp b/src/ovms_lib/customloaderinterface.hpp new file mode 100644 index 0000000000..bd44c2df72 --- /dev/null +++ b/src/ovms_lib/customloaderinterface.hpp @@ -0,0 +1,117 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +namespace ovms { + +enum class CustomLoaderStatus { + OK, /*!< Success */ + MODEL_TYPE_IR, /*!< When model buffers are returned, they belong to IR model */ + MODEL_TYPE_ONNX, /*!< When model buffers are returned, they belong to ONXX model */ + MODEL_TYPE_BLOB, /*!< When model buffers are returned, they belong to Blob */ + MODEL_LOAD_ERROR, /*!< Error while loading the model */ + MODEL_BLACKLISTED, /*!< Model is blacklisted. Do not load */ + INTERNAL_ERROR /*!< generic error */ +}; + +/** + * @brief This class is the custom loader interface base class. + * Custom Loader implementation shall derive from this base calss + * and implement interface functions and define the virtual functions. + * Based on the config file, OVMS loads a model using specified custom loader + */ +class CustomLoaderInterface { +public: + /** + * @brief Constructor + */ + CustomLoaderInterface() { + } + /** + * @brief Destructor + */ + virtual ~CustomLoaderInterface() { + } + + /** + * @brief Initialize the custom loader + * + * @param loader config file defined under custom loader config in the config file + * + * @return status + */ + virtual CustomLoaderStatus loaderInit(const std::string& loaderConfigFile) = 0; + + /** + * @brief Load the model by the custom loader + * + * @param model name required to be loaded - defined under model config in the config file + * @param base path where the required model files are present + * @param version of the model + * @param loader config parameters json as string + * @param vector of uint8_t of model + * @param vector of uint8_t of weights + * @return status (On success, the return value will specify the type of model (IR,ONNX,BLOB) read into vectors) + */ + virtual CustomLoaderStatus loadModel(const std::string& modelName, + const std::string& basePath, + const int version, + const std::string& loaderOptions, + std::vector& modelBuffer, + std::vector& weights) = 0; + + /** + * @brief Get the model black list status + * + * @param model name for which black list status is required + * @param version for which the black list status is required + * @return blacklist status OK or MODEL_BLACKLISTED + */ + virtual CustomLoaderStatus getModelBlacklistStatus(const std::string& modelName, const int version) { + return CustomLoaderStatus::OK; + } + + /** + * @brief Unload model resources by custom loader once model is unloaded by OVMS + * + * @param model name which is been unloaded + * @param version which is been unloaded + * @return status + */ + virtual CustomLoaderStatus unloadModel(const std::string& modelName, const int version) = 0; + + /** + * @brief Retire the model from customloader when OVMS retires the model + * + * @param model name which is being retired + * @return status + */ + virtual CustomLoaderStatus retireModel(const std::string& modelName) = 0; + + /** + * @brief Deinitialize the custom loader + * + */ + virtual CustomLoaderStatus loaderDeInit() = 0; +}; + +// the types of the class factories +typedef CustomLoaderInterface* createCustomLoader_t(); + +} // namespace ovms diff --git a/src/ovms_lib/customloaders.cpp b/src/ovms_lib/customloaders.cpp new file mode 100644 index 0000000000..77aa1193ad --- /dev/null +++ b/src/ovms_lib/customloaders.cpp @@ -0,0 +1,90 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "customloaders.hpp" + +#include + +#include "customloaderinterface.hpp" + +namespace ovms { + +Status CustomLoaders::add(std::string name, std::shared_ptr loaderInterface, void* library) { + auto loaderIt = newCustomLoaderInterfacePtrs.emplace(name, std::make_pair(library, loaderInterface)); + // if the loader already exists, print an error message + if (!loaderIt.second) { + SPDLOG_ERROR("The loader {} already exists in the config file", name); + return StatusCode::CUSTOM_LOADER_EXISTS; + } + + return StatusCode::OK; +} + +Status CustomLoaders::remove(const std::string& name) { + SPDLOG_INFO("Removing loader {} from loaders list", name); + auto loaderIt = customLoaderInterfacePtrs.find(name); + + if (loaderIt == customLoaderInterfacePtrs.end()) { + return StatusCode::CUSTOM_LOADER_NOT_PRESENT; + } + + customLoaderInterfacePtrs.erase(loaderIt); + return StatusCode::OK; +} + +std::shared_ptr CustomLoaders::find(const std::string& name) { + SPDLOG_DEBUG("Looking for loader {} in loaders list", name); + auto loaderIt = customLoaderInterfacePtrs.find(name); + + if (loaderIt == customLoaderInterfacePtrs.end()) { + return nullptr; + } + + return (loaderIt->second).second; +} + +Status CustomLoaders::move(const std::string& name) { + SPDLOG_INFO("Moving loader {} from old to new loaders list", name); + auto loaderIt = customLoaderInterfacePtrs.find(name); + + if (loaderIt == customLoaderInterfacePtrs.end()) { + return StatusCode::INTERNAL_ERROR; + } + + newCustomLoaderInterfacePtrs.insert({name, customLoaderInterfacePtrs[name]}); + customLoaderInterfacePtrs.erase(loaderIt); + return StatusCode::OK; +} + +Status CustomLoaders::finalize() { + // By now the remaining loaders in current list are not there in new config. Delete them + for (auto it = customLoaderInterfacePtrs.begin(); it != customLoaderInterfacePtrs.end(); it++) { + SPDLOG_INFO("Loader {} is not there in new list.. deleting the same", it->first); + auto loaderPtr = (it->second).second; + loaderPtr->loaderDeInit(); + } + + SPDLOG_INFO("Clearing the list"); + customLoaderInterfacePtrs.clear(); + + SPDLOG_INFO("Adding new list to the old list"); + // now assign new map to servicing map. + customLoaderInterfacePtrs.insert(newCustomLoaderInterfacePtrs.begin(), newCustomLoaderInterfacePtrs.end()); + newCustomLoaderInterfacePtrs.clear(); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/customloaders.hpp b/src/ovms_lib/customloaders.hpp new file mode 100644 index 0000000000..7d8b4333e6 --- /dev/null +++ b/src/ovms_lib/customloaders.hpp @@ -0,0 +1,92 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include "customloaderinterface.hpp" +#include "status.hpp" + +namespace ovms { +/** + * @brief Provides all customloaders + */ +class CustomLoaders { +private: + /** + * @brief A default constructor is private + */ + CustomLoaders() = default; + + /** + * @brief Private copying constructor + */ + CustomLoaders(const CustomLoaders&) = delete; + + std::map>> customLoaderInterfacePtrs; + std::map>> newCustomLoaderInterfacePtrs; + + std::vector currentCustomLoaderNames; + +public: + /** + * @brief Gets the instance of the CustomLoaders + */ + static CustomLoaders& instance() { + static CustomLoaders instance; + return instance; + } + + /** + * @brief insert a new customloader + * + * @return status + */ + Status add(std::string name, std::shared_ptr loaderInsterface, void* library); + + /** + * @brief remove an existing customLoader referenced by it's name + * + * @return status + */ + Status remove(const std::string& name); + + /** + * @brief find an existing customLoader referenced by it's name + * + * @return pointer to customloader Interface if found, else NULL + */ + std::shared_ptr find(const std::string& name); + + /** + * @brief move the existing loader from serviced map to new map. + * + * @return status + */ + Status move(const std::string& name); + + /** + * @brief finalize the customloaders list after parsing config + * + * @return status + */ + Status finalize(); +}; +} // namespace ovms diff --git a/src/ovms_lib/customnodesession.cpp b/src/ovms_lib/customnodesession.cpp new file mode 100644 index 0000000000..70b20ac90d --- /dev/null +++ b/src/ovms_lib/customnodesession.cpp @@ -0,0 +1,250 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "customnodesession.hpp" + +#include +#include +#include +#include + +#include "custom_node_output_allocator.hpp" +#include "logging.hpp" +#include "node.hpp" +#include "node_library.hpp" +#include "node_library_utils.hpp" +#include "nodeinputhandler.hpp" +#include "pipelineeventqueue.hpp" +#include "profiler.hpp" +#include "timer.hpp" + +namespace ovms { + +CustomNodeSession::CustomNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : + NodeSession(metadata, nodeName, inputsCount, collapsingDetails) {} + +CustomNodeSession::CustomNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : + NodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails) {} + +CustomNodeSession::~CustomNodeSession() = default; + +static std::unordered_map createOwnedShapesCopy(const TensorMap& tensorMap) { + std::unordered_map tensorsDims; + for (auto& [name, tensor] : tensorMap) { + shape_t tensorDims = tensor.get_shape(); + tensorsDims.emplace(name, std::move(tensorDims)); + } + return tensorsDims; +} + +Status CustomNodeSession::execute(PipelineEventQueue& notifyEndQueue, Node& node, const NodeLibrary& library, std::unique_ptr& parameters, int parametersCount, void* customNodeLibraryInternalManager) { + OVMS_PROFILE_FUNCTION(); + const auto& tensorMap = this->inputHandler->getInputs(); + auto inputTensorsCount = tensorMap.size(); + // this is a hack to overcome OV 1.0 -> 2.0 API change where we do not get reference to + // tensor shape now but a copy. Hence we have to extend the lifetime of dims vector + auto tensorsDims = createOwnedShapesCopy(tensorMap); + auto inputTensors = createCustomNodeTensorArray(tensorMap, tensorsDims); + struct CustomNodeTensor* outputTensors = nullptr; + int outputTensorsCount = 0; + this->timer->start("execution"); + OVMS_PROFILE_SYNC_BEGIN("Custom Node Library execute()"); + int result = library.execute( + inputTensors.get(), + inputTensorsCount, + &outputTensors, + &outputTensorsCount, + parameters.get(), + parametersCount, + customNodeLibraryInternalManager); + OVMS_PROFILE_SYNC_END("Custom Node Library execute()"); + this->timer->stop("execution"); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Custom node execution processing time for node {}; session: {} - {} ms", + this->getName(), + this->getSessionKey(), + this->timer->elapsed("execution") / 1000); + + // If result is not 0, it means execution has failed. + // In this case shared library is responsible for cleaning up resources (memory). + if (result != 0) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has failed custom node execution with return code: {}", getName(), getSessionKey(), result); + notifyEndQueue.push({node, getSessionKey()}); + return StatusCode::NODE_LIBRARY_EXECUTION_FAILED; + } + // In other cases we are responsible of cleaning whatever is possible. + if (outputTensors == nullptr) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has corrupted outputs handle", getName(), getSessionKey()); + notifyEndQueue.push({node, getSessionKey()}); + return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED; + } + + if (outputTensorsCount <= 0) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has corrupted number of outputs", getName(), getSessionKey()); + library.release(outputTensors, customNodeLibraryInternalManager); + notifyEndQueue.push({node, getSessionKey()}); + return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT; + } + + // At this point this is important we do not exit before finishing the loop. + // There will be memory leak if any tensor is not converted into ov::Tensor. + // ov::Tensor destructor is responsible for cleaning up resources. + Status status = StatusCode::OK; + for (int i = 0; i < outputTensorsCount; i++) { + ov::Tensor resultTensor; + auto result = this->createTensor(&outputTensors[i], resultTensor, library, customNodeLibraryInternalManager); + if (outputTensors[i].name == nullptr) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; failed tensor conversion - missing output name", getName(), getSessionKey()); + status = StatusCode::NODE_LIBRARY_OUTPUT_MISSING_NAME; + continue; + } + if (!result.ok()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; failed to convert {}: to tensor", getName(), getSessionKey(), outputTensors[i].name); + if (status.ok()) { + status = result; + } + continue; + } + this->resultTensors.emplace(std::string(outputTensors[i].name), std::move(resultTensor)); + } + + library.release(outputTensors, customNodeLibraryInternalManager); + notifyEndQueue.push({node, getSessionKey()}); + return status; +} + +Status CustomNodeSession::fetchResult(const std::string& name, ov::Tensor& resultTensor) { + auto it = resultTensors.find(name); + if (it == resultTensors.end()) { + return StatusCode::NODE_LIBRARY_MISSING_OUTPUT; + } + resultTensor = it->second; + return StatusCode::OK; +} + +void CustomNodeSession::releaseTensorResources(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) { + if (tensor->data) { + library.release(tensor->data, customNodeLibraryInternalManager); + } + if (tensor->dims) { + library.release(tensor->dims, customNodeLibraryInternalManager); + } +} + +class TensorResourcesGuard { + const struct CustomNodeTensor* tensor; + const NodeLibrary& library; + bool persistData = false; + void* customNodeLibraryInternalManager; + +public: + TensorResourcesGuard(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) : + tensor(tensor), + library(library), + customNodeLibraryInternalManager(customNodeLibraryInternalManager) {} + ~TensorResourcesGuard() { + if (tensor->data && !persistData) { + library.release(tensor->data, customNodeLibraryInternalManager); + } + if (tensor->dims) { + library.release(tensor->dims, customNodeLibraryInternalManager); + } + } + void setPersistData() { + this->persistData = true; + } +}; + +Status CustomNodeSession::createTensor(const struct CustomNodeTensor* tensor, ov::Tensor& resultTensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) { + TensorResourcesGuard tensorResourcesGuard(tensor, library, customNodeLibraryInternalManager); + + auto precision = ovmsPrecisionToIE2Precision(toInferenceEnginePrecision(tensor->precision)); + if (precision == ov::element::Type_t::undefined) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; Unspecified output precision:{} from custom node tensor: {}", + this->getName(), + this->getSessionKey(), + precision, + tensor->name); + return StatusCode::NODE_LIBRARY_INVALID_PRECISION; + } + + if (tensor->dims == nullptr || tensor->dimsCount == 0) { + std::string error; + if (tensor->dims == nullptr) { + error = "shape handle is null"; + } else if (tensor->dimsCount == 0) { + error = "shape dimensions number is equal to 0"; + } + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; error: {}", + this->getName(), + this->getSessionKey(), + error); + return StatusCode::NODE_LIBRARY_INVALID_SHAPE; + } + shape_t shape(tensor->dims, tensor->dims + tensor->dimsCount); + + size_t expectedElementsCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies()); + size_t expectedDataLength = expectedElementsCount *= ov::element::Type(precision).size(); + if (tensor->data == nullptr || tensor->dataBytes != expectedDataLength) { + std::stringstream error; + if (tensor->data == nullptr) { + error << "data handle is null"; + } else if (tensor->dataBytes != expectedDataLength) { + error << "not expected data length: expected: " << expectedDataLength << " vs " << tensor->dataBytes; + } + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; error: {}", + this->getName(), + this->getSessionKey(), + error.str()); + return StatusCode::NODE_LIBRARY_INVALID_CONTENT_SIZE; + } + auto allocatorImpl = std::make_shared(*tensor, library, customNodeLibraryInternalManager); + auto allocator = ov::Allocator(allocatorImpl); + try { + switch (tensor->precision) { + case CustomNodeTensorPrecision::FP32: + case CustomNodeTensorPrecision::I32: + case CustomNodeTensorPrecision::I8: + case CustomNodeTensorPrecision::U8: + case CustomNodeTensorPrecision::FP16: + case CustomNodeTensorPrecision::I16: + case CustomNodeTensorPrecision::U16: + case CustomNodeTensorPrecision::FP64: + case CustomNodeTensorPrecision::I64: + resultTensor = ov::Tensor(ov::element::Type(ovmsPrecisionToIE2Precision(toInferenceEnginePrecision(tensor->precision))), ov::Shape(shape), allocator); + break; + case CustomNodeTensorPrecision::UNSPECIFIED: + return StatusCode::INTERNAL_ERROR; + } + } catch (const ov::Exception& e) { + Status status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_LOGGER_ERROR(dag_executor_logger, "{}: {}", status.string(), e.what()); + return status; + } catch (std::logic_error& e) { + Status status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_LOGGER_ERROR(dag_executor_logger, "{}: {}", status.string(), e.what()); + return status; + } + tensorResourcesGuard.setPersistData(); + return StatusCode::OK; +} + +void CustomNodeSession::clearInputs() { + this->inputHandler->clearInputs(); +} + +void CustomNodeSession::release() { +} + +} // namespace ovms diff --git a/src/ovms_lib/customnodesession.hpp b/src/ovms_lib/customnodesession.hpp new file mode 100644 index 0000000000..2c0db19b85 --- /dev/null +++ b/src/ovms_lib/customnodesession.hpp @@ -0,0 +1,59 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include + +#include "custom_node_interface.h" // NOLINT +#include "nodesession.hpp" +#include "pipelineeventqueue.hpp" +#include "status.hpp" +#include "tensormap.hpp" + +namespace ovms { + +class Node; +class NodeLibrary; + +class CustomNodeSession : public NodeSession { + TensorMap resultTensors; + +public: + CustomNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); + CustomNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); + virtual ~CustomNodeSession(); + + Status execute( + PipelineEventQueue& notifyEndQueue, + Node& node, + const NodeLibrary& library, + std::unique_ptr& parameters, + int parametersCount, + void* customNodeLibraryInternalManager); + + Status fetchResult(const std::string& name, ov::Tensor& resultTensor); + + void clearInputs(); + void release() override; + +private: + static void releaseTensorResources(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager); + Status createTensor(const struct CustomNodeTensor* tensor, ov::Tensor& resultTensor, const NodeLibrary& library, void* customNodeLibraryInternalManager); +}; +} // namespace ovms diff --git a/src/ovms_lib/deserialization.cpp b/src/ovms_lib/deserialization.cpp new file mode 100644 index 0000000000..be4e5a6111 --- /dev/null +++ b/src/ovms_lib/deserialization.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "deserialization.hpp" + +namespace ovms { + +template <> +Status InputSink::give(const std::string& name, ov::Tensor& tensor) { + Status status; + try { + requester.set_tensor(name, tensor); + // OV implementation the ov::Exception is not + // a base class for all other exceptions thrown from OV. + // OV can throw exceptions derived from std::logic_error. + } catch (const ov::Exception& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } catch (std::logic_error& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } + + return status; +} + +ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, + const std::shared_ptr& tensorInfo) { + ov::Shape shape; + for (size_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { + shape.push_back(requestInput.tensor_shape().dim(i).size()); + } + ov::element::Type precision = tensorInfo->getOvPrecision(); + return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.tensor_content().data()))); +} + +ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo, + const std::string& buffer) { + ov::Shape shape; + for (size_t i = 0; i < requestInput.shape_size(); i++) { + shape.push_back(requestInput.shape().at(i)); + } + ov::element::Type precision = tensorInfo->getOvPrecision(); + ov::Tensor tensor(precision, shape, const_cast(reinterpret_cast(buffer.data()))); + return tensor; +} +ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo) { + ov::Shape shape; + for (size_t i = 0; i < requestInput.shape_size(); i++) { + shape.push_back(requestInput.shape().at(i)); + } + ov::element::Type precision = tensorInfo->getOvPrecision(); + ov::Tensor tensor(precision, shape); + return tensor; +} + +} // namespace ovms diff --git a/src/ovms_lib/deserialization.hpp b/src/ovms_lib/deserialization.hpp new file mode 100644 index 0000000000..6cb7f666fd --- /dev/null +++ b/src/ovms_lib/deserialization.hpp @@ -0,0 +1,405 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" + +#include "kfs_grpc_inference_service.hpp" +#pragma GCC diagnostic pop + +#include "binaryutils.hpp" +#include "profiler.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, + const std::shared_ptr& tensorInfo); + +ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo, + const std::string& buffer); +ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo); + +class ConcreteTensorProtoDeserializator { +public: + static ov::Tensor deserializeTensorProto( + const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo, + const std::string* buffer) { + if (nullptr != buffer) { + switch (tensorInfo->getPrecision()) { + case ovms::Precision::FP64: + case ovms::Precision::FP32: + case ovms::Precision::FP16: + case ovms::Precision::I64: + case ovms::Precision::I32: + case ovms::Precision::I16: + case ovms::Precision::I8: + case ovms::Precision::U64: + case ovms::Precision::U32: + case ovms::Precision::U16: + case ovms::Precision::BOOL: + case ovms::Precision::U8: { + return makeTensor(requestInput, tensorInfo, *buffer); + } + case ovms::Precision::U1: + case ovms::Precision::CUSTOM: + case ovms::Precision::UNDEFINED: + case ovms::Precision::DYNAMIC: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + default: + return ov::Tensor(); + } + } else { + switch (tensorInfo->getPrecision()) { + // bool_contents + case ovms::Precision::BOOL: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + bool* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().bool_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + /// int_contents + case ovms::Precision::I8: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + int8_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().int_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + case ovms::Precision::I16: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + int16_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().int_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + case ovms::Precision::I32: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + int32_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().int_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + /// int64_contents + case ovms::Precision::I64: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + int64_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().int64_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + // uint_contents + case ovms::Precision::U8: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + uint8_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().uint_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + case ovms::Precision::U16: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + uint16_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().uint_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + case ovms::Precision::U32: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + uint32_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().uint_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + // uint64_contents + case ovms::Precision::U64: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + uint64_t* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().uint64_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + // fp32_contents + case ovms::Precision::FP32: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + float* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().fp32_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + // fp64_contentes + case ovms::Precision::FP64: { + ov::Tensor tensor = makeTensor(requestInput, tensorInfo); + double* ptr = reinterpret_cast(tensor.data()); + size_t i = 0; + for (auto& number : requestInput.contents().fp64_contents()) { + ptr[i++] = *(const_cast(reinterpret_cast(&number))); + } + return tensor; + break; + } + case ovms::Precision::FP16: + case ovms::Precision::U1: + case ovms::Precision::CUSTOM: + case ovms::Precision::UNDEFINED: + case ovms::Precision::DYNAMIC: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + default: + return ov::Tensor(); + } + } + } + + static ov::Tensor deserializeTensorProto( + const tensorflow::TensorProto& requestInput, + const std::shared_ptr& tensorInfo) { + switch (tensorInfo->getPrecision()) { + case ovms::Precision::FP32: + case ovms::Precision::I32: + case ovms::Precision::FP64: + case ovms::Precision::I64: + case ovms::Precision::U8: + case ovms::Precision::I16: + case ovms::Precision::I8: { + return makeTensor(requestInput, tensorInfo); + } + case ovms::Precision::FP16: { + ov::Shape shape; + for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { + shape.push_back(requestInput.tensor_shape().dim(i).size()); + } + ov::Tensor tensor(ov::element::f16, shape); + // Needs conversion due to zero padding for each value: + // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 + uint16_t* ptr = (uint16_t*)tensor.data(); + auto size = static_cast(requestInput.half_val_size()); + for (size_t i = 0; i < size; i++) { + ptr[i] = requestInput.half_val(i); + } + return tensor; + } + case ovms::Precision::U16: { + ov::Shape shape; + for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { + shape.push_back(requestInput.tensor_shape().dim(i).size()); + } + ov::Tensor tensor(ov::element::u16, shape); + // Needs conversion due to zero padding for each value: + // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 + uint16_t* ptr = (uint16_t*)tensor.data(); + auto size = static_cast(requestInput.int_val_size()); + for (size_t i = 0; i < size; i++) { + ptr[i] = requestInput.int_val(i); + } + return tensor; + } + case ovms::Precision::U32: + case ovms::Precision::U64: + default: + return ov::Tensor(); + } + } +}; + +template +ov::Tensor deserializeTensorProto( + const tensorflow::TensorProto& requestInput, + const std::shared_ptr& tensorInfo) { + return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo); +} + +template +ov::Tensor deserializeTensorProto( + const ::inference::ModelInferRequest::InferInputTensor& requestInput, + const std::shared_ptr& tensorInfo, + const std::string* buffer) { + return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo, buffer); +} + +template +class InputSink { + Requester requester; + +public: + InputSink(Requester requester) : + requester(requester) {} + Status give(const std::string& name, ov::Tensor& tensor); +}; + +template +Status deserializePredictRequest( + const tensorflow::serving::PredictRequest& request, + const tensor_map_t& inputMap, + Sink& inputSink, bool isPipeline) { + OVMS_PROFILE_FUNCTION(); + Status status; + for (const auto& pair : inputMap) { + try { + const auto& name = pair.first; + auto tensorInfo = pair.second; + auto requestInputItr = request.inputs().find(name); + if (requestInputItr == request.inputs().end()) { + SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); + return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); + } + auto& requestInput = requestInputItr->second; + ov::Tensor tensor; + + if (requestInput.dtype() == tensorflow::DataType::DT_STRING) { + SPDLOG_DEBUG("Request contains binary input: {}", name); + status = convertBinaryRequestTensorToOVTensor(requestInput, tensor, tensorInfo); + if (!status.ok()) { + SPDLOG_DEBUG("Binary inputs conversion failed."); + return status; + } + } else { + tensor = deserializeTensorProto( + requestInput, tensorInfo); + } + + if (!tensor) { + status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; + SPDLOG_DEBUG(status.string()); + return status; + } + const std::string ovTensorName = isPipeline ? name : tensorInfo->getName(); + status = inputSink.give(ovTensorName, tensor); + if (!status.ok()) { + SPDLOG_DEBUG("Feeding input:{} to inference performer failed:{}", ovTensorName, status.string()); + return status; + } + // OV implementation the ov::Exception is not + // a base class for all other exceptions thrown from OV. + // OV can throw exceptions derived from std::logic_error. + } catch (const ov::Exception& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } catch (std::logic_error& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } + } + return status; +} + +template +Status deserializePredictRequest( + const ::inference::ModelInferRequest& request, + const tensor_map_t& inputMap, + Sink& inputSink, bool isPipeline) { + OVMS_PROFILE_FUNCTION(); + Status status; + bool deserializeFromSharedInputContents = request.raw_input_contents().size() > 0; + for (const auto& pair : inputMap) { + try { + const auto& name = pair.first; + auto tensorInfo = pair.second; + auto requestInputItr = std::find_if(request.inputs().begin(), request.inputs().end(), [&name](const ::inference::ModelInferRequest::InferInputTensor& tensor) { return tensor.name() == name; }); + if (requestInputItr == request.inputs().end()) { + SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); + return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); + } + ov::Tensor tensor; + + if (requestInputItr->datatype() == "BYTES") { + SPDLOG_DEBUG("Request contains binary input: {}", name); + status = convertBinaryRequestTensorToOVTensor(*requestInputItr, tensor, tensorInfo); + if (!status.ok()) { + SPDLOG_DEBUG("Binary inputs conversion failed."); + return status; + } + } else { + auto inputIndex = requestInputItr - request.inputs().begin(); + auto bufferLocation = deserializeFromSharedInputContents ? &request.raw_input_contents()[inputIndex] : nullptr; + + tensor = deserializeTensorProto(*requestInputItr, tensorInfo, bufferLocation); + if (!tensor) { + status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; + SPDLOG_DEBUG(status.string()); + return status; + } + } + + const std::string ovTensorName = isPipeline ? name : tensorInfo->getName(); + status = inputSink.give(ovTensorName, tensor); + if (!status.ok()) { + SPDLOG_DEBUG("Feeding input:{} to inference performer failed:{}", ovTensorName, status.string()); + return status; + } + // OV implementation the ov::Exception is not + // a base class for all other exceptions thrown from OV. + // OV can throw exceptions derived from std::logic_error. + } catch (const ov::Exception& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } catch (std::logic_error& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } + } + return status; +} +} // namespace ovms diff --git a/src/ovms_lib/dl_node.cpp b/src/ovms_lib/dl_node.cpp new file mode 100644 index 0000000000..fd6f8ce2e4 --- /dev/null +++ b/src/ovms_lib/dl_node.cpp @@ -0,0 +1,134 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "dl_node.hpp" + +#include +#include + +#include "dlnodesession.hpp" +#include "logging.hpp" +#include "modelmanager.hpp" +#include "ov_utils.hpp" +#include "ovinferrequestsqueue.hpp" +#include "prediction_service_utils.hpp" +#include "timer.hpp" + +namespace ovms { + +const uint WAIT_FOR_STREAM_ID_TIMEOUT_MICROSECONDS = 1; + +Status DLNode::execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) { + auto& nodeSession = getNodeSession(sessionKey); + auto& dlNodeSession = static_cast(nodeSession); + return dlNodeSession.execute(notifyEndQueue, WAIT_FOR_STREAM_ID_TIMEOUT_MICROSECONDS, *this); +} + +Status DLNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { + auto& dlNodeSession = static_cast(nodeSession); + const auto& sessionMetadata = nodeSession.getNodeSessionMetadata(); + SessionResult sessionResults{sessionMetadata, {}}; + auto it = nodeSessionOutputs.emplace(sessionMetadata.getSessionKey(), std::move(sessionResults)); + if (!it.second) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to put node: {} session: {} results in node session outputs", + getName(), nodeSession.getSessionKey()); + return StatusCode::INTERNAL_ERROR; + } + auto& metadataTensorResultsPair = it.first->second; + auto& tensorResults = metadataTensorResultsPair.second; + Status status; + const uint waitTimeMicroseconds = 1; + auto& inferRequest = dlNodeSession.getInferRequest(waitTimeMicroseconds); + auto& model = dlNodeSession.getModelInstance(); + status = this->fetchResults(tensorResults, inferRequest, model, nodeSession.getSessionKey()); + return status; +} + +Status DLNode::fetchResults(TensorWithSourceMap& outputs, ov::InferRequest& inferRequest, ModelInstance& model, session_key_t sessionKey) { + ReleaseSessionGuard releaseSessionGuard(this->getNodeSession(sessionKey)); + // Wait for tensor results + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Waiting for infer request to finish", getName(), sessionKey); + try { + inferRequest.wait(); + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} IE exception occured during infer request wait: {}", getName(), sessionKey, e.what()); + return StatusCode::INTERNAL_ERROR; + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} exception occured during infer request wait: {}", getName(), sessionKey, e.what()); + return StatusCode::INTERNAL_ERROR; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} infer request finished", getName(), sessionKey); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Inference processing time for node {}; model name: {}; session: {} - {} ms", + this->getName(), + model.getName(), + sessionKey, + this->getNodeSession(sessionKey).getTimer().elapsed("inference") / 1000); + + static_cast(this->getNodeSession(sessionKey)).clearInputs(); + + // Fill outputs map with result tensors. Fetch only those that are required in following nodes. + for (const auto& node : this->next) { + for (const auto& pair : node.get().getMappingByDependency(*this)) { + const auto& output_name = pair.first; + if (outputs.find(output_name) != outputs.end()) { + continue; + } + + try { + std::string realModelOutputName; + if (!getRealOutputName(model, output_name, &realModelOutputName).ok()) { + SPDLOG_LOGGER_WARN(dag_executor_logger, "Node: {} session: {} Cannot find real model output name for alias: {}", getName(), sessionKey, output_name); + return StatusCode::INTERNAL_ERROR; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Getting tensor from model: {}, inferRequestStreamId: {}, tensorName: {}", + getName(), sessionKey, modelName, sessionKey, realModelOutputName); + const auto tensor = inferRequest.get_tensor(realModelOutputName); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Creating copy of tensor from model: {}, tensorName: {}", + getName(), sessionKey, modelName, realModelOutputName); + ov::Tensor copiedTensor; + auto status = tensorClone(copiedTensor, tensor); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Could not clone result tensor; node: {}; session: {}; model name: {}; output: {}", + getName(), + this->modelName, + realModelOutputName); + return status; + } + outputs.emplace(std::make_pair(output_name, TensorWithSource(std::move(copiedTensor)))); + } catch (const ov::Exception& e) { + Status status = StatusCode::OV_INTERNAL_SERIALIZATION_ERROR; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session:{} Error during getting tensor {}; exception message: {}", getName(), sessionKey, status.string(), e.what()); + return status; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Tensor with name {} has been prepared", getName(), sessionKey, output_name); + } + } + return StatusCode::OK; +} + +void DLNode::release(session_key_t sessionId) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Release node: {} sessionKey: {}", getName(), sessionId); + getNodeSession(sessionId).release(); +} +bool DLNode::tryDisarm(const session_key_t& sessionKey, const uint microseconds) { + return getNodeSession(sessionKey).tryDisarm(microseconds); +} + +std::unique_ptr DLNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { + return std::make_unique(metadata, getName(), previous.size(), collapsingDetails, + this->modelManager, this->modelName, this->modelVersion.value_or(0)); +} + +} // namespace ovms diff --git a/src/ovms_lib/dl_node.hpp b/src/ovms_lib/dl_node.hpp new file mode 100644 index 0000000000..a0b4f219bf --- /dev/null +++ b/src/ovms_lib/dl_node.hpp @@ -0,0 +1,98 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "executingstreamidguard.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelversion.hpp" +#include "node.hpp" +#include "nodestreamidguard.hpp" + +namespace ovms { + +class ModelManager; + +class DLNode : public Node { +protected: + std::string modelName; + std::optional modelVersion; + ModelManager& modelManager; + const std::unordered_map nodeOutputNameAlias; + + std::shared_ptr model; + std::unique_ptr nodeStreamIdGuard; + std::unique_ptr modelUnloadGuard; + +public: + DLNode(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, + ModelManager& modelManager, + std::unordered_map nodeOutputNameAlias = {}, + std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : + Node(nodeName, demultiplyCount, gatherFromNode), + modelName(modelName), + modelVersion(modelVersion), + modelManager(modelManager), + nodeOutputNameAlias(nodeOutputNameAlias) { + } + + Status execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) override; + + Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; + +private: + Status fetchResults(TensorWithSourceMap& outputs, ov::InferRequest& inferRequest, ModelInstance& model, session_key_t sessionKey); + +public: + void release(session_key_t sessionId) override; + +private: + Status getRealInputName(ModelInstance& model, const std::string& alias, std::string* result) const { + auto it = model.getInputsInfo().find(alias); + if (it == model.getInputsInfo().end()) { + return StatusCode::INVALID_MISSING_INPUT; + } + *result = it->second->getName(); + return StatusCode::OK; + } + + Status getRealOutputName(ModelInstance& model, const std::string& alias, std::string* result) const { + auto it = nodeOutputNameAlias.find(alias); + const auto& modelOutputName = it != nodeOutputNameAlias.end() ? it->second : alias; + auto jt = model.getOutputsInfo().find(modelOutputName); + if (jt == model.getOutputsInfo().end()) { + return StatusCode::INVALID_MISSING_OUTPUT; + } + *result = jt->second->getName(); + return StatusCode::OK; + } + + Status executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest& infer_request); + bool tryDisarm(const session_key_t& sessionKey, const uint microseconds = 1) override; + +protected: + std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; +}; + +} // namespace ovms diff --git a/src/ovms_lib/dlnodesession.cpp b/src/ovms_lib/dlnodesession.cpp new file mode 100644 index 0000000000..abc0d37721 --- /dev/null +++ b/src/ovms_lib/dlnodesession.cpp @@ -0,0 +1,345 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "dlnodesession.hpp" + +#include +#include + +#include "logging.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "nodeinputhandler.hpp" +#include "nodeoutputhandler.hpp" +#include "nodestreamidguard.hpp" +#include "ov_utils.hpp" +#include "profiler.hpp" +#include "shape.hpp" +#include "tensorinfo.hpp" +#include "timer.hpp" + +namespace ovms { +DLNodeSession::DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : + NodeSession(metadata, nodeName, inputsCount, collapsingDetails), + modelManager(manager), + modelName(modelName), + modelVersion(modelVersion) {} + +DLNodeSession::DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : + NodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails), + modelManager(manager), + modelName(modelName), + modelVersion(modelVersion) {} + +DLNodeSession::~DLNodeSession() = default; + +void DLNodeSession::clearInputs() { + this->inputHandler->clearInputs(); +} + +ModelInstance& DLNodeSession::getModelInstance() { + return *this->model; +} + +ov::InferRequest& DLNodeSession::getInferRequest(const uint microseconds) { + auto& inferRequestsQueue = this->model->getInferRequestsQueue(); + auto streamIdOpt = this->nodeStreamIdGuard->tryGetId(microseconds); + if (!streamIdOpt) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to get streamId on already executed node: {} session: {}", getName(), getSessionKey()); + throw std::logic_error("Stream id is empty on already executed node"); + } + return inferRequestsQueue.getInferRequest(streamIdOpt.value()); +} + +Status DLNodeSession::requestExecuteRequiredResources() { + OVMS_PROFILE_FUNCTION(); + Status status = modelManager.getModelInstance( + modelName, + modelVersion, + this->model, + this->modelUnloadGuard); + + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Getting modelInstance failed for node: {} session: {} with: {}", getName(), getSessionKey(), status.string()); + return status; + } + + status = prepareInputsAndModelForInference(); + if (!status.ok()) { + return status; + } + this->nodeStreamIdGuard = std::make_unique(model->getInferRequestsQueue()); + return status; +} + +Status DLNodeSession::prepareInputsAndModelForInference() { + OVMS_PROFILE_FUNCTION(); + std::optional requestedBatchSize = std::nullopt; + std::map requestedReshapes; + + // Validate each tensor against its OV tensor info + const auto& inputsInfo = this->model->getInputsInfo(); + for (const auto& kv : this->inputHandler->getInputs()) { + const auto& name = kv.first; + auto& tensor = kv.second; + + auto it = inputsInfo.find(name); + if (it == inputsInfo.end()) { + std::stringstream ss; + ss << "Required input: " << name; + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Missing input with specific name - {}", getName(), details); + return Status(StatusCode::INVALID_MISSING_INPUT, details); + } + auto& inputInfo = *it->second; + auto status = validate(tensor, inputInfo); + if (status.ok()) { + continue; + } + + // If precision is incorrect, perform conversion + if (status == StatusCode::INVALID_PRECISION) { + return status; + } + + // If batch size is incorrect, perform model batch size change if allowed (shape mode=auto or batch size=auto) + if (status == StatusCode::INVALID_BATCH_SIZE) { + if (this->model->getModelConfig().getBatchingMode() == Mode::AUTO) { + requestedBatchSize = tensor.get_shape()[0]; + } else if (this->model->getModelConfig().isShapeAuto(name)) { + requestedReshapes[name] = tensor.get_shape(); + } else { + return status; + } + } + + // If shape is incorrect, perform reshape if allowed (mode=auto) + if (status == StatusCode::INVALID_SHAPE) { + if (!this->model->getModelConfig().isShapeAuto(name)) { + return status; + } + requestedReshapes[name] = tensor.get_shape(); + } + } + if (requestedReshapes.size() > 0) { + auto status = this->model->reloadModel(std::nullopt, requestedReshapes, this->modelUnloadGuard); + if (!status.ok()) { + return status; + } + } else if (requestedBatchSize.has_value()) { + auto status = this->model->reloadModel(requestedBatchSize, {}, this->modelUnloadGuard); + if (!status.ok()) { + return status; + } + } + return StatusCode::OK; +} + +Status DLNodeSession::validate(const ov::Tensor& tensor, const TensorInfo& tensorInfo) { + OVMS_PROFILE_FUNCTION(); + if (ovmsPrecisionToIE2Precision(tensorInfo.getPrecision()) != tensor.get_element_type()) { + std::stringstream ss; + ss << "Node: " << getName() << " input: " << tensorInfo.getName() + << " Invalid precision -" + << " Expected: " << tensorInfo.getPrecisionAsString() + << "; Actual: " << toString(ovElementTypeToOvmsPrecision(tensor.get_element_type())); + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); + return Status(StatusCode::INVALID_PRECISION, details); + } + + // If batch size differs, check if remaining dimensions are equal + const auto& dims = tensor.get_shape(); + const auto batchIndex = tensorInfo.getLayout().getBatchIndex(); + if (!batchIndex.has_value() || batchIndex.value() >= tensorInfo.getShape().size() || batchIndex.value() >= dims.size()) { + std::stringstream ss; + ss << "Node: " << getName() << " input: " << tensorInfo.getName() + << " Invalid batch size index"; + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); + return Status(StatusCode::INVALID_BATCH_DIMENSION, details); + } + if (!tensorInfo.getShape()[batchIndex.value()].match(dims[batchIndex.value()])) { + // If remaining dimensions are equal, it is invalid batch size + std::stringstream ss; + if (tensorInfo.getShape().match(dims, batchIndex.value())) { + ss << "Node: " << getName() << " input: " << tensorInfo.getName() + << " Invalid batch size -" + << " Expected: " << tensorInfo.getShape()[batchIndex.value()].toString() + << "; Actual: " << dims[batchIndex.value()]; + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } else { + // Otherwise whole shape is incorrect + ss << "Node: " << getName() << " input: " << tensorInfo.getName() + << " Invalid shape -" + << " Expected: " << tensorInfo.getShape().toString() + << "; Actual: " << TensorInfo::shapeToString(dims); + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); + return Status(StatusCode::INVALID_SHAPE, details); + } + } + + if (!tensorInfo.getShape().match(dims)) { + std::stringstream ss; + ss << "Node: " << getName() << " input: " << tensorInfo.getName() + << " Invalid shape -" + << " Expected: " << tensorInfo.getShape().toString() + << "; Actual: " << TensorInfo::shapeToString(dims); + const std::string details = ss.str(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); + return Status(StatusCode::INVALID_SHAPE, details); + } + + return StatusCode::OK; +} + +Status DLNodeSession::execute(PipelineEventQueue& notifyEndQueue, uint waitForStreamIdTimeoutMicroseconds, Node& node) { + OVMS_PROFILE_FUNCTION(); + Status status; + if (this->nodeStreamIdGuard == nullptr) { + status = requestExecuteRequiredResources(); + if (!status.ok()) { + notifyEndQueue.push({node, getSessionKey()}); + return status; + } + } + auto streamIdOpt = this->nodeStreamIdGuard->tryGetId(waitForStreamIdTimeoutMicroseconds); + if (!streamIdOpt) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Could not acquire stream Id right away", getName()); + return StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET; + } + auto& inferRequestsQueue = this->model->getInferRequestsQueue(); + auto& inferRequest = inferRequestsQueue.getInferRequest(streamIdOpt.value()); + status = setInputsForInference(inferRequest); + if (!status.ok()) { + notifyEndQueue.push({node, getSessionKey()}); + return status; + } + status = executeInference(notifyEndQueue, inferRequest, node); + if (!status.ok()) { + notifyEndQueue.push({node, getSessionKey()}); + return status; + } + return status; +} + +Status DLNodeSession::getRealInputName(const std::string& alias, std::string* result) const { + auto it = this->model->getInputsInfo().find(alias); + if (it == this->model->getInputsInfo().end()) { + return StatusCode::INVALID_MISSING_INPUT; + } + *result = it->second->getName(); + return StatusCode::OK; +} + +Status DLNodeSession::setInputsForInference(ov::InferRequest& inferRequest) { + OVMS_PROFILE_FUNCTION(); + Status status = StatusCode::OK; + try { + // Prepare inference request, fill with input tensors + for (const auto& [name, tensor] : this->inputHandler->getInputs()) { + std::string realModelInputName; + if (!getRealInputName(name, &realModelInputName).ok()) { + SPDLOG_LOGGER_WARN(dag_executor_logger, "DLNode::{} [Node name: {}]; cannot find real model input name for alias: {}", + __FUNCTION__, getName(), name); + return StatusCode::INTERNAL_ERROR; + } + // Workaround for GPU. + if (this->model->getModelConfig().isDeviceUsed("GPU")) { + ov::Tensor clonedTensor; + status = ovms::tensorClone(clonedTensor, tensor); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor clone error: {}", getName(), status.string()); + return status; + } + OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::set_tensor"); + inferRequest.set_tensor(realModelInputName, clonedTensor); + OVMS_PROFILE_SYNC_END("ov::InferRequest::set_tensor"); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor name: {} cloned before GPU inference", getName(), name); + } else { + OVMS_PROFILE_SCOPE("ov::InferRequest::set_tensor"); + inferRequest.set_tensor(realModelInputName, tensor); + } + } + // OV implementation the ov::Exception is not + // a base class for all other exceptions thrown from OV. + // OV can throw exceptions derived from std::logic_error. + } catch (const ov::Exception& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; exception message: {}", getName(), status.string(), e.what()); + } catch (std::logic_error& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; exception message: {}", getName(), status.string(), e.what()); + } catch (...) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; with unknown exception", getName(), status.string()); + } + return status; +} + +Status DLNodeSession::executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest& inferRequest, Node& node) { + OVMS_PROFILE_FUNCTION(); + try { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Setting completion callback for node name: {}", this->getName()); + inferRequest.set_callback([this, ¬ifyEndQueue, &inferRequest, &node](std::exception_ptr exception_ptr) { + OVMS_PROFILE_ASYNC_END("async inference", this); + this->timer->stop("inference"); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Completion callback received for node name: {}", this->getName()); + // After inference is completed, input tensors are not needed anymore + this->inputHandler->clearInputs(); + notifyEndQueue.push({node, getSessionKey()}); + inferRequest.set_callback([](std::exception_ptr exception_ptr) {}); // reset callback on infer request + }); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Starting infer async for node name: {}", getName()); + this->timer->start("inference"); + OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::start_async"); + inferRequest.start_async(); + OVMS_PROFILE_SYNC_END("ov::InferRequest::start_async"); + OVMS_PROFILE_ASYNC_BEGIN("async inference", this); + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Exception occured when starting async inference or setting completion callback on model: {}, error: {}", + getName(), getModelName(), e.what()); + return StatusCode::OV_INTERNAL_INFERENCE_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Exception occured when starting async inference or setting completion callback on model: {}, error: {}", + getName(), getModelName(), e.what()); + return StatusCode::OV_INTERNAL_INFERENCE_ERROR; + } catch (...) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Unknown exception occured when starting async inference or setting completion callback on model: {}", + getName(), getModelName()); + return StatusCode::OV_INTERNAL_INFERENCE_ERROR; + } + return StatusCode::OK; +} + +void DLNodeSession::release() { + this->nodeStreamIdGuard.reset(); + this->model.reset(); + this->modelUnloadGuard.reset(); +} + +bool DLNodeSession::tryDisarm(uint microseconds) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to disarm stream id guard of node: {}", getName()); + if (this->nodeStreamIdGuard == nullptr) { + return true; + } + return this->nodeStreamIdGuard->tryDisarm(microseconds); +} +} // namespace ovms diff --git a/src/ovms_lib/dlnodesession.hpp b/src/ovms_lib/dlnodesession.hpp new file mode 100644 index 0000000000..ca9a798a3e --- /dev/null +++ b/src/ovms_lib/dlnodesession.hpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include + +#include "modelversion.hpp" +#include "nodesession.hpp" +#include "pipelineeventqueue.hpp" +#include "status.hpp" + +namespace ovms { + +class ModelManager; +class ModelInstance; +class Node; +class NodeStreamIdGuard; +class ModelInstanceUnloadGuard; +class TensorInfo; + +class DLNodeSession : public NodeSession { + std::shared_ptr model; + std::unique_ptr nodeStreamIdGuard; + std::unique_ptr modelUnloadGuard; + + ModelManager& modelManager; + const std::string& modelName; + const model_version_t modelVersion; + +public: + DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); + DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); + virtual ~DLNodeSession(); + + ov::InferRequest& getInferRequest(const uint microseconds); + ModelInstance& getModelInstance(); + +private: + Status requestExecuteRequiredResources(); + +public: + Status prepareInputsAndModelForInference(); + Status validate(const ov::Tensor& tensor, const TensorInfo& info); + Status execute(PipelineEventQueue& notifyEndQueue, uint waitForStreamIdTimeoutMicroseconds, Node& node); + Status executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest&, Node& node); + Status setInputsForInference(ov::InferRequest& inferRequest); + Status getRealInputName(const std::string& alias, std::string* result) const; + void release() override; + + void clearInputs(); + + const std::string& getModelName() { return modelName; } + bool tryDisarm(uint microseconds) override; +}; +} // namespace ovms diff --git a/src/ovms_lib/entry_node.cpp b/src/ovms_lib/entry_node.cpp new file mode 100644 index 0000000000..a01ceea217 --- /dev/null +++ b/src/ovms_lib/entry_node.cpp @@ -0,0 +1,177 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "entry_node.hpp" + +#include +#include +#include +#include +#include + +#include "binaryutils.hpp" +#include "deserialization.hpp" +#include "logging.hpp" +#include "ov_utils.hpp" +#include "predict_request_validation_utils.hpp" +#include "profiler.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +namespace ovms { + +template +Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) { + OVMS_PROFILE_FUNCTION(); + // this should be created in EntryNode::SetInputs, or special method for entry node called + // in event loop can be done in future release while implementing dynamic demultiplexing at + // entry node + NodeSessionMetadata metadata; + auto nodeSession = getNodeSession(metadata); // call to create session + if (!nodeSession) { + notifyEndQueue.push(NodeSessionKeyPair(*this, nodeSession->getSessionKey())); + return StatusCode::INTERNAL_ERROR; + } + notifyEndQueue.push(NodeSessionKeyPair(*this, nodeSession->getSessionKey())); + return StatusCode::OK; +} + +template +Status EntryNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { + OVMS_PROFILE_FUNCTION(); + TensorWithSourceMap outputs; + auto status = fetchResults(outputs); + if (!status.ok()) { + return status; + } + SessionResult metaOutputsPair{nodeSession.getNodeSessionMetadata(), std::move(outputs)}; + auto it = nodeSessionOutputs.emplace(nodeSession.getSessionKey(), std::move(metaOutputsPair)); + if (!it.second) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Failed to set entry node session results."); + return StatusCode::UNKNOWN_ERROR; + } + return StatusCode::OK; +} + +template +Status EntryNode::fetchResults(TensorWithSourceMap& outputs) { + auto status = validate(); + if (!status.ok()) { + return status; + } + InputSink inputSink(outputs); + bool isPipeline = true; + return deserializePredictRequest(*request, inputsInfo, inputSink, isPipeline); +} + +template <> +Status InputSink::give(const std::string& name, ov::Tensor& tensor) { + requester.emplace(std::make_pair(name, TensorWithSource(tensor))); + return StatusCode::OK; +} + +template <> +Status EntryNode::isInputBinary(const std::string& name, bool& isBinary) const { + auto it = request->inputs().find(name); + if (it == request->inputs().end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Error during checking binary input; input: {} does not exist", name); + return StatusCode::INTERNAL_ERROR; + } + isBinary = it->second.string_val_size() > 0; + return StatusCode::OK; +} +template <> +Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const { + auto it = request->inputs().begin(); + while (it != request->inputs().end()) { + if (it->name() == name) { + break; + } + ++it; + } + if (it == request->inputs().end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Error during checking binary input; input: {} does not exist", name); + return StatusCode::INTERNAL_ERROR; + } + isBinary = it->contents().bytes_contents_size() > 0; + return StatusCode::OK; +} + +template +Status EntryNode::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) { + bool isBinary = false; + auto status = this->isInputBinary(tensorName, isBinary); + if (!status.ok()) { + return status; + } + if (isBinary) { + return Node::createShardedTensor(dividedTensor, precision, shape, tensor, i, step, metadata, tensorName); + } + + // if condition is perf optimization + // when demultiplying from entry node from tensor content we can skip allocation for sharded tensors + // and reuse memory from original tensor since its memory is kept for whole duration of predict request + if ((precision == Precision::FP32) || + (precision == Precision::I32) || + (precision == Precision::FP64) || + (precision == Precision::I64) || + (precision == Precision::I8) || + (precision == Precision::U8) || + (precision == Precision::I16)) { + dividedTensor = createSharedTensor(ovmsPrecisionToIE2Precision(precision), shape, (void*)((char*)(tensor.data()) + i * step)); + } else { + return Node::createShardedTensor(dividedTensor, precision, shape, tensor, i, step, metadata, tensorName); + } + return StatusCode::OK; +} + +template <> +const Status EntryNode::validate() { + static const std::set optionalInputNames = {}; + return request_validation_utils::validate( + *request, + inputsInfo, + request->model_spec().name(), + 1, + optionalInputNames); // Pipelines are not versioned and always reports version 1 +} +template <> +const Status EntryNode<::inference::ModelInferRequest>::validate() { + static const std::set optionalInputNames = {}; + return request_validation_utils::validate( + *request, + inputsInfo, + request->model_name(), + 1, + optionalInputNames); // Pipelines are not versioned and always reports version 1 +} + +template Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status EntryNode<::inference::ModelInferRequest>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status EntryNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status EntryNode<::inference::ModelInferRequest>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status EntryNode::fetchResults(TensorWithSourceMap& outputs); +template Status EntryNode<::inference::ModelInferRequest>::fetchResults(TensorWithSourceMap& outputs); +template Status EntryNode::isInputBinary(const std::string& name, bool& isBinary) const; +template Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const; +template Status EntryNode::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); +template Status EntryNode<::inference::ModelInferRequest>::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); +template const Status EntryNode::validate(); +template const Status EntryNode<::inference::ModelInferRequest>::validate(); +} // namespace ovms diff --git a/src/ovms_lib/entry_node.hpp b/src/ovms_lib/entry_node.hpp new file mode 100644 index 0000000000..57c22f92a2 --- /dev/null +++ b/src/ovms_lib/entry_node.hpp @@ -0,0 +1,63 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include + +#include + +#include "logging.hpp" +#include "node.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +const std::string ENTRY_NODE_NAME = "request"; + +template +class EntryNode : public Node { + const RequestType* request; + const tensor_map_t inputsInfo; + +public: + EntryNode(const RequestType* request, + const tensor_map_t& inputsInfo, + std::optional demultiplyCount = std::nullopt) : + Node(ENTRY_NODE_NAME, demultiplyCount), + request(request), + inputsInfo(inputsInfo) {} + + Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override; + + Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; + +protected: + Status fetchResults(TensorWithSourceMap& outputs); + Status createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) override; + +public: + // Entry nodes have no dependency + void addDependency(Node&, const Aliases&) override { + throw std::logic_error("This node cannot have dependency"); + } + + Status isInputBinary(const std::string& name, bool& isBinary) const; + + const Status validate(); +}; + +} // namespace ovms diff --git a/src/ovms_lib/executingstreamidguard.hpp b/src/ovms_lib/executingstreamidguard.hpp new file mode 100644 index 0000000000..c5642afded --- /dev/null +++ b/src/ovms_lib/executingstreamidguard.hpp @@ -0,0 +1,39 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include "ovinferrequestsqueue.hpp" + +namespace ovms { + +struct ExecutingStreamIdGuard { + ExecutingStreamIdGuard(ovms::OVInferRequestsQueue& inferRequestsQueue) : + inferRequestsQueue_(inferRequestsQueue), + id_(inferRequestsQueue_.getIdleStream().get()), + inferRequest(inferRequestsQueue.getInferRequest(id_)) {} + ~ExecutingStreamIdGuard() { + inferRequestsQueue_.returnStream(id_); + } + int getId() { return id_; } + ov::InferRequest& getInferRequest() { return inferRequest; } + +private: + ovms::OVInferRequestsQueue& inferRequestsQueue_; + const int id_; + ov::InferRequest& inferRequest; +}; + +} // namespace ovms diff --git a/src/ovms_lib/exit_node.cpp b/src/ovms_lib/exit_node.cpp new file mode 100644 index 0000000000..5558b2fb1b --- /dev/null +++ b/src/ovms_lib/exit_node.cpp @@ -0,0 +1,77 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "exit_node.hpp" + +#include +#include + +#include "logging.hpp" +#include "ov_utils.hpp" +#include "serialization.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#pragma GCC diagnostic pop + +#include "exitnodesession.hpp" + +namespace ovms { +template +Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { + OVMS_PROFILE_FUNCTION(); + auto& exitNodeSession = static_cast&>(nodeSession); + return this->fetchResults(exitNodeSession.getInputTensors()); +} + +template +Status ExitNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) { + OVMS_PROFILE_FUNCTION(); + notifyEndQueue.push(NodeSessionKeyPair(*this, sessionId)); + return StatusCode::OK; +} + +template <> +Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { + auto it = outputSource.find(name); + if (it == outputSource.end()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Failed to find expected pipeline output when serializing response: {}", name); + return StatusCode::INTERNAL_ERROR; + } + tensor = it->second; + return StatusCode::OK; +} + +template +Status ExitNode::fetchResults(const TensorMap& inputTensors) { + OutputGetter outputGetter(inputTensors); + return serializePredictResponse(outputGetter, this->outputsInfo, this->response, getOutputMapKeyName); +} + +template +std::unique_ptr ExitNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { + return std::make_unique>(metadata, getName(), previous.size(), collapsingDetails, response); +} + +template Status ExitNode<::inference::ModelInferResponse>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); +template Status ExitNode<::inference::ModelInferResponse>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status ExitNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); +template Status ExitNode<::inference::ModelInferResponse>::fetchResults(const TensorMap& inputTensors); +template Status ExitNode::fetchResults(const TensorMap& inputTensors); +template std::unique_ptr ExitNode<::inference::ModelInferResponse>::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); +template std::unique_ptr ExitNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); +} // namespace ovms diff --git a/src/ovms_lib/exit_node.hpp b/src/ovms_lib/exit_node.hpp new file mode 100644 index 0000000000..d3500736f3 --- /dev/null +++ b/src/ovms_lib/exit_node.hpp @@ -0,0 +1,63 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "node.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +const std::string EXIT_NODE_NAME = "response"; + +template +class ExitNode : public Node { + ResponseType* response; + const tensor_map_t outputsInfo; + +public: + ExitNode(ResponseType* response, const tensor_map_t& outputsInfo, std::set gatherFromNode = {}) : + Node(EXIT_NODE_NAME, std::nullopt, gatherFromNode), + response(response), + outputsInfo(outputsInfo) { + } + + // Exit node does not have execute logic. + // It serializes its received input tensors to proto in ::fetchResults + Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override; + +protected: + Status fetchResults(const TensorMap& outputs); + +public: + Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; + + // Exit nodes have no dependants + void addDependant(Node& node) override { + throw std::logic_error("This node cannot have dependant"); + } + + std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; +}; + +} // namespace ovms diff --git a/src/ovms_lib/exitnodesession.cpp b/src/ovms_lib/exitnodesession.cpp new file mode 100644 index 0000000000..715d466608 --- /dev/null +++ b/src/ovms_lib/exitnodesession.cpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "exitnodesession.hpp" + +#include + +#include "gatherexitnodeinputhandler.hpp" + +namespace ovms { + +template +ExitNodeSession::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ResponseType* response) : + NodeSession(metadata, nodeName, inputsCount, collapsingDetails) { + if (collapsingDetails.collapsedSessionNames.size() != 0) { + this->inputHandler = std::make_unique>(inputsCount, collapsingDetails, response); + } +} + +template +const TensorMap& ExitNodeSession::getInputTensors() const { + return this->inputHandler->getInputs(); +} + +template +ExitNodeSession::~ExitNodeSession() = default; + +template ExitNodeSession::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, tensorflow::serving::PredictResponse* response); +template ExitNodeSession<::inference::ModelInferResponse>::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ::inference::ModelInferResponse* response); + +template const TensorMap& ExitNodeSession::getInputTensors() const; +template const TensorMap& ExitNodeSession<::inference::ModelInferResponse>::getInputTensors() const; + +} // namespace ovms diff --git a/src/ovms_lib/exitnodesession.hpp b/src/ovms_lib/exitnodesession.hpp new file mode 100644 index 0000000000..fb7aac0da2 --- /dev/null +++ b/src/ovms_lib/exitnodesession.hpp @@ -0,0 +1,37 @@ +//***************************************************************************** +// Copyright 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include + +#include "nodeinputhandler.hpp" +#include "nodesession.hpp" +#include "nodesessionmetadata.hpp" +#include "tensormap.hpp" + +namespace ovms { + +template +class ExitNodeSession : public NodeSession { +public: + ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ResponseType* response); + virtual ~ExitNodeSession(); + const TensorMap& getInputTensors() const; +}; + +} // namespace ovms diff --git a/src/ovms_lib/filesystem.hpp b/src/ovms_lib/filesystem.hpp new file mode 100644 index 0000000000..8f14be21c4 --- /dev/null +++ b/src/ovms_lib/filesystem.hpp @@ -0,0 +1,196 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include + +#include "model_version_policy.hpp" +#include "status.hpp" + +namespace ovms { + +namespace fs = std::filesystem; + +using files_list_t = std::set; +class FileSystem { +public: + /** + * @brief Destroy the File System object + * + */ + virtual ~FileSystem() {} + + /** + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ + virtual StatusCode fileExists(const std::string& path, bool* exists) = 0; + + /** + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ + virtual StatusCode isDirectory(const std::string& path, bool* is_dir) = 0; + + /** + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ + virtual StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) = 0; + + /** + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ + virtual StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) = 0; + + /** + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ + virtual StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) = 0; + + /** + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ + virtual StatusCode readTextFile(const std::string& path, std::string* contents) = 0; + + /** + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ + virtual StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) = 0; + + /** + * @brief Download model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ + virtual StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) = 0; + + /** + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ + + virtual StatusCode deleteFileFolder(const std::string& path) = 0; + /** + * @brief Create a Temp Path + * + * @param local_path + * @return StatusCode + */ + static StatusCode createTempPath(std::string* local_path) { + std::string file_template = "/tmp/fileXXXXXX"; + char* tmp_folder = mkdtemp(const_cast(file_template.c_str())); + if (tmp_folder == nullptr) { + SPDLOG_ERROR("Failed to create local temp folder: {} {}", file_template, strerror(errno)); + return StatusCode::FILESYSTEM_ERROR; + } + fs::permissions(tmp_folder, + fs::perms::others_all | fs::perms::group_all, + fs::perm_options::remove); + + *local_path = std::string(tmp_folder); + + return StatusCode::OK; + } + + static bool isPathEscaped(const std::string& path) { + return std::string::npos != path.find("../") || std::string::npos != path.find("/.."); + } + + std::string appendSlash(const std::string& name) { + if (name.empty() || (name.back() == '/')) { + return name; + } + + return (name + "/"); + } + + bool isAbsolutePath(const std::string& path) { + return !path.empty() && (path[0] == '/'); + } + + std::string joinPath(std::initializer_list segments) { + std::string joined; + + for (const auto& seg : segments) { + if (joined.empty()) { + joined = seg; + } else if (isAbsolutePath(seg)) { + if (joined[joined.size() - 1] == '/') { + joined.append(seg.substr(1)); + } else { + joined.append(seg); + } + } else { + if (joined[joined.size() - 1] != '/') { + joined.append("/"); + } + joined.append(seg); + } + } + + return joined; + } + + StatusCode CreateLocalDir(const std::string& path) { + int status = + mkdir(const_cast(path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); + if (status == -1) { + SPDLOG_ERROR("Failed to create local folder: {} {} ", path, strerror(errno)); + return StatusCode::PATH_INVALID; + } + return StatusCode::OK; + } + + static const std::vector acceptedFiles; +}; + +} // namespace ovms diff --git a/src/ovms_lib/gatherexitnodeinputhandler.cpp b/src/ovms_lib/gatherexitnodeinputhandler.cpp new file mode 100644 index 0000000000..08e73a81d1 --- /dev/null +++ b/src/ovms_lib/gatherexitnodeinputhandler.cpp @@ -0,0 +1,53 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "gatherexitnodeinputhandler.hpp" + +#include + +#include "logging.hpp" + +namespace ovms { + +Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& bufferOut, const std::string& name, size_t size) { + OVMS_PROFILE_FUNCTION(); + tensorflow::TensorProto tensorProto; + auto [it, isInserted] = response->mutable_outputs()->insert(google::protobuf::MapPair(name, std::move(tensorProto))); + if (!isInserted) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); + return StatusCode::INTERNAL_ERROR; + } + it->second.mutable_tensor_content()->resize(size); + bufferOut = it->second.mutable_tensor_content()->data(); + return StatusCode::OK; +} + +Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& bufferOut, const std::string& name, size_t size) { + OVMS_PROFILE_FUNCTION(); + for (int i = 0; i < response->outputs_size(); i++) { + if (response->mutable_outputs(i)->name() == name) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); + return StatusCode::INTERNAL_ERROR; + } + } + auto* proto = response->add_outputs(); + proto->set_name(name); + auto* content = response->add_raw_output_contents(); + content->resize(size); + bufferOut = content->data(); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/gatherexitnodeinputhandler.hpp b/src/ovms_lib/gatherexitnodeinputhandler.hpp new file mode 100644 index 0000000000..43756052f2 --- /dev/null +++ b/src/ovms_lib/gatherexitnodeinputhandler.hpp @@ -0,0 +1,68 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include + +#include "gathernodeinputhandler.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" + +#include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" +#include "src/kfserving_api/grpc_predict_v2.pb.h" +#pragma GCC diagnostic pop + +#include "logging.hpp" +#include "profiler.hpp" + +namespace ovms { + +Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& tensorOut, const std::string& name, size_t size); +Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& tensorOut, const std::string& name, size_t size); + +template +class GatherExitNodeInputHandler : public GatherNodeInputHandler { + ResponseType* response; + + Status prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const override { + OVMS_PROFILE_FUNCTION(); + auto numOfElements = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + size_t numOfBytes = numOfElements * ov::element::Type(precision).size(); + char* buffer = nullptr; + auto status = prepareConsolidatedTensorImpl(response, buffer, name, numOfBytes); + if (!status.ok()) { + return status; + } + if (!buffer) { + return StatusCode::INTERNAL_ERROR; + } + tensorOut = ov::Tensor(precision, shape, buffer); + return StatusCode::OK; + } + +public: + GatherExitNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails, ResponseType* response) : + GatherNodeInputHandler(inputsMissingCount, collapsingDetails), + response(response) {} +}; + +} // namespace ovms diff --git a/src/ovms_lib/gathernodeinputhandler.cpp b/src/ovms_lib/gathernodeinputhandler.cpp new file mode 100644 index 0000000000..08a79fb059 --- /dev/null +++ b/src/ovms_lib/gathernodeinputhandler.cpp @@ -0,0 +1,116 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "gathernodeinputhandler.hpp" + +#include +#include + +#include "logging.hpp" +#include "nodesessionmetadata.hpp" +#include "ov_utils.hpp" +#include "profiler.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +GatherNodeInputHandler::GatherNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails) : + NodeInputHandler(inputsMissingCount), + collapsingDetails(std::make_unique(collapsingDetails)) { + remainingDependencies = std::accumulate( + collapsingDetails.collapsedSessionSizes.begin(), + collapsingDetails.collapsedSessionSizes.end(), + remainingDependencies, + std::multiplies()); +} + +Status GatherNodeInputHandler::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { + auto inputsShardsIt = shardsStorage.find(inputName); + if (inputsShardsIt == shardsStorage.end()) { + shard_map_t shardMap{{shardId, tensor.getActualTensor()}}; + auto itDidInsertPair = shardsStorage.emplace(inputName, std::move(shardMap)); + if (!itDidInsertPair.second) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to insert the same input: {} twice with the same shardId: {}", inputName, shardId); + return StatusCode::INTERNAL_ERROR; + } + } else { + auto itDidEmplacePair = inputsShardsIt->second.emplace(shardId, tensor.getActualTensor()); + if (!itDidEmplacePair.second) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to put the same input: {} shard: {} twice", inputName, shardId); + return StatusCode::INTERNAL_ERROR; + } + } + if (tensor.hasSource()) { + sourceTensorRefs.push_back(tensor.getSourceTensor()); + } + return StatusCode::OK; +} + +Status GatherNodeInputHandler::notifyFinishedDependency() { + OVMS_PROFILE_FUNCTION(); + NodeInputHandler::notifyFinishedDependency(); + if (remainingDependencies > 0) { + return StatusCode::OK; + } + for (auto& [inputName, shardMap] : shardsStorage) { + OVMS_PROFILE_SCOPE("Gather Tensor"); + const auto shardsCount = shardMap.size(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Consolidating: {} shards for input: {}", shardsCount, inputName); + session_id_t firstShardId = 0; + auto firstShard = shardMap.at(firstShardId); + auto firstShardDims = firstShard.get_shape(); + auto precision = firstShard.get_element_type(); + auto newDims = firstShardDims; + newDims.insert(newDims.begin(), + collapsingDetails->collapsedSessionSizes.begin(), + collapsingDetails->collapsedSessionSizes.end()); + ov::Tensor consolidatedTensor; + auto status = prepareConsolidatedTensor(consolidatedTensor, inputName, precision, newDims); + if (!status.ok()) { + return status; + } + for (auto& [shardId, tensor] : shardMap) { + OVMS_PROFILE_SCOPE("Copy Shard"); + if ((tensor.get_element_type() != precision) || + (tensor.get_shape() != firstShardDims)) { + std::stringstream firstShardShapeStream; + firstShardShapeStream << firstShardDims; + auto currentShardShape = tensor.get_shape(); + std::stringstream currentShardShapeStream; + currentShardShapeStream << currentShardShape; + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to consolidate tensor: {}; shards in gather node. First shard has different tensor precision: {}; or shape: {}; than current shard precision: {}; shape: {};", + inputName, + toString(ovElementTypeToOvmsPrecision(precision)), + firstShardShapeStream.str(), + toString(ovElementTypeToOvmsPrecision(tensor.get_element_type())), + currentShardShapeStream.str()); + return StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS; + } + const auto memstep = tensor.get_byte_size(); + size_t offset = shardId * memstep; + memcpy((char*)consolidatedTensor.data() + offset, + tensor.data(), + memstep); + } + inputTensors.insert({inputName, consolidatedTensor}); + } + return StatusCode::OK; +} + +Status GatherNodeInputHandler::prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const { + return createSharedTensor(tensorOut, precision, shape); +} + +} // namespace ovms diff --git a/src/ovms_lib/gathernodeinputhandler.hpp b/src/ovms_lib/gathernodeinputhandler.hpp new file mode 100644 index 0000000000..4d4f73d8db --- /dev/null +++ b/src/ovms_lib/gathernodeinputhandler.hpp @@ -0,0 +1,48 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include + +#include "nodeinputhandler.hpp" +#include "precision.hpp" +#include "session_id.hpp" +#include "shape.hpp" + +namespace ovms { + +using shard_map_t = std::unordered_map; + +class CollapseDetails; + +class GatherNodeInputHandler : public NodeInputHandler { + std::unordered_map shardsStorage; + std::unique_ptr collapsingDetails; + +public: + GatherNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails); + Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) override; + Status notifyFinishedDependency() override; + +protected: + virtual Status prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const; +}; +} // namespace ovms diff --git a/src/ovms_lib/gcsfilesystem.cpp b/src/ovms_lib/gcsfilesystem.cpp new file mode 100644 index 0000000000..1821228a52 --- /dev/null +++ b/src/ovms_lib/gcsfilesystem.cpp @@ -0,0 +1,407 @@ +// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "gcsfilesystem.hpp" + +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "stringutils.hpp" + +namespace ovms { + +namespace fs = std::filesystem; +namespace gcs = google::cloud::storage; + +const std::string GCSFileSystem::GCS_URL_PREFIX = "gs://"; + +StatusCode GCSFileSystem::parsePath(const std::string& path, + std::string* bucket, std::string* object) { + int bucket_start = path.find(GCS_URL_PREFIX) + GCS_URL_PREFIX.size(); + int bucket_end = path.find("/", bucket_start); + if (bucket_end > bucket_start) { + *bucket = path.substr(bucket_start, bucket_end - bucket_start); + *object = path.substr(bucket_end + 1); + } else { + *bucket = path.substr(bucket_start); + *object = ""; + } + if (bucket->empty()) { + return StatusCode::GCS_BUCKET_NOT_FOUND; + } + return StatusCode::OK; +} + +namespace { + +google::cloud::storage::ClientOptions createDefaultOrAnonymousClientOptions() { + if (std::getenv("GOOGLE_APPLICATION_CREDENTIALS") == nullptr) { + auto credentials = + google::cloud::storage::v1::oauth2::CreateAnonymousCredentials(); + if (!credentials) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to create default GCS credentials"); + throw std::runtime_error("Unable to create default GCS credentials"); + } + auto options = google::cloud::storage::ClientOptions(credentials); + return options; + } else { + auto credentials = + google::cloud::storage::v1::oauth2::GoogleDefaultCredentials(); + if (!credentials) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to create default GCS credentials"); + throw std::runtime_error("Unable to create default GCS credentials"); + } + auto options = google::cloud::storage::ClientOptions(*credentials); + return options; + } +} + +} // namespace + +GCSFileSystem::GCSFileSystem() : + client_{createDefaultOrAnonymousClientOptions()} { + SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem default ctor"); +} + +GCSFileSystem::GCSFileSystem(const gcs::v1::ClientOptions& options) : + client_{options, gcs::StrictIdempotencyPolicy()} { + SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem ctor with custom options"); +} + +GCSFileSystem::~GCSFileSystem() { SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem dtor"); } + +StatusCode GCSFileSystem::fileExists(const std::string& path, bool* exists) { + *exists = false; + std::string bucket, object; + + auto status = this->parsePath(path, &bucket, &object); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to parse path: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + + google::cloud::StatusOr object_metadata = + client_.GetObjectMetadata(bucket, object); + if (object_metadata) { + *exists = true; + return StatusCode::OK; + } + bool is_directory; + auto dir_status = this->isDirectory(path, &is_directory); + if (dir_status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "isDirectory failed: {} -> {}", path, + ovms::Status(status).string()); + return dir_status; + } + *exists = is_directory; + SPDLOG_LOGGER_TRACE(gcs_logger, "fileExits {} -> {}", path, is_directory); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::isDirectory(const std::string& path, + bool* is_directory) { + *is_directory = false; + std::string bucket, object; + auto status = this->parsePath(path, &bucket, &object); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to parse path: {} -> {}", path, + ovms::Status(status).string()); + return status; + } + if (path.empty()) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Path {} is empty", path); + *is_directory = true; + return StatusCode::OK; + } + try { + for (auto&& meta : + client_.ListObjects(bucket, gcs::Prefix(appendSlash(object)))) { + if (meta) { + *is_directory = true; + return StatusCode::OK; + } + } + } catch (std::exception& ex) { + SPDLOG_LOGGER_DEBUG(gcs_logger, "GCS list objects exception {}", ex.what()); + SPDLOG_LOGGER_ERROR(gcs_logger, "Invalid or missing GCS credentials, or directory does not exist - {}", path); + } + + return StatusCode::OK; +} + +StatusCode +GCSFileSystem::getDirectoryContents(const std::string& path, + std::set* contents) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Getting directory contents {}", path); + std::string bucket, directory_path, full_directory; + auto status = this->parsePath(path, &bucket, &directory_path); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to get directory content {} -> {}", path, + ovms::Status(status).string()); + return status; + } + full_directory = appendSlash(directory_path); + try { + for (auto&& meta : client_.ListObjects(bucket, gcs::Prefix(full_directory))) { + if (!meta) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to get directory content -> object metadata " + "is empty. Error: {}", + meta.status().message()); + return StatusCode::GCS_INVALID_ACCESS; + } + // ignore self: + if (meta->name() == full_directory) { + continue; + } + + // keep only basename: + std::string name = meta->name(); + int name_start = name.find(full_directory) + full_directory.size(); + int name_end = name.find("/", name_start); + contents->insert(name.substr(name_start, name_end - name_start)); + } + } catch (std::exception& ex) { + SPDLOG_LOGGER_DEBUG(gcs_logger, "GCS list objects exception {}", ex.what()); + SPDLOG_LOGGER_ERROR(gcs_logger, "Invalid or missing GCS credentials, or directory does not exist - {}", full_directory); + return StatusCode::GCS_INVALID_ACCESS; + } + + SPDLOG_LOGGER_TRACE(gcs_logger, "Directory contents fetched, items: {}", contents->size()); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::getDirectorySubdirs(const std::string& path, + std::set* subdirs) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory subdirs: {}", path); + auto status = this->getDirectoryContents(path, subdirs); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory subdir content {} -> {}", path, + ovms::Status(status).string()); + return status; + } + for (auto item = subdirs->begin(); item != subdirs->end();) { + bool is_directory; + auto status = this->isDirectory(joinPath({path, *item}), &is_directory); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory subdir content {} -> {}", path, + ovms::Status(status).string()); + return status; + } + if (!is_directory) { + item = subdirs->erase(item); + } else { + ++item; + } + } + SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory subdirs ok: {}", path); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::getDirectoryFiles(const std::string& path, + std::set* files) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory: {}", path); + auto status = this->getDirectoryContents(path, files); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory content {} -> {}", path, + ovms::Status(status).string()); + return status; + } + for (auto item = files->begin(); item != files->end();) { + bool is_directory; + auto status = this->isDirectory(joinPath({path, *item}), &is_directory); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory content {} -> {}", path, + ovms::Status(status).string()); + return status; + } + if (is_directory) { + item = files->erase(item); + } else { + ++item; + } + } + SPDLOG_LOGGER_TRACE(gcs_logger, "listing directory ok for {}", path); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::readTextFile(const std::string& path, + std::string* contents) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Downloading file {}", path); + bool exists; + auto status = fileExists(path, &exists); + if (status != StatusCode::OK) { + return status; + } + if (!exists) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Downloading file -> file does not exist at {}", path); + return StatusCode::GCS_FILE_NOT_FOUND; + } + std::string bucket, object; + status = parsePath(path, &bucket, &object); + if (status != StatusCode::OK) { + return status; + } + gcs::ObjectReadStream stream = client_.ReadObject(bucket, object); + if (!stream) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Downloading file has failed: ", path); + return StatusCode::GCS_FILE_INVALID; + } + std::string data = ""; + char c = 0; + while (stream.get(c)) { + data += c; + } + *contents = data; + SPDLOG_LOGGER_TRACE(gcs_logger, "File {} has been downloaded (bytes={})", path, + data.size()); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::downloadFile(const std::string& remote_path, + const std::string& local_path) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Saving file {} to {}", remote_path, local_path); + std::string contents; + auto read_status = this->readTextFile(remote_path, &contents); + if (read_status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to get object at {}", remote_path); + return read_status; + } + std::ofstream output_file(local_path.c_str(), std::ios::binary); + output_file << contents; + output_file.close(); + return StatusCode::OK; +} + +StatusCode GCSFileSystem::downloadModelVersions(const std::string& path, + std::string* local_path, + const std::vector& versions) { + auto sc = createTempPath(local_path); + if (sc != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to create a temporary path {}", sc); + return sc; + } + + StatusCode result = StatusCode::OK; + for (auto& ver : versions) { + std::string versionpath = path; + if (!endsWith(versionpath, "/")) { + versionpath.append("/"); + } + versionpath.append(std::to_string(ver)); + std::string lpath = *local_path; + if (!endsWith(lpath, "/")) { + lpath.append("/"); + } + lpath.append(std::to_string(ver)); + fs::create_directory(lpath); + auto status = downloadFileFolder(versionpath, lpath); + if (status != StatusCode::OK) { + result = status; + SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to download model version {}", versionpath); + } + } + + return result; +} + +StatusCode GCSFileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { + SPDLOG_LOGGER_TRACE(gcs_logger, "Downloading dir {} and saving to {}", path, local_path); + bool is_dir; + auto status = this->isDirectory(path, &is_dir); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "File/folder does not exist at {}", path); + return StatusCode::GCS_FILE_NOT_FOUND; + } + if (!is_dir) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Path is not a directory: {}", path); + return StatusCode::GCS_FILE_NOT_FOUND; + } + + std::set dirs; + status = getDirectorySubdirs(path, &dirs); + if (status != StatusCode::OK) { + return status; + } + + std::set files; + status = getDirectoryFiles(path, &files); + if (status != StatusCode::OK) { + return status; + } + + for (auto&& d : dirs) { + std::string remote_dir_path = joinPath({path, d}); + std::string local_dir_path = joinPath({local_path, d}); + SPDLOG_LOGGER_TRACE(gcs_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, + local_dir_path); + auto mkdir_status = CreateLocalDir(local_dir_path); + if (mkdir_status != StatusCode::OK) { + return status; + } + auto download_dir_status = + this->downloadFileFolder(remote_dir_path, local_dir_path); + if (download_dir_status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to download directory from {} to {}", + remote_dir_path, local_dir_path); + return download_dir_status; + } + } + + for (auto&& f : files) { + if (std::any_of(acceptedFiles.begin(), acceptedFiles.end(), [&f](const std::string& x) { + return f.size() > 0 && endsWith(f, x); + })) { + std::string remote_file_path = joinPath({path, f}); + std::string local_file_path = joinPath({local_path, f}); + SPDLOG_LOGGER_TRACE(gcs_logger, "Processing file {} from {} -> {}", f, remote_file_path, + local_file_path); + auto download_status = + this->downloadFile(remote_file_path, local_file_path); + if (download_status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to save file from {} to {}", remote_file_path, + local_file_path); + return download_status; + } + } + } + return StatusCode::OK; +} + +StatusCode GCSFileSystem::deleteFileFolder(const std::string& path) { + SPDLOG_LOGGER_DEBUG(gcs_logger, "Deleting local file folder {}", path); + if (::remove(path.c_str()) == 0) { + return StatusCode::OK; + } else { + SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to remove local path: {}", path); + return StatusCode::FILE_INVALID; + } +} + +} // namespace ovms diff --git a/src/ovms_lib/gcsfilesystem.hpp b/src/ovms_lib/gcsfilesystem.hpp new file mode 100644 index 0000000000..3af4c4a7fe --- /dev/null +++ b/src/ovms_lib/gcsfilesystem.hpp @@ -0,0 +1,165 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "google/cloud/storage/client.h" + +#include "filesystem.hpp" +#include "status.hpp" + +namespace ovms { + +class GCSFileSystem : public FileSystem { +public: + /** + * @brief Construct a new GCSFileSystem object + * + */ + GCSFileSystem(); + /** + * @brief Construct a new GCSFileSystem object + * + * @param options + */ + GCSFileSystem(const google::cloud::storage::v1::ClientOptions& options); + + /** + * @brief Destroy the GCSFileSystem object + * + */ + virtual ~GCSFileSystem(); + + /** + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ + StatusCode fileExists(const std::string& path, bool* exists) override; + + /** + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ + StatusCode isDirectory(const std::string& path, bool* is_dir) override; + + /** + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode getDirectoryContents(const std::string& path, + files_list_t* contents) override; + + /** + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ + StatusCode getDirectorySubdirs(const std::string& path, + files_list_t* subdirs) override; + + /** + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ + StatusCode getDirectoryFiles(const std::string& path, + files_list_t* files) override; + + /** + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode readTextFile(const std::string& path, + std::string* contents) override; + + /** + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ + StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; + + /** + * @brief Download selected model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ + StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; + + /** + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ + StatusCode deleteFileFolder(const std::string& path) override; + + static const std::string GCS_URL_PREFIX; + +private: + /** + * @brief + * + * @param path + * @param bucket + * @param object + * @return StatusCode + */ + StatusCode parsePath(const std::string& path, std::string* bucket, + std::string* object); + + /** + * + * @brief + * + * @param remote_path + * @param local_path + */ + StatusCode downloadFile(const std::string& remote_path, + const std::string& local_path); + + /** + * @brief + * + */ + google::cloud::storage::Client client_; +}; + +} // namespace ovms diff --git a/src/ovms_lib/get_model_metadata_impl.cpp b/src/ovms_lib/get_model_metadata_impl.cpp new file mode 100644 index 0000000000..33f7fd1e9b --- /dev/null +++ b/src/ovms_lib/get_model_metadata_impl.cpp @@ -0,0 +1,202 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "get_model_metadata_impl.hpp" + +#include + +#include "modelmanager.hpp" +#include "pipelinedefinition.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" +#include "tfs_frontend/tfs_utils.hpp" + +using google::protobuf::util::JsonPrintOptions; +using google::protobuf::util::MessageToJsonString; + +namespace ovms { +GetModelMetadataImpl::GetModelMetadataImpl(ovms::Server& ovmsServer) : + ovmsServer(ovmsServer) {} + +Status GetModelMetadataImpl::getModelStatus( + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response) const { + auto status = validate(request); + if (!status.ok()) { + return status; + } + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return getModelStatus(request, response, manager); +} + +Status GetModelMetadataImpl::getModelStatus( + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response, + ModelManager& manager) { + const auto& name = request->model_spec().name(); + model_version_t version = request->model_spec().has_version() ? request->model_spec().version().value() : 0; + + auto model = manager.findModelByName(name); + if (model == nullptr) { + SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); + auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); + if (!pipelineDefinition) { + return StatusCode::MODEL_NAME_MISSING; + } + return buildResponse(*pipelineDefinition, response, manager); + } + + std::shared_ptr instance = nullptr; + if (version != 0) { + SPDLOG_DEBUG("GetModelMetadata requested model: name {}; version {}", name, version); + instance = model->getModelInstanceByVersion(version); + if (instance == nullptr) { + SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, version); + return StatusCode::MODEL_VERSION_MISSING; + } + } else { + SPDLOG_DEBUG("GetModelMetadata requested model: name {}; default version", name); + instance = model->getDefaultModelInstance(); + if (instance == nullptr) { + SPDLOG_DEBUG("GetModelMetadata requested model {}; default version is missing", name); + return StatusCode::MODEL_VERSION_MISSING; + } + } + + return buildResponse(instance, response); +} + +Status GetModelMetadataImpl::validate( + const tensorflow::serving::GetModelMetadataRequest* request) { + + if (!request->has_model_spec()) { + return StatusCode::MODEL_SPEC_MISSING; + } + + if (request->metadata_field_size() != 1) { + return StatusCode::INVALID_SIGNATURE_DEF; + } + + const auto& signature = request->metadata_field().at(0); + + if (signature != "signature_def") { + return StatusCode::INVALID_SIGNATURE_DEF; + } + + return StatusCode::OK; +} + +void GetModelMetadataImpl::convert( + const tensor_map_t& from, + proto_signature_map_t* to) { + for (const auto& [name, tensor] : from) { + auto& input = (*to)[name]; + + input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); + + // Since this method is used for models and pipelines we cannot rely on tensor getMappedName(). + // In both cases we can rely on tensor_map key values as final names. + *input.mutable_name() = name; + *input.mutable_tensor_shape() = tensorflow::TensorShapeProto(); + + for (auto dim : tensor->getShape()) { + if (dim.isStatic()) { + input.mutable_tensor_shape()->add_dim()->set_size(dim.getStaticValue()); + } else { + input.mutable_tensor_shape()->add_dim()->set_size(DYNAMIC_DIMENSION); + } + } + } +} + +Status GetModelMetadataImpl::buildResponse( + std::shared_ptr instance, + tensorflow::serving::GetModelMetadataResponse* response) { + + std::unique_ptr unloadGuard; + + // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state + auto status = instance->waitForLoaded(0, unloadGuard); + if (!status.ok()) { + return status; + } + + response->Clear(); + response->mutable_model_spec()->set_name(instance->getName()); + response->mutable_model_spec()->mutable_version()->set_value(instance->getVersion()); + + tensorflow::serving::SignatureDefMap def; + convert(instance->getInputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); + convert(instance->getOutputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); + + (*response->mutable_metadata())["signature_def"].PackFrom(def); + return StatusCode::OK; +} + +Status GetModelMetadataImpl::buildResponse( + PipelineDefinition& pipelineDefinition, + tensorflow::serving::GetModelMetadataResponse* response, + const ModelManager& manager) { + + // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state + std::unique_ptr unloadGuard; + auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); + if (!status.ok()) { + return status; + } + + const tensor_map_t& inputs = pipelineDefinition.getInputsInfo(); + const tensor_map_t& outputs = pipelineDefinition.getOutputsInfo(); + + response->Clear(); + response->mutable_model_spec()->set_name(pipelineDefinition.getName()); + response->mutable_model_spec()->mutable_version()->set_value(1); + + tensorflow::serving::SignatureDefMap def; + convert(inputs, ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); + convert(outputs, ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); + + (*response->mutable_metadata())["signature_def"].PackFrom(def); + return StatusCode::OK; +} + +Status GetModelMetadataImpl::createGrpcRequest(std::string model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request) { + request->mutable_model_spec()->set_name(model_name); + if (model_version.has_value()) { + request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); + } + request->mutable_metadata_field()->Add("signature_def"); + return StatusCode::OK; +} + +Status GetModelMetadataImpl::serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output) { + JsonPrintOptions opts; + opts.add_whitespace = true; + opts.always_print_primitive_fields = true; + const auto& status = MessageToJsonString(*response, output, opts); + if (!status.ok()) { + SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); + return StatusCode::JSON_SERIALIZATION_ERROR; + } + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/get_model_metadata_impl.hpp b/src/ovms_lib/get_model_metadata_impl.hpp new file mode 100644 index 0000000000..5a31d054ba --- /dev/null +++ b/src/ovms_lib/get_model_metadata_impl.hpp @@ -0,0 +1,71 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +using proto_signature_map_t = google::protobuf::Map; + +class ModelInstance; +class ModelManager; +class PipelineDefinition; +class Server; + +class GetModelMetadataImpl { + ovms::Server& ovmsServer; + +public: + GetModelMetadataImpl(ovms::Server& ovmsServer); + static Status validate( + const tensorflow::serving::GetModelMetadataRequest* request); + + static void convert( + const tensor_map_t& from, + proto_signature_map_t* to); + + static Status buildResponse( + std::shared_ptr instance, + tensorflow::serving::GetModelMetadataResponse* response); + static Status buildResponse( + PipelineDefinition& pipelineDefinition, + tensorflow::serving::GetModelMetadataResponse* response, + const ModelManager& manager); + + Status getModelStatus( + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response) const; + static Status getModelStatus( + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response, + ModelManager& manager); + static Status createGrpcRequest(std::string model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request); + static Status serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output); +}; + +} // namespace ovms diff --git a/src/ovms_lib/global_sequences_viewer.cpp b/src/ovms_lib/global_sequences_viewer.cpp new file mode 100644 index 0000000000..d0d4f43d78 --- /dev/null +++ b/src/ovms_lib/global_sequences_viewer.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "global_sequences_viewer.hpp" + +#include +#include +#include + +#include "logging.hpp" +#include "model.hpp" +#include "modelversion.hpp" +#include "statefulmodelinstance.hpp" +#include "status.hpp" + +namespace ovms { + +static std::string separator = "_"; + +Status GlobalSequencesViewer::registerForCleanup(std::string modelName, model_version_t modelVersion, std::shared_ptr sequenceManager) { + std::string registration_id = modelName + separator + std::to_string(modelVersion); + std::unique_lock viewerLock(viewerMutex); + if (registeredSequenceManagers.find(registration_id) != registeredSequenceManagers.end()) { + SPDLOG_LOGGER_ERROR(sequence_manager_logger, "Model: {}, version: {}, cannot register model instance in sequence cleaner. Already registered.", modelName, modelVersion); + return StatusCode::INTERNAL_ERROR; + } else { + registeredSequenceManagers.emplace(registration_id, sequenceManager); + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, has been successfully registered in sequence cleaner", modelName, modelVersion); + } + + return StatusCode::OK; +} + +Status GlobalSequencesViewer::unregisterFromCleanup(std::string modelName, model_version_t modelVersion) { + std::string registration_id = modelName + separator + std::to_string(modelVersion); + std::unique_lock viewerLock(viewerMutex); + auto it = registeredSequenceManagers.find(registration_id); + if (it != registeredSequenceManagers.end()) { + registeredSequenceManagers.erase(it); + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, has been successfully unregistered from sequence cleaner", modelName, modelVersion); + } else { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, cannot unregister model instance from sequence cleaner. It has not been registered.", modelName, modelVersion); + return StatusCode::INTERNAL_ERROR; + } + return StatusCode::OK; +} + +Status GlobalSequencesViewer::removeIdleSequences() { + std::unique_lock viewerLock(viewerMutex); + for (auto it = registeredSequenceManagers.begin(); it != registeredSequenceManagers.end();) { + auto sequenceManager = it->second; + auto status = sequenceManager->removeIdleSequences(); + it++; + if (status.getCode() != ovms::StatusCode::OK) + return status; + } + + return ovms::StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/global_sequences_viewer.hpp b/src/ovms_lib/global_sequences_viewer.hpp new file mode 100644 index 0000000000..b15ef71bee --- /dev/null +++ b/src/ovms_lib/global_sequences_viewer.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "sequence_manager.hpp" +#include "status.hpp" + +namespace ovms { +const uint32_t DEFAULT_SEQUENCE_CLEANER_INTERVAL = 5; // in minutes +class GlobalSequencesViewer { +private: + // used to block parallel access to registered sequence managers map + std::mutex viewerMutex; + + std::map> registeredSequenceManagers; + +public: + Status removeIdleSequences(); + + Status registerForCleanup(std::string modelName, model_version_t modelVersion, std::shared_ptr sequenceManager); + + Status unregisterFromCleanup(std::string modelName, model_version_t modelVersion); +}; +} // namespace ovms diff --git a/src/ovms_lib/grpcservermodule.cpp b/src/ovms_lib/grpcservermodule.cpp new file mode 100644 index 0000000000..6a18f5bd02 --- /dev/null +++ b/src/ovms_lib/grpcservermodule.cpp @@ -0,0 +1,183 @@ +//**************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "grpcservermodule.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.hpp" +#include "kfs_grpc_inference_service.hpp" +#include "logging.hpp" +#include "model_service.hpp" +#include "modelmanager.hpp" +#include "prediction_service.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" +#include "stringutils.hpp" +#include "version.hpp" + +using grpc::ServerBuilder; + +namespace ovms { +} // namespace ovms +using namespace ovms; +static const int GIGABYTE = 1024 * 1024 * 1024; + +bool isPortAvailable(uint64_t port) { + struct sockaddr_in addr; + int s = socket(AF_INET, SOCK_STREAM, 0); + if (s == -1) { + return false; + } + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(port); + + if (bind(s, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + close(s); + return false; + } + close(s); + return true; +} + +struct GrpcChannelArgument { + std::string key; + std::string value; +}; + +// Parses a comma separated list of gRPC channel arguments into list of +// ChannelArgument. +Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std::vector& result) { + const std::vector channel_arguments = tokenize(channel_arguments_str, ','); + + for (const std::string& channel_argument : channel_arguments) { + std::vector key_val = tokenize(channel_argument, '='); + if (key_val.size() != 2) { + return StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT; + } + erase_spaces(key_val[0]); + erase_spaces(key_val[1]); + result.push_back({key_val[0], key_val[1]}); + } + + return StatusCode::OK; +} + +uint getGRPCServersCount(const ovms::Config& config) { + const char* environmentVariableBuffer = std::getenv("GRPC_SERVERS"); + if (environmentVariableBuffer) { + auto result = stou32(environmentVariableBuffer); + if (result && result.value() > 0) { + return result.value(); + } + } + + return std::max(1, config.grpcWorkers()); +} + +GRPCServerModule::GRPCServerModule(Server& server) : + server(server), + tfsPredictService(this->server), + tfsModelService(this->server), + kfsGrpcInferenceService(this->server) {} +int GRPCServerModule::start(const ovms::Config& config) { + state = ModuleState::STARTED_INITIALIZE; + SPDLOG_INFO("{} starting", GRPC_SERVER_MODULE_NAME); + std::vector channel_arguments; + auto status = parseGrpcChannelArgs(config.grpcChannelArguments(), channel_arguments); + if (!status.ok()) { + SPDLOG_ERROR("grpc channel arguments passed in wrong format: {}", config.grpcChannelArguments()); + return EXIT_FAILURE; + } + + ServerBuilder builder; + builder.SetMaxReceiveMessageSize(GIGABYTE); + builder.SetMaxSendMessageSize(GIGABYTE); + builder.AddListeningPort(config.grpcBindAddress() + ":" + std::to_string(config.port()), grpc::InsecureServerCredentials()); + builder.RegisterService(&tfsPredictService); + builder.RegisterService(&tfsModelService); + builder.RegisterService(&kfsGrpcInferenceService); + for (const GrpcChannelArgument& channel_argument : channel_arguments) { + // gRPC accept arguments of two types, int and string. We will attempt to + // parse each arg as int and pass it on as such if successful. Otherwise we + // will pass it as a string. gRPC will log arguments that were not accepted. + SPDLOG_DEBUG("setting grpc channel argument {}: {}", channel_argument.key, channel_argument.value); + try { + int i = std::stoi(channel_argument.value); + builder.AddChannelArgument(channel_argument.key, i); + } catch (std::invalid_argument const& e) { + builder.AddChannelArgument(channel_argument.key, channel_argument.value); + } catch (std::out_of_range const& e) { + SPDLOG_WARN("Out of range parameter {} : {}", channel_argument.key, channel_argument.value); + } + } + uint grpcServersCount = getGRPCServersCount(config); + servers.reserve(grpcServersCount); + SPDLOG_DEBUG("Starting gRPC servers: {}", grpcServersCount); + + if (!isPortAvailable(config.port())) { + SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); + return EXIT_FAILURE; + } + for (uint i = 0; i < grpcServersCount; ++i) { + std::unique_ptr server = builder.BuildAndStart(); + if (server == nullptr) { + SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); + return EXIT_FAILURE; + } + servers.push_back(std::move(server)); + } + state = ModuleState::INITIALIZED; + SPDLOG_INFO("{} started", GRPC_SERVER_MODULE_NAME); + // FIXME this should be reenabled when functional tests are switched to wait + // for servablemanager module start log + // #KFS_CLEANUP + // SPDLOG_INFO("Server started on port {}", config.port()); + return EXIT_SUCCESS; +} + +void GRPCServerModule::shutdown() { + state = ModuleState::STARTED_SHUTDOWN; + SPDLOG_INFO("{} shutting down", GRPC_SERVER_MODULE_NAME); + for (const auto& server : servers) { + server->Shutdown(); + SPDLOG_INFO("Shutdown gRPC server"); + } + state = ModuleState::SHUTDOWN; + SPDLOG_INFO("{} shutdown", GRPC_SERVER_MODULE_NAME); +} + +const GetModelMetadataImpl& GRPCServerModule::getTFSModelMetadataImpl() const { + return this->tfsPredictService.getTFSModelMetadataImpl(); +} +KFSInferenceServiceImpl& GRPCServerModule::getKFSGrpcImpl() const { + return this->kfsGrpcInferenceService; +} diff --git a/src/ovms_lib/grpcservermodule.hpp b/src/ovms_lib/grpcservermodule.hpp new file mode 100644 index 0000000000..8ab7be17d0 --- /dev/null +++ b/src/ovms_lib/grpcservermodule.hpp @@ -0,0 +1,46 @@ +//**************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include + +#include + +#include "model_service.hpp" +#include "prediction_service.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" + +namespace ovms { +class Config; + +class GRPCServerModule : public Module { + Server& server; + PredictionServiceImpl tfsPredictService; + ModelServiceImpl tfsModelService; + mutable KFSInferenceServiceImpl kfsGrpcInferenceService; + std::vector> servers; + +public: + GRPCServerModule(Server& server); + int start(const ovms::Config& config) override; + void shutdown() override; + + const GetModelMetadataImpl& getTFSModelMetadataImpl() const; + KFSInferenceServiceImpl& getKFSGrpcImpl() const; +}; +} // namespace ovms diff --git a/src/ovms_lib/http_rest_api_handler.cpp b/src/ovms_lib/http_rest_api_handler.cpp new file mode 100644 index 0000000000..1628d22fb6 --- /dev/null +++ b/src/ovms_lib/http_rest_api_handler.cpp @@ -0,0 +1,606 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "http_rest_api_handler.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "config.hpp" +#include "filesystem.hpp" +#include "get_model_metadata_impl.hpp" +#include "grpcservermodule.hpp" +#include "kfs_grpc_inference_service.hpp" +#include "model_service.hpp" +#include "modelinstanceunloadguard.hpp" +#include "pipelinedefinition.hpp" +#include "prediction_service_utils.hpp" +#include "rest_parser.hpp" +#include "rest_utils.hpp" +#include "server.hpp" +#include "timer.hpp" + +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; + +namespace ovms { + +const std::string HttpRestApiHandler::predictionRegexExp = + R"((.?)\/v1\/models\/([^\/:]+)(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?:(classify|regress|predict))"; +const std::string HttpRestApiHandler::modelstatusRegexExp = + R"((.?)\/v1\/models(?:\/([^\/:]+))?(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?(?:\/(metadata))?)"; +const std::string HttpRestApiHandler::configReloadRegexExp = R"((.?)\/v1\/config\/reload)"; +const std::string HttpRestApiHandler::configStatusRegexExp = R"((.?)\/v1\/config)"; + +const std::string HttpRestApiHandler::kfs_modelreadyRegexExp = + R"(/v2/models/([^/]+)(?:/versions/([0-9]+))?(?:/(ready)))"; +const std::string HttpRestApiHandler::kfs_modelmetadataRegexExp = + R"(/v2/models/([^/]+)(?:/versions/([0-9]+))?(?:/)?)"; +HttpRestApiHandler::HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ms) : + predictionRegex(predictionRegexExp), + modelstatusRegex(modelstatusRegexExp), + configReloadRegex(configReloadRegexExp), + configStatusRegex(configStatusRegexExp), + kfs_modelreadyRegex(kfs_modelreadyRegexExp), + kfs_modelmetadataRegex(kfs_modelmetadataRegexExp), + timeout_in_ms(timeout_in_ms), + ovmsServer(ovmsServer), + + kfsGrpcImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getKFSGrpcImpl()), + grpcGetModelMetadataImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getTFSModelMetadataImpl()) { + registerAll(); +} + +Status HttpRestApiHandler::parseModelVersion(std::string& model_version_str, std::optional& model_version) { + if (!model_version_str.empty()) { + try { + model_version = std::stoll(model_version_str.c_str()); + } catch (std::exception& e) { + SPDLOG_ERROR("Couldn't parse model version {}", model_version_str); + return StatusCode::REST_COULD_NOT_PARSE_VERSION; + } + } + return StatusCode::OK; +} + +void HttpRestApiHandler::registerHandler(RequestType type, std::function f) { + handlers[type] = f; +} + +void HttpRestApiHandler::registerAll() { + registerHandler(Predict, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + if (request_components.processing_method == "predict") { + return processPredictRequest(request_components.model_name, request_components.model_version, + request_components.model_version_label, request_body, &response); + } else { + SPDLOG_WARN("Requested REST resource not found"); + return (Status)StatusCode::REST_NOT_FOUND; + } + }); + + registerHandler(GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + return processModelMetadataRequest(request_components.model_name, request_components.model_version, + request_components.model_version_label, &response); + }); + registerHandler(GetModelStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + return processModelStatusRequest(request_components.model_name, request_components.model_version, + request_components.model_version_label, &response); + }); + registerHandler(ConfigReload, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& manager = servableManagerModule->getServableManager(); + return processConfigReloadRequest(response, manager); + }); + registerHandler(ConfigStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& manager = servableManagerModule->getServableManager(); + return processConfigStatusRequest(response, manager); + }); + registerHandler(KFS_GetModelReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + return processModelReadyKFSRequest(request_components, response, request_body); + }); + registerHandler(KFS_GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { + return processModelReadyKFSRequest(request_components, response, request_body); + }); +} + +Status HttpRestApiHandler::dispatchToProcessor( + const std::string& request_body, + std::string* response, + const HttpRequestComponents& request_components) { + + auto handler = handlers.find(request_components.type); + if (handler != handlers.end()) { + return handler->second(request_components, *response, request_body); + } else { + return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; + } + return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; +} + +Status HttpRestApiHandler::processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + ::inference::ModelReadyRequest grpc_request; + ::inference::ModelReadyResponse grpc_response; + Status status; + std::string modelName(request_components.model_name); + std::string modelVersion(std::to_string(request_components.model_version.value_or(0))); + grpc_request.set_name(modelName); + grpc_request.set_version(modelVersion); + SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", modelName, modelVersion); + + kfsGrpcImpl.ModelReady(nullptr, &grpc_request, &grpc_response); + std::string output; + google::protobuf::util::JsonPrintOptions opts; + google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); + response = output; + return StatusCode::OK; +} + +Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { + ::inference::ModelMetadataRequest grpc_request; + ::inference::ModelMetadataResponse grpc_response; + Status status; + std::string modelName(request_components.model_name); + std::string modelVersion(std::to_string(request_components.model_version.value_or(0))); + grpc_request.set_name(modelName); + grpc_request.set_version(modelVersion); + SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", modelName, modelVersion); + kfsGrpcImpl.ModelMetadata(nullptr, &grpc_request, &grpc_response); + std::string output; + google::protobuf::util::JsonPrintOptions opts; + google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); + response = output; + return StatusCode::OK; +} + +Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& requestComponents, + const std::string_view http_method, + const std::string& request_path) { + std::smatch sm; + requestComponents.http_method = http_method; + + if (http_method != "POST" && http_method != "GET") { + return StatusCode::REST_UNSUPPORTED_METHOD; + } + + if (FileSystem::isPathEscaped(request_path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", request_path); + return StatusCode::PATH_INVALID; + } + + if (http_method == "POST") { + if (std::regex_match(request_path, sm, predictionRegex)) { + requestComponents.type = Predict; + requestComponents.model_name = sm[2]; + + std::string model_version_str = sm[3]; + auto status = parseModelVersion(model_version_str, requestComponents.model_version); + if (!status.ok()) + return status; + + std::string model_version_label_str = sm[4]; + if (!model_version_label_str.empty()) { + requestComponents.model_version_label = model_version_label_str; + } + + requestComponents.processing_method = sm[5]; + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, configReloadRegex)) { + requestComponents.type = ConfigReload; + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, modelstatusRegex)) + return StatusCode::REST_UNSUPPORTED_METHOD; + } else if (http_method == "GET") { + if (std::regex_match(request_path, sm, modelstatusRegex)) { + requestComponents.model_name = sm[2]; + + std::string model_version_str = sm[3]; + auto status = parseModelVersion(model_version_str, requestComponents.model_version); + if (!status.ok()) + return status; + + std::string model_version_label_str = sm[4]; + if (!model_version_label_str.empty()) { + requestComponents.model_version_label = model_version_label_str; + } + + requestComponents.model_subresource = sm[5]; + if (!requestComponents.model_subresource.empty() && requestComponents.model_subresource == "metadata") { + requestComponents.type = GetModelMetadata; + } else { + requestComponents.type = GetModelStatus; + } + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, configStatusRegex)) { + requestComponents.type = ConfigStatus; + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, kfs_modelmetadataRegex)) { + requestComponents.model_name = sm[1]; + std::string model_version_str = sm[2]; + auto status = parseModelVersion(model_version_str, requestComponents.model_version); + if (!status.ok()) + return status; + requestComponents.type = KFS_GetModelMetadata; + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, kfs_modelreadyRegex)) { + requestComponents.model_name = sm[1]; + std::string model_version_str = sm[2]; + auto status = parseModelVersion(model_version_str, requestComponents.model_version); + if (!status.ok()) + return status; + requestComponents.type = KFS_GetModelReady; + return StatusCode::OK; + } + if (std::regex_match(request_path, sm, predictionRegex)) + return StatusCode::REST_UNSUPPORTED_METHOD; + } + return StatusCode::REST_INVALID_URL; +} + +Status HttpRestApiHandler::processRequest( + const std::string_view http_method, + const std::string_view request_path, + const std::string& request_body, + std::vector>* headers, + std::string* response) { + + std::smatch sm; + std::string request_path_str(request_path); + if (FileSystem::isPathEscaped(request_path_str)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", request_path); + return StatusCode::PATH_INVALID; + } + + headers->clear(); + response->clear(); + headers->push_back({"Content-Type", "application/json"}); + + HttpRequestComponents requestComponents; + auto status = parseRequestComponents(requestComponents, http_method, request_path_str); + if (!status.ok()) + return status; + return dispatchToProcessor(request_body, response, requestComponents); +} + +Status HttpRestApiHandler::processPredictRequest( + const std::string& modelName, + const std::optional& modelVersion, + const std::optional& modelVersionLabel, + const std::string& request, + std::string* response) { + // model_version_label currently is not in use + + Timer timer; + timer.start("total"); + using std::chrono::microseconds; + + SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", + modelName, modelVersion.value_or(0)); + + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& modelManager = servableManagerModule->getServableManager(); + Order requestOrder; + tensorflow::serving::PredictResponse responseProto; + Status status; + + if (modelManager.modelExists(modelName)) { + SPDLOG_DEBUG("Found model with name: {}. Searching for requested version...", modelName); + status = processSingleModelRequest(modelName, modelVersion, request, requestOrder, responseProto); + } else if (modelManager.pipelineDefinitionExists(modelName)) { + SPDLOG_DEBUG("Found pipeline with name: {}", modelName); + status = processPipelineRequest(modelName, request, requestOrder, responseProto); + } else { + SPDLOG_WARN("Model or pipeline matching request parameters not found - name: {}, version: {}", modelName, modelVersion.value_or(0)); + status = StatusCode::MODEL_NAME_MISSING; + } + if (!status.ok()) + return status; + + status = makeJsonFromPredictResponse(responseProto, response, requestOrder); + if (!status.ok()) + return status; + + timer.stop("total"); + SPDLOG_DEBUG("Total REST request processing time: {} ms", timer.elapsed("total") / 1000); + return StatusCode::OK; +} + +Status HttpRestApiHandler::processSingleModelRequest(const std::string& modelName, + const std::optional& modelVersion, + const std::string& request, + Order& requestOrder, + tensorflow::serving::PredictResponse& responseProto) { + + std::shared_ptr modelInstance; + std::unique_ptr modelInstanceUnloadGuard; + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& modelManager = servableManagerModule->getServableManager(); + auto status = modelManager.getModelInstance( + modelName, + modelVersion.value_or(0), + modelInstance, + modelInstanceUnloadGuard); + + if (!status.ok()) { + SPDLOG_WARN("Requested model instance - name: {}, version: {} - does not exist.", modelName, modelVersion.value_or(0)); + return status; + } + Timer timer; + timer.start("parse"); + RestParser requestParser(modelInstance->getInputsInfo()); + status = requestParser.parse(request.c_str()); + if (!status.ok()) { + return status; + } + requestOrder = requestParser.getOrder(); + timer.stop("parse"); + SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed("parse") / 1000); + + tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); + requestProto.mutable_model_spec()->set_name(modelName); + if (modelVersion.has_value()) { + requestProto.mutable_model_spec()->mutable_version()->set_value(modelVersion.value()); + } + status = modelInstance->infer(&requestProto, &responseProto, modelInstanceUnloadGuard); + return status; +} + +Status HttpRestApiHandler::getPipelineInputs(const std::string& modelName, ovms::tensor_map_t& inputs) { + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& modelManager = servableManagerModule->getServableManager(); + auto pipelineDefinition = modelManager.getPipelineFactory().findDefinitionByName(modelName); + if (!pipelineDefinition) { + return StatusCode::MODEL_MISSING; + } + std::unique_ptr unloadGuard; + Status status = pipelineDefinition->waitForLoaded(unloadGuard); + if (!status.ok()) { + return status; + } + + inputs = pipelineDefinition->getInputsInfo(); + return StatusCode::OK; +} + +Status HttpRestApiHandler::processPipelineRequest(const std::string& modelName, + const std::string& request, + Order& requestOrder, + tensorflow::serving::PredictResponse& responseProto) { + + std::unique_ptr pipelinePtr; + + Timer timer; + timer.start("parse"); + ovms::tensor_map_t inputs; + auto status = getPipelineInputs(modelName, inputs); + if (!status.ok()) { + return status; + } + + RestParser requestParser(inputs); + status = requestParser.parse(request.c_str()); + if (!status.ok()) { + return status; + } + requestOrder = requestParser.getOrder(); + timer.stop("parse"); + SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed("parse") / 1000); + + tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); + requestProto.mutable_model_spec()->set_name(modelName); + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& manager = servableManagerModule->getServableManager(); + status = manager.createPipeline(pipelinePtr, modelName, &requestProto, &responseProto); + if (!status.ok()) { + return status; + } + status = pipelinePtr->execute(); + return status; +} + +Status HttpRestApiHandler::processModelMetadataRequest( + const std::string_view model_name, + const std::optional& model_version, + const std::optional& model_version_label, + std::string* response) { + // model_version_label currently is not in use + tensorflow::serving::GetModelMetadataRequest grpc_request; + tensorflow::serving::GetModelMetadataResponse grpc_response; + Status status; + std::string modelName(model_name); + status = grpcGetModelMetadataImpl.createGrpcRequest(modelName, model_version, &grpc_request); + if (!status.ok()) { + return status; + } + status = grpcGetModelMetadataImpl.getModelStatus(&grpc_request, &grpc_response); + if (!status.ok()) { + return status; + } + status = grpcGetModelMetadataImpl.serializeResponse2Json(&grpc_response, response); + if (!status.ok()) { + return status; + } + return StatusCode::OK; +} + +Status HttpRestApiHandler::processModelStatusRequest( + const std::string_view model_name, + const std::optional& model_version, + const std::optional& model_version_label, + std::string* response) { + // model_version_label currently is not in use + SPDLOG_DEBUG("Processing model status request"); + tensorflow::serving::GetModelStatusRequest grpc_request; + tensorflow::serving::GetModelStatusResponse grpc_response; + Status status; + std::string modelName(model_name); + status = GetModelStatusImpl::createGrpcRequest(modelName, model_version, &grpc_request); + if (!status.ok()) { + return status; + } + // TODO #KFS_CLEANUP + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO #KFS_CLEANUP + auto& manager = servableManagerModule->getServableManager(); + status = GetModelStatusImpl::getModelStatus(&grpc_request, &grpc_response, manager); + if (!status.ok()) { + return status; + } + status = GetModelStatusImpl::serializeResponse2Json(&grpc_response, response); + if (!status.ok()) { + return status; + } + return StatusCode::OK; +} + +std::string createErrorJsonWithMessage(std::string message) { + return "{\n\t\"error\": \"" + message + "\"\n}"; +} + +Status HttpRestApiHandler::processConfigReloadRequest(std::string& response, ModelManager& manager) { + SPDLOG_DEBUG("Processing config reload request started."); + Status status; + auto& config = ovms::Config::instance(); + + bool reloadNeeded = false; + if (manager.getConfigFilename() != "") { + status = manager.configFileReloadNeeded(reloadNeeded); + if (!reloadNeeded) { + if (status == StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED) { + response = createErrorJsonWithMessage("Config file not found or cannot open."); + return status; + } + } + } + + if (reloadNeeded) { + status = manager.loadConfig(config.configPath()); + if (!status.ok()) { + response = createErrorJsonWithMessage("Reloading config file failed. Check server logs for more info."); + return status; + } + } else { + if (!status.ok()) { + status = manager.loadConfig(config.configPath()); + if (!status.ok()) { + response = createErrorJsonWithMessage("Reloading config file failed. Check server logs for more info."); + return status; + } + reloadNeeded = true; + } + } + + status = manager.updateConfigurationWithoutConfigFile(); + if (!status.ok()) { + response = createErrorJsonWithMessage("Reloading models versions failed. Check server logs for more info."); + return status; + } + if (status == StatusCode::OK_RELOADED) { + reloadNeeded = true; + } + + std::map modelsStatuses; + status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager); + if (!status.ok()) { + response = createErrorJsonWithMessage("Retrieving all model statuses failed. Check server logs for more info."); + return status; + } + + status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); + if (!status.ok()) { + response = createErrorJsonWithMessage("Serializing model statuses to json failed. Check server logs for more info."); + return status; + } + + if (!reloadNeeded) { + SPDLOG_DEBUG("Config file reload was not needed."); + return StatusCode::OK_NOT_RELOADED; + } + return StatusCode::OK_RELOADED; +} + +Status HttpRestApiHandler::processConfigStatusRequest(std::string& response, ModelManager& manager) { + SPDLOG_DEBUG("Processing config status request started."); + Status status; + + std::map modelsStatuses; + status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager); + if (!status.ok()) { + response = createErrorJsonWithMessage("Retrieving all model statuses failed."); + return status; + } + + status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); + if (!status.ok()) { + response = createErrorJsonWithMessage("Serializing model statuses to json failed."); + return status; + } + + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/http_rest_api_handler.hpp b/src/ovms_lib/http_rest_api_handler.hpp new file mode 100644 index 0000000000..4d76bbfa87 --- /dev/null +++ b/src/ovms_lib/http_rest_api_handler.hpp @@ -0,0 +1,190 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "modelmanager.hpp" +#include "rest_parser.hpp" +#include "status.hpp" + +namespace ovms { +class KFSInferenceServiceImpl; +class GetModelMetadataImpl; +class Server; +enum RequestType { Predict, + GetModelStatus, + GetModelMetadata, + ConfigReload, + ConfigStatus, + KFS_GetModelReady, + KFS_GetModelMetadata }; +struct HttpRequestComponents { + RequestType type; + std::string_view http_method; + std::string model_name; + std::optional model_version; + std::optional model_version_label; + std::string processing_method; + std::string model_subresource; +}; + +class HttpRestApiHandler { +public: + static const std::string predictionRegexExp; + static const std::string modelstatusRegexExp; + static const std::string configReloadRegexExp; + static const std::string configStatusRegexExp; + + static const std::string kfs_modelreadyRegexExp; + static const std::string kfs_modelmetadataRegexExp; + /** + * @brief Construct a new HttpRest Api Handler + * + * @param timeout_in_ms + */ + HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ms); + + Status parseRequestComponents(HttpRequestComponents& components, + const std::string_view http_method, + const std::string& request_path); + + Status parseModelVersion(std::string& model_version_str, std::optional& model_version); + + void registerHandler(RequestType type, std::function); + void registerAll(); + + Status dispatchToProcessor( + const std::string& request_body, + std::string* response, + const HttpRequestComponents& request_components); + + /** + * @brief Process Request + * + * @param http_method + * @param request_path + * @param request_body + * @param headers + * @param resposnse + * + * @return StatusCode + */ + Status processRequest( + const std::string_view http_method, + const std::string_view request_path, + const std::string& request_body, + std::vector>* headers, + std::string* response); + + /** + * @brief Process predict request + * + * @param modelName + * @param modelVersion + * @param modelVersionLabel + * @param request + * @param response + * + * @return StatusCode + */ + Status processPredictRequest( + const std::string& modelName, + const std::optional& modelVersion, + const std::optional& modelVersionLabel, + const std::string& request, + std::string* response); + + Status processSingleModelRequest( + const std::string& modelName, + const std::optional& modelVersion, + const std::string& request, + Order& requestOrder, + tensorflow::serving::PredictResponse& responseProto); + + Status processPipelineRequest( + const std::string& modelName, + const std::string& request, + Order& requestOrder, + tensorflow::serving::PredictResponse& responseProto); + + /** + * @brief Process Model Metadata request + * + * @param model_name + * @param model_version + * @param model_version_label + * @param response + * + * @return StatusCode + */ + Status processModelMetadataRequest( + const std::string_view model_name, + const std::optional& model_version, + const std::optional& model_version_label, + std::string* response); + + /** + * @brief Process Model Status request + * + * @param model_name + * @param model_version + * @param model_version_label + * @param response + * @return StatusCode + */ + Status processModelStatusRequest( + const std::string_view model_name, + const std::optional& model_version, + const std::optional& model_version_label, + std::string* response); + + Status processConfigReloadRequest(std::string& response, ModelManager& manager); + + Status processConfigStatusRequest(std::string& response, ModelManager& manager); + Status processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); + Status processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); + +private: + const std::regex predictionRegex; + const std::regex modelstatusRegex; + const std::regex configReloadRegex; + const std::regex configStatusRegex; + + const std::regex kfs_modelreadyRegex; + const std::regex kfs_modelmetadataRegex; + + std::map> handlers; + int timeout_in_ms; + + ovms::Server& ovmsServer; + ovms::KFSInferenceServiceImpl& kfsGrpcImpl; + const GetModelMetadataImpl& grpcGetModelMetadataImpl; + + Status getPipelineInputs(const std::string& modelName, ovms::tensor_map_t& inputs); +}; + +} // namespace ovms diff --git a/src/ovms_lib/http_server.cpp b/src/ovms_lib/http_server.cpp new file mode 100644 index 0000000000..1304d4e425 --- /dev/null +++ b/src/ovms_lib/http_server.cpp @@ -0,0 +1,131 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "http_server.hpp" + +#include +#include +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/util/net_http/server/public/httpserver.h" +#include "tensorflow_serving/util/net_http/server/public/response_code_enum.h" +#include "tensorflow_serving/util/net_http/server/public/server_request_interface.h" +#include "tensorflow_serving/util/threadpool_executor.h" +#pragma GCC diagnostic pop + +#include "http_rest_api_handler.hpp" +#include "status.hpp" + +namespace ovms { + +namespace net_http = tensorflow::serving::net_http; + +class RequestExecutor final : public net_http::EventExecutor { +public: + explicit RequestExecutor(int num_threads) : + executor_(tensorflow::Env::Default(), "httprestserver", num_threads) {} + + void Schedule(std::function fn) override { executor_.Schedule(fn); } + +private: + tensorflow::serving::ThreadPoolExecutor executor_; +}; + +class RestApiRequestDispatcher { +public: + RestApiRequestDispatcher(ovms::Server& ovmsServer, int timeout_in_ms) { + handler_ = std::make_unique(ovmsServer, timeout_in_ms); + } + + net_http::RequestHandler dispatch(net_http::ServerRequestInterface* req) { + return [this](net_http::ServerRequestInterface* req) { + this->processRequest(req); + }; + } + +private: + void processRequest(net_http::ServerRequestInterface* req) { + SPDLOG_DEBUG("REST request {}", req->uri_path()); + std::string body; + int64_t num_bytes = 0; + auto request_chunk = req->ReadRequestBytes(&num_bytes); + while (request_chunk != nullptr) { + body.append(std::string_view(request_chunk.get(), num_bytes)); + request_chunk = req->ReadRequestBytes(&num_bytes); + } + + std::vector> headers; + std::string output; + SPDLOG_DEBUG("Processing HTTP request: {} {} body: {} bytes", + req->http_method(), + req->uri_path(), + body.size()); + const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output); + if (!status.ok() && output.empty()) { + output.append("{\"error\": \"" + status.string() + "\"}"); + } + const auto http_status = status.http(); + for (const auto& kv : headers) { + req->OverwriteResponseHeader(kv.first, kv.second); + } + req->WriteResponseString(output); + if (http_status != net_http::HTTPStatusCode::OK && http_status != net_http::HTTPStatusCode::CREATED) { + SPDLOG_DEBUG("Processing HTTP/REST request failed: {} {}. Reason: {}", + req->http_method(), + req->uri_path(), + status.string()); + } + req->ReplyWithStatus(http_status); + } + + std::unique_ptr handler_; +}; + +std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms) { + auto options = std::make_unique(); + options->AddPort(static_cast(port)); + options->SetAddress(address); + options->SetExecutor(std::make_unique(num_threads)); + + auto server = net_http::CreateEvHTTPServer(std::move(options)); + if (server == nullptr) { + SPDLOG_ERROR("Failed to create http server"); + return nullptr; + } + + std::shared_ptr dispatcher = + std::make_shared(ovmsServer, timeout_in_ms); + + net_http::RequestHandlerOptions handler_options; + server->RegisterRequestDispatcher( + [dispatcher](net_http::ServerRequestInterface* req) { + return dispatcher->dispatch(req); + }, + handler_options); + + if (server->StartAcceptingRequests()) { + SPDLOG_INFO("REST server listening on port {} with {} threads", port, num_threads); + return server; + } + + return nullptr; +} +} // namespace ovms diff --git a/src/ovms_lib/http_server.hpp b/src/ovms_lib/http_server.hpp new file mode 100644 index 0000000000..88117e3048 --- /dev/null +++ b/src/ovms_lib/http_server.hpp @@ -0,0 +1,41 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/util/net_http/server/public/httpserver_interface.h" +#pragma GCC diagnostic pop + +namespace ovms { +class Server; + +using http_server = tensorflow::serving::net_http::HTTPServerInterface; + +/** + * @brief Creates a and starts Http Server + * + * @param port + * @param num_threads + * @param timeout_in_m not implemented + * + * @return std::unique_ptr + */ +std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms = -1); +} // namespace ovms diff --git a/src/ovms_lib/kfs_grpc_inference_service.cpp b/src/ovms_lib/kfs_grpc_inference_service.cpp new file mode 100644 index 0000000000..4d803f2756 --- /dev/null +++ b/src/ovms_lib/kfs_grpc_inference_service.cpp @@ -0,0 +1,343 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "kfs_grpc_inference_service.hpp" + +#include +#include +#include + +#include "deserialization.hpp" +#include "modelinstance.hpp" +#include "modelmanager.hpp" +#include "ovinferrequestsqueue.hpp" +#include "pipelinedefinition.hpp" +#include "prediction_service_utils.hpp" +#include "serialization.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" +#include "tensorinfo.hpp" +#include "timer.hpp" +#include "version.hpp" + +namespace ovms { + +Status KFSInferenceServiceImpl::getModelInstance(const ::inference::ModelInferRequest* request, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; // TODO consider other + add details + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + model_version_t requestedVersion = 0; + if (!request->model_version().empty()) { + auto versionRead = stoi64(request->model_version()); + if (versionRead) { + requestedVersion = versionRead.value(); + } else { + SPDLOG_DEBUG("requested model: name {}; with version in invalid format: {}", request->model_name(), request->model_version()); + return StatusCode::MODEL_VERSION_INVALID_FORMAT; + } + } + return manager.getModelInstance(request->model_name(), requestedVersion, modelInstance, modelInstanceUnloadGuardPtr); +} + +Status KFSInferenceServiceImpl::getPipeline(const ::inference::ModelInferRequest* request, + ::inference::ModelInferResponse* response, + std::unique_ptr& pipelinePtr) { + OVMS_PROFILE_FUNCTION(); + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; // TODO consider other + add details + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return manager.createPipeline(pipelinePtr, request->model_name(), request, response); +} + +const std::string PLATFORM = "OpenVINO"; + +::grpc::Status KFSInferenceServiceImpl::ServerLive(::grpc::ServerContext* context, const ::inference::ServerLiveRequest* request, ::inference::ServerLiveResponse* response) { + (void)context; + (void)request; + (void)response; + bool isLive = this->ovmsServer.isLive(); + SPDLOG_DEBUG("Requested Server liveness state: {}", isLive); + response->set_live(isLive); + return grpc::Status::OK; +} + +::grpc::Status KFSInferenceServiceImpl::ServerReady(::grpc::ServerContext* context, const ::inference::ServerReadyRequest* request, ::inference::ServerReadyResponse* response) { + (void)context; + (void)request; + (void)response; + bool isReady = this->ovmsServer.isReady(); + SPDLOG_DEBUG("Requested Server readiness state: {}", isReady); + response->set_ready(isReady); + return grpc::Status::OK; +} + +Status KFSInferenceServiceImpl::getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager) { + // Return in response true/false + // if no version requested give response for default version + const auto& name = request->name(); + const auto& versionString = request->version(); + auto model = manager.findModelByName(name); + SPDLOG_DEBUG("ModelReady requested name: {}, version: {}", name, versionString); + if (model == nullptr) { + SPDLOG_DEBUG("ModelReady requested model {} is missing, trying to find pipeline with such name", name); + auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); + if (!pipelineDefinition) { + return Status(StatusCode::MODEL_NAME_MISSING); + } + return buildResponse(*pipelineDefinition, response); + } + std::shared_ptr instance = nullptr; + if (!versionString.empty()) { + SPDLOG_DEBUG("ModelReady requested model: name {}; version {}", name, versionString); + model_version_t requestedVersion = 0; + auto versionRead = stoi64(versionString); + if (versionRead) { + requestedVersion = versionRead.value(); + } else { + SPDLOG_DEBUG("ModelReady requested model: name {}; with version in invalid format: {}", name, versionString); + return Status(StatusCode::MODEL_VERSION_INVALID_FORMAT); + } + instance = model->getModelInstanceByVersion(requestedVersion); + if (instance == nullptr) { + SPDLOG_DEBUG("ModelReady requested model {}; version {} is missing", name, versionString); + return Status(StatusCode::MODEL_VERSION_MISSING); + } + } else { + SPDLOG_DEBUG("ModelReady requested model: name {}; default version", name); + instance = model->getDefaultModelInstance(); + if (instance == nullptr) { + SPDLOG_DEBUG("ModelReady requested model {}; version {} is missing", name, versionString); + return Status(StatusCode::MODEL_VERSION_MISSING); + } + } + return buildResponse(instance, response); +} + +::grpc::Status KFSInferenceServiceImpl::ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) { + (void)context; + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return this->getModelReady(request, response, manager).grpc(); +} + +::grpc::Status KFSInferenceServiceImpl::ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) { + (void)context; + (void)request; + (void)response; + response->set_name(PROJECT_NAME); + response->set_version(PROJECT_VERSION); + return grpc::Status::OK; +} + +::grpc::Status KFSInferenceServiceImpl::ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) { + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + const auto& name = request->name(); + const auto& versionString = request->version(); + + auto model = manager.findModelByName(name); + if (model == nullptr) { + SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); + auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); + if (!pipelineDefinition) { + return Status(StatusCode::MODEL_NAME_MISSING).grpc(); + } + return buildResponse(*pipelineDefinition, response); + } + std::shared_ptr instance = nullptr; + if (!versionString.empty()) { + SPDLOG_DEBUG("GetModelMetadata requested model: name {}; version {}", name, versionString); + model_version_t requestedVersion = 0; + auto versionRead = stoi64(versionString); + if (versionRead) { + requestedVersion = versionRead.value(); + } else { + SPDLOG_DEBUG("GetModelMetadata requested model: name {}; with version in invalid format: {}", name, versionString); + return Status(StatusCode::MODEL_VERSION_INVALID_FORMAT).grpc(); + } + instance = model->getModelInstanceByVersion(requestedVersion); + if (instance == nullptr) { + SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, versionString); + return Status(StatusCode::MODEL_VERSION_MISSING).grpc(); + } + } else { + SPDLOG_DEBUG("GetModelMetadata requested model: name {}; default version", name); + instance = model->getDefaultModelInstance(); + if (instance == nullptr) { + SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, versionString); + return Status(StatusCode::MODEL_VERSION_MISSING).grpc(); + } + } + return buildResponse(*model, *instance, response).grpc(); +} + +::grpc::Status KFSInferenceServiceImpl::ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) { + (void)context; + OVMS_PROFILE_FUNCTION(); + Timer timer; + timer.start("total"); + using std::chrono::microseconds; + SPDLOG_DEBUG("Processing gRPC request for model: {}; version: {}", + request->model_name(), + request->model_version()); + + std::shared_ptr modelInstance; + std::unique_ptr pipelinePtr; + + std::unique_ptr modelInstanceUnloadGuard; + auto status = getModelInstance(request, modelInstance, modelInstanceUnloadGuard); + if (status == StatusCode::MODEL_NAME_MISSING) { + SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", request->model_name()); + status = getPipeline(request, response, pipelinePtr); + } + if (!status.ok()) { + SPDLOG_DEBUG("Getting modelInstance or pipeline failed. {}", status.string()); + return status.grpc(); + } + + if (pipelinePtr) { + status = pipelinePtr->execute(); + } else { + status = modelInstance->infer(request, response, modelInstanceUnloadGuard); + } + + if (!status.ok()) { + return status.grpc(); + } + + timer.stop("total"); + SPDLOG_DEBUG("Total gRPC request processing time: {} ms", timer.elapsed("total") / 1000); + return grpc::Status::OK; +} + +Status KFSInferenceServiceImpl::buildResponse( + std::shared_ptr instance, + ::inference::ModelReadyResponse* response) { + response->set_ready(instance->getStatus().getState() == ModelVersionState::AVAILABLE); + return StatusCode::OK; +} + +Status KFSInferenceServiceImpl::buildResponse( + PipelineDefinition& pipelineDefinition, + ::inference::ModelReadyResponse* response) { + response->set_ready(pipelineDefinition.getStatus().isAvailable()); + return StatusCode::OK; +} + +void addReadyVersions(Model& model, + ::inference::ModelMetadataResponse* response) { + auto modelVersions = model.getModelVersionsMapCopy(); + for (auto& [modelVersion, modelInstance] : modelVersions) { + if (modelInstance.getStatus().getState() == ModelVersionState::AVAILABLE) + response->add_versions(std::to_string(modelVersion)); + } +} + +Status KFSInferenceServiceImpl::buildResponse( + Model& model, + ModelInstance& instance, + ::inference::ModelMetadataResponse* response) { + + std::unique_ptr unloadGuard; + + // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state + auto status = instance.waitForLoaded(0, unloadGuard); + if (!status.ok()) { + return status; + } + + response->Clear(); + response->set_name(instance.getName()); + addReadyVersions(model, response); + response->set_platform(PLATFORM); + + for (const auto& input : instance.getInputsInfo()) { + convert(input, response->add_inputs()); + } + + for (const auto& output : instance.getOutputsInfo()) { + convert(output, response->add_outputs()); + } + + return StatusCode::OK; +} + +KFSInferenceServiceImpl::KFSInferenceServiceImpl(const Server& server) : + ovmsServer(server) {} + +Status KFSInferenceServiceImpl::buildResponse( + PipelineDefinition& pipelineDefinition, + ::inference::ModelMetadataResponse* response) { + + std::unique_ptr unloadGuard; + + // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state + auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); + if (!status.ok()) { + return status; + } + + response->Clear(); + response->set_name(pipelineDefinition.getName()); + response->add_versions("1"); + response->set_platform(PLATFORM); + + for (const auto& input : pipelineDefinition.getInputsInfo()) { + convert(input, response->add_inputs()); + } + + for (const auto& output : pipelineDefinition.getOutputsInfo()) { + convert(output, response->add_outputs()); + } + + return StatusCode::OK; +} + +void KFSInferenceServiceImpl::convert( + const std::pair>& from, + ::inference::ModelMetadataResponse::TensorMetadata* to) { + to->set_name(from.first); + to->set_datatype(from.second->getPrecisionAsKFSPrecision()); + for (auto dim : from.second->getShape()) { + if (dim.isStatic()) { + to->add_shape(dim.getStaticValue()); + } else { + to->add_shape(DYNAMIC_DIMENSION); + } + } +} + +} // namespace ovms diff --git a/src/ovms_lib/kfs_grpc_inference_service.hpp b/src/ovms_lib/kfs_grpc_inference_service.hpp new file mode 100644 index 0000000000..24c4b486fc --- /dev/null +++ b/src/ovms_lib/kfs_grpc_inference_service.hpp @@ -0,0 +1,68 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" +#include "src/kfserving_api/grpc_predict_v2.pb.h" +#include "status.hpp" + +namespace ovms { + +using inference::GRPCInferenceService; +class Model; +class ModelManager; +class ModelInstance; +class ModelInstanceUnloadGuard; +class Pipeline; +class Server; +class Status; +class TensorInfo; +class PipelineDefinition; + +class KFSInferenceServiceImpl final : public GRPCInferenceService::Service { + const Server& ovmsServer; + +public: + KFSInferenceServiceImpl(const Server& server); + ::grpc::Status ServerLive(::grpc::ServerContext* context, const ::inference::ServerLiveRequest* request, ::inference::ServerLiveResponse* response) override; + ::grpc::Status ServerReady(::grpc::ServerContext* context, const ::inference::ServerReadyRequest* request, ::inference::ServerReadyResponse* response) override; + ::grpc::Status ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) override; + ::grpc::Status ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) override; + ::grpc::Status ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) override; + ::grpc::Status ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) override; + static Status buildResponse(Model& model, ModelInstance& instance, ::inference::ModelMetadataResponse* response); + static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelMetadataResponse* response); + static Status buildResponse(std::shared_ptr instance, ::inference::ModelReadyResponse* response); + static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelReadyResponse* response); + static void convert(const std::pair>& from, ::inference::ModelMetadataResponse::TensorMetadata* to); + static Status getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager); + +protected: + Status getModelInstance(const ::inference::ModelInferRequest* request, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr); + Status getPipeline(const ::inference::ModelInferRequest* request, + ::inference::ModelInferResponse* response, + std::unique_ptr& pipelinePtr); +}; + +} // namespace ovms diff --git a/src/ovms_lib/layout.cpp b/src/ovms_lib/layout.cpp new file mode 100644 index 0000000000..92b7ea2903 --- /dev/null +++ b/src/ovms_lib/layout.cpp @@ -0,0 +1,169 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "layout.hpp" + +#include +#include +#include + +#include "logging.hpp" + +const char* DEFAULT_LAYOUT = "N..."; +const char* UNSPECIFIED_LAYOUT = "..."; + +namespace ovms { +const Layout& Layout::getDefaultLayout() { + static const Layout defaultLayout{DEFAULT_LAYOUT}; + return defaultLayout; +} + +const Layout& Layout::getUnspecifiedLayout() { + static const Layout unspecifiedLayout{UNSPECIFIED_LAYOUT}; + return unspecifiedLayout; +} + +Layout::Layout(const std::string& str) : + std::string(str) { + this->batchIndex = retrieveBatchIndex(); +} + +const std::optional& Layout::getBatchIndex() const { + return this->batchIndex; +} + +std::optional Layout::retrieveBatchIndex() const { + auto status = validate(); + if (!status.ok()) { + return std::nullopt; + } + auto batchPos = this->find(BATCH_DIMENSION_LETTER); + auto etcPos = this->find(ETC_LAYOUT_DELIMETER); + if (static_cast(*this) == UNSPECIFIED_LAYOUT) { + // we want to treat ANY layout as having BS on 0 position + // otherwise in any case we extract this we have to check + // against layout ... + return 0; + } + if (batchPos == std::string::npos) { + return std::nullopt; + } + if (etcPos != std::string::npos && batchPos > etcPos) { + return std::nullopt; + } + return batchPos; +} + +Status Layout::validate() const { + if (this->find_first_not_of(ALLOWED_DIMENSION_LETTERS_AND_CHARS) != std::string::npos) + return StatusCode::LAYOUT_WRONG_FORMAT; // Cannot contain other letters + + for (char c : ALLOWED_DIMENSION_LETTERS) { + if (std::count(this->begin(), this->end(), c) > 1) { + return StatusCode::LAYOUT_WRONG_FORMAT; // Can contain NCHWD only single time + } + } + + size_t dotCount = 0; + bool firstEtcAppeared = false; + bool fullEtcAppeared = false; + for (char c : (*this)) { + if (c == ETC_CHAR) { + if (fullEtcAppeared) { + return StatusCode::LAYOUT_WRONG_FORMAT; // Cannot appear multiple times + } + firstEtcAppeared = true; + dotCount++; + if (dotCount >= 3) { + fullEtcAppeared = true; + firstEtcAppeared = false; + } + } else if (firstEtcAppeared) { + return StatusCode::LAYOUT_WRONG_FORMAT; // Dots separated + } + } + if (firstEtcAppeared && !fullEtcAppeared) { + return StatusCode::LAYOUT_WRONG_FORMAT; // Dots not completed + } + + return StatusCode::OK; +} + +std::optional Layout::createIntersection(const Layout& other, size_t numberOfDimensions) const { + Layout lhs = (*this); + Layout rhs = other; + + if (lhs.containsEtc()) { + size_t knownDimensions = std::count_if(lhs.begin(), lhs.end(), [](char c) { return c != ETC_CHAR; }); + if (knownDimensions > numberOfDimensions) + return std::nullopt; + size_t unknownDimensions = numberOfDimensions - knownDimensions; + lhs.replace(lhs.find(ETC_LAYOUT_DELIMETER), ETC_LAYOUT_DELIMETER.size(), std::string(unknownDimensions, UNDEFINED_DIMENSION_CHAR)); + } + + if (rhs.containsEtc()) { + size_t knownDimensions = std::count_if(rhs.begin(), rhs.end(), [](char c) { return c != ETC_CHAR; }); + if (knownDimensions > numberOfDimensions) + return std::nullopt; + size_t unknownDimensions = numberOfDimensions - knownDimensions; + rhs.replace(rhs.find(ETC_LAYOUT_DELIMETER), ETC_LAYOUT_DELIMETER.size(), std::string(unknownDimensions, UNDEFINED_DIMENSION_CHAR)); + } + + if (lhs.size() != rhs.size() || lhs.size() != numberOfDimensions) + return std::nullopt; + + for (size_t i = 0; i < lhs.size(); i++) { + if (lhs[i] == rhs[i]) + continue; + if (rhs[i] != UNDEFINED_DIMENSION_CHAR && lhs.find(rhs[i]) != std::string::npos) + return std::nullopt; + if (lhs[i] == UNDEFINED_DIMENSION_CHAR) { + lhs[i] = rhs[i]; + continue; + } + if (rhs[i] == UNDEFINED_DIMENSION_CHAR) + continue; + return std::nullopt; + } + + return lhs; +} + +Layout Layout::fromOvLayout(const ov::Layout& layout) { + std::string strCopy = layout.to_string(); + strCopy.erase(std::remove_if(strCopy.begin(), strCopy.end(), + [](char c) -> bool { + return c == '[' || c == ']' || c == ','; + }), + strCopy.end()); + return Layout(strCopy); +} + +bool Layout::containsEtc() const { + return this->find(ETC_LAYOUT_DELIMETER) != std::string::npos; +} + +std::string::size_type Layout::getNumberOfKnownDimensions() const { + return std::count_if(this->begin(), this->end(), [](char c) { return ALLOWED_DIMENSION_LETTERS.find(c) != std::string::npos || c == UNDEFINED_DIMENSION_CHAR; }); +} + +bool Layout::isCompatible(const Shape& shape) const { + if (this->containsEtc()) { + return this->getNumberOfKnownDimensions() <= shape.size(); + } + return this->getNumberOfKnownDimensions() == shape.size(); +} + +} // namespace ovms diff --git a/src/ovms_lib/layout.hpp b/src/ovms_lib/layout.hpp new file mode 100644 index 0000000000..42fb215ba3 --- /dev/null +++ b/src/ovms_lib/layout.hpp @@ -0,0 +1,59 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "shape.hpp" +#include "status.hpp" + +namespace ovms { + +static const std::string ALLOWED_DIMENSION_LETTERS = "NCHWD"; +static const char ETC_CHAR = '.'; +static const char UNDEFINED_DIMENSION_CHAR = '?'; +static const std::string ALLOWED_DIMENSION_LETTERS_AND_CHARS = ALLOWED_DIMENSION_LETTERS + ETC_CHAR + UNDEFINED_DIMENSION_CHAR; +static const std::string ETC_LAYOUT_DELIMETER = "..."; +static const std::string BATCH_DIMENSION_LETTER = "N"; + +class Layout : public std::string { + std::optional batchIndex = std::nullopt; + + std::optional retrieveBatchIndex() const; + bool containsEtc() const; + + std::string::size_type getNumberOfKnownDimensions() const; + +public: + Layout() = default; + Layout(const std::string& str); + + static Layout fromOvLayout(const ov::Layout& layout); + + const std::optional& getBatchIndex() const; + Status validate() const; + std::optional createIntersection(const Layout& other, size_t numberOfDimensions) const; + static const Layout& getDefaultLayout(); + static const Layout& getUnspecifiedLayout(); + + bool isCompatible(const Shape& shape) const; +}; + +} // namespace ovms diff --git a/src/ovms_lib/layout_configuration.cpp b/src/ovms_lib/layout_configuration.cpp new file mode 100644 index 0000000000..7462e3f76d --- /dev/null +++ b/src/ovms_lib/layout_configuration.cpp @@ -0,0 +1,98 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "layout_configuration.hpp" + +#include +#include +#include + +#include "stringutils.hpp" + +namespace ovms { + +LayoutConfiguration::LayoutConfiguration(const char* layout) : + LayoutConfiguration(std::string(layout)) { +} + +LayoutConfiguration::LayoutConfiguration(const std::string& layout) : + LayoutConfiguration(layout, layout) { +} + +LayoutConfiguration::LayoutConfiguration(const std::string& tensorLayout, const std::string& modelLayout) : + tensor(Layout(tensorLayout)), + model(Layout(modelLayout)) { +} + +bool LayoutConfiguration::isSet() const { + return !tensor.empty() || !model.empty(); +} + +Status LayoutConfiguration::fromString(const std::string& configurationStr, LayoutConfiguration& configOut) { + std::string configurationCopy = configurationStr; + erase_spaces(configurationCopy); + + std::transform(configurationCopy.begin(), configurationCopy.end(), configurationCopy.begin(), ::toupper); + + if (configurationCopy.find_first_not_of(ALLOWED_DIMENSION_LETTERS_AND_CHARS + LAYOUT_CONFIGURATION_DELIMETER) != std::string::npos) + return StatusCode::LAYOUT_WRONG_FORMAT; + + size_t delimCount = std::count(configurationCopy.begin(), configurationCopy.end(), LAYOUT_CONFIGURATION_DELIMETER); + if (delimCount > 1) + return StatusCode::LAYOUT_WRONG_FORMAT; + + if (delimCount == 0) { + configOut = LayoutConfiguration(configurationCopy); + } else { + std::vector tokens = tokenize(configurationCopy, LAYOUT_CONFIGURATION_DELIMETER); + if (tokens.size() > 2) + return StatusCode::LAYOUT_WRONG_FORMAT; + else if (tokens.size() == 2) + configOut = LayoutConfiguration(tokens[0], tokens[1]); + else if (tokens.size() == 1) + configOut = LayoutConfiguration(tokens[0]); + else + return StatusCode::LAYOUT_WRONG_FORMAT; + } + return StatusCode::OK; +} + +std::string LayoutConfiguration::toString() const { + std::stringstream ss; + if (tensor.empty()) { + ss << model; + } else { + ss << tensor << LAYOUT_CONFIGURATION_DELIMETER << model; + } + return ss.str(); +} + +const Layout& LayoutConfiguration::getTensorLayout() const { + return this->tensor; +} + +const Layout& LayoutConfiguration::getModelLayout() const { + return this->model; +} + +bool LayoutConfiguration::operator==(const LayoutConfiguration& rhs) const { + return this->tensor == rhs.tensor && this->model == rhs.model; +} + +bool LayoutConfiguration::operator!=(const LayoutConfiguration& rhs) const { + return !(this->operator==(rhs)); +} + +} // namespace ovms diff --git a/src/ovms_lib/layout_configuration.hpp b/src/ovms_lib/layout_configuration.hpp new file mode 100644 index 0000000000..11d56cffba --- /dev/null +++ b/src/ovms_lib/layout_configuration.hpp @@ -0,0 +1,52 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include "layout.hpp" +#include "status.hpp" + +namespace ovms { + +static const char LAYOUT_CONFIGURATION_DELIMETER = ':'; + +class LayoutConfiguration { + Layout tensor; + Layout model; + +public: + LayoutConfiguration() = default; + LayoutConfiguration(const char* layout); + LayoutConfiguration(const std::string& layout); + LayoutConfiguration(const std::string& tensorLayout, const std::string& modelLayout); + + const Layout& getTensorLayout() const; + const Layout& getModelLayout() const; + + bool isSet() const; + + bool operator==(const LayoutConfiguration& rhs) const; + bool operator!=(const LayoutConfiguration& rhs) const; + + static Status fromString(const std::string& configurationStr, LayoutConfiguration& configOut); + std::string toString() const; +}; + +using layout_configurations_map_t = std::unordered_map; + +} // namespace ovms diff --git a/src/ovms_lib/localfilesystem.cpp b/src/ovms_lib/localfilesystem.cpp new file mode 100644 index 0000000000..3620abb5e1 --- /dev/null +++ b/src/ovms_lib/localfilesystem.cpp @@ -0,0 +1,173 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "localfilesystem.hpp" + +#include +#include +#include +#include + +#include + +#if defined(__APPLE__) || defined(__NetBSD__) +#define st_mtim st_mtimespec +#endif + +namespace ovms { + +namespace fs = std::filesystem; +constexpr uint64_t NANOS_PER_SECOND = 1000000000; + +const std::vector FileSystem::acceptedFiles = {".bin", ".onnx", ".xml", "mapping_config.json"}; + +StatusCode LocalFileSystem::fileExists(const std::string& path, bool* exists) { + try { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + *exists = fs::exists(path); + } catch (fs::filesystem_error& e) { + SPDLOG_DEBUG("Couldn't access path {}", e.what()); + return StatusCode::PATH_INVALID; + } + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::isDirectory(const std::string& path, bool* is_dir) { + try { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + *is_dir = fs::is_directory(path); + } catch (fs::filesystem_error& e) { + SPDLOG_DEBUG("Couldn't access path {}", e.what()); + return StatusCode::PATH_INVALID; + } + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::getDirectoryContents(const std::string& path, files_list_t* contents) { + try { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + for (const auto& entry : fs::directory_iterator(path)) { + contents->insert(entry.path().string()); + } + } catch (fs::filesystem_error& e) { + SPDLOG_DEBUG("Couldn't access path {}", e.what()); + return StatusCode::PATH_INVALID; + } + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::getDirectorySubdirs(const std::string& path, files_list_t* subdirs) { + try { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + for (const auto& entry : fs::directory_iterator(path)) { + if (entry.is_directory()) { + subdirs->insert(entry.path().filename().string()); + } + } + } catch (fs::filesystem_error& e) { + SPDLOG_DEBUG("Couldn't access path {}", e.what()); + return StatusCode::PATH_INVALID; + } + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::getDirectoryFiles(const std::string& path, files_list_t* files) { + try { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + for (const auto& entry : fs::directory_iterator(path)) { + if (!entry.is_directory()) { + files->insert(entry.path().string()); + } + } + } catch (fs::filesystem_error& e) { + SPDLOG_DEBUG("Couldn't access path {}", e.what()); + return StatusCode::PATH_INVALID; + } + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::readTextFile(const std::string& path, std::string* contents) { + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + std::ifstream input(path, std::ios::in | std::ios::binary); + if (!input) { + SPDLOG_DEBUG("Couldn't access path {}", path); + return StatusCode::PATH_INVALID; + } + + input.seekg(0, std::ios::end); + contents->resize(input.tellg()); + input.seekg(0, std::ios::beg); + input.read(&(*contents)[0], contents->size()); + input.close(); + + return StatusCode::OK; +} + +StatusCode LocalFileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { + // For LocalFileSystem there is no need to download + return StatusCode::OK; +} + +StatusCode LocalFileSystem::downloadModelVersions(const std::string& path, + std::string* local_path, + const std::vector& versions) { + *local_path = path; + return StatusCode::OK; +} + +StatusCode LocalFileSystem::deleteFileFolder(const std::string& path) { + std::error_code errorCode; + std::filesystem::path p = path; + std::filesystem::path parentPath = p.parent_path(); + if (isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + if (!std::filesystem::remove_all(path, errorCode)) { + return StatusCode::PATH_INVALID; + } + // delete empty folder with model version + if (std::filesystem::is_empty(parentPath)) { + SPDLOG_DEBUG("Deleting empty folder: ()", parentPath.string()); + std::filesystem::remove(parentPath); + } + + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/localfilesystem.hpp b/src/ovms_lib/localfilesystem.hpp new file mode 100644 index 0000000000..08f4affaea --- /dev/null +++ b/src/ovms_lib/localfilesystem.hpp @@ -0,0 +1,116 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include "filesystem.hpp" +#include "status.hpp" + +namespace ovms { + +class LocalFileSystem : public FileSystem { +public: + /** + * @brief Destroy the Local File System object + * + */ + ~LocalFileSystem() {} + + /** + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ + StatusCode fileExists(const std::string& path, bool* exists) override; + + /** + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ + StatusCode isDirectory(const std::string& path, bool* is_dir) override; + + /** + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; + + /** + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ + StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; + + /** + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ + StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; + + /** + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode readTextFile(const std::string& path, std::string* contents) override; + + /** + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ + StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; + + /** + * @brief Download selected model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ + StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; + + /** + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ + StatusCode deleteFileFolder(const std::string& path) override; +}; + +} // namespace ovms diff --git a/src/ovms_lib/log b/src/ovms_lib/log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/ovms_lib/logging.cpp b/src/ovms_lib/logging.cpp new file mode 100644 index 0000000000..009bfd73e6 --- /dev/null +++ b/src/ovms_lib/logging.cpp @@ -0,0 +1,87 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "logging.hpp" + +#include + +namespace ovms { + +std::shared_ptr gcs_logger = std::make_shared("gcs"); +std::shared_ptr azurestorage_logger = std::make_shared("azurestorage"); +std::shared_ptr s3_logger = std::make_shared("s3"); +std::shared_ptr modelmanager_logger = std::make_shared("modelmanager"); +std::shared_ptr dag_executor_logger = std::make_shared("dag_executor"); +std::shared_ptr sequence_manager_logger = std::make_shared("sequence_manager"); + +const std::string default_pattern = "[%Y-%m-%d %T.%e][%t][%n][%l][%s:%#] %v"; + +void set_log_level(const std::string log_level, std::shared_ptr logger) { + logger->set_level(spdlog::level::info); + if (!log_level.empty()) { + if (log_level == "DEBUG") { + logger->set_level(spdlog::level::debug); + logger->flush_on(spdlog::level::trace); + } else if (log_level == "ERROR") { + logger->set_level(spdlog::level::err); + logger->flush_on(spdlog::level::err); + } else if (log_level == "TRACE") { + logger->set_level(spdlog::level::trace); + logger->flush_on(spdlog::level::trace); + } + } +} + +void register_loggers(const std::string log_level, std::vector sinks) { + auto serving_logger = std::make_shared("serving", begin(sinks), end(sinks)); + serving_logger->set_pattern(default_pattern); + gcs_logger->set_pattern(default_pattern); + azurestorage_logger->set_pattern(default_pattern); + s3_logger->set_pattern(default_pattern); + modelmanager_logger->set_pattern(default_pattern); + dag_executor_logger->set_pattern(default_pattern); + sequence_manager_logger->set_pattern(default_pattern); + for (auto sink : sinks) { + gcs_logger->sinks().push_back(sink); + azurestorage_logger->sinks().push_back(sink); + s3_logger->sinks().push_back(sink); + modelmanager_logger->sinks().push_back(sink); + dag_executor_logger->sinks().push_back(sink); + sequence_manager_logger->sinks().push_back(sink); + } + set_log_level(log_level, serving_logger); + set_log_level(log_level, gcs_logger); + set_log_level(log_level, azurestorage_logger); + set_log_level(log_level, s3_logger); + set_log_level(log_level, modelmanager_logger); + set_log_level(log_level, dag_executor_logger); + set_log_level(log_level, sequence_manager_logger); + spdlog::set_default_logger(serving_logger); +} + +void configure_logger(const std::string log_level, const std::string log_path) { + static bool wasRun = false; + if (wasRun) + return; + wasRun = true; + std::vector sinks; + sinks.push_back(std::make_shared()); + if (!log_path.empty()) { + sinks.push_back(std::make_shared(log_path)); + } + register_loggers(log_level, sinks); +} + +} // namespace ovms diff --git a/src/ovms_lib/logging.hpp b/src/ovms_lib/logging.hpp new file mode 100644 index 0000000000..31a481f440 --- /dev/null +++ b/src/ovms_lib/logging.hpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include +#include +#include + +namespace ovms { + +extern std::shared_ptr gcs_logger; +extern std::shared_ptr azurestorage_logger; +extern std::shared_ptr s3_logger; +extern std::shared_ptr modelmanager_logger; +extern std::shared_ptr dag_executor_logger; +extern std::shared_ptr sequence_manager_logger; + +void configure_logger(const std::string log_level, const std::string log_path); + +} // namespace ovms diff --git a/src/ovms_lib/main.cpp b/src/ovms_lib/main.cpp new file mode 100644 index 0000000000..3f3d9b8c58 --- /dev/null +++ b/src/ovms_lib/main.cpp @@ -0,0 +1,24 @@ +//***************************************************************************** +// Copyright 2018-2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "server.hpp" + +using ovms::Server; + +int main(int argc, char** argv) { + Server& server = Server::instance(); + return server.start(argc, argv); +} diff --git a/src/ovms_lib/main2.cpp b/src/ovms_lib/main2.cpp new file mode 100644 index 0000000000..0c60c1a2a6 --- /dev/null +++ b/src/ovms_lib/main2.cpp @@ -0,0 +1,15 @@ +#include +#include + +#include "server.hpp" + +using ovms::Server; + +int main(int argc, char** argv) { + Server& server = Server::instance(); + std::thread t([&server, &argv, &argc](){ + std::cout << server.start(argc, argv) << std::endl; + }); + t.join(); + return 0; +} diff --git a/src/ovms_lib/model.cpp b/src/ovms_lib/model.cpp new file mode 100644 index 0000000000..9a751a37df --- /dev/null +++ b/src/ovms_lib/model.cpp @@ -0,0 +1,305 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "model.hpp" + +#include +#include +#include +#include +#include + +#include "customloaders.hpp" +#include "localfilesystem.hpp" +#include "logging.hpp" +#include "pipelinedefinition.hpp" + +namespace ovms { + +StatusCode downloadModels(std::shared_ptr& fs, ModelConfig& config, std::shared_ptr versions) { + if (versions->size() == 0) { + return StatusCode::OK; + } + + std::string localPath; + SPDLOG_INFO("Getting model from {}", config.getBasePath()); + auto sc = fs->downloadModelVersions(config.getBasePath(), &localPath, *versions); + if (sc != StatusCode::OK) { + SPDLOG_ERROR("Couldn't download model from {}", config.getBasePath()); + return sc; + } + config.setLocalPath(localPath); + SPDLOG_INFO("Model downloaded to {}", config.getLocalPath()); + + return StatusCode::OK; +} + +void Model::subscribe(PipelineDefinition& pd) { + subscriptionManager.subscribe(pd); +} + +void Model::unsubscribe(PipelineDefinition& pd) { + subscriptionManager.unsubscribe(pd); +} + +bool Model::isAnyVersionSubscribed() const { + if (subscriptionManager.isSubscribed()) { + return true; + } + for (const auto& [name, instance] : modelVersions) { + if (instance->getSubscribtionManager().isSubscribed()) { + return true; + } + } + return false; +} + +const std::map Model::getModelVersionsMapCopy() const { + std::shared_lock lock(modelVersionsMtx); + std::map modelInstancesMapCopy; + for (auto& [modelVersion, modelInstancePtr] : modelVersions) { + modelInstancesMapCopy.insert({modelVersion, *modelInstancePtr}); + } + return modelInstancesMapCopy; +} + +const std::map>& Model::getModelVersions() const { + return modelVersions; +} + +void Model::updateDefaultVersion(int ignoredVersion) { + model_version_t newDefaultVersion = 0; + SPDLOG_INFO("Updating default version for model: {}, from: {}", getName(), defaultVersion); + for (const auto& [version, versionInstance] : modelVersions) { + if (version != ignoredVersion && + version > newDefaultVersion && + ModelVersionState::AVAILABLE == versionInstance->getStatus().getState()) { + newDefaultVersion = version; + } + } + defaultVersion = newDefaultVersion; + if (newDefaultVersion) { + SPDLOG_INFO("Updated default version for model: {}, to: {}", getName(), newDefaultVersion); + } else { + SPDLOG_INFO("Model: {} will not have default version since no version is available.", getName()); + } +} + +const std::shared_ptr Model::getDefaultModelInstance() const { + std::shared_lock lock(modelVersionsMtx); + auto defaultVersion = getDefaultVersion(); + const auto modelInstanceIt = modelVersions.find(defaultVersion); + + if (modelVersions.end() == modelInstanceIt) { + SPDLOG_WARN("Default version: {} for model: {} not found", defaultVersion, getName()); + return nullptr; + } + return modelInstanceIt->second; +} + +std::shared_ptr Model::modelInstanceFactory(const std::string& modelName, const model_version_t modelVersion, ov::Core& ieCore) { + if (isStateful()) { + SPDLOG_DEBUG("Creating new stateful model instance - model name: {}; model version: {};", modelName, modelVersion); + return std::move(std::static_pointer_cast( + std::make_shared(modelName, modelVersion, ieCore, this->globalSequencesViewer))); + } else { + SPDLOG_DEBUG("Creating new model instance - model name: {}; model version: {};", modelName, modelVersion); + return std::move(std::make_shared(modelName, modelVersion, ieCore)); + } +} + +Status Model::addVersion(const ModelConfig& config, ov::Core& ieCore) { + const auto& version = config.getVersion(); + std::shared_ptr modelInstance = modelInstanceFactory(config.getName(), version, ieCore); + + std::unique_lock lock(modelVersionsMtx); + modelVersions.emplace(version, modelInstance); + lock.unlock(); + auto status = modelInstance->loadModel(config); + if (!status.ok()) { + return status; + } + updateDefaultVersion(); + subscriptionManager.notifySubscribers(); + return StatusCode::OK; +} + +Status Model::addVersions(std::shared_ptr versionsToStart, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed) { + Status result = StatusCode::OK; + downloadModels(fs, config, versionsToStart); + versionsFailed->clear(); + for (const auto version : *versionsToStart) { + SPDLOG_INFO("Will add model: {}; version: {} ...", getName(), version); + config.setVersion(version); + config.parseModelMapping(); + auto status = addVersion(config, ieCore); + if (!status.ok()) { + SPDLOG_ERROR("Error occurred while loading model: {}; version: {}; error: {}", + getName(), + version, + status.string()); + versionsFailed->push_back(version); + result = status; + cleanupModelTmpFiles(config); + } + } + return result; +} + +Status Model::retireVersions(std::shared_ptr versionsToRetire) { + Status result = StatusCode::OK; + for (const auto version : *versionsToRetire) { + SPDLOG_INFO("Will unload model: {}; version: {} ...", getName(), version); + auto modelVersion = getModelInstanceByVersion(version); + if (!modelVersion) { + Status status = StatusCode::UNKNOWN_ERROR; + SPDLOG_ERROR("Error occurred while unloading model: {}; version: {}; error: {}", + getName(), + version, + status.string()); + result = status; + continue; + } + cleanupModelTmpFiles(modelVersion->getModelConfig()); + updateDefaultVersion(version); + modelVersion->retireModel(); + } + subscriptionManager.notifySubscribers(); + return result; +} + +Status Model::cleanupFailedLoad(std::shared_ptr versionsToCleanUp) { + Status result = StatusCode::OK; + for (const auto version : *versionsToCleanUp) { + SPDLOG_INFO("Will clean up model: {}; version: {} ...", getName(), version); + auto modelVersion = getModelInstanceByVersion(version); + if (!modelVersion) { + Status status = StatusCode::UNKNOWN_ERROR; + SPDLOG_ERROR("Error occurred while cleaning up model: {}; version: {}; error: {}", + getName(), + version, + status.string()); + result = status; + continue; + } + cleanupModelTmpFiles(modelVersion->getModelConfig()); + updateDefaultVersion(version); + modelVersion->cleanupFailedLoad(); + } + subscriptionManager.notifySubscribers(); + return result; +} + +void Model::retireAllVersions() { + if (!(customLoaderName.empty())) { + auto& customloaders = ovms::CustomLoaders::instance(); + auto loaderPtr = customloaders.find(customLoaderName); + if (loaderPtr != nullptr) { + loaderPtr->retireModel(name); + } else { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Could not find custom loader for model: {} but it is using custom loader: {}", getName(), customLoaderName); + } + } + + for (const auto versionModelInstancePair : modelVersions) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Will unload model: {}; version: {} ...", getName(), versionModelInstancePair.first); + cleanupModelTmpFiles(versionModelInstancePair.second->getModelConfig()); + versionModelInstancePair.second->retireModel(); + updateDefaultVersion(); + } + subscriptionManager.notifySubscribers(); +} + +void Model::cleanupAllVersions() { + if (!(customLoaderName.empty())) { + auto& customloaders = ovms::CustomLoaders::instance(); + auto loaderPtr = customloaders.find(customLoaderName); + if (loaderPtr != nullptr) { + loaderPtr->retireModel(name); + } else { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Could not find custom loader for model: {} but it is using custom loader: {}", getName(), customLoaderName); + } + } + + for (const auto versionModelInstancePair : modelVersions) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Will unload model: {}; version: {} ...", getName(), versionModelInstancePair.first); + cleanupModelTmpFiles(versionModelInstancePair.second->getModelConfig()); + versionModelInstancePair.second->cleanupFailedLoad(); + updateDefaultVersion(); + } + subscriptionManager.notifySubscribers(); +} + +Status Model::reloadVersions(std::shared_ptr versionsToReload, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed) { + Status result = StatusCode::OK; + for (const auto version : *versionsToReload) { + SPDLOG_INFO("Will reload model: {}; version: {} ...", getName(), version); + config.setVersion(version); + auto status = config.parseModelMapping(); + if ((!status.ok()) && (status != StatusCode::FILE_INVALID)) { + SPDLOG_ERROR("Error while parsing model mapping for model {}; error: {}", getName(), status.string()); + } + + auto modelVersion = getModelInstanceByVersion(version); + + if (!modelVersion) { + SPDLOG_ERROR("Error occurred while reloading model: {}; version: {}; Model version was not found", + getName(), + version); + result = StatusCode::UNKNOWN_ERROR; + continue; + } + if (modelVersion->getStatus().getState() == ModelVersionState::END || + modelVersion->getStatus().getState() == ModelVersionState::LOADING || + modelVersion->getModelConfig().getBasePath() != config.getBasePath()) { + downloadModels(fs, config, versionsToReload); + } else { + config.setLocalPath(modelVersion->getModelConfig().getLocalPath()); + } + status = modelVersion->reloadModel(config); + if (!status.ok()) { + SPDLOG_ERROR("Error occurred while loading model: {}; version: {}; error: {}", + getName(), + version, + status.string()); + result = status; + versionsFailed->push_back(version); + continue; + } + updateDefaultVersion(); + } + subscriptionManager.notifySubscribers(); + return result; +} + +Status Model::cleanupModelTmpFiles(const ModelConfig& config) { + auto lfstatus = StatusCode::OK; + + if (config.isCloudStored()) { + LocalFileSystem lfs; + lfstatus = lfs.deleteFileFolder(config.getPath()); + if (lfstatus != StatusCode::OK) { + SPDLOG_ERROR("Error occurred while deleting local copy of cloud model: {} reason: {}", + config.getLocalPath(), + lfstatus); + } else { + SPDLOG_DEBUG("Model removed from: {}", config.getPath()); + } + } + + return lfstatus; +} + +} // namespace ovms diff --git a/src/ovms_lib/model.hpp b/src/ovms_lib/model.hpp new file mode 100644 index 0000000000..5d513072e6 --- /dev/null +++ b/src/ovms_lib/model.hpp @@ -0,0 +1,246 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "filesystem.hpp" +#include "modelchangesubscription.hpp" +#include "modelinstance.hpp" +#include "statefulmodelinstance.hpp" + +namespace ovms { +class PipelineDefinition; +/* * @brief This class represent inference models + */ +class Model { +private: + /** + * @brief Mutex for protecting concurrent modfying and accessing modelVersions + */ + mutable std::shared_mutex modelVersionsMtx; + + /** + * @brief Flag indicating whether model is stateful or not + */ + bool stateful; + + GlobalSequencesViewer* globalSequencesViewer; + + /** + * @brief Update default version + * + * @param ignoredVersion Version to exclude from being selected as the default version + */ + void updateDefaultVersion(int ignoredVersion = 0); + +protected: + /** + * @brief Model name + */ + std::string name; + + /** + * @brief Holds different versions of model + */ + std::map> modelVersions; + + /** + * @brief Model default version + * + */ + model_version_t defaultVersion = 0; + + /** + * @brief Get default version + * + * @return default version + */ + const model_version_t getDefaultVersion() const { + SPDLOG_DEBUG("Getting default version for model: {}, {}", getName(), defaultVersion); + return defaultVersion; + } + + /** + * @brief Adds a new version of ModelInstance to the list of versions + * + * @param config model configuration + * + * @return status + */ + virtual Status addVersion(const ModelConfig& config, ov::Core& ieCore); + + /** + * @brief ModelInstances factory + * + * @return modelInstance + */ + virtual std::shared_ptr modelInstanceFactory(const std::string& modelName, const model_version_t modelVersion, ov::Core& ieCore); + + ModelChangeSubscription subscriptionManager; + + /** + * @brief Holds the custom loader interface Name + * + */ + std::string customLoaderName; + +public: + /** + * @brief Constructor + */ + Model(const std::string& name, bool stateful, GlobalSequencesViewer* globalSequencesViewer) : + stateful(stateful), + globalSequencesViewer(globalSequencesViewer), + name(name), + defaultVersion(0), + subscriptionManager(std::string("model: ") + name) {} + + /** + * @brief Destroy the Model object + * + */ + virtual ~Model() {} + + /** + * @brief Gets the model name + * + * @return model name + */ + const std::string& getName() const { + return name; + } + + const bool isStateful() const { + return stateful; + } + + /** + * @brief Gets the default ModelInstance + * + * @return ModelInstance + */ + const std::shared_ptr getDefaultModelInstance() const; + + /** + * @brief Gets model versions instances + * + * @return model versions instances + */ + const std::map>& getModelVersions() const; + + /** + * @brief Gets model versions instances + * + * @return model versions instances + */ + const std::map getModelVersionsMapCopy() const; + + /** + * @brief Finds ModelInstance with specific version + * + * @param version of the model to search for + * + * @return specific model version + */ + const std::shared_ptr getModelInstanceByVersion(const model_version_t& version) const { + std::shared_lock lock(modelVersionsMtx); + auto it = modelVersions.find(version); + return it != modelVersions.end() ? it->second : nullptr; + } + + /** + * @brief Adds new versions of ModelInstance + * + * @param config model configuration + * + * @return status + */ + Status addVersions(std::shared_ptr versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed); + + /** + * @brief Retires versions of Model + * + * @param versions versions to retire + * + * @return status + */ + Status retireVersions(std::shared_ptr versions); + + /** + * @brief Cleans up versions of Model + * + * @param versions versions to clean up + * + * @return status + */ + Status cleanupFailedLoad(std::shared_ptr versions); + + /** + * @brief Retires all versions of Model + */ + void retireAllVersions(); + + /** + * @brief Cleans up all versions of Model + */ + void cleanupAllVersions(); + + /** + * @brief Reloads versions of Model + * + * @param config model configuration + * + * @return status + */ + Status reloadVersions(std::shared_ptr versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed); + + void subscribe(PipelineDefinition& pd); + void unsubscribe(PipelineDefinition& pd); + /** + * @brief Set the custom loader name + * + * @param custom loader name + * + */ + + bool isAnyVersionSubscribed() const; + + void setCustomLoaderName(const std::string name) { + customLoaderName = name; + } + + /** + * @brief Reset the custom loader name + * + */ + void resetCustomLoaderName() { + customLoaderName.clear(); + } + + /** + * @brief Delete temporary model files + * + */ + static Status cleanupModelTmpFiles(const ModelConfig& config); +}; +} // namespace ovms diff --git a/src/ovms_lib/model_service.cpp b/src/ovms_lib/model_service.cpp new file mode 100644 index 0000000000..5cfbb7105f --- /dev/null +++ b/src/ovms_lib/model_service.cpp @@ -0,0 +1,220 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "model_service.hpp" + +#include +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/get_model_status.pb.h" +#include "tensorflow_serving/apis/model_service.grpc.pb.h" +#include "tensorflow_serving/apis/model_service.pb.h" +#pragma GCC diagnostic pop + +#include "modelmanager.hpp" +#include "pipelinedefinition.hpp" +#include "servablemanagermodule.hpp" +#include "status.hpp" + +using google::protobuf::util::JsonPrintOptions; +using google::protobuf::util::MessageToJsonString; + +namespace ovms { + +void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status) { + SPDLOG_DEBUG("add_status_to_response version={} status={}", version, model_version_status.getStateString()); + auto status_to_fill = response->add_model_version_status(); + status_to_fill->set_state(static_cast(static_cast(model_version_status.getState()))); + status_to_fill->set_version(version); + status_to_fill->clear_status(); + status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(model_version_status.getErrorCode()))); + status_to_fill->mutable_status()->set_error_message(model_version_status.getErrorMsg()); +} + +void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, const model_version_t version, const PipelineDefinitionStatus& pipeline_status) { + auto [state, error_code] = pipeline_status.convertToModelStatus(); + SPDLOG_DEBUG("add_status_to_response state={} error_code", state, error_code); + auto status_to_fill = response->add_model_version_status(); + status_to_fill->set_state(static_cast(static_cast(state))); + status_to_fill->set_version(version); + status_to_fill->clear_status(); + status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(error_code))); + status_to_fill->mutable_status()->set_error_message(ModelVersionStatusErrorCodeToString(error_code)); +} + +::grpc::Status ModelServiceImpl::GetModelStatus( + ::grpc::ServerContext* context, const tensorflow::serving::GetModelStatusRequest* request, + tensorflow::serving::GetModelStatusResponse* response) { + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return GetModelStatusImpl::getModelStatus(request, response, manager).grpc(); +} + +Status GetModelStatusImpl::createGrpcRequest(std::string model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request) { + request->mutable_model_spec()->set_name(model_name); + if (model_version.has_value()) { + request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); + } + return StatusCode::OK; +} + +Status GetModelStatusImpl::serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output) { + JsonPrintOptions opts; + opts.add_whitespace = true; + opts.always_print_primitive_fields = true; + const auto& status = MessageToJsonString(*response, output, opts); + if (!status.ok()) { + SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); + return StatusCode::JSON_SERIALIZATION_ERROR; + } + return StatusCode::OK; +} + +Status GetModelStatusImpl::getModelStatus( + const tensorflow::serving::GetModelStatusRequest* request, + tensorflow::serving::GetModelStatusResponse* response, + ModelManager& manager) { + SPDLOG_DEBUG("model_service: request: {}", request->DebugString()); + + bool has_requested_version = request->model_spec().has_version(); + auto requested_version = request->model_spec().version().value(); + std::string requested_model_name = request->model_spec().name(); + auto model_ptr = manager.findModelByName(requested_model_name); + if (!model_ptr) { + SPDLOG_DEBUG("GetModelStatus: Model {} is missing, trying to find pipeline with such name", requested_model_name); + auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(requested_model_name); + if (!pipelineDefinition) { + return StatusCode::MODEL_NAME_MISSING; + } + + addStatusToResponse(response, pipelineDefinition->getVersion(), pipelineDefinition->getStatus()); + SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); + SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); + return StatusCode::OK; + } + + SPDLOG_DEBUG("requested model: {}, has_version: {} (version: {})", requested_model_name, has_requested_version, requested_version); + if (has_requested_version && requested_version != 0) { + // return details only for a specific version of requested model; NOT_FOUND otherwise. If requested_version == 0, default is returned. + std::shared_ptr model_instance = model_ptr->getModelInstanceByVersion(requested_version); + if (!model_instance) { + SPDLOG_DEBUG("requested model {} in version {} was not found.", requested_model_name, requested_version); + return StatusCode::MODEL_VERSION_MISSING; + } + const auto& status = model_instance->getStatus(); + SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, requested_version, status.getStateString()); + addStatusToResponse(response, requested_version, status); + } else { + // return status details of all versions of a requested model. + auto modelVersionsInstances = model_ptr->getModelVersionsMapCopy(); + for (const auto& [modelVersion, modelInstance] : modelVersionsInstances) { + const auto& status = modelInstance.getStatus(); + SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, modelVersion, status.getStateString()); + addStatusToResponse(response, modelVersion, status); + } + } + SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); + SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); + return StatusCode::OK; +} + +Status GetModelStatusImpl::getAllModelsStatuses(std::map& modelsStatuses, ModelManager& manager) { + std::shared_lock lock(manager.modelsMtx); + std::map modelsStatusesTmp; + + const std::map>& models = manager.getModels(); + for (auto const& model : models) { + std::optional noValueModelVersion; + tensorflow::serving::GetModelStatusRequest request; + GetModelStatusImpl::createGrpcRequest(model.first, noValueModelVersion, &request); + tensorflow::serving::GetModelStatusResponse response; + auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager); + if (status != StatusCode::OK) { + // For now situation when getModelStatus return status other than OK cannot occur because we never remove models and pipelines from model manager. + // However, if something in this matter will change we should handle this somehow. + continue; + } + modelsStatusesTmp.insert({model.first, response}); + } + lock.unlock(); + + const std::vector& pipelinesNames = manager.getPipelineFactory().getPipelinesNames(); + for (auto const& pipelineName : pipelinesNames) { + std::optional noValueModelVersion; + tensorflow::serving::GetModelStatusRequest request; + GetModelStatusImpl::createGrpcRequest(pipelineName, noValueModelVersion, &request); + tensorflow::serving::GetModelStatusResponse response; + auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager); + if (status != StatusCode::OK) { + // Same situation like with models. + continue; + } + modelsStatusesTmp.insert({pipelineName, response}); + } + + modelsStatuses.merge(modelsStatusesTmp); + return StatusCode::OK; +} + +Status GetModelStatusImpl::serializeModelsStatuses2Json(const std::map& modelsStatuses, std::string& output) { + std::string outputTmp; + if (modelsStatuses.begin() == modelsStatuses.end()) { + output = "{}"; + return StatusCode::OK; + } + + outputTmp += "{\n"; + for (auto modelStatus = modelsStatuses.begin(); modelStatus != modelsStatuses.end(); modelStatus++) { + outputTmp += ("\"" + modelStatus->first + "\" : \n"); + std::string responseStr; + auto status = GetModelStatusImpl::serializeResponse2Json(&modelStatus->second, &responseStr); + if (status != StatusCode::OK) { + return status; + } + responseStr.pop_back(); + outputTmp += responseStr; + if (std::next(modelStatus) != modelsStatuses.end()) { + outputTmp += (",\n"); + } + } + outputTmp += "\n}"; + output = outputTmp; + + return StatusCode::OK; +} + +ModelServiceImpl::ModelServiceImpl(ovms::Server& ovmsServer) : + ovmsServer(ovmsServer) {} + +::grpc::Status ModelServiceImpl::HandleReloadConfigRequest( + ::grpc::ServerContext* context, const tensorflow::serving::ReloadConfigRequest* request, + tensorflow::serving::ReloadConfigResponse* response) { + SPDLOG_INFO("Requested HandleReloadConfigRequest - but this service is reloading config automatically by itself, therefore this operation has no *EXTRA* affect."); + return grpc::Status::OK; // we're reloading config all the time; for a total client compatibility, this means returning success here. +} + +} // namespace ovms diff --git a/src/ovms_lib/model_service.hpp b/src/ovms_lib/model_service.hpp new file mode 100644 index 0000000000..d2df3d702c --- /dev/null +++ b/src/ovms_lib/model_service.hpp @@ -0,0 +1,65 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/get_model_status.pb.h" +#include "tensorflow_serving/apis/model_service.grpc.pb.h" +#include "tensorflow_serving/apis/model_service.pb.h" +#pragma GCC diagnostic pop + +#include "modelmanager.hpp" +#include "status.hpp" + +namespace ovms { +class Server; + +void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status); + +class ModelServiceImpl final : public tensorflow::serving::ModelService::Service { + ovms::Server& ovmsServer; + +public: + ModelServiceImpl(ovms::Server& ovmsServer); + ::grpc::Status GetModelStatus(::grpc::ServerContext* context, + const tensorflow::serving::GetModelStatusRequest* request, + tensorflow::serving::GetModelStatusResponse* response) override; + ::grpc::Status HandleReloadConfigRequest(::grpc::ServerContext* context, + const tensorflow::serving::ReloadConfigRequest* request, + tensorflow::serving::ReloadConfigResponse* response) override; +}; + +class GetModelStatusImpl { + ovms::Server& ovmsServer; + +public: + GetModelStatusImpl(ovms::Server& ovmsServer); + static Status getModelStatus(const tensorflow::serving::GetModelStatusRequest* request, tensorflow::serving::GetModelStatusResponse* response, ModelManager& manager); + static Status createGrpcRequest(std::string model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request); + static Status serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output); + + static Status getAllModelsStatuses(std::map& models_versions, ModelManager& manager); + static Status serializeModelsStatuses2Json(const std::map& models_versions, std::string& output); +}; + +} // namespace ovms diff --git a/src/ovms_lib/model_version_policy.cpp b/src/ovms_lib/model_version_policy.cpp new file mode 100644 index 0000000000..600d5aa365 --- /dev/null +++ b/src/ovms_lib/model_version_policy.cpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "model_version_policy.hpp" + +#include +#include +#include +#include +#include + +namespace ovms { + +std::shared_ptr ModelVersionPolicy::getDefaultVersionPolicy() { + return std::make_shared(1); +} + +AllModelVersionPolicy::operator std::string() const { + return std::string("all"); +} + +std::vector SpecificModelVersionPolicy::filter(std::vector versions) const { + std::vector result; + std::sort(versions.begin(), versions.end()); + std::set_intersection( + versions.begin(), + versions.end(), + specificVersions.begin(), + specificVersions.end(), + std::back_inserter(result)); + return result; +} + +SpecificModelVersionPolicy::operator std::string() const { + std::stringstream versionStream; + versionStream << "specific: "; + std::copy(specificVersions.begin(), specificVersions.end(), std::ostream_iterator(versionStream, " ")); + return versionStream.str(); +} + +std::vector LatestModelVersionPolicy::filter(std::vector versions) const { + std::vector result; + std::sort(versions.begin(), versions.end(), std::greater()); + for (size_t i = 0; i < numVersions && i < versions.size(); i++) { + result.push_back(versions[i]); + } + return result; +} + +LatestModelVersionPolicy::operator std::string() const { + return std::string("latest: ") + std::to_string(numVersions); +} + +} // namespace ovms diff --git a/src/ovms_lib/model_version_policy.hpp b/src/ovms_lib/model_version_policy.hpp new file mode 100644 index 0000000000..f213126bf5 --- /dev/null +++ b/src/ovms_lib/model_version_policy.hpp @@ -0,0 +1,135 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include "modelversion.hpp" + +namespace ovms { + +/** + * @brief Base class for model version policy types + */ +class ModelVersionPolicy { +protected: + ModelVersionPolicy() {} + virtual ~ModelVersionPolicy() {} + +public: + /** + * @brief Strips out model versions list passed by parameter depending on internal state + * + * @param versions model versions to filter + * @return Filtered version list + */ + virtual model_versions_t filter(model_versions_t versions) const = 0; + + /** + * @brief Creates default model version policy, by default only one version (highest) should be served + * + * @param highestVersion highest version + * @return default version policy + */ + static std::shared_ptr getDefaultVersionPolicy(); + + /** + * @brief Converts ModelVersionPolicy to readable string + */ + virtual operator std::string() const = 0; +}; + +/** + * @brief Model version policy that enables all available versions + */ +class AllModelVersionPolicy : public ModelVersionPolicy { +public: + /** + * @brief Default constructor, nothing needs to be specified since all versions will be served + */ + AllModelVersionPolicy() {} + + /** + * @brief Filters passed versions depending on internal state + * + * @param versions model versions to filter + * @return Filtered version list + */ + model_versions_t filter(model_versions_t versions) const override { + return versions; + } + + operator std::string() const override; +}; + +/** + * @brief Model version policy for explicitely specifying which versions should be enabled + */ +class SpecificModelVersionPolicy : public ModelVersionPolicy { + model_versions_t specificVersions; + +public: + /** + * @brief Default constructor + * + * @param versions list of all model versions that should be served + */ + SpecificModelVersionPolicy(const model_versions_t& versions) : + specificVersions(versions) { + std::sort(specificVersions.begin(), specificVersions.end()); + } + + /** + * @brief Filters passed versions depending on internal state + * + * @param versions model versions to filter + * @return Filtered version list + */ + model_versions_t filter(model_versions_t versions) const override; + + operator std::string() const override; +}; + +/** + * @brief Model version policy for serving only X latest versions + */ +class LatestModelVersionPolicy : public ModelVersionPolicy { + size_t numVersions; + +public: + /** + * @brief Default constructor + * + * @param numVersions number of latest versions to be served + */ + LatestModelVersionPolicy(size_t numVersions = 1) : + numVersions(numVersions) {} + + /** + * @brief Filters passed versions depending on internal state + * + * @param versions model versions to filter + * @return Filtered version list + */ + model_versions_t filter(model_versions_t versions) const override; + + operator std::string() const override; +}; + +} // namespace ovms diff --git a/src/ovms_lib/modelchangesubscription.cpp b/src/ovms_lib/modelchangesubscription.cpp new file mode 100644 index 0000000000..804850a3b4 --- /dev/null +++ b/src/ovms_lib/modelchangesubscription.cpp @@ -0,0 +1,57 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelchangesubscription.hpp" + +#include +#include + +#include "pipelinedefinition.hpp" + +namespace ovms { +void ModelChangeSubscription::subscribe(PipelineDefinition& pd) { + SPDLOG_INFO("Subscription to {} from {}", ownerName, pd.getName()); + if (subscriptions.find(pd.getName()) != subscriptions.end()) { + std::stringstream ss; + ss << "Tried to subscribe pipeline:" << pd.getName() << " to:" << ownerName; + ss << ", but this pipeline was already subscribed"; + SPDLOG_ERROR(ss.str().c_str()); + throw std::logic_error(ss.str()); + } + subscriptions.insert({pd.getName(), pd}); +} + +void ModelChangeSubscription::unsubscribe(PipelineDefinition& pd) { + SPDLOG_INFO("Subscription to {} from {} removed", ownerName, pd.getName()); + auto numberOfErased = subscriptions.erase(pd.getName()); + if (0 == numberOfErased) { + std::stringstream ss; + ss << "Tried to unsubscribe pipeline:" << pd.getName() << " to:" << ownerName; + ss << ", but this pipeline was never subscribed"; + SPDLOG_ERROR(ss.str().c_str()); + throw std::logic_error(ss.str()); + } +} + +void ModelChangeSubscription::notifySubscribers() { + if (subscriptions.size() == 0) { + return; + } + SPDLOG_INFO("Notified subscribers of: {}", ownerName); + for (auto& [pipelineName, pipelineDefinition] : subscriptions) { + pipelineDefinition.notifyUsedModelChanged(ownerName); + } +} +} // namespace ovms diff --git a/src/ovms_lib/modelchangesubscription.hpp b/src/ovms_lib/modelchangesubscription.hpp new file mode 100644 index 0000000000..427b3cea03 --- /dev/null +++ b/src/ovms_lib/modelchangesubscription.hpp @@ -0,0 +1,42 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include +#include + +namespace ovms { + +class PipelineDefinition; + +class ModelChangeSubscription { + const std::string ownerName; + std::unordered_map subscriptions; + +public: + ModelChangeSubscription(const std::string& ownerName) : + ownerName(ownerName) {} + + void subscribe(PipelineDefinition& pd); + + void unsubscribe(PipelineDefinition& pd); + + void notifySubscribers(); + + bool isSubscribed() const { return subscriptions.size() > 0; } +}; +} // namespace ovms diff --git a/src/ovms_lib/modelconfig.cpp b/src/ovms_lib/modelconfig.cpp new file mode 100644 index 0000000000..b3c95ff4d3 --- /dev/null +++ b/src/ovms_lib/modelconfig.cpp @@ -0,0 +1,669 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelconfig.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "logging.hpp" +#include "schema.hpp" +#include "stringutils.hpp" + +namespace ovms { + +bool ModelConfig::isDeviceUsed(const std::string& device) const { + if (this->isSingleDeviceUsed(device)) + return true; + if (targetDevice.find(device) != std::string::npos) + return true; + if (targetDevice == "AUTO") + return true; + return false; +} + +bool ModelConfig::isReloadRequired(const ModelConfig& rhs) const { + if (this->name != rhs.name) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to name mismatch", this->name); + return true; + } + if (this->stateful != rhs.stateful) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to stateful mismatch", this->name); + return true; + } + if (this->idleSequenceCleanup != rhs.idleSequenceCleanup) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to idleSequenceCleanup mismatch", this->name); + return true; + } + if (this->maxSequenceNumber != rhs.maxSequenceNumber) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to maxSequenceNumber mismatch", this->name); + return true; + } + if (this->lowLatencyTransformation != rhs.lowLatencyTransformation) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to lowLatencyTransformation mismatch", this->name); + return true; + } + if (this->basePath != rhs.basePath) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to original base path mismatch", this->name); + return true; + } + if (this->targetDevice != rhs.targetDevice) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to target device mismatch", this->name); + return true; + } + if (this->batchingMode != rhs.batchingMode) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to batching mode mismatch", this->name); + return true; + } + if (!isBatchSizeConfigurationEqual(rhs)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to batch size mismatch", this->name); + return true; + } + if (this->nireq != rhs.nireq) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to nireq mismatch", this->name); + return true; + } + if (this->pluginConfig != rhs.pluginConfig) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to plugin config mismatch", this->name); + return true; + } + if (!isLayoutConfigurationEqual(rhs)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to named layout mismatch", this->name); + return true; + } + if (!isShapeConfigurationEqual(rhs)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to shape configuration mismatch", this->name); + return true; + } + if (isCustomLoaderConfigChanged(rhs)) { + return true; + } + if (this->isAllowCacheSetToTrue() != rhs.isAllowCacheSetToTrue()) { + return true; + } + return false; +} + +bool ModelConfig::isCustomLoaderConfigChanged(const ModelConfig& rhs) const { + if (this->customLoaderOptionsConfigMap.size() != rhs.customLoaderOptionsConfigMap.size()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to custom loader config mismatch", this->name); + return true; + } + if (this->customLoaderOptionsConfigMap.size() > 0 && rhs.customLoaderOptionsConfigMap.size() > 0) { + if (!(this->customLoaderOptionsConfigMap == rhs.customLoaderOptionsConfigMap)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to custom loader config mismatch", this->name); + return true; + } + } + return false; +} + +bool ModelConfig::isBatchSizeConfigurationEqual(const ModelConfig& rhs) const { + if (this->batchSize.has_value() != rhs.batchSize.has_value()) { + return false; + } + if (this->batchSize.has_value() && this->batchSize.value() != rhs.batchSize.value()) { + return false; + } + return true; +} + +bool ModelConfig::isLayoutConfigurationEqual(const ModelConfig& rhs) const { + if (this->layout != rhs.layout) { + return false; + } + + if (this->layouts.size() != rhs.layouts.size()) { + return false; + } + for (const auto& [name, layoutConfig] : this->layouts) { + auto it = rhs.layouts.find(name); + if (it == rhs.layouts.end()) { + return false; + } + if (layoutConfig != it->second) { + return false; + } + } + return true; +} + +bool ModelConfig::isShapeConfigurationEqual(const ModelConfig& rhs) const { + if (this->shapes.size() != rhs.shapes.size()) { + return false; + } + for (const auto& [name, shape] : this->shapes) { + auto it = rhs.shapes.find(name); + if (it == rhs.shapes.end()) { + return false; + } + if (shape != it->second) { + return false; + } + } + return true; +} + +std::tuple> ModelConfig::extractBatchingParams(std::string configBatchSize) { + Mode batchingMode = FIXED; + std::optional effectiveBatchSize = std::nullopt; + if (configBatchSize == "auto") { + batchingMode = AUTO; + } else if (configBatchSize == "0") { + // do nothing + } else { + Dimension dim; + auto status = Dimension::fromString(configBatchSize, dim); + if (!status.ok()) { + SPDLOG_WARN("Wrong batch size parameter provided. Model batch size will be set to default."); + } else { + effectiveBatchSize = dim; + } + } + return std::tuple>{batchingMode, effectiveBatchSize}; +} + +Status ModelConfig::parseModelVersionPolicy(std::string command) { + rapidjson::Document node; + if (command == "") { + modelVersionPolicy = ModelVersionPolicy::getDefaultVersionPolicy(); + return StatusCode::OK; + } + + if (node.Parse(command.c_str()).HasParseError()) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + + if (!node.IsObject()) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + if (node.MemberCount() != 1) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + + auto m = node.FindMember("all"); + if (m != node.MemberEnd()) { + modelVersionPolicy = std::make_shared(); + return StatusCode::OK; + } + + m = node.FindMember("specific"); + if (m != node.MemberEnd()) { + auto& specific = m->value; + if (specific.MemberCount() != 1) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + m = specific.FindMember("versions"); + if (m == specific.MemberEnd()) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + std::vector versions; + for (auto& version : m->value.GetArray()) { + if (version.IsUint64() && version.GetUint64() > 0) { + versions.push_back(version.GetUint64()); + } else { + SPDLOG_WARN("Model policy specified in config contains invalid version. Version should be a number greater than 0."); + } + } + modelVersionPolicy = std::make_shared(versions); + return StatusCode::OK; + } + + m = node.FindMember("latest"); + if (m != node.MemberEnd()) { + auto& latest = m->value; + if (latest.MemberCount() != 1) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + m = latest.FindMember("num_versions"); + if (m == latest.MemberEnd()) { + return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; + } + modelVersionPolicy = std::make_shared(m->value.GetInt64()); + return StatusCode::OK; + } + + return StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY; +} + +Status ModelConfig::parsePluginConfig(const rapidjson::Value& node) { + if (!node.IsObject()) { + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + + for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { + if (it->value.IsString()) { + pluginConfig[it->name.GetString()] = it->value.GetString(); + } else if (it->value.IsInt64()) { + pluginConfig[it->name.GetString()] = std::to_string(it->value.GetInt64()); + } else if (it->value.IsDouble()) { + pluginConfig[it->name.GetString()] = std::to_string(it->value.GetDouble()); + } else { + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + } + + return StatusCode::OK; +} + +Status ModelConfig::parseShapeParameter(const rapidjson::Value& node) { + if (!node.IsObject()) { + return StatusCode::SHAPE_WRONG_FORMAT; + } + + shapes_info_map_t shapes; + for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { + if (!it->value.IsString()) { + return StatusCode::SHAPE_WRONG_FORMAT; + } + ShapeInfo shapeInfo; + auto status = parseShape(shapeInfo, it->value.GetString()); + if (!status.ok()) { + return status; + } + shapes[it->name.GetString()] = shapeInfo; + } + this->shapes = shapes; + + return StatusCode::OK; +} + +Status ModelConfig::parseShapeParameter(const std::string& command) { + this->shapes.clear(); + + if (command.empty()) { + return StatusCode::OK; + } + + // parse as string + if (command.front() == shapeLeft || command == "auto") { + ShapeInfo shapeInfo; + auto status = parseShape(shapeInfo, command); + if (!status.ok()) { + return status; + } + this->addShape(ANONYMOUS_INPUT_NAME, shapeInfo); + return StatusCode::OK; + } + + // parse as json + rapidjson::Document node; + if (command.empty()) { + return StatusCode::OK; + } + if (node.Parse(command.c_str()).HasParseError()) { + return StatusCode::SHAPE_WRONG_FORMAT; + } + return parseShapeParameter(node); +} + +Status ModelConfig::parseLayoutParameter(const rapidjson::Value& node) { + if (!node.IsObject()) { + return StatusCode::LAYOUT_WRONG_FORMAT; + } + layout_configurations_map_t layouts; + for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { + if (!it->value.IsString()) { + return StatusCode::LAYOUT_WRONG_FORMAT; + } + std::string layoutStr = it->value.GetString(); + std::transform(layoutStr.begin(), layoutStr.end(), layoutStr.begin(), ::toupper); + + LayoutConfiguration layout; + auto status = LayoutConfiguration::fromString(layoutStr, layout); + if (!status.ok()) { + return status; + } + layouts[it->name.GetString()] = layout; + } + setLayouts(layouts); + + return StatusCode::OK; +} + +Status ModelConfig::parseLayoutParameter(const std::string& command) { + this->layouts.clear(); + this->layout = LayoutConfiguration(); + if (command.empty()) { + return StatusCode::OK; + } + + std::string upperCaseCommand; + std::transform(command.begin(), command.end(), std::back_inserter(upperCaseCommand), ::toupper); + + erase_spaces(upperCaseCommand); + + if (*upperCaseCommand.begin() != '{') { + LayoutConfiguration layout; + auto status = LayoutConfiguration::fromString(upperCaseCommand, layout); + if (!status.ok()) { + return status; + } + setLayout(layout); + return StatusCode::OK; + } + + // parse as json + rapidjson::Document node; + if (node.Parse(command.c_str()).HasParseError()) { + return StatusCode::LAYOUT_WRONG_FORMAT; + } + return parseLayoutParameter(node); +} + +Status ModelConfig::parseShape(ShapeInfo& shapeInfo, const std::string& str) { + if (str == "auto") { + shapeInfo.shapeMode = AUTO; + return StatusCode::OK; + } + + shapeInfo.shapeMode = FIXED; + shapeInfo.shape = Shape(); + return Shape::fromString(str, shapeInfo.shape); +} + +Status ModelConfig::parseModelMapping() { + SPDLOG_DEBUG("Parsing model: {} mapping from path: {}", getName(), getPath()); + mappingInputs.clear(); + mappingOutputs.clear(); + std::filesystem::path path = this->getPath(); + path.append(MAPPING_CONFIG_JSON); + + std::ifstream ifs(path.c_str()); + if (!ifs.good()) { + return StatusCode::FILE_INVALID; + } + + rapidjson::Document doc; + rapidjson::IStreamWrapper isw(ifs); + if (doc.ParseStream(isw).HasParseError()) { + SPDLOG_ERROR("Configuration file is not a valid JSON file."); + return StatusCode::JSON_INVALID; + } + + if (validateJsonAgainstSchema(doc, MODELS_MAPPING_INPUTS_SCHEMA) != StatusCode::OK) { + SPDLOG_WARN("Couldn't load inputs object from file {}", path.c_str()); + } else { + // Process inputs + const auto itr = doc.FindMember("inputs"); + for (const auto& key : itr->value.GetObject()) { + SPDLOG_DEBUG("Loaded input mapping {} => {}", key.name.GetString(), key.value.GetString()); + mappingInputs[key.name.GetString()] = key.value.GetString(); + reversedMappingInputs[key.value.GetString()] = key.name.GetString(); + } + } + + if (validateJsonAgainstSchema(doc, MODELS_MAPPING_OUTPUTS_SCHEMA) != StatusCode::OK) { + SPDLOG_WARN("Couldn't load outputs object from file {}", path.c_str()); + } else { + // Process outputs + const auto it = doc.FindMember("outputs"); + for (const auto& key : it->value.GetObject()) { + SPDLOG_DEBUG("Loaded output mapping {} => {}", key.name.GetString(), key.value.GetString()); + mappingOutputs[key.name.GetString()] = key.value.GetString(); + reversedMappingOutputs[key.value.GetString()] = key.name.GetString(); + } + } + + return StatusCode::OK; +} + +Status ModelConfig::parseNode(const rapidjson::Value& v) { + this->setName(v["name"].GetString()); + this->setBasePath(v["base_path"].GetString()); + Status firstErrorStatus = StatusCode::OK; + + // Check for optional parameters + if (v.HasMember("batch_size")) { + if (v["batch_size"].IsString()) { + this->setBatchingParams(v["batch_size"].GetString()); + } else { + this->setBatchingParams(v["batch_size"].GetUint64()); + } + } + if (v.HasMember("target_device")) + this->setTargetDevice(v["target_device"].GetString()); + if (v.HasMember("version")) { + this->setVersion(v["version"].GetUint64()); + } + if (v.HasMember("nireq")) + this->setNireq(v["nireq"].GetUint64()); + + if (v.HasMember("shape")) { + // Legacy format as string + if (v["shape"].IsString()) { + ShapeInfo shapeInfo; + auto status = parseShape(shapeInfo, v["shape"].GetString()); + if (!status.ok()) { + if (!firstErrorStatus.ok()) { + firstErrorStatus = status; + } + SPDLOG_WARN("There was an error parsing shape {}", v["shape"].GetString()); + } else { + this->addShape(ANONYMOUS_INPUT_NAME, shapeInfo); + } + } else { + // Map of shapes + for (auto& s : v["shape"].GetObject()) { + ShapeInfo shapeInfo; + bool valid = true; + // check if legacy format is used + if (s.value.IsString()) { + auto status = ModelConfig::parseShape(shapeInfo, s.value.GetString()); + if (!status.ok()) { + if (!firstErrorStatus.ok()) { + firstErrorStatus = status; + } + SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); + valid = false; + } + } else { + for (auto& sh : s.value.GetArray()) { + if (sh.IsUint64()) { + size_t dim = sh.GetUint64(); + if (dim > std::numeric_limits::max()) { + SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); + if (!firstErrorStatus.ok()) { + firstErrorStatus = StatusCode::SHAPE_WRONG_FORMAT; + } + valid = false; + break; + } + shapeInfo.shape.add(Dimension{static_cast(dim)}); + } else { + SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); + if (!firstErrorStatus.ok()) { + firstErrorStatus = StatusCode::SHAPE_WRONG_FORMAT; + } + valid = false; + break; + } + } + } + if (s.name.GetString() != ANONYMOUS_INPUT_NAME) { + if (valid) { + this->addShape(s.name.GetString(), shapeInfo); + } + } else { + SPDLOG_WARN("Provided shape name: {} is forbidden and will be omitted", ANONYMOUS_INPUT_NAME); + } + } + } + } + + if (v.HasMember("layout")) { + if (v["layout"].IsString()) { + Status status = this->parseLayoutParameter(v["layout"].GetString()); + if (!status.ok()) { + return status; + } + } else { + Status status = this->parseLayoutParameter(v["layout"]); + if (!status.ok()) { + return status; + } + } + } + + if (v.HasMember("plugin_config")) { + auto status = parsePluginConfig(v["plugin_config"]); + if (!status.ok()) { + if (!firstErrorStatus.ok()) { + firstErrorStatus = status; + } + SPDLOG_ERROR("Couldn't parse plugin config"); + return status; + } + } + + if (v.HasMember("stateful")) + this->setStateful(v["stateful"].GetBool()); + + if (v.HasMember("low_latency_transformation")) { + if (!this->isStateful()) { + SPDLOG_ERROR("Low latency transformation parameter was set for non stateful model {}.", v["name"].GetString()); + return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; + } + this->setLowLatencyTransformation(v["low_latency_transformation"].GetBool()); + } + + if (v.HasMember("idle_sequence_cleanup")) { + if (!this->isStateful()) { + SPDLOG_ERROR("Idle sequence cleanup parameter was set for non stateful model {}.", v["name"].GetString()); + return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; + } + this->setIdleSequenceCleanup(v["idle_sequence_cleanup"].GetBool()); + } + + if (v.HasMember("max_sequence_number")) { + if (!this->isStateful()) { + SPDLOG_ERROR("Max sequence number parameter was set for non stateful model {}.", v["name"].GetString()); + return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; + } + if (!v["max_sequence_number"].IsUint()) { + SPDLOG_ERROR("Sequence maximum number parameter was set above unsigned int value for model {}.", v["name"].GetString()); + return StatusCode::INVALID_MAX_SEQUENCE_NUMBER; + } + this->setMaxSequenceNumber(v["max_sequence_number"].GetUint()); + } + + if (v.HasMember("model_version_policy")) { + rapidjson::StringBuffer buffer; + buffer.Clear(); + rapidjson::Writer writer(buffer); + v["model_version_policy"].Accept(writer); + const auto& status = parseModelVersionPolicy(buffer.GetString()); + if (!status.ok()) { + if (!firstErrorStatus.ok()) { + firstErrorStatus = status; + } + SPDLOG_WARN("Couldn't parse model version policy. {}", status.string()); + } + } else { + modelVersionPolicy = ModelVersionPolicy::getDefaultVersionPolicy(); + } + + SPDLOG_DEBUG("Specified model parameters:"); + SPDLOG_DEBUG("model_basepath: {}", getBasePath()); + SPDLOG_DEBUG("model_name: {}", getName()); + SPDLOG_DEBUG("batch_size: {}", getBatchSize().has_value() ? getBatchSize().value().toString() : "not configured"); + if (isShapeAnonymous()) { + SPDLOG_DEBUG("shape: {}", std::string(getShapes().begin()->second)); + } else { + SPDLOG_DEBUG("shape:"); + for (auto& [shapeInput, shapeValue] : getShapes()) { + SPDLOG_DEBUG(" {}: {}", shapeInput, std::string(shapeValue)); + } + } + if (getModelVersionPolicy()) { + SPDLOG_DEBUG("model_version_policy: {}", std::string(*getModelVersionPolicy())); + } + SPDLOG_DEBUG("nireq: {}", getNireq()); + SPDLOG_DEBUG("target_device: {}", getTargetDevice()); + SPDLOG_DEBUG("plugin_config:"); + for (auto& [pluginParameter, pluginValue] : getPluginConfig()) { + SPDLOG_DEBUG(" {}: {}", pluginParameter, pluginValue.as()); + } + + bool batchSizeSet = (getBatchingMode() != FIXED || getBatchSize().has_value()); + bool shapeSet = (getShapes().size() > 0); + + SPDLOG_DEBUG("Batch size set: {}, shape set: {}", batchSizeSet, shapeSet); + if (batchSizeSet && shapeSet) { + SPDLOG_WARN("Both shape and batch size have been defined. Batch size parameter will be ignored."); + setBatchingMode(FIXED); + setBatchSize(std::nullopt); + } + + SPDLOG_DEBUG("stateful: {}", isStateful()); + if (isStateful()) { + SPDLOG_DEBUG("idle_sequence_cleanup: {}", getIdleSequenceCleanup()); + SPDLOG_DEBUG("max_sequence_number: {}", getMaxSequenceNumber()); + SPDLOG_DEBUG("low_latency_transformation: {}", isLowLatencyTransformationUsed()); + } + + // Model Cache options + if (v.HasMember("allow_cache")) { + setAllowCache(v["allow_cache"].GetBool()); + SPDLOG_DEBUG("allow_cache: {}", v["allow_cache"].GetBool()); + } + + // if the config has models which require custom loader to be used, then load the same here + if (v.HasMember("custom_loader_options")) { + if (!parseCustomLoaderOptionsConfig(v["custom_loader_options"]).ok()) { + SPDLOG_ERROR("Couldn't parse custom loader options config"); + } + } + return StatusCode::OK; +} + +Status ModelConfig::parseCustomLoaderOptionsConfig(const rapidjson::Value& node) { + if (!node.IsObject()) { + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { + if (!it->value.IsString()) { + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + addCustomLoaderOption(it->name.GetString(), it->value.GetString()); + } + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + node.Accept(writer); + customLoaderOptionsStr = buffer.GetString(); + + return StatusCode::OK; +} + +std::string ModelConfig::layoutConfigurationToString() const { + if (getLayout().isSet()) { + return getLayout().toString(); + } + std::stringstream ss; + for (const auto& [name, layoutCfg] : getLayouts()) { + ss << name << " " << layoutCfg.toString() << "; "; + } + return ss.str(); +} + +} // namespace ovms diff --git a/src/ovms_lib/modelconfig.hpp b/src/ovms_lib/modelconfig.hpp new file mode 100644 index 0000000000..21a4be8c98 --- /dev/null +++ b/src/ovms_lib/modelconfig.hpp @@ -0,0 +1,992 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "layout_configuration.hpp" +#include "model_version_policy.hpp" +#include "shape.hpp" +#include "status.hpp" + +namespace ovms { + +using mapping_config_t = std::unordered_map; +using plugin_config_t = std::map; +using custom_loader_options_config_t = std::map; + +const std::string ANONYMOUS_INPUT_NAME = "ANONYMOUS_INPUT_NAME"; +const std::string MAPPING_CONFIG_JSON = "mapping_config.json"; +const uint32_t DEFAULT_MAX_SEQUENCE_NUMBER = 500; + +/** + * @brief This class represents model configuration + */ +class ModelConfig { +private: + /** + * @brief Model name + */ + std::string name; + + /** + * @brief Model uri path + */ + std::string basePath; + + /** + * @brief Model local destination path on disk after downloading from online storage + */ + std::string localPath; + + /** + * @brief Target device + */ + std::string targetDevice; + + /** + * @brief Batching mode + */ + Mode batchingMode; + + /** + * @brief Batch size + */ + std::optional batchSize; + + /** + * @brief Model version policy + */ + std::shared_ptr modelVersionPolicy; + + /** + * @brief Nireq + */ + uint64_t nireq; + + /** + * @brief Flag determining if model is stateful + */ + bool stateful; + + /** + * @brief Flag determining if model will be a subject to sequence cleaner scans + */ + bool idleSequenceCleanup; + + /** + * @brief Flag determining if model will use low latency transformation + */ + bool lowLatencyTransformation; + + /** + * @brief Number of maximum frames in one sequence + */ + uint32_t maxSequenceNumber; + + /** + * @brief Model cache directory + */ + std::string cacheDir; + + /** + * @brief Flag determining if allow cache option is set to true + */ + bool isAllowCacheTrue = false; + + /** + * @brief Model version + */ + model_version_t version = -1; + + /** + * @brief Plugin config + */ + plugin_config_t pluginConfig; + + /** + * @brief Layout for single input + */ + LayoutConfiguration layout; + + /** + * @brief Map of shapes + */ + shapes_info_map_t shapes; + + /** + * @brief Map of layouts + */ + layout_configurations_map_t layouts; + + /** + * @brief Input mapping configuration + */ + mapping_config_t mappingInputs; + + /** + * @brief Output mapping configuration + */ + mapping_config_t mappingOutputs; + + /** + * @brief Reversed input mapping configuration + */ + mapping_config_t reversedMappingInputs; + /** + * @brief Reversed output mapping configuration + */ + mapping_config_t reversedMappingOutputs; + + /** + * @brief Shape left opening bracket in string format + */ + static const char shapeLeft = '('; + + /** + * @brief Shape right opening bracket in string format + */ + static const char shapeRight = ')'; + + /** + * @brief Shape delimeter in string format + */ + static const char shapeDelimeter = ','; + + /** + * @brief Allowed configurable layouts + */ + static const std::set configAllowedLayouts; + + /** + * @brief custom_loader_options config as map + */ + custom_loader_options_config_t customLoaderOptionsConfigMap; + + /** + * @brief custom_loader_options config as string + */ + std::string customLoaderOptionsStr; + +public: + /** + * @brief Construct a new Model Config object + * + * @param name + * @param basePath + * @param targetDevice + * @param configBatchSize + * @param nireq + */ + ModelConfig(const std::string& name = "", + const std::string& basePath = "", + const std::string& targetDevice = "CPU", + const std::string& configBatchSize = "0", + uint64_t nireq = 0, + bool stateful = false, + bool idleSequenceCleanup = true, + bool lowLatencyTransformation = false, + uint32_t maxSequenceNumber = DEFAULT_MAX_SEQUENCE_NUMBER, + const std::string& cacheDir = "", + model_version_t version = 0, + const std::string& localPath = "") : + name(name), + basePath(basePath), + localPath(localPath), + targetDevice(targetDevice), + modelVersionPolicy(ModelVersionPolicy::getDefaultVersionPolicy()), + nireq(nireq), + stateful(stateful), + idleSequenceCleanup(idleSequenceCleanup), + lowLatencyTransformation(lowLatencyTransformation), + maxSequenceNumber(maxSequenceNumber), + cacheDir(cacheDir), + version(version), + pluginConfig({}), + layout(""), + shapes({}), + layouts({}), + mappingInputs({}), + mappingOutputs({}) { + setBatchingParams(configBatchSize); + } + + /** + * @brief Compares two ModelConfig instances and decides if models should be reloaded + * + * @param rhs + * + * @return true if configs are equal false otherwise + */ + bool isReloadRequired(const ModelConfig& rhs) const; + + /** + * @brief Compares two ModelConfig instances and decides if customloader configuration changed + * + * @param rhs + * + * @return true if customloader configuration has changed + */ + bool isCustomLoaderConfigChanged(const ModelConfig& rhs) const; + + /** + * @brief Compares two ModelConfig instances for batch size configuration + * + * @param rhs + * + * @return true if configurations are equal false otherwise + */ + bool isBatchSizeConfigurationEqual(const ModelConfig& rhs) const; + + /** + * @brief Compares two ModelConfig instances for layout configuration + * + * @param rhs + * + * @return true if configurations are equal false otherwise + */ + bool isLayoutConfigurationEqual(const ModelConfig& rhs) const; + + /** + * @brief Compares two ModelConfig instances for shape configuration + * + * @param rhs + * + * @return true if configurations are equal false otherwise + */ + bool isShapeConfigurationEqual(const ModelConfig& rhs) const; + + /** + * @brief Get the name + * + * @return const std::string& + */ + const std::string& getName() const { + return this->name; + } + + /** + * @brief Set the name + * + * @param name + */ + void setName(const std::string& name) { + this->name = name; + } + + /** + * @brief Get local path to specific model version where .xml and .bin is located for loading + * + * @return std::string + * */ + const std::string getPath() const { + return getLocalPath() + "/" + std::to_string(version); + } + + /** + * @brief Get the base path + * + * @return const std::string& + */ + const std::string& getBasePath() const { + return this->basePath; + } + + /** + * @brief Set the base path + * + * @param basePath + */ + void setBasePath(const std::string& basePath) { + this->basePath = basePath; + } + + /** + * @brief Get the local path + * + * @return const std::string& + */ + const std::string& getLocalPath() const { + return this->localPath; + } + + /** + * @brief Set the local path + * + * @param localPath + */ + void setLocalPath(const std::string& localPath) { + this->localPath = localPath; + } + + /** + * @brief Get the target device + * + * @return const std::string& + */ + const std::string& getTargetDevice() const { + return this->targetDevice; + } + + /** + * @brief Set the target device + * + * @param target device + */ + void setTargetDevice(const std::string& targetDevice) { + this->targetDevice = targetDevice; + } + + /** + * @brief Get the cache directory + * + * @return const std::string& + */ + const std::string& getCacheDir() const { + return this->cacheDir; + } + + /** + * @brief Set the cache directory + * + * @param cache directory + */ + void setCacheDir(const std::string& cacheDir) { + this->cacheDir = cacheDir; + } + + /** + * @brief Get the allow cache flag + * + * @return bool + */ + bool isAllowCacheSetToTrue() const { + return this->isAllowCacheTrue; + } + + /** + * @brief Set the allow cache flag + * + * @param allow cache flag + */ + void setAllowCache(const bool& allowCache) { + this->isAllowCacheTrue = allowCache; + } + + /** + * @brief Checks if given device is used as single target device. + * + * @param bool + */ + bool isSingleDeviceUsed(const std::string& device) const { + return this->targetDevice == device; + } + + bool isDeviceUsed(const std::string& device) const; + + /** + * @brief Get the batching mode + * + * @return ovms::Mode + */ + Mode getBatchingMode() const { + return this->batchingMode; + } + + bool isDynamicParameterEnabled() const { + return this->getBatchingMode() == Mode::AUTO || this->anyShapeSetToAuto(); + } + + /** + * @brief Get the batch size + * + * @return size_t + */ + std::optional getBatchSize() const { + return this->batchSize; + } + + /** + * @brief Set the batching mode + * + * @param batchingMode + */ + void setBatchingMode(Mode batchingMode) { + this->batchingMode = batchingMode; + } + + /** + * @brief Set the batch size + * + * @param batchSize + */ + void setBatchSize(std::optional batchSize) { + this->batchSize = batchSize; + } + + /** + * @brief Set batching mode to FIXED and batch size to value provided in a parameter. + * + * @param configBatchSize + */ + void setBatchingParams(size_t configBatchSize) { + setBatchingMode(FIXED); + setBatchSize(configBatchSize); + } + + /** + * @brief Extracts batching mode and batch size value from string provided in a parameter. + * + * @param configBatchSize + */ + void setBatchingParams(const std::string& configBatchSize) { + auto [batchingMode, effectiveBatchSize] = extractBatchingParams(configBatchSize); + setBatchingMode(batchingMode); + setBatchSize(effectiveBatchSize); + } + + /** + * @brief Extract batching mode and effective batch size from string. + * + * @param configBatchSize + */ + static std::tuple> extractBatchingParams(std::string configBatchSize); + + /** + * @brief Get the model version policy + * + * @return std::shared_ptr + */ + std::shared_ptr getModelVersionPolicy() { + return this->modelVersionPolicy; + } + + /** + * @brief Set the model version policy + * + * @param modelVersionPolicy + */ + void setModelVersionPolicy(std::shared_ptr modelVersionPolicy) { + this->modelVersionPolicy = modelVersionPolicy; + } + + /** + * @brief Parses string for model version policy + * + * @param string representing model version policy configuration + + * @return status + */ + Status parseModelVersionPolicy(std::string command); + + /** + * @brief Get the nireq + * + * @return uint64_t + */ + uint64_t getNireq() const { + return this->nireq; + } + + /** + * @brief Set the nireq + * + * @param nireq + */ + void setNireq(const uint64_t nireq) { + this->nireq = nireq; + } + + /** + * @brief Get the plugin config + * + * @return const std::string& + */ + const plugin_config_t& getPluginConfig() const { + return this->pluginConfig; + } + + /** + * @brief Set the plugin config + * + * @param pluginConfig + */ + void setPluginConfig(const plugin_config_t& pluginConfig) { + this->pluginConfig = pluginConfig; + } + + /** + * @brief Get stateful model flag + * + * @return bool + */ + const bool isStateful() const { + return this->stateful; + } + + /** + * @brief Set stateful model flag + * + * @return bool + */ + void setStateful(bool stateful) { + this->stateful = stateful; + } + + /** + * @brief Set stateful low latency transformation flag + * + * @return bool + */ + void setLowLatencyTransformation(bool lowLatencyTransformation) { + this->lowLatencyTransformation = lowLatencyTransformation; + } + + /** + * @brief Get stateful low latency transformation flag + * + * @return bool + */ + const bool isLowLatencyTransformationUsed() const { + return this->lowLatencyTransformation; + } + + /** + * @brief Get max number of sequences handled concurrently by the model + * + * @return uint + */ + uint64_t getMaxSequenceNumber() const { + return this->maxSequenceNumber; + } + + /** + * @brief Set max number of sequences handled concurrently by the model + * + * @return uint + */ + void setMaxSequenceNumber(const uint32_t maxSequenceNumber) { + this->maxSequenceNumber = maxSequenceNumber; + } + + /** + * @brief Get stateful sequence timeout + * + * @return uint + */ + bool getIdleSequenceCleanup() const { + return this->idleSequenceCleanup; + } + + /** + * @brief Set stateful sequence timeout + * + * @return uint + */ + void setIdleSequenceCleanup(const bool idleSequenceCleanup) { + this->idleSequenceCleanup = idleSequenceCleanup; + } + + /** + * @brief Parses json node for plugin config keys and values + * + * @param json node representing plugin_config + * + * @return status + */ + Status parsePluginConfig(const rapidjson::Value& node); + + /** + * @brief Parses string for plugin config keys and values + * + * @param string representing plugin_config + * + * @return status + */ + Status parsePluginConfig(std::string command) { + rapidjson::Document node; + if (command.empty()) { + return StatusCode::OK; + } + if (node.Parse(command.c_str()).HasParseError()) { + return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; + } + + return parsePluginConfig(node); + } + + /** + * @brief Parses value from json and extracts shapes info + * + * @param rapidjson::Value& node + * + * @return status + */ + Status parseShapeParameter(const rapidjson::Value& node); + + /** + * @brief Parses value from string and extracts shapes info + * + * @param string + * + * @return status + */ + Status parseShapeParameter(const std::string& command); + + /** + * @brief Parses value from json and extracts layouts info + * + * @param rapidjson::Value& node + * + * @return status + */ + Status parseLayoutParameter(const rapidjson::Value& node); + + /** + * @brief Parses value from string and extracts layouts info + * + * @param string + * + * @return status + */ + Status parseLayoutParameter(const std::string& command); + + /** + * @brief Returns true if any input shape specified in shapes map is in AUTO mode + * + * @return bool + */ + bool anyShapeSetToAuto() const { + for (const auto& [name, shapeInfo] : getShapes()) { + if (shapeInfo.shapeMode == AUTO) + return true; + } + return false; + } + + /** + * @brief Get the shapes + * + * @return const shapes_map_t& + */ + const shapes_info_map_t& getShapes() const { + return this->shapes; + } + + /** + * @brief Set the shapes + * + * @param shapes + */ + void setShapes(const shapes_info_map_t& shapes) { + this->shapes = shapes; + } + + /** + * @brief Returns true if shape with certain name is in AUTO mode + * + * @return bool + */ + bool isShapeAuto(const std::string& name) const { + auto it = getShapes().find(name); + if (it == getShapes().end()) { + it = getShapes().find(ANONYMOUS_INPUT_NAME); + } + if (it == getShapes().end()) { + return false; + } + return it->second.shapeMode == Mode::AUTO; + } + + bool isShapeAnonymous() const { + return getShapes().size() == 1 && getShapes().begin()->first == ANONYMOUS_INPUT_NAME; + } + + bool isShapeAnonymousFixed() const { + return isShapeAnonymous() && !isShapeAuto(ANONYMOUS_INPUT_NAME); + } + + bool isCloudStored() const { + return getLocalPath() != getBasePath(); + } + + /** + * @brief Sets the shape from the string representation + * + * @param shape + * @param str + * @return Status + */ + static Status parseShape(ShapeInfo& shapeInfo, const std::string& str); + + /** + * @brief Add a single named shape + * + * @param name + * @param shape + */ + void addShape(const std::string& name, const ShapeInfo& shapeInfo) { + this->shapes[name] = shapeInfo; + } + + void removeShape(const std::string& name) { + this->shapes.erase(name); + } + + /** + * @brief Get the layouts + * + * @return const std::string& + */ + const LayoutConfiguration& getLayout() const { + return this->layout; + } + + /** + * @brief Set the layout + * + * @param layout + */ + void setLayout(const LayoutConfiguration& layout) { + this->layout = layout; + this->layouts.clear(); + } + + /** + * @brief Get the layouts + * + * @return const layout_configurations_map_t& + */ + const layout_configurations_map_t& getLayouts() const { + return this->layouts; + } + + /** + * @brief Set the layouts + * + * @param layouts + */ + void setLayouts(const layout_configurations_map_t& layouts) { + this->layouts = layouts; + this->layout = LayoutConfiguration(); + } + + /** + * @brief Get the version + * + * @return const model_version_t& + */ + const model_version_t& getVersion() const { + return this->version; + } + + /** + * @brief Set the version + * + * @param version + */ + void setVersion(const model_version_t& version) { + this->version = version; + } + + /** + * @brief Get the mapping for inputs + * + * @return const mapping_config_t& + */ + const mapping_config_t& getMappingInputs() const { + return this->mappingInputs; + } + + /** + * @brief Get the mapping for outputs + * + * @return const mapping_config_t& + */ + const mapping_config_t& getMappingOutputs() const { + return this->mappingOutputs; + } + + /** + * @brief Get the reversed mapping for inputs + * + * @return const mapping_config_t& + */ + const mapping_config_t& getRealMappingInputs() const { + return this->reversedMappingInputs; + } + + /** + * @brief Get the reversed mapping for outputs + * + * @return const mapping_config_t& + */ + const mapping_config_t& getRealMappingOutputs() const { + return this->reversedMappingOutputs; + } + + /** + * @brief Get the mapping inputs by key + * + * @param key + * @return const std::string + */ + const std::string getMappingInputByKey(const std::string& key) const { + auto it = mappingInputs.find(key); + return it != mappingInputs.end() ? it->second : ""; + } + + /** + * @brief Get the mapping outputs by key + * + * @param key + * @return const std::string + */ + const std::string getMappingOutputByKey(const std::string& key) const { + auto it = mappingOutputs.find(key); + return it != mappingOutputs.end() ? it->second : ""; + } + + /** + * @brief Get the real inputs by value + * + * @param value + * @return const std::string + */ + const std::string getRealInputNameByValue(const std::string& value) const { + auto it = reversedMappingInputs.find(value); + return it != reversedMappingInputs.end() ? it->second : ""; + } + + /** + * @brief Get the real outputs by value + * + * @param value + * @return const std::string + */ + const std::string getRealOutputNameByValue(const std::string& value) const { + auto it = reversedMappingOutputs.find(value); + return it != reversedMappingOutputs.end() ? it->second : ""; + } + + /** + * @brief Set the mapping inputs + * + * @param mapping + */ + void setMappingInputs(const mapping_config_t& mapping) { + this->mappingInputs = mapping; + } + + /** + * @brief Set the mapping outputs + * + * @param mapping + */ + void setMappingOutputs(const mapping_config_t& mapping) { + this->mappingOutputs = mapping; + } + + /** + * @brief Set the reversed mapping inputs + * + * @param mapping + */ + void setRealMappingInputs(const mapping_config_t& mapping) { + this->reversedMappingInputs = mapping; + } + + /** + * @brief Set the reversed mapping outputs + * + * @param mapping + */ + void setRealMappingOutputs(const mapping_config_t& mapping) { + this->reversedMappingOutputs = mapping; + } + + /** + * @brief Parses mapping_config.json for mapping input/outputs in the model + * + * @return Status + */ + Status parseModelMapping(); + + /** + * @brief Parses all settings from a JSON node + * + * @return Status + */ + Status parseNode(const rapidjson::Value& v); + + /** + * @brief Returns true if model requires a custom loader to load + * + * @return bool + */ + const bool isCustomLoaderRequiredToLoadModel() const { + return (this->customLoaderOptionsConfigMap.size() > 0); + } + + /** + * @brief Get the custom loader option config + * + * @return const custom_loader_options_config_t& + */ + const custom_loader_options_config_t& getCustomLoaderOptionsConfigMap() const { + return this->customLoaderOptionsConfigMap; + } + + /** + * @brief Add custom loader option + * + * @param name + * @param value + */ + void addCustomLoaderOption(const std::string& name, const std::string& value) { + customLoaderOptionsConfigMap[name] = value; + } + + /** + * @brief Get the custom loader option config + * + * @return const std::string + */ + const std::string& getCustomLoaderOptionsConfigStr() const { + return this->customLoaderOptionsStr; + } + + /** + * @brief Parses json node for custom_loader_options config keys and values + * + * @param json node representing custom_loader_options_config + * + * @return status + */ + Status parseCustomLoaderOptionsConfig(const rapidjson::Value& node); + + std::string layoutConfigurationToString() const; +}; +} // namespace ovms diff --git a/src/ovms_lib/modelinstance.cpp b/src/ovms_lib/modelinstance.cpp new file mode 100644 index 0000000000..39104d70b7 --- /dev/null +++ b/src/ovms_lib/modelinstance.cpp @@ -0,0 +1,1225 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelinstance.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "config.hpp" +#include "customloaders.hpp" +#include "deserialization.hpp" +#include "executingstreamidguard.hpp" +#include "filesystem.hpp" +#include "layout.hpp" +#include "layout_configuration.hpp" +#include "logging.hpp" +#include "ov_utils.hpp" +#include "predict_request_validation_utils.hpp" +#include "prediction_service_utils.hpp" +#include "profiler.hpp" +#include "serialization.hpp" +#include "shape.hpp" +#include "stringutils.hpp" +#include "tensorinfo.hpp" +#include "timer.hpp" + +namespace ovms { + +const char* CPU_THROUGHPUT_STREAMS = "CPU_THROUGHPUT_STREAMS"; +const char* NIREQ = "NIREQ"; + +const uint MAX_NIREQ_COUNT = 100000; + +const int DEFAULT_OV_STREAMS = std::thread::hardware_concurrency() / 4; + +const uint UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS = 10; + +void ModelInstance::subscribe(PipelineDefinition& pd) { + subscriptionManager.subscribe(pd); +} + +void ModelInstance::unsubscribe(PipelineDefinition& pd) { + subscriptionManager.unsubscribe(pd); +} + +Status getRequestedShape(const ModelConfig& config, const DynamicModelParameter& parameter, const std::string& name, Shape& shapeOut) { + Shape shape; + auto mappedName = config.getMappingInputByKey(name); + if (config.getBatchSize().has_value() || parameter.isBatchSizeRequested()) { + // leave shape untouched + } else if (config.isShapeAuto(name) && parameter.isShapeRequested(name)) { + auto status = Shape::fromFlatShape(parameter.getShape(name), shape); + if (!status.ok()) { + return status; + } + } else if (mappedName == "" && config.getShapes().count(name) && config.getShapes().at(name).shape.size()) { + shape = config.getShapes().at(name).shape; + } else if (config.getShapes().count(mappedName) && config.getShapes().at(mappedName).shape.size()) { + shape = config.getShapes().at(mappedName).shape; + } else if (config.getShapes().count(ANONYMOUS_INPUT_NAME) && config.getShapes().at(ANONYMOUS_INPUT_NAME).shape.size()) { + shape = config.getShapes().at(ANONYMOUS_INPUT_NAME).shape; + } + shapeOut = shape; + return StatusCode::OK; +} + +bool hasInputWithName(std::shared_ptr& model, const std::string& name) { + try { + model->input(name); + return true; + } catch (ov::Exception& e) { + return false; + } +} + +bool hasOutputWithName(std::shared_ptr& model, const std::string& name) { + try { + model->output(name); + return true; + } catch (ov::Exception& e) { + return false; + } +} + +Status validateConfigurationAgainstNetwork(const ModelConfig& config, std::shared_ptr& model) { + if (config.isShapeAnonymousFixed() && model->inputs().size() > 1) { + Status status = StatusCode::ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED; + SPDLOG_LOGGER_WARN(modelmanager_logger, status.string()); + return status; + } + if (config.getLayout().isSet() && model->inputs().size() > 1) { + Status status = StatusCode::ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED; + SPDLOG_LOGGER_WARN(modelmanager_logger, status.string()); + return status; + } + for (const auto& [name, _] : config.getShapes()) { + if (name == ANONYMOUS_INPUT_NAME) { + continue; + } + if (hasInputWithName(model, name) && config.getMappingInputByKey(name) != "") { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Config shape - {} is mapped by {}. Changes will not apply", name, config.getMappingInputByKey(name)); + return StatusCode::CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME; + } else if (!hasInputWithName(model, name) && !hasInputWithName(model, config.getRealInputNameByValue(name))) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Config shape - {} not found in model", name); + return StatusCode::CONFIG_SHAPE_IS_NOT_IN_MODEL; + } + } + for (const auto& [name, _] : config.getLayouts()) { + if (hasInputWithName(model, name) && config.getMappingInputByKey(name) != "") { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} is mapped by {}. Changes will not apply", name, config.getMappingInputByKey(name)); + return StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME; + } else if (hasOutputWithName(model, name) && config.getMappingOutputByKey(name) != "") { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} is mapped by {}. Changes will not apply", name, config.getMappingOutputByKey(name)); + return StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME; + } else if (!hasInputWithName(model, name) && !hasOutputWithName(model, name) && !hasInputWithName(model, config.getRealInputNameByValue(name)) && !hasOutputWithName(model, config.getRealOutputNameByValue(name))) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} not found in model", name); + return StatusCode::CONFIG_LAYOUT_IS_NOT_IN_MODEL; + } + } + return StatusCode::OK; +} + +const Layout ModelInstance::getReportedTensorLayout(const ModelConfig& config, const std::string& name, bool isInput) { + if (isInput) { + const auto& input = this->model->input(name); + auto networkSpecifiedLayout = getLayoutFromRTMap(input.get_rt_info()); + if (networkSpecifiedLayout.has_value()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting input layout from RTMap: {}; for tensor name: {}", networkSpecifiedLayout.value().to_string(), name); + return Layout::fromOvLayout(networkSpecifiedLayout.value()); + } + } else { + const auto& output = this->model->output(name); + auto networkSpecifiedLayout = getLayoutFromRTMap(output.get_rt_info()); + if (networkSpecifiedLayout.has_value()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting output layout from RTMap: {}; for tensor name: {}", networkSpecifiedLayout.value().to_string(), name); + return Layout::fromOvLayout(networkSpecifiedLayout.value()); + } + } + auto layout = Layout::getDefaultLayout(); + if (isInput && config.getLayout().isSet()) { + layout = config.getLayout().getTensorLayout(); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting layout from ModelConfig: {}; for tensor name: {}", layout, name); + return layout; + } else if (config.getLayouts().size() > 0) { + auto mappedName = config.getMappingInputByKey(name); + auto it = config.getLayouts().find(mappedName == "" ? name : mappedName); + if (it != config.getLayouts().end()) { + layout = it->second.getTensorLayout(); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting layout from ModelConfig: {}; for tensor name: {}", layout, name); + return layout; + } + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting default layout: {}; for tensor name: {}", layout, name); + return layout; +} + +Status applyLayoutConfiguration(const ModelConfig& config, std::shared_ptr& model, const std::string& modelName, model_version_t modelVersion) { + ov::preprocess::PrePostProcessor preproc(model); + + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Applying layout configuration: {}", config.layoutConfigurationToString()); + + for (const ov::Output& input : model->inputs()) { + try { + std::string name = input.get_any_name(); + std::string mappedName = config.getMappingInputByKey(name).empty() ? name : config.getMappingInputByKey(name); + if (config.getLayout().isSet()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding preprocessing step: Tensor Layout:{}; Network Layout:{}; single input", + modelName, + modelVersion, + config.getLayout().getTensorLayout(), + config.getLayout().getModelLayout()); + + preproc.input().tensor().set_layout(ov::Layout(config.getLayout().getTensorLayout())); + preproc.input().model().set_layout(ov::Layout(config.getLayout().getModelLayout())); + } else if (config.getLayouts().count(mappedName) > 0) { + auto& layout = config.getLayouts().at(mappedName); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding preprocessing step: Tensor Layout:{}; Network Layout:{}; input name: {}", + modelName, + modelVersion, + layout.getTensorLayout(), + layout.getModelLayout(), + mappedName); + + preproc.input(name).tensor().set_layout(ov::Layout(layout.getTensorLayout())); + preproc.input(name).model().set_layout(ov::Layout(layout.getModelLayout())); + } else { + auto inheritedModelLayout = getLayoutFromRTMap(input.get_rt_info()); + auto guessedModelLayout = Layout::getDefaultLayout(); + + ov::Layout targetModelLayout = inheritedModelLayout.has_value() ? inheritedModelLayout.value() : ov::Layout(guessedModelLayout); + + if (inheritedModelLayout.has_value()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (inherited from network); input name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); + } else { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (default); input name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); + } + preproc.input(name).model().set_layout(targetModelLayout); + } + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO with error:{}", + modelName, + modelVersion, + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO with error:{}", + modelName, + modelVersion, + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO", + modelName, + modelVersion); + return StatusCode::UNKNOWN_ERROR; + } + } + + for (const ov::Output& output : model->outputs()) { + try { + std::string name = output.get_any_name(); + std::string mappedName = config.getMappingOutputByKey(name).empty() ? name : config.getMappingOutputByKey(name); + if (config.getLayouts().count(mappedName) > 0) { + auto& layout = config.getLayouts().at(mappedName); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding postprocessing step: Tensor Layout:{}; Network Layout:{}; output name: {}", + modelName, + modelVersion, + layout.getTensorLayout(), + layout.getModelLayout(), + mappedName); + preproc.output(name).tensor().set_layout(ov::Layout(layout.getTensorLayout())); + preproc.output(name).model().set_layout(ov::Layout(layout.getModelLayout())); + } else { + auto inheritedModelLayout = getLayoutFromRTMap(output.get_rt_info()); + auto guessedModelLayout = Layout::getDefaultLayout(); + + ov::Layout targetModelLayout = inheritedModelLayout.has_value() ? inheritedModelLayout.value() : ov::Layout(guessedModelLayout); + + if (inheritedModelLayout.has_value()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (inherited from network); output name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); + } else { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (default); output name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); + } + preproc.output(name).model().set_layout(targetModelLayout); + } + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO with error:{}", + modelName, + modelVersion, + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO with error:{}", + modelName, + modelVersion, + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO", + modelName, + modelVersion); + return StatusCode::UNKNOWN_ERROR; + } + } + + try { + model = preproc.build(); + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot change layout"); + return StatusCode::MODEL_NOT_LOADED; + } + return StatusCode::OK; +} + +Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLayoutConfiguration, const DynamicModelParameter& parameter) { + Status status = validateConfigurationAgainstNetwork(config, this->model); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during configuration validation against model"); + return status; + } + if (needsToApplyLayoutConfiguration) { + status = applyLayoutConfiguration(config, this->model, getName(), getVersion()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during layout configuration"); + return status; + } + } + status = loadInputTensors(config, parameter); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during loading input tensors"); + return status; + } + status = loadOutputTensors(config); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during loading output tensors"); + return status; + } + return StatusCode::OK; +} + +Status ModelInstance::loadInputTensors(const ModelConfig& config, const DynamicModelParameter& parameter) { + this->inputsInfo.clear(); + + std::map modelShapes; + bool reshapeRequired = false; + + // First pass, gather reshape info. + for (const ov::Output& input : this->model->inputs()) { + std::string name; + try { + std::string name = input.get_any_name(); + ov::PartialShape shape = input.get_partial_shape(); + + Shape requestedShape; + auto status = getRequestedShape(config, parameter, name, requestedShape); + if (!status.ok()) { + return status; + } + if (requestedShape.size() > 0) { + shape = requestedShape.createPartialShape(); + } + + modelShapes[name] = shape; + if (input.get_partial_shape() != shape) { + reshapeRequired = true; + } + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO", + getName(), + getVersion()); + return StatusCode::UNKNOWN_ERROR; + } + } + + if (reshapeRequired) { + SPDLOG_DEBUG("model: {}, version: {}; reshaping inputs", getName(), getVersion()); + try { + model->reshape(modelShapes); + } catch (const ov::Exception& e) { + SPDLOG_WARN("OV does not support reshaping model: {} with provided shape", getName()); + SPDLOG_DEBUG("Description: {}", e.what()); + return StatusCode::RESHAPE_ERROR; + } catch (const std::exception& e) { + SPDLOG_WARN("OV does not support reshaping model: {} with provided shape", getName()); + SPDLOG_DEBUG("Description: {}", e.what()); + return StatusCode::RESHAPE_ERROR; + } + } else { + SPDLOG_DEBUG("model: {}, version: {}; reshaping inputs is not required", getName(), getVersion()); + } + + configureBatchSize(this->config, parameter); + + for (const ov::Output& input : this->model->inputs()) { + try { + std::string name = input.get_any_name(); + + ovms::Precision precision = ovElementTypeToOvmsPrecision(input.get_element_type()); + Shape shape(input.get_partial_shape()); + std::string mappingName = config.getMappingInputByKey(name); + const Layout layout = getReportedTensorLayout(config, name, true); + + if (!layout.isCompatible(shape)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Layout: {}; incompatible with shape: {}; for input name: {}", layout, shape.toString(), name); + return StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE; + } + + std::shared_ptr info = std::make_shared( + name, + mappingName, + precision, + shape, + layout); + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Input {}", info->asString()); + + this->inputsInfo[info->getMappedName()] = std::move(info); + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO", + getName(), + getVersion()); + return StatusCode::UNKNOWN_ERROR; + } + } + return StatusCode::OK; +} + +Status ModelInstance::loadOutputTensors(const ModelConfig& config) { + this->outputsInfo.clear(); + + for (const ov::Output& output : this->model->outputs()) { + try { + std::string name = output.get_any_name(); + + ovms::Precision precision = ovElementTypeToOvmsPrecision(output.get_element_type()); + Shape shape(output.get_partial_shape()); + std::string mappingName = config.getMappingOutputByKey(name); + const Layout layout = getReportedTensorLayout(config, name, false); + + if (!layout.isCompatible(shape)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Layout: {}; incompatible with shape: {}; for output name: {}", layout, shape.toString(), name); + return StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE; + } + + std::shared_ptr info = std::make_shared( + name, + mappingName, + precision, + shape, + layout); + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Output {}", info->asString()); + + this->outputsInfo[info->getMappedName()] = std::move(info); + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO with error:{}", + getName(), + getVersion(), + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO", + getName(), + getVersion()); + return StatusCode::UNKNOWN_ERROR; + } + } + + return StatusCode::OK; +} + +// Temporary methods. To be replaces with proper storage class. +bool dirExists(const std::string& path) { + if (FileSystem::isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return false; + } + DIR* dir = opendir(path.c_str()); + if (dir) { + closedir(dir); + return true; + } + + return false; +} + +std::string findFilePathWithExtension(const std::string& path, const std::string& extension) { + struct dirent* entry; + if (FileSystem::isPathEscaped(path)) { + SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); + return std::string(); + } + DIR* dir = opendir(path.c_str()); + if (!dir) { + SPDLOG_WARN("Failed to opendir: {}", path); + return std::string(); + } + + while ((entry = readdir(dir)) != nullptr) { + auto name = std::string(entry->d_name); + if (endsWith(name, extension)) { + closedir(dir); + if (endsWith(name, "/")) { + return path + name; + } else { + return path + '/' + name; + } + } + } + closedir(dir); + + return std::string(); +} + +std::string ModelInstance::findModelFilePathWithExtension(const std::string& extension) const { + return findFilePathWithExtension(path, extension); +} + +uint ModelInstance::getNumOfParallelInferRequestsUnbounded(const ModelConfig& modelConfig) { + uint numberOfParallelInferRequests = 0; + if (modelConfig.getNireq() > 0) { + return modelConfig.getNireq(); + } + auto& ovmsConfig = ovms::Config::instance(); + if (ovmsConfig.nireq() > 0) { + // nireq is set globally for all models in ovms startup parameters + return ovmsConfig.nireq(); + } + std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS); + try { + numberOfParallelInferRequests = compiledModel->get_property(key).as(); + } catch (const ov::Exception& ex) { + SPDLOG_WARN("Failed to query OPTIMAL_NUMBER_OF_INFER_REQUESTS with error {}. Using 1 nireq.", ex.what()); + numberOfParallelInferRequests = 1u; + } + return numberOfParallelInferRequests; +} + +uint ModelInstance::getNumOfParallelInferRequests(const ModelConfig& modelConfig) { + uint nireq = getNumOfParallelInferRequestsUnbounded(modelConfig); + if (nireq > MAX_NIREQ_COUNT) { + SPDLOG_WARN("Invalid nireq because its value was too high: {}. Maximum value: {}", nireq, MAX_NIREQ_COUNT); + return 0; + } else if (nireq < 1u) { + SPDLOG_WARN("Ignored configured nireq because it has to be above 0 and was: {}. Set to 1", nireq); + return 1u; + } + return nireq; +} + +std::shared_ptr ModelInstance::loadOVModelPtr(const std::string& modelFile) { + return ieCore.read_model(modelFile); +} + +Status ModelInstance::loadOVModel() { + auto& modelFile = modelFiles[0]; + SPDLOG_DEBUG("Try reading model file: {}", modelFile); + try { + model = loadOVModelPtr(modelFile); + } catch (std::exception& e) { + SPDLOG_ERROR("Error: {}; occurred during loading ov::Model model: {} version: {}", e.what(), getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } + return StatusCode::OK; +} + +Status ModelInstance::loadOVModelUsingCustomLoader() { + SPDLOG_DEBUG("Try reading model using a custom loader"); + try { + std::vector modelBinary; + std::vector weights; + + SPDLOG_INFO("loading ov::Model for model: {} basepath: {} <> {} version: {}", getName(), getPath(), this->config.getBasePath().c_str(), getVersion()); + + custom_loader_options_config_t customLoaderOptionsConfig = this->config.getCustomLoaderOptionsConfigMap(); + const std::string loaderName = customLoaderOptionsConfig["loader_name"]; + + auto& customloaders = ovms::CustomLoaders::instance(); + auto customLoaderInterfacePtr = customloaders.find(loaderName); + if (customLoaderInterfacePtr == nullptr) { + SPDLOG_INFO("Loader {} is not in loaded customloaders list", loaderName); + throw std::invalid_argument("customloader not exisiting"); + } + + CustomLoaderStatus res = customLoaderInterfacePtr->loadModel(this->config.getName(), + this->config.getBasePath(), + getVersion(), + this->config.getCustomLoaderOptionsConfigStr(), modelBinary, weights); + + if (res == CustomLoaderStatus::MODEL_LOAD_ERROR) { + return StatusCode::FILE_INVALID; + } + + if ((res == CustomLoaderStatus::INTERNAL_ERROR) || (res == CustomLoaderStatus::MODEL_BLACKLISTED)) { + return StatusCode::INTERNAL_ERROR; + } + + std::string strModel(modelBinary.begin(), modelBinary.end()); + + if (res == CustomLoaderStatus::MODEL_TYPE_IR) { + ov::Tensor tensorWts(ov::element::u8, ov::Shape{weights.size()}); + std::memcpy(tensorWts.data(), weights.data(), weights.size()); + model = ieCore.read_model(strModel, tensorWts); + } else if (res == CustomLoaderStatus::MODEL_TYPE_ONNX) { + model = ieCore.read_model(strModel, ov::Tensor()); + } else if (res == CustomLoaderStatus::MODEL_TYPE_BLOB) { + return StatusCode::INTERNAL_ERROR; + } + } catch (ov::Exception& e) { + SPDLOG_ERROR("Error: {}; occurred during loading ov::Model for model: {} version: {}", e.what(), getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } catch (std::exception& e) { + SPDLOG_ERROR("Error: {}; occurred during loading ov::Model for model: {} version: {}", e.what(), getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } + return StatusCode::OK; +} + +void ModelInstance::loadCompiledModelPtr(const plugin_config_t& pluginConfig) { + compiledModel = std::make_shared(ieCore.compile_model(model, targetDevice, pluginConfig)); +} + +plugin_config_t ModelInstance::prepareDefaultPluginConfig(const ModelConfig& config) { + plugin_config_t pluginConfig = config.getPluginConfig(); + // Do not add CPU_THROUGHPUT_AUTO when performance hint is specified. + bool isPerformanceHintSpecified = pluginConfig.count("PERFORMANCE_HINT") > 0; + if (isPerformanceHintSpecified) { + return pluginConfig; + } + // For CPU and GPU, if user did not specify, calculate CPU_THROUGHPUT_STREAMS automatically + if (config.isSingleDeviceUsed("CPU")) { + if (pluginConfig.count("CPU_THROUGHPUT_STREAMS") == 0) { + pluginConfig["CPU_THROUGHPUT_STREAMS"] = "CPU_THROUGHPUT_AUTO"; + } + } + if (config.isSingleDeviceUsed("GPU")) { + if (pluginConfig.count("GPU_THROUGHPUT_STREAMS") == 0) { + pluginConfig["GPU_THROUGHPUT_STREAMS"] = "GPU_THROUGHPUT_AUTO"; + } + } + return pluginConfig; +} + +Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { + plugin_config_t pluginConfig = prepareDefaultPluginConfig(config); + try { + loadCompiledModelPtr(pluginConfig); + } catch (ov::Exception& e) { + Status status = StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE; + SPDLOG_LOGGER_ERROR(modelmanager_logger, "{}; error: {}; model: {}; version: {}; device: {}", + status.string(), + e.what(), + getName(), + getVersion(), + config.getTargetDevice()); + return status; + } catch (std::exception& e) { + Status status = StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE; + SPDLOG_LOGGER_ERROR(modelmanager_logger, "{}; error: {}; model: {}; version: {}; device: {}", + status.string(), + e.what(), + getName(), + getVersion(), + config.getTargetDevice()); + return status; + } + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Plugin config for device: {}", targetDevice); + for (const auto pair : pluginConfig) { + const auto key = pair.first; + const auto value = pair.second; + SPDLOG_LOGGER_INFO(modelmanager_logger, "OVMS set plugin settings key: {}; value: {};", key, value.as()); + } + + const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); + std::vector supportedConfigKeys; + try { + std::vector supportedConfigKeys2 = compiledModel->get_property(supportedConfigKey).as>(); + supportedConfigKeys = std::move(supportedConfigKeys2); + } catch (std::exception& e) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}; Error: {}", targetDevice, supportedConfigKey, e.what()); + return StatusCode::OK; + } catch (...) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}", targetDevice, supportedConfigKey); + return StatusCode::OK; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging model:{}; version: {};target device: {}; CompiledModel configuration", getName(), getVersion(), targetDevice); + for (auto& key : supportedConfigKeys) { + std::string value; + try { + auto paramValue = compiledModel->get_property(key); + value = paramValue.as(); + } catch (std::exception& e) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel config key: {}; Error: {}", targetDevice, key, e.what()); + continue; + } catch (...) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel config key: {}", targetDevice, key); + continue; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {}; version: {}; target device: {}, CompiledModel config key: {}, value: {}", getName(), getVersion(), targetDevice, key, value); + } + return StatusCode::OK; +} + +Status ModelInstance::fetchModelFilepaths() { + if (this->config.isCustomLoaderRequiredToLoadModel()) { + // not required if the model is loaded using a custom loader and can be returned from here + return StatusCode::OK; + } + + SPDLOG_DEBUG("Getting model files from path: {}", path); + if (!dirExists(path)) { + SPDLOG_ERROR("Missing model directory {}", path); + return StatusCode::PATH_INVALID; + } + + bool found = true; + for (auto extension : OV_MODEL_FILES_EXTENSIONS) { + auto file = findModelFilePathWithExtension(extension); + if (file.empty()) { + found = false; + } + modelFiles.push_back(file); + } + if (!found) { + found = true; + modelFiles.clear(); + for (auto extension : ONNX_MODEL_FILES_EXTENSIONS) { + auto file = findModelFilePathWithExtension(extension); + if (file.empty()) { + found = false; + } + modelFiles.push_back(file); + } + } + + if (!found) { + SPDLOG_ERROR("Could not find file for model: {} version: {} in path: {}", getName(), getVersion(), path); + return StatusCode::FILE_INVALID; + } + + return StatusCode::OK; +} + +Status ModelInstance::prepareInferenceRequestsQueue(const ModelConfig& config) { + uint numberOfParallelInferRequests = getNumOfParallelInferRequests(config); + if (numberOfParallelInferRequests == 0) { + return Status(StatusCode::INVALID_NIREQ, "Exceeded allowed nireq value"); + } + inferRequestsQueue = std::make_unique(*compiledModel, numberOfParallelInferRequests); + SPDLOG_INFO("Loaded model {}; version: {}; batch size: {}; No of InferRequests: {}", + getName(), + getVersion(), + getBatchSize().toString(), + numberOfParallelInferRequests); + return StatusCode::OK; +} + +void ModelInstance::configureBatchSize(const ModelConfig& config, const DynamicModelParameter& parameter) { + if (parameter.isBatchSizeRequested()) { + ov::set_batch(model, parameter.getBatchSize()); + } else if (config.getBatchSize().has_value()) { + ov::set_batch(model, config.getBatchSize().value().createPartialDimension()); + } +} + +Status ModelInstance::loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter) { + bool isLayoutConfigurationChanged = !config.isLayoutConfigurationEqual(this->config); + bool needsToApplyLayoutConfiguration = isLayoutConfigurationChanged || !this->model; + + subscriptionManager.notifySubscribers(); + this->path = config.getPath(); + this->targetDevice = config.getTargetDevice(); + this->config = config; + auto status = fetchModelFilepaths(); + + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + try { + status = setCacheOptions(this->config); + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + + if (!this->model || isLayoutConfigurationChanged) { + if (this->config.isCustomLoaderRequiredToLoadModel()) { + // loading the model using the custom loader + status = loadOVModelUsingCustomLoader(); + } else { + status = loadOVModel(); + } + } + + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + + status = loadTensors(this->config, needsToApplyLayoutConfiguration, parameter); + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + status = loadOVCompiledModel(this->config); + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + status = prepareInferenceRequestsQueue(this->config); + if (!status.ok()) { + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return status; + } + } catch (const ov::Exception& e) { + SPDLOG_ERROR("exception occurred while loading model: {}", e.what()); + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return StatusCode::MODEL_NOT_LOADED; + } catch (const std::exception& e) { + SPDLOG_ERROR("exception occurred while loading model: {}", e.what()); + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + return StatusCode::MODEL_NOT_LOADED; + } + this->status.setAvailable(); + modelLoadedNotify.notify_all(); + return status; +} + +Status ModelInstance::setCacheOptions(const ModelConfig& config) { + if (!config.getCacheDir().empty()) { + if (!config.isAllowCacheSetToTrue() && (config.isCustomLoaderRequiredToLoadModel() || config.anyShapeSetToAuto() || (config.getBatchingMode() == Mode::AUTO))) { + this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), ""}}); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has disabled caching", this->getName()); + this->cacheDisabled = true; + } else if (config.isAllowCacheSetToTrue() && config.isCustomLoaderRequiredToLoadModel()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model: {} has allow cache set to true while using custom loader", this->getName()); + return StatusCode::ALLOW_CACHE_WITH_CUSTOM_LOADER; + } else { + this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), config.getCacheDir()}}); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has enabled caching", this->getName()); + } + } + return StatusCode::OK; +} + +Status ModelInstance::loadModel(const ModelConfig& config) { + std::lock_guard loadingLock(loadingMutex); + SPDLOG_INFO("Loading model: {}, version: {}, from path: {}, with target device: {} ...", + config.getName(), config.getVersion(), config.getPath(), config.getTargetDevice()); + if (config.getBatchingMode() == AUTO) { + SPDLOG_INFO("Batch size mode for model {} is set to auto", config.getName()); + } else if (config.anyShapeSetToAuto()) { + SPDLOG_INFO("Some inputs shapes for model {} are set to auto", config.getName()); + } + this->status = ModelVersionStatus(config.getName(), config.getVersion()); + this->status.setLoading(); + return loadModelImpl(config); +} + +Status ModelInstance::reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter) { + std::lock_guard loadingLock(loadingMutex); + this->status.setLoading(); + while (!canUnloadInstance()) { + SPDLOG_INFO("Waiting to reload model: {} version: {}. Blocked by: {} inferences in progress.", + getName(), getVersion(), predictRequestsHandlesCount); + std::this_thread::sleep_for(std::chrono::milliseconds(UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS)); + } + if ((this->config.isCustomLoaderRequiredToLoadModel()) && (isCustomLoaderConfigChanged)) { + // unloading and the loading back the model + isCustomLoaderConfigChanged = false; + retireModel(isCustomLoaderConfigChanged); + } + return loadModelImpl(config, parameter); +} + +Status ModelInstance::recoverFromReloadingError(const Status& status) { + SPDLOG_WARN("Failed to perform complete reload with requested dynamic parameter. Model: {} version: {} with error: {}. Reloading to previous configuration", + getName(), getVersion(), status.string()); + bool changeStatus{false}; + retireModel(changeStatus); + + auto recoveryStatus = reloadModel(config); + if (!recoveryStatus.ok()) { + SPDLOG_WARN("Failed to recover model: {} version: {} to previous configuration with error: {}", + getName(), getVersion(), recoveryStatus.string()); + } + return status; +} + +Status ModelInstance::reshapeWithFullReload(const Status& status, const DynamicModelParameter& parameter) { + SPDLOG_WARN("Failed to reload model: {} version: {} with error: {}. Trying to perform complete reload with requested dynamic parameter", + getName(), getVersion(), status.string()); + bool changeStatus{false}; + retireModel(changeStatus); + + auto recoveryStatus = reloadModel(config, parameter); + if (!recoveryStatus.ok()) { + SPDLOG_WARN("Failed to reload model: {} version: {} to previous configuration with error: {}", + getName(), getVersion(), recoveryStatus.string()); + } + return recoveryStatus; +} + +Status ModelInstance::reloadModel(std::optional batchSize, std::map requestShapes, std::unique_ptr& unloadGuard) { + // temporarily release current predictRequest lock on model loading + unloadGuard.reset(); + // block concurrent requests for reloading/unloading - assure that after reload predict request + // will block further requests for reloading/unloading until inference is performed + std::lock_guard loadingLock(loadingMutex); + SPDLOG_INFO("Will reload model: {} version: {}", getName(), getVersion()); + + DynamicModelParameter parameter; + if (batchSize.has_value() && batchSize.value().isStatic()) { + parameter = DynamicModelParameter(batchSize.value().getStaticValue()); + } else if (requestShapes.size() > 0) { + parameter = DynamicModelParameter(requestShapes); + } else { + SPDLOG_DEBUG("Error: requested model: {} version: {} reload with no batchsize and shapes set.", getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } + + auto status = reloadModel(config, parameter); + if (!status.ok()) { + status = this->reshapeWithFullReload(status, parameter); + if (!status.ok()) { + return this->recoverFromReloadingError(status); + } + } + unloadGuard = std::make_unique(*this); + return status; +} + +Status ModelInstance::reloadModelIfRequired( + Status validationStatus, + const std::optional& requestedBatchSize, + const std::map& requestedShapes, + std::unique_ptr& modelUnloadGuardPtr) { + Status status = validationStatus; + if (status.batchSizeChangeRequired()) { + try { + status = reloadModel(requestedBatchSize, {}, modelUnloadGuardPtr); + } catch (const std::exception& e) { + status = Status(StatusCode::INVALID_BATCH_DIMENSION, e.what()); + } + if (!status.ok()) { + SPDLOG_ERROR("Model: {}, version: {} reload (batch size change) failed. Status Code: {}, Error {}", + getName(), getVersion(), status.getCode(), status.string()); + } + } else if (status.reshapeRequired()) { + status = reloadModel(std::nullopt, requestedShapes, modelUnloadGuardPtr); + if (!status.ok() && status != StatusCode::RESHAPE_ERROR) { + SPDLOG_ERROR("Model: {}, version: {} reload (reshape) failed. Status Code: {}, Error: {}", + getName(), getVersion(), status.getCode(), status.string()); + } + } else if (!status.ok()) { + SPDLOG_DEBUG("Model: {}, version: {} validation of inferRequest failed. Status Code: {}, Error: {}", + getName(), getVersion(), status.getCode(), status.string()); + } + return status; +} + +Status ModelInstance::waitForLoaded(const uint waitForModelLoadedTimeoutMilliseconds, + std::unique_ptr& modelInstanceUnloadGuard) { + // order is important here for performance reasons + // assumption: model is already loaded for most of the calls + modelInstanceUnloadGuard = std::make_unique(*this); + if (getStatus().getState() == ModelVersionState::AVAILABLE) { + SPDLOG_DEBUG("Model: {}, version: {} already loaded", getName(), getVersion()); + return StatusCode::OK; + } + modelInstanceUnloadGuard.reset(); + + // wait several time since no guarantee that cv wakeup will be triggered before calling wait_for + const uint waitLoadedTimestepMilliseconds = 100; + const uint waitCheckpoints = waitForModelLoadedTimeoutMilliseconds / waitLoadedTimestepMilliseconds; + uint waitCheckpointsCounter = waitCheckpoints; + SPDLOG_DEBUG("Waiting for loaded state for model: {} version: {} with timestep: {} timeout: {} check count: {}", getName(), getVersion(), + waitLoadedTimestepMilliseconds, waitForModelLoadedTimeoutMilliseconds, waitCheckpointsCounter); + std::mutex cv_mtx; + std::unique_lock cv_lock(cv_mtx); + while (waitCheckpointsCounter-- > 0) { + if (modelLoadedNotify.wait_for(cv_lock, + std::chrono::milliseconds(waitLoadedTimestepMilliseconds), + [this]() { + return this->getStatus().getState() > ModelVersionState::LOADING; + })) { + SPDLOG_INFO("Waiting for model: {} version: {} loaded state for: {} time", + getName(), getVersion(), waitCheckpoints - waitCheckpointsCounter); + } + modelInstanceUnloadGuard = std::make_unique(*this); + if (getStatus().getState() == ModelVersionState::AVAILABLE) { + SPDLOG_INFO("Succesfully waited for model: {}, version: {}", getName(), getVersion()); + return StatusCode::OK; + } + modelInstanceUnloadGuard.reset(); + if (ModelVersionState::AVAILABLE < getStatus().getState()) { + SPDLOG_INFO("Stopped waiting for model: {} version: {} since it is unloading.", getName(), getVersion()); + return StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE; + } + } + SPDLOG_INFO("Waiting for loaded state reached timeout for model: {} version: {}", + getName(), getVersion()); + if (getStatus().getState() > ModelVersionState::AVAILABLE) { + SPDLOG_DEBUG("Waiting for model: {}, version: {} ended since it started unloading.", getName(), getVersion()); + return StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE; + } else { + SPDLOG_DEBUG("Waiting for model: {}, version: {} ended due to timeout.", getName(), getVersion()); + return StatusCode::MODEL_VERSION_NOT_LOADED_YET; + } +} + +void ModelInstance::retireModel(bool isPermanent) { + std::lock_guard loadingLock(loadingMutex); + if (isPermanent) { + this->status.setUnloading(); + } else { + this->status.setLoading(); + } + unloadModelComponents(); + if (isPermanent) { + status.setEnd(); + } +} + +void ModelInstance::cleanupFailedLoad() { + std::lock_guard loadingLock(loadingMutex); + this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); + unloadModelComponents(); +} + +void ModelInstance::unloadModelComponents() { + subscriptionManager.notifySubscribers(); + while (!canUnloadInstance()) { + SPDLOG_DEBUG("Waiting to unload model: {} version: {}. Blocked by: {} inferences in progres.", + getName(), getVersion(), predictRequestsHandlesCount); + std::this_thread::sleep_for(std::chrono::milliseconds(UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS)); + } + inferRequestsQueue.reset(); + compiledModel.reset(); + model.reset(); + outputsInfo.clear(); + inputsInfo.clear(); + modelFiles.clear(); + + if (this->config.isCustomLoaderRequiredToLoadModel()) { + custom_loader_options_config_t customLoaderOptionsConfig = this->config.getCustomLoaderOptionsConfigMap(); + const std::string loaderName = customLoaderOptionsConfig["loader_name"]; + auto& customloaders = ovms::CustomLoaders::instance(); + auto customLoaderInterfacePtr = customloaders.find(loaderName); + if (customLoaderInterfacePtr == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "The loader {} is no longer available for model: {} version : {}", + loaderName, getName(), getVersion()); + } else { + // once model is unloaded, notify custom loader object about the unload + customLoaderInterfacePtr->unloadModel(getName(), getVersion()); + } + } +} + +template +const Status ModelInstance::validate(const RequestType* request) { + OVMS_PROFILE_FUNCTION(); + static const std::set optionalInputNames = {}; + return request_validation_utils::validate( + *request, + getInputsInfo(), + getName(), + getVersion(), + optionalInputNames, + getModelConfig().getBatchingMode(), + getModelConfig().getShapes()); +} + +template const Status ModelInstance::validate(const ::inference::ModelInferRequest* request); +template const Status ModelInstance::validate(const tensorflow::serving::PredictRequest* request); + +Status ModelInstance::performInference(ov::InferRequest& inferRequest) { + OVMS_PROFILE_FUNCTION(); + try { + OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::start_async"); + inferRequest.start_async(); + OVMS_PROFILE_SYNC_END("ov::InferRequest::start_async"); + OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::wait"); + inferRequest.wait(); + OVMS_PROFILE_SYNC_END("ov::InferRequest::wait"); + } catch (const ov::Exception& e) { + Status status = StatusCode::OV_INTERNAL_INFERENCE_ERROR; + SPDLOG_ERROR("Async caught an exception {}: {}", status.string(), e.what()); + return status; + } + return StatusCode::OK; +} + +Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, + tensorflow::serving::PredictResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + Timer timer; + using std::chrono::microseconds; + + auto status = validate(requestProto); + auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); + auto requestShapes = getRequestShapes(requestProto); + status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); + if (!status.ok()) + return status; + timer.start("get infer request"); + ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); + int executingInferId = executingStreamIdGuard.getId(); + ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); + timer.stop("get infer request"); + SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); + + timer.start("deserialize"); + InputSink inputSink(inferRequest); + bool isPipeline = false; + status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); + timer.stop("deserialize"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); + + timer.start("prediction"); + status = performInference(inferRequest); + timer.stop("prediction"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); + + timer.start("serialize"); + OutputGetter outputGetter(inferRequest); + status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); + timer.stop("serialize"); + if (!status.ok()) + return status; + + SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); + + return StatusCode::OK; +} + +Status ModelInstance::infer(const ::inference::ModelInferRequest* requestProto, + ::inference::ModelInferResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + Timer timer; + using std::chrono::microseconds; + + auto status = validate(requestProto); + auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); + auto requestShapes = getRequestShapes(requestProto); + status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); + if (!status.ok()) + return status; + timer.start("get infer request"); + ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); + int executingInferId = executingStreamIdGuard.getId(); + ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); + timer.stop("get infer request"); + SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); + + timer.start("deserialize"); + InputSink inputSink(inferRequest); + bool isPipeline = false; + status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); + timer.stop("deserialize"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); + + timer.start("prediction"); + status = performInference(inferRequest); + timer.stop("prediction"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); + + timer.start("serialize"); + OutputGetter outputGetter(inferRequest); + status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); + timer.stop("serialize"); + if (!status.ok()) + return status; + + SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); + + return StatusCode::OK; +} + +const size_t ModelInstance::getBatchSizeIndex() const { + const auto& inputItr = this->inputsInfo.cbegin(); + if (inputItr == this->inputsInfo.cend()) { + throw std::logic_error("model has no inputs"); + } + const auto& input = inputItr->second; + const auto batchIndex = input->getLayout().getBatchIndex(); + if (!batchIndex.has_value()) { + throw std::logic_error("cannot get batch index"); + } + return batchIndex.value(); +} + +} // namespace ovms diff --git a/src/ovms_lib/modelinstance.hpp b/src/ovms_lib/modelinstance.hpp new file mode 100644 index 0000000000..5f9316f910 --- /dev/null +++ b/src/ovms_lib/modelinstance.hpp @@ -0,0 +1,556 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "customloaderconfig.hpp" +#include "customloaderinterface.hpp" +#include "modelchangesubscription.hpp" +#include "modelconfig.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelversionstatus.hpp" +#include "ovinferrequestsqueue.hpp" +#include "sequence_processing_spec.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace inference { +class ModelInferRequest; +class ModelInferResponse; +} // namespace inference + +namespace ovms { + +class DynamicModelParameter { +public: + DynamicModelParameter() : + batchSize(0), + shapes({}) {} + DynamicModelParameter(int batchSize) : + batchSize(batchSize), + shapes({}) {} + DynamicModelParameter(const std::map& shapes) : + batchSize(0), + shapes(shapes) {} + + bool isBatchSizeRequested() const { return batchSize > 0; } + bool isShapeRequested(const std::string& name) const { return shapes.count(name) && shapes.at(name).size() > 0; } + + int getBatchSize() const { return batchSize; } + const shape_t& getShape(const std::string& name) const { return shapes.at(name); } + +private: + int batchSize; + std::map shapes; +}; + +class PipelineDefinition; + +/** + * @brief This class contains all the information about model + */ +class ModelInstance { +protected: + /** + * @brief Performs model loading + * + * @return status + */ + virtual Status loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); + + /** + * @brief OpenVINO Runtime Core object reference + */ + ov::Core& ieCore; + + /** + * @brief OpenVINO Runtime Model object + */ + std::shared_ptr model; + + /** + * @brief OpenVINO Runtime CompiledModel object + */ + std::shared_ptr compiledModel; + + /** + * @brief Model name + */ + const std::string name; + + /** + * @brief A path for the model + */ + std::string path; + + /** + * @brief A model version + */ + const model_version_t version = -1; + + /** + * @brief A model subscription manager + */ + ModelChangeSubscription subscriptionManager; + + /** + * @brief A model status + */ + ModelVersionStatus status; + + /** + * @brief Target device to run model + */ + std::string targetDevice; + + /** + * @brief Model batch size + */ + size_t batchSize = 0; + + /** + * @brief Stores required OpenVINO model files extensions to be able to load model + * Order is important, first file on the list is passed to LoadNetwork + */ + static constexpr std::array OV_MODEL_FILES_EXTENSIONS{".xml", ".bin"}; + + /** + * @brief Stores required onnx model files extensions to be able to load model + */ + static constexpr std::array ONNX_MODEL_FILES_EXTENSIONS{".onnx"}; + + /** + * @brief Notifies model instance users who wait for loading + */ + std::condition_variable modelLoadedNotify; + + /** + * @brief Holds currently loaded model configuration + */ + ModelConfig config; + + /** + * @brief Load OV CNNNetwork ptr + * + * @return CNNNetwork ptr + */ + virtual std::shared_ptr loadOVModelPtr(const std::string& modelFile); + + /** + * @brief Lock to disable concurrent modelinstance load/unload/reload + */ + std::recursive_mutex loadingMutex; + + /** + * @brief Load OV Engine + */ + void loadOVEngine(); + + /** + * @brief Loads OV CNNNetwork + * + * @return Status + */ + Status loadOVModel(); + + /** + * @brief Sets OV CompiledModelPtr + */ + virtual void loadCompiledModelPtr(const plugin_config_t& pluginConfig); + + /** + * @brief Loads OV CompiledModel + * + * @return Status + */ + virtual Status loadOVCompiledModel(const ModelConfig& config); + + /** + * @brief Prepares inferenceRequestsQueue + */ + Status prepareInferenceRequestsQueue(const ModelConfig& config); + + /** + * @brief Fetch model file paths + * + * @return Status + */ + Status fetchModelFilepaths(); + + const Layout getReportedTensorLayout(const ModelConfig& config, const std::string& name, bool isInput); + + /** + * @brief Find file path with extension in model path + * + * @return Returns filename with desired extension if exists otherwise empty string + */ + std::string findModelFilePathWithExtension(const std::string& extension) const; + + /** + * @brief Loads OV CNNNetwork Using the Custom Loader + * + * @return Status + */ + Status loadOVModelUsingCustomLoader(); + + template + const Status validate(const RequestType* request); + +private: + /** + * @brief Holds the information about inputs and it's parameters + */ + tensor_map_t inputsInfo; + + /** + * @brief Holds the information about outputs and it's parameters + */ + tensor_map_t outputsInfo; + + /** + * @brief Holds model required file names. First is loaded + */ + std::vector modelFiles; + + /** + * @brief OpenVINO inference execution stream pool + */ + std::unique_ptr inferRequestsQueue; + + /** + * @brief Holds current usage count in predict requests + * + * Needed for gating model unloading. + */ + std::atomic predictRequestsHandlesCount = 0; + + /** + * @brief Internal method for loading tensors + * + * @param config + */ + Status loadTensors(const ModelConfig& config, bool needsToApplyLayoutConfiguration, const DynamicModelParameter& parameter = DynamicModelParameter()); + + /** + * @brief Internal method for loading inputs + * + * @param config + */ + Status loadInputTensors(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); + + /** + * @brief Internal method for loading outputs + * + * @param config + */ + Status loadOutputTensors(const ModelConfig& config); + + /** + * @brief Flag determining if cache is disabled + */ + bool cacheDisabled = false; + + /** + * @brief Configures batchsize + */ + void configureBatchSize(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); + + uint32_t getNumOfParallelInferRequests(const ModelConfig& config); + uint32_t getNumOfParallelInferRequestsUnbounded(const ModelConfig& config); + + /** + * @brief Recover from any state model is put into when reload is requested + * + * @param status returned from reload operation + * + * @return Status + */ + Status recoverFromReloadingError(const Status& status); + + /** + * @brief Perform full model reload with dynamic parameter + * + * @param status returned from reload operation + * @param parameter requested dynamic parameter + * + * @return Status + */ + Status reshapeWithFullReload(const Status& status, const DynamicModelParameter& parameter); + + /** + * Variable to tell reload is due to customloader config change + */ + bool isCustomLoaderConfigChanged; + +public: + /** + * @brief A default constructor + */ + ModelInstance(const std::string& name, model_version_t version, ov::Core& ieCore) : + ieCore(ieCore), + name(name), + version(version), + subscriptionManager(std::string("model: ") + name + std::string(" version: ") + std::to_string(version)), + status(name, version) { isCustomLoaderConfigChanged = false; } + + /** + * @brief Destroy the Model Instance object + */ + virtual ~ModelInstance() = default; + + /** + * @brief Increases predict requests usage count + */ + void increasePredictRequestsHandlesCount() { + ++predictRequestsHandlesCount; + } + + /** + * @brief sets the flag in model instance indicating change in custom loader configuration. + */ + void setCustomLoaderConfigChangeFlag() { + isCustomLoaderConfigChanged = true; + } + + /** + * @brief Decreases predict requests usage count + */ + void decreasePredictRequestsHandlesCount() { + --predictRequestsHandlesCount; + } + + /** + * @brief Gets the model name + * + * @return model name + */ + virtual const std::string& getName() const { + return name; + } + + /** + * @brief Gets path for the model + * + * @return path + */ + const std::string& getPath() const { + return path; + } + + /** + * @brief Gets version + * + * @return version + */ + virtual model_version_t getVersion() const { + return version; + } + + /** + * @brief Gets model status + * + * @return status + */ + const ModelVersionStatus& getStatus() const { + return status; + } + + /** + * @brief Gets executing target device name + * + * @return target device name + */ + const std::string& getTargetDevice() { + return targetDevice; + } + + /** + * @brief Internal method for setting cache options + */ + Status setCacheOptions(const ModelConfig& config); + + /** + * @brief Check if cache is disabled + * + * @return cache disabled + */ + const bool isCacheDisabled() { + return cacheDisabled; + } + + /** + * @brief Gets batch size + * + * @return batch size + */ + virtual Dimension getBatchSize() const { + return Dimension(ov::get_batch(model)); + } + + const size_t getBatchSizeIndex() const; + + /** + * @brief Gets model config + * + * @return model config + */ + virtual const ModelConfig& getModelConfig() const { + return config; + } + + /** + * @brief Get the Inputs Info object + * + * @return const tensor_map_t& + */ + virtual const tensor_map_t& getInputsInfo() const { + return inputsInfo; + } + + /** + * @brief Get the Outputs Info object + * + * @return const tensor_map_t& + */ + virtual const tensor_map_t& getOutputsInfo() const { + return outputsInfo; + } + + /** + * @brief Check if can unload infer requests + * + * @return bool + */ + virtual bool canUnloadInstance() const { + return 0 == predictRequestsHandlesCount; + } + + /** + * @brief Get OV streams pool + * + * @return OVStreamsQueue + */ + OVInferRequestsQueue& getInferRequestsQueue() { + return *inferRequestsQueue; + } + + /** + * @brief Combines plugin config from user with default config calculated at runtime + * + * @return plugin config + */ + static plugin_config_t prepareDefaultPluginConfig(const ModelConfig& config); + + /** + * @brief Loads model version + * + * @param config model configuration + * + * @return Status + */ + virtual Status loadModel(const ModelConfig& config); + + /** + * @brief Reloads model version + * + * @param config model configuration + * + * @return Status + */ + virtual Status reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); + + /** + * @brief Reloads model version with different batch size or shape + * + * @param batchSize new batch size + * @param shape new shape + * @param unloadGuard unloadGuardPtr + * + * @return Status + */ + virtual Status reloadModel(std::optional batchSize, std::map shape, std::unique_ptr& unloadGuardPtr); + + /** + * @brief Reloads model version if status of request validation indicates there's a need for reshape or batch size change + * + * @param validationStatus status of request validation + * @param requestProto pointer to the request proto + * @param modelUnloadGuardPtr unloadGuardPtr + * + * @return Status + */ + Status reloadModelIfRequired( + Status validationStatus, + const std::optional& requestedBatchSize, + const std::map& requestedShapes, + std::unique_ptr& modelUnloadGuardPtr); + + /** + * @brief Unloads model version + * @param isPermanent defines if the unload operation should be permanent and should change instance state to End after it is completed + * otherwise model might be unloaded temporarily so the instance state should be preserved as Loading + */ + virtual void retireModel(bool isPermanent = true); + + /** + * @brief Cleans model version that failed to load + */ + virtual void cleanupFailedLoad(); + + void unloadModelComponents(); + + /** + * @brief Wait for model to change to AVAILABLE state + * + * @param waitForModelLoadedTimeoutMilliseconds + * @param modelInstanceUnloadGuard + * + * @return Status + */ + Status waitForLoaded(const uint waitForModelLoadedTimeoutMilliseconds, + std::unique_ptr& modelInstanceUnloadGuard); + + void subscribe(PipelineDefinition& pd); + + void unsubscribe(PipelineDefinition& pd); + + const ModelChangeSubscription& getSubscribtionManager() const { return subscriptionManager; } + + Status performInference(ov::InferRequest& inferRequest); + + virtual Status infer(const tensorflow::serving::PredictRequest* requestProto, + tensorflow::serving::PredictResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr); + virtual Status infer(const ::inference::ModelInferRequest* requestProto, + ::inference::ModelInferResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr); +}; +} // namespace ovms diff --git a/src/ovms_lib/modelinstanceunloadguard.cpp b/src/ovms_lib/modelinstanceunloadguard.cpp new file mode 100644 index 0000000000..0636e64d51 --- /dev/null +++ b/src/ovms_lib/modelinstanceunloadguard.cpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelinstanceunloadguard.hpp" + +#include "modelinstance.hpp" + +namespace ovms { +ModelInstanceUnloadGuard::ModelInstanceUnloadGuard(ModelInstance& modelInstance) : + modelInstance(modelInstance) { + modelInstance.increasePredictRequestsHandlesCount(); +} + +ModelInstanceUnloadGuard::~ModelInstanceUnloadGuard() { + modelInstance.decreasePredictRequestsHandlesCount(); +} +} // namespace ovms diff --git a/src/ovms_lib/modelinstanceunloadguard.hpp b/src/ovms_lib/modelinstanceunloadguard.hpp new file mode 100644 index 0000000000..55e5a0ba56 --- /dev/null +++ b/src/ovms_lib/modelinstanceunloadguard.hpp @@ -0,0 +1,30 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +namespace ovms { +class ModelInstance; + +class ModelInstanceUnloadGuard { +public: + ModelInstanceUnloadGuard() = delete; + ModelInstanceUnloadGuard(ModelInstance& modelInstance); + ~ModelInstanceUnloadGuard(); + +private: + ModelInstance& modelInstance; +}; +} // namespace ovms diff --git a/src/ovms_lib/modelmanager.cpp b/src/ovms_lib/modelmanager.cpp new file mode 100644 index 0000000000..a0c77acccd --- /dev/null +++ b/src/ovms_lib/modelmanager.cpp @@ -0,0 +1,1316 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelmanager.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "azurefilesystem.hpp" +#include "cleaner_utils.hpp" +#include "config.hpp" +#include "custom_node_library_manager.hpp" +#include "customloaders.hpp" +#include "entry_node.hpp" // need for ENTRY_NODE_NAME +#include "exit_node.hpp" // need for EXIT_NODE_NAME +#include "filesystem.hpp" +#include "gcsfilesystem.hpp" +#include "localfilesystem.hpp" +#include "logging.hpp" +#include "node_library.hpp" +#include "openssl/md5.h" +#include "ov_utils.hpp" +#include "pipeline.hpp" +#include "pipeline_factory.hpp" +#include "pipelinedefinition.hpp" +#include "s3filesystem.hpp" +#include "schema.hpp" +#include "stringutils.hpp" + +namespace ovms { + +static uint16_t MAX_CONFIG_JSON_READ_RETRY_COUNT = 2; + +ModelManager::ModelManager(const std::string& modelCacheDirectory) : + ieCore(std::make_unique()), + waitForModelLoadedTimeoutMs(DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS), + modelCacheDirectory(modelCacheDirectory) { + // Take --cache_dir from CLI + if (this->modelCacheDirectory.empty()) { + this->modelCacheDirectory = ovms::Config::instance().cacheDir(); + } + // If not enabled via CLI, check for /opt/cache existence. + if (this->modelCacheDirectory.empty()) { + if (std::filesystem::exists(DEFAULT_MODEL_CACHE_DIRECTORY)) { + this->modelCacheDirectory = DEFAULT_MODEL_CACHE_DIRECTORY; + } + } + // If cache dir enabled, check for write access. + if (!this->modelCacheDirectory.empty()) { + // Create directory if does not exist + if (!std::filesystem::exists(this->modelCacheDirectory)) { + std::filesystem::create_directories(this->modelCacheDirectory); + SPDLOG_LOGGER_WARN(modelmanager_logger, "Cache directory {} did not exist, created", this->modelCacheDirectory); + } + int result = access(this->modelCacheDirectory.c_str(), W_OK); + if (result != 0) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Cache directory {} is not writable; access() result: {}", this->modelCacheDirectory, result); + } else { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Model cache is enabled: {}", this->modelCacheDirectory); + } + } + this->customNodeLibraryManager = std::make_unique(); + if (ovms::Config::instance().cpuExtensionLibraryPath() != "") { + SPDLOG_INFO("Loading custom CPU extension from {}", ovms::Config::instance().cpuExtensionLibraryPath()); + try { + ieCore->add_extension(ovms::Config::instance().cpuExtensionLibraryPath()); + SPDLOG_INFO("Extension added."); + } catch (std::exception& ex) { + SPDLOG_CRITICAL("Custom CPU extension loading has failed! Reason: {}", ex.what()); + throw; + } catch (...) { + SPDLOG_CRITICAL("Custom CPU extension loading has failed with an unknown error!"); + throw; + } + } + this->logPluginConfiguration(); +} + +void ModelManager::logPluginConfiguration() { + auto availableDevices = ieCore->get_available_devices(); + SPDLOG_LOGGER_INFO(modelmanager_logger, "Available devices for Open VINO: {}", joins(availableDevices, std::string(", "))); + auto availablePlugins = availableDevices; + const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); + for (const auto& plugin : availablePlugins) { + std::vector supportedConfigKeys; + try { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging plugin: {}; configuration", plugin); + std::vector supportedConfigKeys2 = ieCore->get_property(plugin, supportedConfigKey).as>(); + supportedConfigKeys = std::move(supportedConfigKeys2); + } catch (std::exception& e) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", plugin, supportedConfigKey, e.what()); + } catch (...) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", plugin, supportedConfigKey); + } + for (auto& key : supportedConfigKeys) { + std::string value; + try { + auto paramValue = ieCore->get_property(plugin, key); + value = paramValue.as(); + } catch (std::exception& e) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; config key: {}; Error: {}", plugin, key, e.what()); + continue; + } catch (...) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; config key: {}", plugin, key); + continue; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Global plugin: {}; key: {}; value: {}", plugin, key, value); + } + } +} + +ModelManager::~ModelManager() = default; + +Status ModelManager::start(const Config& config) { + watcherIntervalSec = config.filesystemPollWaitSeconds(); + sequenceCleaupIntervalMinutes = config.sequenceCleanerPollWaitMinutes(); + resourcesCleanupIntervalSec = config.resourcesCleanerPollWaitSeconds(); + if (resourcesCleanupIntervalSec < 1) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Parameter: custom_node_resources_cleaner_interval has to be greater than 0. Applying default value(1 second)"); + resourcesCleanupIntervalSec = 1; + } + Status status; + if (config.configPath() != "") { + status = startFromFile(config.configPath()); + } else { + status = startFromConfig(); + } + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't start model manager"); + return status; + } + + startWatcher(); + startCleaner(); + return status; +} + +void ModelManager::startWatcher() { + if ((!watcherStarted) && (watcherIntervalSec > 0)) { + std::future exitSignal = exitTrigger.get_future(); + std::thread t(std::thread(&ModelManager::watcher, this, std::move(exitSignal))); + watcherStarted = true; + monitor = std::move(t); + } +} + +void ModelManager::startCleaner() { + if ((!cleanerStarted)) { + std::future exitSignal = cleanerExitTrigger.get_future(); + std::thread t(std::thread(&ModelManager::cleanerRoutine, this, resourcesCleanupIntervalSec, sequenceCleaupIntervalMinutes, std::move(exitSignal))); + cleanerStarted = true; + cleanerThread = std::move(t); + } +} + +Status ModelManager::startFromConfig() { + auto& config = ovms::Config::instance(); + + auto [it, success] = servedModelConfigs.emplace( + config.modelName(), + ModelConfig{ + config.modelName(), + config.modelPath(), + config.targetDevice(), + config.batchSize(), + config.nireq(), + config.stateful(), + config.idleSequenceCleanup(), + config.lowLatencyTransformation(), + config.maxSequenceNumber(), + this->modelCacheDirectory}); + + if (!success) { + return StatusCode::UNKNOWN_ERROR; + } + + ModelConfig& modelConfig = it->second; + + auto status = modelConfig.parsePluginConfig(config.pluginConfig()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse plugin config"); + return status; + } + + status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); + return status; + } + + status = modelConfig.parseModelVersionPolicy(config.modelVersionPolicy()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse model version policy. {}", status.string()); + return status; + } + + status = modelConfig.parseShapeParameter(config.shape()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse shape parameter"); + return status; + } + + status = modelConfig.parseLayoutParameter(config.layout()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse layout parameter"); + return status; + } + + bool batchSizeSet = (modelConfig.getBatchingMode() != FIXED || modelConfig.getBatchSize() != 0); + bool shapeSet = (modelConfig.getShapes().size() > 0); + + SPDLOG_DEBUG("Batch size set: {}, shape set: {}", batchSizeSet, shapeSet); + if (batchSizeSet && shapeSet) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Both shape and batch size have been defined. Batch size parameter will be ignored."); + modelConfig.setBatchingMode(FIXED); + modelConfig.setBatchSize(std::nullopt); + } + + return reloadModelWithVersions(modelConfig); +} + +Status ModelManager::startFromFile(const std::string& jsonFilename) { + Status status = loadConfig(jsonFilename); + if (status == StatusCode::CONFIG_FILE_INVALID || status == StatusCode::JSON_INVALID) { + return status; + } + + return StatusCode::OK; +} + +void processNodeInputs(const std::string nodeName, const rapidjson::Value::ConstMemberIterator& itro, pipeline_connections_t& connections) { + for (const auto& nodeInput : itro->value.GetArray()) { + for (const auto& objectNameValue : nodeInput.GetObject()) { + const std::string inputName = objectNameValue.name.GetString(); + const std::string sourceNodeName = objectNameValue.value.GetObject()["node_name"].GetString(); + const std::string sourceOutputName = objectNameValue.value.GetObject()["data_item"].GetString(); + SPDLOG_DEBUG("Creating node dependencies mapping request. Node: {} input: {} <- SourceNode: {} output: {}", + nodeName, inputName, sourceNodeName, sourceOutputName); + if (connections.find(nodeName) == connections.end()) { + connections[nodeName] = { + {sourceNodeName, + {{sourceOutputName, inputName}}}}; + } else { + if (connections[nodeName].find(sourceNodeName) == connections[nodeName].end()) { + connections[nodeName].insert({sourceNodeName, + {{sourceOutputName, inputName}}}); + } else { + connections[nodeName][sourceNodeName].push_back({sourceOutputName, inputName}); + } + } + } + } +} + +void processPipelineInputs(const rapidjson::Value::ConstMemberIterator& pipelineInputsPtr, const std::string& nodeName, std::unordered_map& nodeOutputNameAlias, const std::string& pipelineName) { + for (const auto& pipelineInput : pipelineInputsPtr->value.GetArray()) { + const std::string pipelineInputName = pipelineInput.GetString(); + SPDLOG_DEBUG("Mapping node:{} output:{}, under alias:{}", + nodeName, pipelineInputName, pipelineInputName); + auto result = nodeOutputNameAlias.insert({pipelineInputName, pipelineInputName}); + if (!result.second) { + SPDLOG_ERROR("Pipeline {} has duplicated input declaration", pipelineName); + } + } +} + +void processNodeOutputs(const rapidjson::Value::ConstMemberIterator& nodeOutputsItr, const std::string& nodeName, const std::string& modelName, std::unordered_map& nodeOutputNameAlias) { + for (const auto& nodeOutput : nodeOutputsItr->value.GetArray()) { + const std::string modelOutputName = nodeOutput.GetObject()["data_item"].GetString(); + const std::string nodeOutputName = nodeOutput.GetObject()["alias"].GetString(); + SPDLOG_DEBUG("Mapping node: {} model_name: {} output: {}, under alias: {}", + nodeName, modelName, modelOutputName, nodeOutputName); + nodeOutputNameAlias[nodeOutputName] = modelOutputName; + } +} + +void processDLNodeConfig(const rapidjson::Value& nodeConfig, DLNodeInfo& info) { + info.modelName = nodeConfig["model_name"].GetString(); + if (nodeConfig.HasMember("version")) { + info.modelVersion = nodeConfig["version"].GetUint64(); + } +} + +#define IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status) \ + if (firstErrorStatus.ok()) { \ + firstErrorStatus = status; \ + } + +Status processCustomNodeConfig(const rapidjson::Value& nodeConfig, CustomNodeInfo& info, const std::string& pipelineName, ModelManager& manager) { + std::string libraryName = nodeConfig["library_name"].GetString(); + auto status = manager.getCustomNodeLibraryManager().getLibrary(libraryName, info.library); + if (!status.ok()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {} refers to non existing custom node library: {}", pipelineName, libraryName); + } + if (nodeConfig.HasMember("params")) { + for (const auto& param : nodeConfig["params"].GetObject()) { + info.parameters.emplace(param.name.GetString(), param.value.GetString()); + } + } + return StatusCode::OK; +} + +Status processPipelineConfig(rapidjson::Document& configJson, const rapidjson::Value& pipelineConfig, std::set& pipelinesInConfigFile, PipelineFactory& factory, ModelManager& manager) { + const std::string pipelineName = pipelineConfig["name"].GetString(); + if (pipelinesInConfigFile.find(pipelineName) != pipelinesInConfigFile.end()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated pipeline names: {} defined in config file. Only first definition will be loaded.", pipelineName); + return StatusCode::OK; + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Reading pipeline: {} configuration", pipelineName); + std::set demultiplexerNodes; + std::set gatheredDemultiplexerNodes; + std::optional demultiplyCountEntry = std::nullopt; + auto demultiplyCountEntryIt = pipelineConfig.FindMember("demultiply_count"); + if (demultiplyCountEntryIt != pipelineConfig.MemberEnd()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} does have demultiply at entry node", pipelineName); + int32_t parsedDemultiplyCount = pipelineConfig["demultiply_count"].GetInt(); + if (parsedDemultiplyCount == 0) { + parsedDemultiplyCount = -1; + SPDLOG_LOGGER_WARN(modelmanager_logger, "demultiply_count 0 will be deprecated. For dynamic count use -1."); + } + demultiplyCountEntry = parsedDemultiplyCount; + demultiplexerNodes.insert(ENTRY_NODE_NAME); + } else { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} does not have demultiply at entry node", pipelineName); + } + + std::vector info; + NodeInfo entryInfo{NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {}, demultiplyCountEntry}; + info.emplace_back(std::move(entryInfo)); + processPipelineInputs(pipelineConfig.FindMember("inputs"), ENTRY_NODE_NAME, info[0].outputNameAliases, pipelineName); + pipeline_connections_t connections; + + auto nodesItr = pipelineConfig.FindMember("nodes"); + for (const auto& nodeConfig : nodesItr->value.GetArray()) { + std::string nodeName; + nodeName = nodeConfig["name"].GetString(); + + const std::string nodeKindStr = nodeConfig["type"].GetString(); + NodeKind nodeKind; + auto status = toNodeKind(nodeKindStr, nodeKind); + if (!status.ok()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Parsing node kind failed: {} for pipeline: {}", nodeKindStr, pipelineName); + return status; + } + + DLNodeInfo dlNodeInfo; + CustomNodeInfo customNodeInfo; + if (nodeKind == NodeKind::DL) { + processDLNodeConfig(nodeConfig, dlNodeInfo); + } else if (nodeKind == NodeKind::CUSTOM) { + status = processCustomNodeConfig(nodeConfig, customNodeInfo, pipelineName, manager); + if (!status.ok()) { + return status; + } + } else { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline {} contains unknown node kind", pipelineName); + throw std::invalid_argument("unknown node kind"); + } + + auto nodeOutputsItr = nodeConfig.FindMember("outputs"); + if (nodeOutputsItr == nodeConfig.MemberEnd() || !nodeOutputsItr->value.IsArray()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {} does not have valid outputs configuration", pipelineName); + return status; + } + std::unordered_map nodeOutputNameAlias; // key:alias, value realName + processNodeOutputs(nodeOutputsItr, nodeName, dlNodeInfo.modelName, nodeOutputNameAlias); + std::optional demultiplyCount; + if (nodeConfig.HasMember("demultiply_count")) { + int32_t parsedDemultiplyCount = nodeConfig["demultiply_count"].GetInt(); + if (parsedDemultiplyCount == 0) { + parsedDemultiplyCount = -1; + SPDLOG_LOGGER_WARN(modelmanager_logger, "demultiply_count 0 will be deprecated. For dynamic count use -1."); + } + demultiplyCount = parsedDemultiplyCount; + demultiplexerNodes.insert(nodeName); + } + std::set gatherFromNode; + if (nodeConfig.HasMember("gather_from_node")) { + std::string nodeToGatherFrom = nodeConfig["gather_from_node"].GetString(); + gatherFromNode.insert(nodeToGatherFrom); + gatheredDemultiplexerNodes.insert(nodeToGatherFrom); + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Creating node: {} type: {} model_name: {} modelVersion: {}", + nodeName, nodeKindStr, dlNodeInfo.modelName, dlNodeInfo.modelVersion.value_or(0)); + info.emplace_back( + nodeKind, + nodeName, + dlNodeInfo.modelName, + dlNodeInfo.modelVersion, + nodeOutputNameAlias, + demultiplyCount, + gatherFromNode, + customNodeInfo.library, + customNodeInfo.parameters); + auto nodeInputItr = nodeConfig.FindMember("inputs"); + processNodeInputs(nodeName, nodeInputItr, connections); + } + const auto iteratorOutputs = pipelineConfig.FindMember("outputs"); + // pipeline outputs are node exit inputs + processNodeInputs(EXIT_NODE_NAME, iteratorOutputs, connections); + std::set nonGatheredDemultiplexerNodes; + std::set_difference(demultiplexerNodes.begin(), demultiplexerNodes.end(), + gatheredDemultiplexerNodes.begin(), gatheredDemultiplexerNodes.end(), + std::inserter(nonGatheredDemultiplexerNodes, nonGatheredDemultiplexerNodes.begin())); + info.emplace_back(std::move(NodeInfo(NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, nonGatheredDemultiplexerNodes))); + if (!factory.definitionExists(pipelineName)) { + SPDLOG_DEBUG("Pipeline:{} was not loaded so far. Triggering load", pipelineName); + auto status = factory.createDefinition(pipelineName, info, connections, manager); + pipelinesInConfigFile.insert(pipelineName); + return status; + } + SPDLOG_DEBUG("Pipeline:{} is already loaded. Triggering reload", pipelineName); + auto status = factory.reloadDefinition(pipelineName, + std::move(info), + std::move(connections), + manager); + pipelinesInConfigFile.insert(pipelineName); + return status; +} + +Status ModelManager::loadCustomNodeLibrariesConfig(rapidjson::Document& configJson) { + const auto doc = configJson.FindMember("custom_node_library_config_list"); + if (doc == configJson.MemberEnd()) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Configuration file doesn't have custom node libraries property."); + return StatusCode::OK; + } + std::set librariesInConfig; + for (const auto& libraryConfig : doc->value.GetArray()) { + librariesInConfig.emplace(libraryConfig.FindMember("name")->value.GetString()); + this->customNodeLibraryManager->loadLibrary( + libraryConfig.FindMember("name")->value.GetString(), + libraryConfig.FindMember("base_path")->value.GetString()); + } + this->customNodeLibraryManager->unloadLibrariesRemovedFromConfig(librariesInConfig); + return StatusCode::OK; +} + +Status ModelManager::loadPipelinesConfig(rapidjson::Document& configJson) { + const auto itrp = configJson.FindMember("pipeline_config_list"); + if (itrp == configJson.MemberEnd() || !itrp->value.IsArray()) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Configuration file doesn't have pipelines property."); + pipelineFactory.retireOtherThan({}, *this); + return StatusCode::OK; + } + std::set pipelinesInConfigFile; + Status firstErrorStatus = StatusCode::OK; + for (const auto& pipelineConfig : itrp->value.GetArray()) { + auto status = processPipelineConfig(configJson, pipelineConfig, pipelinesInConfigFile, pipelineFactory, *this); + if (status != StatusCode::OK) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + } + pipelineFactory.retireOtherThan(std::move(pipelinesInConfigFile), *this); + return firstErrorStatus; +} + +Status ModelManager::createCustomLoader(CustomLoaderConfig& loaderConfig) { + auto& customloaders = ovms::CustomLoaders::instance(); + std::string loaderName = loaderConfig.getLoaderName(); + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Check if loader is already loaded"); + if (customloaders.find(loaderName) == nullptr) { + // this is where library or custom loader is loaded + if (FileSystem::isPathEscaped(loaderConfig.getLibraryPath())) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", loaderConfig.getLibraryPath()); + return StatusCode::PATH_INVALID; + } + void* handleCL = dlopen(const_cast(loaderConfig.getLibraryPath().c_str()), RTLD_LAZY | RTLD_LOCAL); + if (!handleCL) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot open library: {} {}", loaderConfig.getLibraryPath(), dlerror()); + return StatusCode::CUSTOM_LOADER_LIBRARY_INVALID; + } + + // load the symbols + createCustomLoader_t* customObj = (createCustomLoader_t*)dlsym(handleCL, "createCustomLoader"); + const char* dlsym_error = dlerror(); + if (dlsym_error || (customObj == nullptr)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot load symbol create: {} ", dlsym_error); + return StatusCode::CUSTOM_LOADER_LIBRARY_LOAD_FAILED; + } + + std::shared_ptr customLoaderIfPtr{customObj()}; + try { + customLoaderIfPtr->loaderInit(loaderConfig.getLoaderConfigFile()); + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot create or initialize the custom loader. Failed with error {}", e.what()); + return StatusCode::CUSTOM_LOADER_INIT_FAILED; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot create or initialize the custom loader"); + return StatusCode::CUSTOM_LOADER_INIT_FAILED; + } + customloaders.add(loaderName, customLoaderIfPtr, handleCL); + } else { + // Loader is already in the existing loaders. Move it to new loaders. + // Reload of customloader is not supported yet + customloaders.move(loaderName); + } + return StatusCode::OK; +} + +Status ModelManager::loadCustomLoadersConfig(rapidjson::Document& configJson) { + const auto itrp = configJson.FindMember("custom_loader_config_list"); + if (itrp == configJson.MemberEnd() || !itrp->value.IsArray()) { + return StatusCode::OK; + } + + Status firstErrorStatus = StatusCode::OK; + // Load Customer Loaders as per the configuration + SPDLOG_DEBUG("Using Customloader"); + for (const auto& configs : itrp->value.GetArray()) { + const std::string loaderName = configs["config"]["loader_name"].GetString(); + SPDLOG_INFO("Reading Custom Loader: {} configuration", loaderName); + + CustomLoaderConfig loaderConfig; + auto status = loaderConfig.parseNode(configs["config"]); + if (status != StatusCode::OK) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + SPDLOG_ERROR("Parsing loader: {} config failed", loaderName); + } + + auto retVal = createCustomLoader(loaderConfig); + if (retVal != StatusCode::OK) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(retVal); + SPDLOG_ERROR("Creation of loader: {} failed", loaderName); + } + } + // All loaders are the done. Finalize the list by deleting removed loaders in config + auto& customloaders = ovms::CustomLoaders::instance(); + customloaders.finalize(); + return firstErrorStatus; +} + +Status ModelManager::loadModelsConfig(rapidjson::Document& configJson, std::vector& gatedModelConfigs) { + const auto itr = configJson.FindMember("model_config_list"); + if (itr == configJson.MemberEnd() || !itr->value.IsArray()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file doesn't have models property."); + return StatusCode::JSON_INVALID; + } + Status firstErrorStatus = StatusCode::OK; + std::set modelsInConfigFile; + std::set modelsWithInvalidConfig; + std::unordered_map newModelConfigs; + for (const auto& configs : itr->value.GetArray()) { + ModelConfig modelConfig; + auto status = modelConfig.parseNode(configs["config"]); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_CONFIG_INVALID); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Parsing model: {} config failed due to error: {}", modelConfig.getName(), status.string()); + modelsWithInvalidConfig.emplace(modelConfig.getName()); + continue; + } + + status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); + return status; + } + modelConfig.setCacheDir(this->modelCacheDirectory); + + const auto modelName = modelConfig.getName(); + if (pipelineDefinitionExists(modelName)) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model name: {} is already occupied by pipeline definition.", modelName); + continue; + } + if (modelsInConfigFile.find(modelName) != modelsInConfigFile.end()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); + SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated model names: {} defined in config file. Only first definition will be loaded.", modelName); + continue; + } + status = reloadModelWithVersions(modelConfig); + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + + modelsInConfigFile.emplace(modelName); + if (!status.ok()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cannot reload model: {} with versions due to error: {}", modelName, status.string()); + } + if (status == StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL || status == StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Will retry to reload model({}) after pipelines are revalidated", modelName); + auto it = this->servedModelConfigs.find(modelName); + if (it == this->servedModelConfigs.end()) { + continue; + } + gatedModelConfigs.emplace_back(std::move(modelConfig)); + newModelConfigs.emplace(modelName, std::move(it->second)); + this->servedModelConfigs.erase(modelName); + } else { + newModelConfigs.emplace(modelName, std::move(modelConfig)); + } + } + this->servedModelConfigs = std::move(newModelConfigs); + retireModelsRemovedFromConfigFile(modelsInConfigFile, modelsWithInvalidConfig); + return firstErrorStatus; +} + +Status ModelManager::tryReloadGatedModelConfigs(std::vector& gatedModelConfigs) { + Status firstErrorStatus = StatusCode::OK; + for (auto& modelConfig : gatedModelConfigs) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Trying to reload model({}) configuration", modelConfig.getName()); + auto status = reloadModelWithVersions(modelConfig); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + continue; + } + auto it = this->servedModelConfigs.find(modelConfig.getName()); + if (it == this->servedModelConfigs.end()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + continue; + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Successfully retried to load new model({}) configuration after unsubscribed from pipeline", modelConfig.getName()); + this->servedModelConfigs.at(modelConfig.getName()) = std::move(modelConfig); + } + return firstErrorStatus; +} + +class LoudFileInfoReporter { + std::stringstream ss; + +public: + LoudFileInfoReporter(const std::string& filename, std::ifstream& file) { + struct stat statTime; + + if (stat(filename.c_str(), &statTime) != 0) { + SPDLOG_ERROR("Failed to debug-read fileconfig"); + return; + } + ss << "FileInfoReporter: " << filename + << " time modification [s]: " << statTime.st_ctim.tv_sec + << " [ns]: " << statTime.st_ctim.tv_nsec << std::endl; + std::string some; + file.clear(); + file.seekg(0); + while (file) { + file >> some; + ss << some << std::endl; + } + file.clear(); + file.seekg(0); + } + void log() { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, ss.str()); + } +}; + +Status ModelManager::loadConfig(const std::string& jsonFilename) { + std::lock_guard loadingLock(configMtx); + configFilename = jsonFilename; + lastConfigFileMD5 = getConfigFileMD5(); + rapidjson::Document configJson; + + uint16_t counter = 0; + Status intermediateStatus; + do { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Loading configuration from {} for: {} time", jsonFilename, counter + 1); + std::ifstream ifs(jsonFilename.c_str()); + LoudFileInfoReporter loud(jsonFilename, ifs); + if (!ifs.good()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is invalid {}", jsonFilename); + intermediateStatus = StatusCode::CONFIG_FILE_INVALID; + loud.log(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + rapidjson::IStreamWrapper isw(ifs); + rapidjson::ParseResult parseResult = configJson.ParseStream(isw); + if (!parseResult) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is not a valid JSON file. Error: {}", + rapidjson::GetParseError_En(parseResult.Code())); + intermediateStatus = StatusCode::JSON_INVALID; + loud.log(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + continue; + } + intermediateStatus = StatusCode::OK; + break; + } while (++counter < MAX_CONFIG_JSON_READ_RETRY_COUNT && !intermediateStatus.ok()); + if (!intermediateStatus.ok()) { + lastLoadConfigStatus = intermediateStatus; + return lastLoadConfigStatus; + } + + if (validateJsonAgainstSchema(configJson, MODELS_CONFIG_SCHEMA) != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is not in valid configuration format"); + lastLoadConfigStatus = StatusCode::JSON_INVALID; + return lastLoadConfigStatus; + } + Status status; + Status firstErrorStatus = StatusCode::OK; + // load the custom loader config, if available + status = loadCustomLoadersConfig(configJson); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + std::vector gatedModelConfigs; + status = loadModelsConfig(configJson, gatedModelConfigs); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + status = loadCustomNodeLibrariesConfig(configJson); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + status = loadPipelinesConfig(configJson); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + status = tryReloadGatedModelConfigs(gatedModelConfigs); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + + lastLoadConfigStatus = firstErrorStatus; + return firstErrorStatus; +} + +void ModelManager::retireModelsRemovedFromConfigFile(const std::set& modelsExistingInConfigFile, const std::set& modelsWithInvalidConfig) { + std::set modelsCurrentlyLoaded; + for (auto& nameModelPair : getModels()) { + modelsCurrentlyLoaded.insert(nameModelPair.first); + } + std::vector modelsToUnloadAllVersions(getModels().size()); + auto it = std::set_difference( + modelsCurrentlyLoaded.begin(), modelsCurrentlyLoaded.end(), + modelsExistingInConfigFile.begin(), modelsExistingInConfigFile.end(), + modelsToUnloadAllVersions.begin()); + modelsToUnloadAllVersions.resize(it - modelsToUnloadAllVersions.begin()); + for (auto& modelName : modelsToUnloadAllVersions) { + if (modelsWithInvalidConfig.find(modelName) == modelsWithInvalidConfig.end()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Retiring all versions of model: {}", modelName); + try { + models.at(modelName)->retireAllVersions(); + } catch (const std::out_of_range& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unknown error occurred when tried to retire all versions of model: {}", modelName); + } + } else { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cleaning up all versions of model: {}", modelName); + try { + models.at(modelName)->cleanupAllVersions(); + } catch (const std::out_of_range& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unknown error occurred when tried to clean up all versions of model: {}", modelName); + } + } + } +} + +Status ModelManager::updateConfigurationWithoutConfigFile() { + std::lock_guard loadingLock(configMtx); + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Checking if something changed with model versions"); + bool reloadNeeded = false; + Status firstErrorStatus = StatusCode::OK; + Status status; + for (auto& [name, config] : servedModelConfigs) { + status = reloadModelWithVersions(config); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } else if (status == StatusCode::OK_RELOADED) { + reloadNeeded = true; + } + } + status = pipelineFactory.revalidatePipelines(*this); + if (!status.ok()) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + + if (!firstErrorStatus.ok()) { + return firstErrorStatus; + } + + if (reloadNeeded) { + return StatusCode::OK_RELOADED; + } else { + return StatusCode::OK_NOT_RELOADED; + } +} + +std::string ModelManager::getConfigFileMD5() { + std::ifstream ifs; + ifs.open(configFilename); + std::stringstream strStream; + strStream << ifs.rdbuf(); + std::string str = strStream.str(); + ifs.close(); + + unsigned char result[MD5_DIGEST_LENGTH]; + MD5((unsigned char*)str.c_str(), str.size(), result); + std::string md5sum(reinterpret_cast(result), MD5_DIGEST_LENGTH); + return (md5sum); +} + +Status ModelManager::configFileReloadNeeded(bool& isNeeded) { + std::lock_guard loadingLock(configMtx); + + if (!std::ifstream(configFilename)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Config file not found or cannot open."); + isNeeded = false; + return StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED; + } + + std::string newmd5 = getConfigFileMD5(); + bool configFileModified = false; + if (lastConfigFileMD5 != newmd5) { + configFileModified = true; + } + + if (configFilename == "" || !configFileModified) { + isNeeded = false; + return lastLoadConfigStatus; + } else { + isNeeded = true; + } + + return StatusCode::OK; +} + +void ModelManager::watcher(std::future exitSignal) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Started model manager thread"); + + while (exitSignal.wait_for(std::chrono::seconds(watcherIntervalSec)) == std::future_status::timeout) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle begin"); + std::lock_guard loadingLock(configMtx); + bool isNeeded; + configFileReloadNeeded(isNeeded); + if (isNeeded) { + loadConfig(configFilename); + } + updateConfigurationWithoutConfigFile(); + + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle end"); + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped model manager thread"); +} + +void ModelManager::cleanerRoutine(uint32_t resourcesCleanupIntervalSec, uint32_t sequenceCleanerIntervalMinutes, std::future cleanerExitSignal) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Started cleaner thread"); + + uint32_t resourcesCleanupIntervalMiliseconds = resourcesCleanupIntervalSec * 1000; + uint32_t sequenceCleanerIntervalMiliseconds = sequenceCleanerIntervalMinutes * 60 * 1000; + + FunctorResourcesCleaner functorResourcesCleaner{*this}; + FunctorSequenceCleaner functorSequenceCleaner{globalSequencesViewer}; + + ovms::cleanerRoutine(resourcesCleanupIntervalMiliseconds, functorResourcesCleaner, sequenceCleanerIntervalMiliseconds, functorSequenceCleaner, cleanerExitSignal); + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped cleaner thread"); +} + +void cleanerRoutine(uint32_t resourcesCleanupInterval, FunctorResourcesCleaner& functorResourcesCleaner, uint32_t sequenceCleanerInterval, FunctorSequenceCleaner& functorSequenceCleaner, std::future& cleanerExitSignal) { + uint32_t currentResourcesWaitTime = resourcesCleanupInterval; + uint32_t currentSequenceWaitTime = sequenceCleanerInterval; + bool shouldCheckForSequenceCleanup = sequenceCleanerInterval != 0; + uint32_t currentWaitTime = (!shouldCheckForSequenceCleanup || currentResourcesWaitTime < currentSequenceWaitTime) ? currentResourcesWaitTime : currentSequenceWaitTime; + + while (cleanerExitSignal.wait_for(std::chrono::milliseconds(currentWaitTime)) == std::future_status::timeout) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Cleanup check cycle begin"); + + currentResourcesWaitTime = (currentResourcesWaitTime - currentWaitTime) == 0 ? resourcesCleanupInterval : currentResourcesWaitTime - currentWaitTime; + if (shouldCheckForSequenceCleanup) + currentSequenceWaitTime = (currentSequenceWaitTime - currentWaitTime) == 0 ? sequenceCleanerInterval : currentSequenceWaitTime - currentWaitTime; + currentWaitTime = (!shouldCheckForSequenceCleanup || currentResourcesWaitTime < currentSequenceWaitTime) ? currentResourcesWaitTime : currentSequenceWaitTime; + + if (currentResourcesWaitTime == resourcesCleanupInterval) + functorResourcesCleaner.cleanup(); + if (currentSequenceWaitTime == sequenceCleanerInterval && shouldCheckForSequenceCleanup) + functorSequenceCleaner.cleanup(); + + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Cleanup check cycle end"); + } +} + +void ModelManager::cleanupResources() { + std::vector> toBeRemoved; + std::unique_lock resourcesLock(resourcesMtx); + // Move all resources that should be destroyed to temporary container + std::copy_if(resources.begin(), + resources.end(), + std::back_inserter(toBeRemoved), + [](auto& resource) { return resource.use_count() == 1; }); + resources.erase( + std::remove_if( + resources.begin(), + resources.end(), + [toBeRemoved](auto& resource) { return std::find(toBeRemoved.begin(), toBeRemoved.end(), resource) != toBeRemoved.end(); }), + resources.end()); + // Unlock mutex so new resources can be put into container owned by ModelManager + resourcesLock.unlock(); + // Temporary container will fall out of scope and therefore deinitialize should be called on every resource inside of it +} + +void ModelManager::join() { + if (watcherStarted) + exitTrigger.set_value(); + if (cleanerStarted) + cleanerExitTrigger.set_value(); + + if (watcherStarted) { + if (monitor.joinable()) { + monitor.join(); + watcherStarted = false; + SPDLOG_INFO("Shutdown model manager"); + } + } + + if (cleanerStarted) { + if (cleanerThread.joinable()) { + cleanerThread.join(); + cleanerStarted = false; + SPDLOG_INFO("Shutdown cleaner thread"); + } + } +} + +void ModelManager::getVersionsToChange( + const ModelConfig& newModelConfig, + const std::map>& modelVersionsInstances, + model_versions_t requestedVersions, + std::shared_ptr& versionsToStartIn, + std::shared_ptr& versionsToReloadIn, + std::shared_ptr& versionsToRetireIn) { + std::sort(requestedVersions.begin(), requestedVersions.end()); + model_versions_t registeredModelVersions; + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Currently registered model: {} versions count: {}", newModelConfig.getName(), modelVersionsInstances.size()); + for (const auto& [version, versionInstance] : modelVersionsInstances) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "model: {} version: {} state: {}", newModelConfig.getName(), version, ovms::ModelVersionStateToString(versionInstance->getStatus().getState())); + registeredModelVersions.push_back(version); + } + + if (newModelConfig.isCustomLoaderRequiredToLoadModel()) { + custom_loader_options_config_t customLoaderOptionsConfig = newModelConfig.getCustomLoaderOptionsConfigMap(); + const std::string loaderName = customLoaderOptionsConfig["loader_name"]; + + auto& customloaders = ovms::CustomLoaders::instance(); + auto loaderPtr = customloaders.find(loaderName); + if (loaderPtr != nullptr) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom Loader to be used : {}", loaderName); + + // check existing version for blacklist + for (const auto& [version, versionInstance] : modelVersionsInstances) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "The model {} checking for blacklist", versionInstance->getName()); + CustomLoaderStatus bres = loaderPtr->getModelBlacklistStatus(versionInstance->getName(), version); + if (bres != CustomLoaderStatus::OK) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "The model {} is blacklisted", versionInstance->getName()); + requestedVersions.erase(std::remove(requestedVersions.begin(), requestedVersions.end(), version), requestedVersions.end()); + } + } + } + } + + model_versions_t alreadyRegisteredVersionsWhichAreRequested(requestedVersions.size()); + model_versions_t::iterator it = std::set_intersection( + requestedVersions.begin(), requestedVersions.end(), + registeredModelVersions.begin(), registeredModelVersions.end(), + alreadyRegisteredVersionsWhichAreRequested.begin()); + alreadyRegisteredVersionsWhichAreRequested.resize(it - alreadyRegisteredVersionsWhichAreRequested.begin()); + + std::shared_ptr versionsToReload = std::make_shared(); + for (const auto& version : alreadyRegisteredVersionsWhichAreRequested) { + try { + if (modelVersionsInstances.at(version)->getStatus().willEndUnloaded() || + modelVersionsInstances.at(version)->getStatus().isFailedLoading() || + modelVersionsInstances.at(version)->getModelConfig().isReloadRequired(newModelConfig)) { + if (modelVersionsInstances.at(version)->getModelConfig().isCustomLoaderConfigChanged(newModelConfig)) { + modelVersionsInstances.at(version)->setCustomLoaderConfigChangeFlag(); + } + versionsToReload->push_back(version); + } + } catch (std::out_of_range& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Data race occured during versions update. Could not found version. Details: {}", e.what()); + } + } + + std::shared_ptr versionsToRetire = std::make_shared(registeredModelVersions.size()); + it = std::set_difference( + registeredModelVersions.begin(), registeredModelVersions.end(), + requestedVersions.begin(), requestedVersions.end(), + versionsToRetire->begin()); + versionsToRetire->resize(it - versionsToRetire->begin()); + try { + it = std::remove_if(versionsToRetire->begin(), + versionsToRetire->end(), + [&modelVersionsInstances](model_version_t version) { + return modelVersionsInstances.at(version)->getStatus().willEndUnloaded(); + }); + } catch (std::out_of_range& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Data race occured during versions update. Could not found version. Details: {}", e.what()); + } + versionsToRetire->resize(it - versionsToRetire->begin()); + + std::shared_ptr versionsToStart = std::make_shared(requestedVersions.size()); + it = std::set_difference( + requestedVersions.begin(), requestedVersions.end(), + registeredModelVersions.begin(), registeredModelVersions.end(), + versionsToStart->begin()); + versionsToStart->resize(it - versionsToStart->begin()); + + versionsToStartIn = std::move(versionsToStart); + versionsToReloadIn = std::move(versionsToReload); + versionsToRetireIn = std::move(versionsToRetire); +} + +Status ModelManager::checkStatefulFlagChange(const std::string& modelName, bool configStatefulFlag) { + std::unique_lock modelsLock(modelsMtx); + auto modelIt = models.find(modelName); + if (models.end() == modelIt) + return StatusCode::OK; // Model has not been loaded yet, so there are no restrictions regarding stateful flag setup + + auto model = models[modelName]; + if (model->isStateful() != configStatefulFlag) + return StatusCode::REQUESTED_MODEL_TYPE_CHANGE; + return StatusCode::OK; +} + +std::shared_ptr ModelManager::getModelIfExistCreateElse(const std::string& modelName, const bool isStateful) { + std::unique_lock modelsLock(modelsMtx); + auto modelIt = models.find(modelName); + if (models.end() == modelIt) { + models.insert({modelName, modelFactory(modelName, isStateful)}); + } + return models[modelName]; +} + +std::shared_ptr ModelManager::getFilesystem(const std::string& basePath) { + if (basePath.rfind(S3FileSystem::S3_URL_PREFIX, 0) == 0) { + Aws::SDKOptions options; + Aws::InitAPI(options); + return std::make_shared(options, basePath); + } + if (basePath.rfind(GCSFileSystem::GCS_URL_PREFIX, 0) == 0) { + return std::make_shared(); + } + if (basePath.rfind(AzureFileSystem::AZURE_URL_FILE_PREFIX, 0) == 0) { + return std::make_shared(); + } + if (basePath.rfind(AzureFileSystem::AZURE_URL_BLOB_PREFIX, 0) == 0) { + return std::make_shared(); + } + return std::make_shared(); +} + +Status ModelManager::readAvailableVersions(std::shared_ptr& fs, const std::string& base, model_versions_t& versions) { + files_list_t dirs; + + bool is_directory = false; + if (FileSystem::isPathEscaped(base)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", base); + return StatusCode::PATH_INVALID; + } + + auto status = fs->isDirectory(base, &is_directory); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't check directory: {}", base); + return status; + } + if (!is_directory) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Directory does not exist: {}", base); + return StatusCode::PATH_INVALID; + } + + status = fs->getDirectorySubdirs(base, &dirs); + + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't list directories in path: {}", base); + return status; + } + + for (const auto& entry : dirs) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Detected version folder: {}", entry); + try { + ovms::model_version_t version = std::stoll(entry); + if (version <= 0) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Expected version directory name to be a number greater than 0. Got: {}", version); + continue; + } + versions.push_back(version); + } catch (const std::invalid_argument& e) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Expected version directory name to be in number format. Got: {}", entry); + } catch (const std::out_of_range& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Directory name is out of range for supported version format. Got: {}", entry); + } + } + + if (0 == versions.size()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "No version found for model in path: {}", base); + } + + return StatusCode::OK; +} + +Status ModelManager::addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr versionsFailed) { + Status status = StatusCode::OK; + try { + status = model->addVersions(versionsToStart, config, fs, *ieCore, versionsFailed); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while loading model: {} versions; error: {}", + config.getName(), + status.string()); + return status; + } + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Exception occurred while loading model: {};", e.what()); + } + return status; +} + +Status ModelManager::reloadModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToReload, std::shared_ptr versionsFailed) { + Status status = StatusCode::OK; + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reloading model versions"); + try { + auto status = model->reloadVersions(versionsToReload, config, fs, *ieCore, versionsFailed); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while reloading model: {}; versions; error: {}", + config.getName(), + status.string()); + + return status; + } + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Exception occurred while reloading model: {};", e.what()); + } + + return status; +} + +Status ModelManager::reloadModelWithVersions(ModelConfig& config) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Started applying config changes to model: {}", config.getName()); + + if (config.isStateful() && config.isDynamicParameterEnabled()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested setting dynamic parameters for stateful model {}. Dynamic shape and dynamic batch size not supported for stateful models.", config.getName()); + return StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL; + } + if (!config.isStateful()) { + if (config.isLowLatencyTransformationUsed()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested low latency transformation parameter for non stateful model {}.", config.getName()); + return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; + } + } + + auto status = checkStatefulFlagChange(config.getName(), config.isStateful()); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested model type change on model: {}. Stateful flag cannot be changed after model is loaded", config.getName()); + return StatusCode::REQUESTED_MODEL_TYPE_CHANGE; + } + + auto model = getModelIfExistCreateElse(config.getName(), config.isStateful()); + if (model->isAnyVersionSubscribed()) { + if (config.isDynamicParameterEnabled()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested setting dynamic parameters for model {} but it is used in pipeline. Cannot reload model configuration.", config.getName()); + return StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL; + } + if (config.isStateful()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested using stateful model {} but it is used in pipeline. Stateful model cannot be subscribed to pipeline.", config.getName()); + return StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL; + } + } + + auto fs = ModelManager::getFilesystem(config.getBasePath()); + std::vector availableVersions; + Status blocking_status = readAvailableVersions(fs, config.getBasePath(), availableVersions); + if (!blocking_status.ok()) { + return blocking_status; + } + auto requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); + std::shared_ptr versionsToStart; + std::shared_ptr versionsToReload; + std::shared_ptr versionsToRetire; + std::shared_ptr versionsFailed = std::make_shared(); + // first reset custom loader name to empty string so that any changes to name can be captured + model->resetCustomLoaderName(); + + if (config.isCustomLoaderRequiredToLoadModel()) { + custom_loader_options_config_t customLoaderOptionsConfig = config.getCustomLoaderOptionsConfigMap(); + const std::string loaderName = customLoaderOptionsConfig["loader_name"]; + + auto& customloaders = ovms::CustomLoaders::instance(); + auto loaderPtr = customloaders.find(loaderName); + if (loaderPtr != nullptr) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom Loader to be used : {}", loaderName); + model->setCustomLoaderName(loaderName); + } else { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Specified custom loader {} not found. In case any models are loaded, will be unloading them", loaderName); + model->retireAllVersions(); + return StatusCode::OK; + } + } + getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); + bool reloadNeeded = false; + if (versionsToStart->size() > 0 || versionsToReload->size() > 0 || versionsToRetire->size() > 0) { + reloadNeeded = true; + } + std::set allFailedVersions; + while (versionsToStart->size() > 0) { + blocking_status = addModelVersions(model, fs, config, versionsToStart, versionsFailed); + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Adding new versions. Status: {};", blocking_status.string()); + if (!blocking_status.ok()) { + for (const auto version : *versionsFailed) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Removing available version {} due to load failure; ", version); + if (std::binary_search(availableVersions.begin(), availableVersions.end(), version)) { + availableVersions.erase(std::remove(availableVersions.begin(), availableVersions.end(), version), availableVersions.end()); + } + allFailedVersions.insert(version); + } + requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); + getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); + } else { + break; + } + } + + if (versionsToReload->size() > 0) { + auto status = reloadModelVersions(model, fs, config, versionsToReload, versionsFailed); + if (!status.ok()) { + blocking_status = status; + } + } + + for (const auto version : *versionsFailed) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Removing available version {} due to load failure.", version); + if (std::binary_search(availableVersions.begin(), availableVersions.end(), version)) { + availableVersions.erase(std::remove(availableVersions.begin(), availableVersions.end(), version), availableVersions.end()); + } + allFailedVersions.insert(version); + } + // refresh versions to retire based on failed reloads + requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); + getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); + std::shared_ptr versionsToCleanup = std::make_shared(); + std::copy_if(versionsToRetire->begin(), versionsToRetire->end(), std::back_inserter(*versionsToCleanup), [&](auto& version) { return allFailedVersions.find(version) != allFailedVersions.end(); }); + versionsToRetire->erase(std::remove_if(versionsToRetire->begin(), versionsToRetire->end(), [&](auto& version) { return allFailedVersions.find(version) != allFailedVersions.end(); }), versionsToRetire->end()); + if (versionsToRetire->size() > 0) { + auto status = model->retireVersions(versionsToRetire); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while unloading model: {}; versions; error: {}", + config.getName(), + status.string()); + return status; + } + } + if (versionsToCleanup->size() > 0) { + auto status = model->cleanupFailedLoad(versionsToCleanup); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while cleaning up model that failed to load: {}; versions; error: {}", + config.getName(), + status.string()); + return status; + } + } + + if (blocking_status.ok() && reloadNeeded) { + return StatusCode::OK_RELOADED; + } + + return blocking_status; +} + +const std::shared_ptr ModelManager::findModelByName(const std::string& name) const { + std::shared_lock lock(modelsMtx); + auto it = models.find(name); + return it != models.end() ? it->second : nullptr; +} + +Status ModelManager::getModelInstance(const std::string& modelName, + ovms::model_version_t modelVersionId, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr) { + SPDLOG_DEBUG("Requesting model: {}; version: {}.", modelName, modelVersionId); + + auto model = findModelByName(modelName); + if (model == nullptr) { + return StatusCode::MODEL_NAME_MISSING; + } + if (modelVersionId != 0) { + modelInstance = model->getModelInstanceByVersion(modelVersionId); + if (modelInstance == nullptr) { + return StatusCode::MODEL_VERSION_MISSING; + } + } else { + modelInstance = model->getDefaultModelInstance(); + if (modelInstance == nullptr) { + return StatusCode::MODEL_VERSION_MISSING; + } + } + + return modelInstance->waitForLoaded(waitForModelLoadedTimeoutMs, modelInstanceUnloadGuardPtr); +} + +const CustomNodeLibraryManager& ModelManager::getCustomNodeLibraryManager() const { + return *customNodeLibraryManager; +} + +} // namespace ovms diff --git a/src/ovms_lib/modelmanager.hpp b/src/ovms_lib/modelmanager.hpp new file mode 100644 index 0000000000..437a2025d0 --- /dev/null +++ b/src/ovms_lib/modelmanager.hpp @@ -0,0 +1,427 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "custom_node_library_internal_manager_wrapper.hpp" +#include "customloaders.hpp" +#include "filesystem.hpp" +#include "global_sequences_viewer.hpp" +#include "model.hpp" +#include "pipeline.hpp" +#include "pipeline_factory.hpp" + +namespace ovms { + +const uint32_t DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS = 10000; +const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; + +class Config; +class IVersionReader; +class CustomNodeLibraryManager; +struct FunctorSequenceCleaner; +struct FunctorResourcesCleaner; +/** + * @brief Model manager is managing the list of model topologies enabled for serving and their versions. + */ +class ModelManager { +public: + /** + * @brief A default constructor is private + */ + ModelManager(const std::string& modelCacheDirectory = ""); + +protected: + void logPluginConfiguration(); + + Status checkStatefulFlagChange(const std::string& modelName, bool configStatefulFlag); + + std::shared_ptr getModelIfExistCreateElse(const std::string& name, const bool isStateful); + + /** + * @brief A collection of models + * + */ + std::map> models; + std::unique_ptr ieCore; + + PipelineFactory pipelineFactory; + + std::unique_ptr customNodeLibraryManager; + + std::vector> resources = {}; + + GlobalSequencesViewer globalSequencesViewer; + uint32_t waitForModelLoadedTimeoutMs; + bool watcherStarted = false; + bool cleanerStarted = false; + +private: + /** + * @brief Private copying constructor + */ + ModelManager(const ModelManager&) = delete; + + Status lastLoadConfigStatus = StatusCode::OK; + + std::string getConfigFileMD5(); + Status cleanupModelTmpFiles(ModelConfig& config); + Status reloadModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToReload, std::shared_ptr versionsFailed); + Status addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr versionsFailed); + Status loadModelsConfig(rapidjson::Document& configJson, std::vector& gatedModelConfigs); + Status tryReloadGatedModelConfigs(std::vector& gatedModelConfigs); + Status loadCustomNodeLibrariesConfig(rapidjson::Document& configJson); + Status loadPipelinesConfig(rapidjson::Document& configJson); + Status loadCustomLoadersConfig(rapidjson::Document& configJson); + + /** + * @brief creates customloader from the loader configuration + */ + Status createCustomLoader(CustomLoaderConfig& loaderConfig); + + /** + * @brief Watcher thread for monitor changes in config + */ + void watcher(std::future exitSignal); + + /** + * @brief Cleaner thread for sequence and resources cleanup + */ + void cleanerRoutine(uint32_t resourcesCleanupIntervalSec, uint32_t sequenceCleanerIntervalMinutes, std::future cleanerExitSignal); + + /** + * @brief Mutex for blocking concurrent add & remove of resources + */ + std::shared_mutex resourcesMtx; + + /** + * @brief A JSON configuration filename + */ + std::string configFilename; + + /** + * @brief A thread object used for monitoring changes in config + */ + std::thread monitor; + + /** + * @brief A thread object used for cleanup + */ + std::thread cleanerThread; + + /** + * @brief An exit trigger to notify watcher thread to exit + */ + std::promise exitTrigger; + + /** + * @brief An exit trigger to notify cleaner thread to exit + */ + std::promise cleanerExitTrigger; + + /** + * @brief A current configurations of models + * + */ + std::unordered_map servedModelConfigs; + + /** + * @brief Retires models non existing in config file + * + * @param modelsExistingInConfigFile + */ + void retireModelsRemovedFromConfigFile(const std::set& modelsExistingInConfigFile, const std::set& modelsWithInvalidConfig); + + /** + * @brief Mutex for protecting concurrent reloading config + */ + mutable std::recursive_mutex configMtx; + + /** + * Time interval between each config file check + */ + uint watcherIntervalSec = 1; + + /** + * Time interval between two consecutive sequence cleanup scans (in minutes) + */ + uint32_t sequenceCleaupIntervalMinutes = 5; + + /** + * Time interval between two consecutive resources cleanup scans (in seconds) + */ + uint32_t resourcesCleanupIntervalSec = 1; + + /** + * @brief last md5sum of configfile + */ + std::string lastConfigFileMD5; + + /** + * @brief Directory for OpenVINO to store cache files. + */ + std::string modelCacheDirectory; + +public: + /** + * @brief Mutex for blocking concurrent add & find of model + */ + mutable std::shared_mutex modelsMtx; + + /** + * @brief Gets the instance of ModelManager + */ + static ModelManager& getInstance() { + static ModelManager instance; + return instance; + } + + /** + * @brief Gets the watcher interval timestep in seconds + */ + uint getWatcherIntervalSec() { + return watcherIntervalSec; + } + + /** + * @brief Gets the cleaner resources interval timestep in seconds + */ + uint32_t getResourcesCleanupIntervalSec() { + return resourcesCleanupIntervalSec; + } + + /** + * @brief Adds new resource to watch by the cleaner thread + */ + void addResourceToCleaner(std::shared_ptr resource) { + std::unique_lock resourcesLock(resourcesMtx); + resources.emplace(resources.end(), std::move(resource)); + } + + /** + * @brief Destroy the Model Manager object + * + */ + virtual ~ModelManager(); + + /** + * @brief Gets config filename + * + * @return config filename + */ + const std::string& getConfigFilename() { + return configFilename; + } + + /** + * @brief Gets models collection + * + * @return models collection + */ + const std::map>& getModels() { + return models; + } + + /** + * @brief Starts monitoring cleanup as new thread + */ + void startCleaner(); + + const PipelineFactory& getPipelineFactory() const { + return pipelineFactory; + } + + const CustomNodeLibraryManager& getCustomNodeLibraryManager() const; + + /** + * @brief Finds model with specific name + * + * @param name of the model to search for + * + * @return pointer to Model or nullptr if not found + */ + const std::shared_ptr findModelByName(const std::string& name) const; + + Status getModelInstance(const std::string& modelName, + ovms::model_version_t modelVersionId, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr); + + const bool modelExists(const std::string& name) const { + if (findModelByName(name) == nullptr) + return false; + else + return true; + } + + /** + * @brief Finds model instance with specific name and version, returns default if version not specified + * + * @param name of the model to search for + * @param version of the model to search for or 0 if default + * + * @return pointer to ModelInstance or nullptr if not found + */ + const std::shared_ptr findModelInstance(const std::string& name, model_version_t version = 0) const { + auto model = findModelByName(name); + if (!model) { + return nullptr; + } + + if (version == 0) { + return model->getDefaultModelInstance(); + } else { + return model->getModelInstanceByVersion(version); + } + } + + template + Status createPipeline(std::unique_ptr& pipeline, + const std::string name, + const RequestType* request, + ResponseType* response) { + return pipelineFactory.create(pipeline, name, request, response, *this); + } + + const bool pipelineDefinitionExists(const std::string& name) const { + return pipelineFactory.definitionExists(name); + } + + /** + * @brief Starts model manager using provided config file + * + * @param filename + * @return status + */ + Status startFromFile(const std::string& jsonFilename); + + /** + * @brief Starts model manager using command line arguments + * + * @return Status + */ + Status startFromConfig(); + + /** + * @brief Reload model versions located in base path + * + * @param ModelConfig config + * + * @return status + */ + Status reloadModelWithVersions(ModelConfig& config); + + /** + * @brief Starts model manager using ovms::Config + * + * @return status + */ + Status start(const Config& config); + + /** + * @brief Starts monitoring as new thread + * + */ + void startWatcher(); + + /** + * @brief Gracefully finish the thread + */ + void join(); + + /** + * @brief Factory for creating a model + * + * @return std::shared_ptr + */ + virtual std::shared_ptr modelFactory(const std::string& name, const bool isStateful) { + return std::make_shared(name, isStateful, &this->globalSequencesViewer); + } + + /** + * @brief Reads available versions from given filesystem + * + * @param fs + * @param base + * @param versions + * @return Status + */ + virtual Status readAvailableVersions( + std::shared_ptr& fs, + const std::string& base, + model_versions_t& versions); + + /** + * @brief Checks what versions needs to be started, reloaded, retired based on currently served ones + * + * @param modelVersionsInstances map with currently served versions + * @param requestedVersions container with requested versions + * @param versionsToRetireIn cointainer for versions to retire + * @param versionsToReloadIn cointainer for versions to reload + * @param versionsToStartIn cointainer for versions to start + */ + static void getVersionsToChange( + const ModelConfig& newModelConfig, + const std::map>& modelVersionsInstances, + std::vector requestedVersions, + std::shared_ptr& versionsToRetireIn, + std::shared_ptr& versionsToReloadIn, + std::shared_ptr& versionsToStartIn); + + static std::shared_ptr getFilesystem(const std::string& basePath); + + /** + * @brief Check if configuration file reload is needed. + */ + Status configFileReloadNeeded(bool& isNeeded); + + /** + * @brief Reads models from configuration file + * + * @param jsonFilename configuration file + * @return Status + */ + Status loadConfig(const std::string& jsonFilename); + + /** + * @brief Updates OVMS configuration with cached configuration file. Will check for newly added model versions + */ + Status updateConfigurationWithoutConfigFile(); + + /** + * @brief Cleaner thread procedure to cleanup resources that are not used + */ + void cleanupResources(); +}; + +void cleanerRoutine(uint32_t resourcesCleanupInterval, FunctorResourcesCleaner& functorResourcesCleaner, uint32_t sequenceCleanerInterval, FunctorSequenceCleaner& functorSequenceCleaner, std::future& cleanerExitSignal); + +} // namespace ovms diff --git a/src/ovms_lib/modelversion.hpp b/src/ovms_lib/modelversion.hpp new file mode 100644 index 0000000000..fcabe46aa4 --- /dev/null +++ b/src/ovms_lib/modelversion.hpp @@ -0,0 +1,24 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { + +using model_version_t = int64_t; +using model_versions_t = std::vector; +} // namespace ovms diff --git a/src/ovms_lib/modelversionstatus.cpp b/src/ovms_lib/modelversionstatus.cpp new file mode 100644 index 0000000000..6936dc67b3 --- /dev/null +++ b/src/ovms_lib/modelversionstatus.cpp @@ -0,0 +1,122 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "modelversionstatus.hpp" + +#include +#include +#include + +#include + +// note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. + +namespace ovms { + +// those values have to match tensorflow-serving state: +static const std::unordered_map versionStatesStrings = { + {ModelVersionState::START, "START"}, + {ModelVersionState::LOADING, "LOADING"}, + {ModelVersionState::AVAILABLE, "AVAILABLE"}, + {ModelVersionState::UNLOADING, "UNLOADING"}, + {ModelVersionState::END, "END"}}; +const std::string& ModelVersionStateToString(ModelVersionState state) { + return versionStatesStrings.at(state); +} + +static const std::unordered_map versionsStatesErrors = { + {ModelVersionStatusErrorCode::OK, "OK"}, + {ModelVersionStatusErrorCode::UNKNOWN, "UNKNOWN"}, + {ModelVersionStatusErrorCode::FAILED_PRECONDITION, "FAILED_PRECONDITION"}}; + +const std::string& ModelVersionStatusErrorCodeToString(ModelVersionStatusErrorCode code) { + return versionsStatesErrors.at(code); +} + +ModelVersionStatus::ModelVersionStatus(const std::string& model_name, model_version_t version, ModelVersionState state) : + modelName(model_name), + version(version), + state(state), + errorCode(ModelVersionStatusErrorCode::OK) { + logStatus(); +} + +ModelVersionState ModelVersionStatus::getState() const { + return this->state; +} + +const std::string& ModelVersionStatus::getStateString() const { + return ModelVersionStateToString(this->state); +} + +ModelVersionStatusErrorCode ModelVersionStatus::getErrorCode() const { + return this->errorCode; +} + +const std::string& ModelVersionStatus::getErrorMsg() const { + return ModelVersionStatusErrorCodeToString(this->errorCode); +} + +bool ModelVersionStatus::willEndUnloaded() const { + return ovms::ModelVersionState::UNLOADING <= this->state; +} + +bool ModelVersionStatus::isFailedLoading() const { + return this->state == ovms::ModelVersionState::LOADING && this->errorCode == ovms::ModelVersionStatusErrorCode::UNKNOWN; +} + +void ModelVersionStatus::setLoading(ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + state = ModelVersionState::LOADING; + errorCode = error_code; + logStatus(); +} + +void ModelVersionStatus::setAvailable(ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + state = ModelVersionState::AVAILABLE; + errorCode = error_code; + logStatus(); +} + +void ModelVersionStatus::setUnloading(ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + state = ModelVersionState::UNLOADING; + errorCode = error_code; + logStatus(); +} + +void ModelVersionStatus::setEnd(ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + state = ModelVersionState::END; + errorCode = error_code; + logStatus(); +} + +void ModelVersionStatus::logStatus() { + SPDLOG_INFO("STATUS CHANGE: Version {} of model {} status change. New status: ( \"state\": \"{}\", \"error_code\": \"{}\" )", + this->version, + this->modelName, + ModelVersionStateToString(state), + ModelVersionStatusErrorCodeToString(errorCode)); +} + +void ModelVersionStatus::setState(ModelVersionState state, ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + this->state = state; + errorCode = error_code; + logStatus(); +} +} // namespace ovms diff --git a/src/ovms_lib/modelversionstatus.hpp b/src/ovms_lib/modelversionstatus.hpp new file mode 100644 index 0000000000..69456ef3b3 --- /dev/null +++ b/src/ovms_lib/modelversionstatus.hpp @@ -0,0 +1,105 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "modelconfig.hpp" + +// note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. + +namespace ovms { + +// those values have to match tensorflow-serving state: +enum class ModelVersionState : int { + START = 10, + LOADING = 20, + AVAILABLE = 30, + UNLOADING = 40, + END = 50 +}; + +const std::string& ModelVersionStateToString(ModelVersionState state); + +enum class ModelVersionStatusErrorCode : int { + OK = 0, + // CANCELLED = 1, + UNKNOWN = 2, + // INVALID_ARGUMENT = 3, + // DEADLINE_EXCEEDED = 4, + // NOT_FOUND = 5, + // ALREADY_EXISTS = 6, + // PERMISSION_DENIED = 7, + // UNAUTHENTICATED = 16, + // RESOURCE_EXHAUSTED = 8, + FAILED_PRECONDITION = 9, + // ABORTED = 10, + // OUT_OF_RANGE = 11, + // UNIMPLEMENTED = 12, + // INTERNAL = 13, + // UNAVAILABLE = 14, + // DATA_LOSS = 15, + // DO_NOT_USE_RESERVED_FOR_FUTURE_EXPANSION_USE_DEFAULT_IN_SWITCH_INSTEAD_ + // = 20 +}; + +const std::string& ModelVersionStatusErrorCodeToString(ModelVersionStatusErrorCode code); + +class ModelVersionStatus { + std::string modelName; + model_version_t version; + ModelVersionState state; + ModelVersionStatusErrorCode errorCode; + +public: + ModelVersionStatus() = delete; + + ModelVersionStatus(const std::string& model_name, model_version_t version, ModelVersionState state = ModelVersionState::START); + + ModelVersionState getState() const; + + const std::string& getStateString() const; + + ModelVersionStatusErrorCode getErrorCode() const; + + const std::string& getErrorMsg() const; + + /** + * @brief Check if current state is state that is either transforming to END or already in that state. + * + * @return + */ + bool willEndUnloaded() const; + + bool isFailedLoading() const; + void setLoading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + + void setAvailable(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + + void setUnloading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + + void setEnd(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + void setState(ModelVersionState state, ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + +private: + void logStatus(); +}; + +} // namespace ovms diff --git a/src/ovms_lib/my_cc_combine.bzl b/src/ovms_lib/my_cc_combine.bzl new file mode 100644 index 0000000000..27113911f2 --- /dev/null +++ b/src/ovms_lib/my_cc_combine.bzl @@ -0,0 +1,70 @@ +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +def _combine_impl(ctx): + cc_toolchain = find_cpp_toolchain(ctx) + + target_list = [] + for dep_target in ctx.attr.deps: + # CcInfo, InstrumentedFilesInfo, OutputGroupInfo + cc_info_linker_inputs = dep_target[CcInfo].linking_context.linker_inputs + + target_dirname_list = [] + for linker_in in cc_info_linker_inputs.to_list(): + for linker_in_lib in linker_in.libraries: + if linker_in_lib.pic_static_library != None: + target_list += [linker_in_lib.pic_static_library] + if linker_in_lib.static_library != None: + target_list += [linker_in_lib.static_library] + + output = ctx.outputs.output + if ctx.attr.genstatic: + cp_command = "" + processed_list = [] + processed_path_list = [] + for dep in target_list: + cp_command += "cp -a "+ dep.path +" "+ output.dirname + "/&&" + processed = ctx.actions.declare_file(dep.basename) + processed_list += [processed] + processed_path_list += [dep.path] + cp_command += "echo'starting to run shell'" + processed_path_list += [output.path] + + ctx.actions.run_shell( + outputs = processed_list, + inputs = target_list, + command = cp_command, + ) + + command = "cd {} && ar -x {} {}".format( + output.dirname, + "&& ar -x ".join([dep.basename for dep in target_list]), + "&& ar -rc libauto.a *.o" + ) + print("command = ", command) + ctx.actions.run_shell( + outputs = [output], + inputs = processed_list, + command = command, + ) + else: + command = "export PATH=$PATH:{} && {} -shared -fPIC -Wl,--whole-archive {} -Wl,--no-whole-archive -Wl,-soname -o {}".format ( + cc_toolchain.ld_executable, + cc_toolchain.compiler_executable, + "".join([dep.path for dep in target_list]), + output.path) + print("command = ", command) + ctx.actions.run_shell( + outputs = [output], + inputs = target_list, + command = command, + ) + +my_cc_combine = rule( + implementation = _combine_impl, + attrs = { + "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), + "genstatic": attr.bool(default = False), + "deps": attr.label_list(), + "output": attr.output() + }, +) diff --git a/src/ovms_lib/node.cpp b/src/ovms_lib/node.cpp new file mode 100644 index 0000000000..dfa6fc150a --- /dev/null +++ b/src/ovms_lib/node.cpp @@ -0,0 +1,272 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "node.hpp" + +#include +#include +#include +#include + +#include "logging.hpp" +#include "nodesession.hpp" +#include "ov_utils.hpp" +#include "profiler.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +const uint64_t DEMULTIPLY_LIMIT = 10'000; + +namespace ovms { + +static std::string demultiplyCountSettingToString(std::optional demultiplyCount) { + if (!demultiplyCount) { + return "NA"; + } + if (demultiplyCount.value() == -1) { + return "dynamic"; + } + return std::to_string(demultiplyCount.value()); +} + +Node::Node(const std::string& nodeName, std::optional demultiplyCount, std::set gatherFromNode) : + nodeName(nodeName), + demultiplexCount(demultiplyCount), + gatherFrom(!gatherFromNode.empty() ? std::optional>(gatherFromNode) : std::nullopt) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will create node: {} with demultiply: {}, gatherFrom: {}.", + getName(), + demultiplyCountSettingToString(demultiplexCount), + std::accumulate(gatherFromNode.begin(), gatherFromNode.end(), std::string("NA"), [](const std::string& lhs, const std::string& rhs) { + if (lhs == "NA") { + return rhs; + } else { + return lhs + ", " + rhs; + } })); +} + +Status Node::fetchResults(session_key_t sessionId, SessionResults& nodeSessionOutputs) { + OVMS_PROFILE_FUNCTION(); + auto it = nodeSessions.find(sessionId); + + auto& nodeSession = it->second; + if (it == nodeSessions.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Could not find session: {} for node: {}", sessionId, getName()); + return StatusCode::UNKNOWN_ERROR; + } + auto status = fetchResults(*nodeSession, nodeSessionOutputs); + if (status.ok() && demultiplexCount) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will demultiply node: {} outputs with demultiplyCount: {}", getName(), demultiplyCountSettingToString(demultiplexCount)); + status = demultiplyOutputs(nodeSessionOutputs); + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will remove node: {} session: {}", getName(), sessionId); + nodeSessions.erase(sessionId); + return status; +} + +void Node::printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs) { + std::stringstream ss; + ss << "Links from:" << sourceNode << " to:" << nodeName << ":\n"; + for (auto& pair : pairs) { + ss << "\t" << nodeName << "[" << pair.second << "]=" << sourceNode << "[" << pair.first << "]\n"; + } + SPDLOG_DEBUG(ss.str()); +} + +Status Node::setInputs(const Node& dependency, SessionResults& sessionResults) { + OVMS_PROFILE_FUNCTION(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} set inputs from node: {}", getName(), dependency.getName()); + for (auto& [sessionKey, metadataInputsPair] : sessionResults) { + auto& [metadata, inputs] = metadataInputsPair; + auto status = this->setInputs(dependency, inputs, metadata); + if (!status.ok()) { + return status; + } + } + return StatusCode::OK; +} + +Status Node::setInputs(const Node& dependency, TensorWithSourceMap& inputs, NodeSessionMetadata& metadata) { + // mapping for dependency - keeps mapping between dependency output name and this node input name + const auto& mapping_for_dependency = this->getMappingByDependency(dependency); + NodeSession* nodeSession = getNodeSession(metadata); + if (!nodeSession) { + return StatusCode::INTERNAL_ERROR; + } + session_id_t shardId; + try { + static const std::set emptySet; + shardId = metadata.getShardId(gatherFrom.value_or(emptySet)); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to get shardId for node: {}", getName()); + return StatusCode::INTERNAL_ERROR; + } + // assign all input tensors from inputs that are required by this node for future inference + for (const auto& pair : mapping_for_dependency) { + const auto& dependency_output_name = pair.first; + const auto& current_node_input_name = pair.second; + + // possibly incorrectly constructed pipeline - required input missing from previous node + auto it = inputs.find(dependency_output_name); + if (it == inputs.end()) { + SPDLOG_LOGGER_WARN(dag_executor_logger, "node: {} error setting required input from node: {} dependency is missing output name: {}", + getName(), + dependency.getName(), + dependency_output_name); + return StatusCode::INVALID_MISSING_INPUT; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} setting required input from node: {}, input name: {}, dependency output name: {}", + getName(), + dependency.getName(), + current_node_input_name, + dependency_output_name); + auto status = nodeSession->setInput(current_node_input_name, it->second, shardId); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "node: {} failed to set input: {}, shard: {}", getName(), current_node_input_name, shardId); + return status; + } + } + return nodeSession->notifyFinishedDependency(); +} + +NodeSession& Node::getNodeSession(const session_key_t& sessionKey) const { + auto it = nodeSessions.find(sessionKey); + if (it == nodeSessions.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to get non-existing node: {} session: {}.", getName(), sessionKey); + throw std::runtime_error("Tried to get non existing session"); + } + return *it->second; +} + +NodeSession* Node::getNodeSession(const NodeSessionMetadata& metadata) { + session_key_t sessionKey; + if (gatherFrom) { + try { + sessionKey = metadata.getSessionKey(gatherFrom.value()); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to create collapsed metadata session key for node: {}, incomming session key: {}", + getName(), metadata.getSessionKey()); + return nullptr; + } + } else { + sessionKey = metadata.getSessionKey(); + } + auto it = nodeSessions.find(sessionKey); + if (it != nodeSessions.end()) { + return it->second.get(); + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will create new session: {} for node: {}", + sessionKey, getName()); + NodeSessionMetadata newSessionMetadata; + CollapseDetails collapsingDetails; + if (gatherFrom) { + try { + std::tie(newSessionMetadata, collapsingDetails) = metadata.getCollapsedSessionMetadata(gatherFrom.value()); + } catch (const std::exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to create collapsed metadata for node: {}", getName()); + return nullptr; + } + } else { + newSessionMetadata = metadata; + } + std::unique_ptr nodeSession = createNodeSession(newSessionMetadata, collapsingDetails); + auto emplacePair = nodeSessions.emplace(sessionKey, std::move(nodeSession)); + return emplacePair.first->second.get(); +} + +std::unique_ptr Node::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { + return std::make_unique(metadata, getName(), previous.size(), collapsingDetails); +} + +std::vector Node::getReadySessions() const { + std::vector readySessions; + for (auto& [sessionKey, nodeSession] : nodeSessions) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Checking readiness of node: {} session: {}", getName(), nodeSession->getSessionKey()); + if (nodeSession->isReady()) { + readySessions.emplace_back(sessionKey); + } + } + return readySessions; +} + +Status Node::demultiplyOutputs(SessionResults& nodeSessionOutputs) { + OVMS_PROFILE_FUNCTION(); + if (!demultiplexCount) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} called demultiplyOutputs but node does not have demultiplexCount set", getName()); + return StatusCode::INTERNAL_ERROR; + } + auto& [metadata, tensorMap] = nodeSessionOutputs.begin()->second; + auto firstTensorShape = tensorMap.begin()->second.getActualTensor().get_shape(); + uint32_t resultsDemultiplyCount = firstTensorShape[0]; + if (firstTensorShape[0] > DEMULTIPLY_LIMIT) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} - too large dim[0] size: {} of tensor: {}. Maximum allowed is: {}", + getName(), firstTensorShape[0], tensorMap.begin()->first, DEMULTIPLY_LIMIT); + return StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will demultiply node: {} outputs to: {} shards", getName(), resultsDemultiplyCount); + std::vector newSessionMetadatas; + try { + newSessionMetadatas = std::move(metadata.generateSubsessions(getName(), resultsDemultiplyCount)); + } catch (std::exception& e) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} failed to generate subsessions due to error: {}", getName(), e.what()); + return StatusCode::INTERNAL_ERROR; + } + for (auto& [tensorName, tensorWithSource] : tensorMap) { + auto& tensor = tensorWithSource.getActualTensor(); + OVMS_PROFILE_SCOPE("Demultiply Tensor"); + auto newDims = tensor.get_shape(); + if (newDims.size() < 3) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Wrong number of dimensions: {} to demultiply. Must be at least 3", newDims.size()); + return StatusCode::PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY; + } + if ((demultiplexCount.value() != -1) && + (newDims[0] != demultiplexCount.value())) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Wrong dim[0] size: {} of tensor: {} expected: {} to demultiply", + newDims[0], tensorName, demultiplexCount.value()); + return StatusCode::PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY; + } + if (resultsDemultiplyCount == 0) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} has no results. Dynamic demultiplexer with demultiply == 0 is not supported yet.", this->getName()); + nodeSessionOutputs.erase(metadata.getSessionKey()); + return StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS; + } + + newDims.erase(newDims.begin()); + const auto step = tensor.get_byte_size() / resultsDemultiplyCount; + for (size_t i = 0; i < newSessionMetadatas.size(); ++i) { + OVMS_PROFILE_SCOPE("Create Shard"); + ov::Tensor dividedTensor; + this->createShardedTensor(dividedTensor, ovElementTypeToOvmsPrecision(tensor.get_element_type()), newDims, tensor, i, step, metadata, tensorName); + std::stringstream ss; + ss << "Node: " << getName() << " input demultiplied: " << tensorName + << "; Actual: " << TensorInfo::shapeToString(dividedTensor.get_shape()); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "{}", ss.str()); + auto sessionKey = newSessionMetadatas[i].getSessionKey(); + auto it = nodeSessionOutputs.find(sessionKey); + if (it == nodeSessionOutputs.end()) { + nodeSessionOutputs.emplace(sessionKey, SessionResult{newSessionMetadatas[i], TensorWithSourceMap{{tensorName, TensorWithSource{dividedTensor, tensor}}}}); + } else { + it->second.second.emplace(tensorName, TensorWithSource{dividedTensor, tensor}); + } + } + } + nodeSessionOutputs.erase(metadata.getSessionKey()); + return StatusCode::OK; +} + +Status Node::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) { + dividedTensor = createSharedTensor(tensor.get_element_type(), shape, (char*)(tensor.data()) + i * step); + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/node.hpp b/src/ovms_lib/node.hpp new file mode 100644 index 0000000000..3a6cdc724b --- /dev/null +++ b/src/ovms_lib/node.hpp @@ -0,0 +1,102 @@ +//***************************************************************************** +// Copyright 2020,2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "aliases.hpp" +#include "nodesession.hpp" +#include "nodesessionresult.hpp" +#include "pipelineeventqueue.hpp" +#include "precision.hpp" +#include "shape.hpp" +#include "status.hpp" +#include "tensormap.hpp" + +namespace ovms { + +using TensorNames = std::vector; +using session_key_t = std::string; + +class Node { +protected: + std::string nodeName; + + std::vector> previous; + std::vector> next; + + // Tensors ready and waiting for execution + std::unordered_map> nodeSessions; + + // Input/Output name mapping and list of required inputs from previous nodes + std::unordered_map tensorNamesMapping; + + const std::optional demultiplexCount; + const std::optional> gatherFrom; + +public: + Node(const std::string& nodeName, std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}); + + virtual ~Node() = default; + + const std::string& getName() const { return this->nodeName; } + + virtual Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) = 0; + Status fetchResults(session_key_t sessionId, SessionResults& nodeSessionOutputs); + +protected: + virtual Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) = 0; + Status demultiplyOutputs(SessionResults& nodeSessionOutputs); + virtual Status createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); + +public: + Status setInputs(const Node& dependency, TensorWithSourceMap& inputs, NodeSessionMetadata& metadata); + Status setInputs(const Node& dependency, SessionResults& inputs); + + virtual void addDependency(Node& node, const Aliases& tensorNamesMapping) { + this->previous.emplace_back(node); + this->tensorNamesMapping[node.getName()] = tensorNamesMapping; + } + + virtual void addDependant(Node& node) { this->next.emplace_back(node); } + + const Aliases& getMappingByDependency(const Node& dependency) { + return tensorNamesMapping.at(dependency.getName()); + } + + std::vector getReadySessions() const; + const std::vector>& getNextNodes() { + return next; + } + virtual void release(session_key_t sessionId) {} + virtual bool tryDisarm(const session_key_t& sessionKey, const uint microseconds = 1) { return true; } + + static void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs); + +protected: + NodeSession* getNodeSession(const NodeSessionMetadata& metadata); + NodeSession& getNodeSession(const session_key_t& sessionKey) const; + virtual std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); +}; + +} // namespace ovms diff --git a/src/ovms_lib/node_library.cpp b/src/ovms_lib/node_library.cpp new file mode 100644 index 0000000000..495b14e306 --- /dev/null +++ b/src/ovms_lib/node_library.cpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "node_library.hpp" + +namespace ovms { + +bool NodeLibrary::isValid() const { + return execute != nullptr && + getInputsInfo != nullptr && + getOutputsInfo != nullptr && + release != nullptr && + initialize != nullptr && + deinitialize != nullptr; +} + +} // namespace ovms diff --git a/src/ovms_lib/node_library.hpp b/src/ovms_lib/node_library.hpp new file mode 100644 index 0000000000..bb29d52a5e --- /dev/null +++ b/src/ovms_lib/node_library.hpp @@ -0,0 +1,52 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "custom_node_interface.h" // NOLINT + +namespace ovms { + +typedef int (*initialize_fn)(void**, const struct CustomNodeParam*, int); +typedef int (*deinitialize_fn)(void*); +typedef int (*execute_fn)(const struct CustomNodeTensor*, int, struct CustomNodeTensor**, int*, const struct CustomNodeParam*, int, void*); +typedef int (*metadata_fn)(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void*); +typedef int (*release_fn)(void*, void*); + +struct NodeLibrary { + initialize_fn initialize = nullptr; + deinitialize_fn deinitialize = nullptr; + execute_fn execute = nullptr; + metadata_fn getInputsInfo = nullptr; + metadata_fn getOutputsInfo = nullptr; + release_fn release = nullptr; + + std::string basePath = ""; + + bool isValid() const; + bool operator==(const NodeLibrary& other) const { + return (initialize == other.initialize) && + (deinitialize == other.deinitialize) && + (execute == other.execute) && + (getInputsInfo == other.getInputsInfo) && + (getOutputsInfo == other.getOutputsInfo) && + (release == other.release) && + (basePath == other.basePath); + } +}; + +} // namespace ovms diff --git a/src/ovms_lib/node_library_utils.cpp b/src/ovms_lib/node_library_utils.cpp new file mode 100644 index 0000000000..a1fe82d892 --- /dev/null +++ b/src/ovms_lib/node_library_utils.cpp @@ -0,0 +1,143 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "node_library_utils.hpp" + +#include + +#include "ov_utils.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +CustomNodeTensorPrecision toCustomNodeTensorPrecision(ov::element::Type_t precision) { + switch (precision) { + case ov::element::Type_t::f32: + return CustomNodeTensorPrecision::FP32; + case ov::element::Type_t::f64: + return CustomNodeTensorPrecision::FP64; + case ov::element::Type_t::i32: + return CustomNodeTensorPrecision::I32; + case ov::element::Type_t::i64: + return CustomNodeTensorPrecision::I64; + case ov::element::Type_t::i8: + return CustomNodeTensorPrecision::I8; + case ov::element::Type_t::u8: + return CustomNodeTensorPrecision::U8; + case ov::element::Type_t::f16: + return CustomNodeTensorPrecision::FP16; + case ov::element::Type_t::i16: + return CustomNodeTensorPrecision::I16; + case ov::element::Type_t::u16: + return CustomNodeTensorPrecision::U16; + default: + return CustomNodeTensorPrecision::UNSPECIFIED; + } +} + +Precision toInferenceEnginePrecision(CustomNodeTensorPrecision precision) { + static std::unordered_map precisionMap{ + {CustomNodeTensorPrecision::FP32, Precision::FP32}, + {CustomNodeTensorPrecision::FP64, Precision::FP64}, + {CustomNodeTensorPrecision::I32, Precision::I32}, + {CustomNodeTensorPrecision::I64, Precision::I64}, + {CustomNodeTensorPrecision::I8, Precision::I8}, + {CustomNodeTensorPrecision::U8, Precision::U8}, + {CustomNodeTensorPrecision::FP16, Precision::FP16}, + {CustomNodeTensorPrecision::I16, Precision::I16}, + {CustomNodeTensorPrecision::U16, Precision::U16}}; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +std::unique_ptr createCustomNodeParamArray(const std::unordered_map& paramMap) { + if (paramMap.size() == 0) { + return nullptr; + } + auto libraryParameters = std::make_unique(paramMap.size()); + int i = 0; + for (const auto& [key, value] : paramMap) { + libraryParameters[i].key = key.c_str(); + libraryParameters[i].value = value.c_str(); + i++; + } + return libraryParameters; +} + +std::unique_ptr createCustomNodeTensorArray(const TensorMap& tensorMap, const std::unordered_map& tensorsDims) { + if (tensorMap.size() == 0) { + return nullptr; + } + auto inputTensors = std::make_unique(tensorMap.size()); + int i = 0; + for (const auto& [name, tensor] : tensorMap) { + auto dimsIt = tensorsDims.find(name); + if (dimsIt == tensorsDims.end()) { + return nullptr; + } + static_assert(sizeof(size_t) == sizeof(uint64_t)); + const auto& dims = dimsIt->second; + inputTensors[i].name = static_cast(name.c_str()); + inputTensors[i].data = static_cast(tensor.data()); + inputTensors[i].dataBytes = static_cast(tensor.get_byte_size()); + inputTensors[i].dims = const_cast(dims.data()); + inputTensors[i].dimsCount = static_cast(dims.size()); + inputTensors[i].precision = toCustomNodeTensorPrecision(tensor.get_element_type()); + i++; + } + return inputTensors; +} + +Status createTensorInfoMap(struct CustomNodeTensorInfo* info, int infoCount, std::map>& out, release_fn freeCallback, void* customNodeLibraryInternalManager) { + if (info == nullptr) { + return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED; + } + if (infoCount <= 0) { + freeCallback(info, customNodeLibraryInternalManager); + return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT; + } + // At this point it is important to not exit before we iterate over every info object. + // This is due to a fact that we need to ensure to free resources allocated by shared library using freeCallback. + for (int i = 0; i < infoCount; i++) { + if (info[i].dims == nullptr) { + continue; + } + if (info[i].dimsCount == 0) { + freeCallback(info[i].dims, customNodeLibraryInternalManager); + continue; + } + if (info[i].name == nullptr) { + continue; + } + + std::string name = std::string(info[i].name); + auto precision = toInferenceEnginePrecision(info[i].precision); + ovms::Shape shape; + for (int j = 0; j < info[i].dimsCount; ++j) { + auto dim = info[i].dims[j]; + shape.add(dim ? Dimension(dim) : Dimension::any()); + } + + freeCallback(info[i].dims, customNodeLibraryInternalManager); + out.emplace(name, std::make_shared(name, precision, std::move(shape), Layout::getUnspecifiedLayout())); + } + freeCallback(info, customNodeLibraryInternalManager); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/node_library_utils.hpp b/src/ovms_lib/node_library_utils.hpp new file mode 100644 index 0000000000..86c55a893b --- /dev/null +++ b/src/ovms_lib/node_library_utils.hpp @@ -0,0 +1,40 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include "custom_node_interface.h" // NOLINT +#include "node_library.hpp" +#include "precision.hpp" +#include "shape.hpp" +#include "status.hpp" +#include "tensormap.hpp" + +namespace ovms { + +class TensorInfo; + +CustomNodeTensorPrecision toCustomNodeTensorPrecision(ov::element::Type_t precision); +Precision toInferenceEnginePrecision(CustomNodeTensorPrecision precision); +std::unique_ptr createCustomNodeParamArray(const std::unordered_map& paramMap); +std::unique_ptr createCustomNodeTensorArray(const TensorMap& tensorMap, const std::unordered_map& tensorDims); +Status createTensorInfoMap(struct CustomNodeTensorInfo* info, int infoCount, std::map>& out, release_fn freeCallback, void* customNodeLibraryInternalManager); + +} // namespace ovms diff --git a/src/ovms_lib/nodeinfo.hpp b/src/ovms_lib/nodeinfo.hpp new file mode 100644 index 0000000000..20a4c2b3fe --- /dev/null +++ b/src/ovms_lib/nodeinfo.hpp @@ -0,0 +1,92 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "aliases.hpp" +#include "modelversion.hpp" +#include "node_library.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +class ModelManager; +class Pipeline; + +using pipeline_connections_t = std::unordered_map>; +using parameters_t = std::unordered_map; + +enum class NodeKind { + ENTRY, + DL, + CUSTOM, + EXIT +}; + +const std::string DL_NODE_CONFIG_TYPE = "DL model"; +const std::string CUSTOM_NODE_CONFIG_TYPE = "custom"; + +Status toNodeKind(const std::string& str, NodeKind& nodeKind); + +struct DLNodeInfo { + std::string modelName; + std::optional modelVersion; +}; + +struct CustomNodeInfo { + NodeLibrary library; + parameters_t parameters; +}; + +struct NodeInfo { + NodeKind kind; + std::string nodeName; + std::string modelName; + std::optional modelVersion; + std::unordered_map outputNameAliases; + std::optional demultiplyCount; + std::set gatherFromNode; + NodeLibrary library; + parameters_t parameters; + + NodeInfo(NodeKind kind, + const std::string& nodeName, + const std::string& modelName = "", + std::optional modelVersion = std::nullopt, + std::unordered_map outputNameAliases = {}, + std::optional demultiplyCount = std::nullopt, + const std::set& gatherFromNode = {}, + const NodeLibrary& library = {}, + const parameters_t& parameters = {}) : + kind(kind), + nodeName(nodeName), + modelName(modelName), + modelVersion(modelVersion), + outputNameAliases(outputNameAliases), + demultiplyCount(demultiplyCount), + gatherFromNode(gatherFromNode), + library(library), + parameters(parameters) {} +}; +} // namespace ovms diff --git a/src/ovms_lib/nodeinputhandler.cpp b/src/ovms_lib/nodeinputhandler.cpp new file mode 100644 index 0000000000..4ee038e6ac --- /dev/null +++ b/src/ovms_lib/nodeinputhandler.cpp @@ -0,0 +1,60 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "nodeinputhandler.hpp" + +#include "logging.hpp" + +namespace ovms { + +NodeInputHandler::NodeInputHandler(uint32_t inputsMissingCount) : + remainingDependencies(inputsMissingCount) { +} + +Status NodeInputHandler::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Setting input: {}, shardId: {}", inputName, shardId); + if (shardId > 0) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to set input: {}, with shardId: {} >0 in NodeInputHandler.", inputName, shardId); + return StatusCode::PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER; + } + if (inputTensors.find(inputName) != inputTensors.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to set the same input: {}, shardId: {} twice for the NodeInputHandler.", inputName, shardId); + return StatusCode::PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE; + } + inputTensors.emplace(inputName, tensor.getActualTensor()); + if (tensor.hasSource()) { + sourceTensorRefs.push_back(tensor.getSourceTensor()); + } + return StatusCode::OK; +} + +void NodeInputHandler::clearInputs() { + inputTensors.clear(); + sourceTensorRefs.clear(); +} + +bool NodeInputHandler::isReady() { + if (this->isUsed) { + return false; + } + return remainingDependencies == 0; +} + +Status NodeInputHandler::notifyFinishedDependency() { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Remaining dependencies count for input handler decreased from: {} to: {}", remainingDependencies, remainingDependencies - 1); + --remainingDependencies; + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/nodeinputhandler.hpp b/src/ovms_lib/nodeinputhandler.hpp new file mode 100644 index 0000000000..12665794dd --- /dev/null +++ b/src/ovms_lib/nodeinputhandler.hpp @@ -0,0 +1,54 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include + +#include "session_id.hpp" +#include "status.hpp" +#include "tensor_utils.hpp" +#include "tensormap.hpp" + +namespace ovms { + +// This class encapsulates input tensor gathering and preprocessing before node execution. +// It is resposible for gathering multiple tensors into one (in case of demultiplexers) +// and taking care of source tensor lifetime if source tensor is present. +class NodeInputHandler { +protected: + TensorMap inputTensors; + TensorVector sourceTensorRefs; + uint32_t remainingDependencies; + bool isUsed = false; + +public: + NodeInputHandler(uint32_t inputsMissingCount); + virtual Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId); + const TensorMap& getInputs() { + isUsed = true; + return inputTensors; + } + void clearInputs(); + bool isReady(); + virtual Status notifyFinishedDependency(); + virtual ~NodeInputHandler() = default; +}; +} // namespace ovms diff --git a/src/ovms_lib/nodeoutputhandler.cpp b/src/ovms_lib/nodeoutputhandler.cpp new file mode 100644 index 0000000000..066d0c9097 --- /dev/null +++ b/src/ovms_lib/nodeoutputhandler.cpp @@ -0,0 +1,20 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "nodeoutputhandler.hpp" + +namespace ovms { +void NodeOutputHandler::setInput(const std::string& name, ov::Tensor& tensorPtr) {} +} // namespace ovms diff --git a/src/ovms_lib/nodeoutputhandler.hpp b/src/ovms_lib/nodeoutputhandler.hpp new file mode 100644 index 0000000000..b3e2a79718 --- /dev/null +++ b/src/ovms_lib/nodeoutputhandler.hpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include + +namespace ovms { +class NodeOutputHandler { +public: + void setInput(const std::string& inputName, ov::Tensor& tensorPtr); +}; +} // namespace ovms diff --git a/src/ovms_lib/nodesession.cpp b/src/ovms_lib/nodesession.cpp new file mode 100644 index 0000000000..99de8ae274 --- /dev/null +++ b/src/ovms_lib/nodesession.cpp @@ -0,0 +1,72 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "nodesession.hpp" + +#include "gathernodeinputhandler.hpp" +#include "logging.hpp" +#include "nodeinputhandler.hpp" +#include "nodeoutputhandler.hpp" +#include "tensor_utils.hpp" +#include "timer.hpp" + +namespace ovms { +NodeSession::~NodeSession() = default; + +const NodeSessionMetadata& NodeSession::getNodeSessionMetadata() const { + return this->metadata; +} + +Status NodeSession::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { + return inputHandler->setInput(inputName, tensor, shardId); +} + +std::unique_ptr createNodeInputHandler(uint32_t inputsCount, const CollapseDetails& collapsingDetails) { + if (collapsingDetails.collapsedSessionNames.size() == 0) { + return std::make_unique(inputsCount); + } else { + return std::make_unique(inputsCount, collapsingDetails); + } +} + +NodeSession::NodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : + metadata(metadata), + sessionKey(metadata.getSessionKey()), + nodeName(nodeName), + timer(std::make_unique()), + inputHandler(createNodeInputHandler(inputsCount, collapsingDetails)), + outputHandler(std::make_unique()) {} + +bool NodeSession::isReady() const { + bool isReady = inputHandler->isReady(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} session: {} isReady: {}", getName(), getSessionKey(), isReady); + return isReady; +} + +Status NodeSession::notifyFinishedDependency() { + return this->inputHandler->notifyFinishedDependency(); +} + +Timer& NodeSession::getTimer() const { + return *this->timer; +} + +ReleaseSessionGuard::ReleaseSessionGuard(NodeSession& nodeSession) : + nodeSession(nodeSession) {} + +ReleaseSessionGuard::~ReleaseSessionGuard() { + nodeSession.release(); +} +} // namespace ovms diff --git a/src/ovms_lib/nodesession.hpp b/src/ovms_lib/nodesession.hpp new file mode 100644 index 0000000000..7eb5028f0b --- /dev/null +++ b/src/ovms_lib/nodesession.hpp @@ -0,0 +1,62 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "nodesessionmetadata.hpp" +#include "status.hpp" + +namespace ovms { +struct NodeInputHandler; +struct NodeOutputHandler; +class TensorWithSource; +class Timer; + +class NodeSession { + NodeSessionMetadata metadata; + session_key_t sessionKey; + const std::string& nodeName; + +protected: + std::unique_ptr timer; + std::unique_ptr inputHandler; + std::unique_ptr outputHandler; + +public: + NodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); + virtual ~NodeSession(); + const std::string& getName() const { return nodeName; } + Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId); + const NodeSessionMetadata& getNodeSessionMetadata() const; + const session_key_t& getSessionKey() const { return sessionKey; } + bool isReady() const; + virtual void release() {} + virtual bool tryDisarm(uint microseconds) { return true; } + Status notifyFinishedDependency(); + Timer& getTimer() const; +}; + +class ReleaseSessionGuard { + NodeSession& nodeSession; + +public: + ReleaseSessionGuard(NodeSession& nodeSession); + ~ReleaseSessionGuard(); +}; +} // namespace ovms diff --git a/src/ovms_lib/nodesessionmetadata.cpp b/src/ovms_lib/nodesessionmetadata.cpp new file mode 100644 index 0000000000..3374229ef3 --- /dev/null +++ b/src/ovms_lib/nodesessionmetadata.cpp @@ -0,0 +1,197 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "nodesessionmetadata.hpp" + +#include +#include +#include +#include +#include + +#include "logging.hpp" + +namespace ovms { + +std::vector NodeSessionMetadata::generateSubsessions(const std::string& nodeName, session_id_t subsessionSize) const { + if (nodeName.size() == 0) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to generate subsession with empty node name"); + throw std::logic_error("Cannot generate subsession with empty parent name"); + } + if (details.find(nodeName) != details.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to generate subsession with node name: {} but it already spawned subsession.", nodeName); + throw std::logic_error("Cannot generate subsession with already used name"); + } + if (subsessionSize == 0) { + return {}; + } + std::vector metas(subsessionSize); + uint counter = 0; + for (auto& meta : metas) { + meta.details = this->details; + meta.details.insert({nodeName, {counter, subsessionSize}}); + meta.sessionsLevels = this->sessionsLevels; + meta.sessionsLevels.push_back(nodeName); + ++counter; + } + SPDLOG_LOGGER_TRACE(dag_executor_logger, "Generated subsession levels: {}", + std::accumulate(metas[0].sessionsLevels.begin(), metas[0].sessionsLevels.end(), + std::string(), [](const std::string& lhs, const std::string& rhs) { + if (lhs.empty()) { + return rhs; + } + return lhs + ", " + rhs; })); + return metas; +} + +std::string NodeSessionMetadata::getSessionKey(const std::set& ignoredNodeNames) const { + if (details.size() == 0) { + return ""; + } + if (std::any_of(ignoredNodeNames.begin(), + ignoredNodeNames.end(), + [this](auto& ignoredNodeName) { + bool notFound = (this->details.find(ignoredNodeName) == this->details.end()); + if (notFound) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to create session key ignoring subsession name: {} but it does not exist", ignoredNodeName); + } + return notFound; + })) { + throw std::logic_error("Tried to create session key ignoring non-existing subsession"); + } + std::stringstream ss; + size_t j = 0; + for (int32_t i = sessionsLevels.size() - 1; i >= 0; --i, ++j) { + if ((ignoredNodeNames.size() > 0) && + (ignoredNodeNames.size() > j) && + ((sessionsLevels.size() - ignoredNodeNames.size()) >= 0) && + (ignoredNodeNames.find(sessionsLevels[i]) == ignoredNodeNames.end())) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first", sessionsLevels[i]); + throw std::logic_error("Cannot collapse sessions not in LIFO order"); + } else { + if (j < ignoredNodeNames.size()) { + continue; + } + if (ss.tellp() > 0) { + ss << "_"; + } + ss << sessionsLevels[i] << "_" << std::get<0>(details.at(sessionsLevels[i])); + } + if (i == 0) { + break; + } + } + return ss.str(); +} + +std::pair NodeSessionMetadata::getCollapsedSessionMetadata(const std::set& ignoredNodeNames) const { + if (ignoredNodeNames.size() == 0) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse subsession with emtpy set"); + throw std::logic_error("Tried to collapse sessions with empty set"); + } + if (std::any_of( + ignoredNodeNames.begin(), + ignoredNodeNames.end(), + [this](auto& ignoredNodeName) { + bool notFound = (this->details.find(ignoredNodeName) == this->details.end()); + if (notFound) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse subsession: {} but it does not exist", ignoredNodeName); + } + return notFound; + })) { + throw std::logic_error("Tried to collapse nonexisting subsession"); + } + for (size_t i = sessionsLevels.size() - 1; i > sessionsLevels.size() - 1 - ignoredNodeNames.size(); --i) { + if (ignoredNodeNames.find(sessionsLevels[i]) == ignoredNodeNames.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first", sessionsLevels[i]); + throw std::logic_error("Cannot collapse sessions not in LIFO order"); + } + } + + NodeSessionMetadata newMeta; + std::copy_if( + std::begin(details), + std::end(details), + std::inserter(newMeta.details, newMeta.details.begin()), + [&ignoredNodeNames](auto& keyValuePair) { + return ignoredNodeNames.find(keyValuePair.first) == ignoredNodeNames.end(); + }); + CollapseDetails collapsingDetails; + for (auto& sessionLevel : sessionsLevels) { + if (ignoredNodeNames.find(sessionLevel) != ignoredNodeNames.end()) { + collapsingDetails.collapsedSessionNames.emplace_back(sessionLevel); + collapsingDetails.collapsedSessionSizes.emplace_back(getSubsessionSize(sessionLevel)); + } else { + newMeta.sessionsLevels.emplace_back(sessionLevel); + } + } + return {newMeta, std::move(collapsingDetails)}; +} + +session_id_t NodeSessionMetadata::getSubsessionSize(const std::string& subsessionName) const { + auto it = details.find(subsessionName); + if (it == details.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to get non-existing subsession: {} size", subsessionName); + throw std::logic_error("Tried to take non existing subsession size"); + } + return std::get<1>(it->second); +} + +session_id_t NodeSessionMetadata::getShardId(const std::set& collapsedNames) const { + if (collapsedNames.size() == 0) { + return 0; + } + if (collapsedNames.size() > sessionsLevels.size()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse more subsession levels than exists"); + throw std::logic_error("Tried to collapse more subsession levels than exists"); + } + for (size_t i = sessionsLevels.size() - 1; i > sessionsLevels.size() - 1 - collapsedNames.size(); --i) { + if (collapsedNames.find(sessionsLevels[i]) == collapsedNames.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first, but tried to: {}. SubsessionLevels: {}", + sessionsLevels[i], + std::accumulate(collapsedNames.begin(), + collapsedNames.end(), + std::string(), + [](const std::string& lhs, const std::string rhs) { + if (lhs.empty()) { + return rhs; + } + return lhs + ", " + rhs; + }), + std::accumulate(sessionsLevels.begin(), + sessionsLevels.end(), + std::string(), + [](const std::string& lhs, const std::string rhs) { + if (lhs.empty()) { + return rhs; + } + return lhs + ", " + rhs; + })); + throw std::logic_error("Cannot collapse sessions not in LIFO order"); + } + } + session_id_t multiplyFactor = 1; + session_id_t shardId = 0; + for (size_t i = 0; i < collapsedNames.size(); ++i) { + const auto& subsessionDetails = details.at(*(sessionsLevels.rbegin() + i)); + const auto& [id, sessionSize] = subsessionDetails; + shardId += multiplyFactor * id; + multiplyFactor *= sessionSize; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "getShardId calculation step shardId: {}, multiplyFactor: {}, subsessionId: {}, sessionSize: {}", + shardId, multiplyFactor, id, sessionSize); + } + return shardId; +} +} // namespace ovms diff --git a/src/ovms_lib/nodesessionmetadata.hpp b/src/ovms_lib/nodesessionmetadata.hpp new file mode 100644 index 0000000000..bf42bb3472 --- /dev/null +++ b/src/ovms_lib/nodesessionmetadata.hpp @@ -0,0 +1,48 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "session_id.hpp" + +namespace ovms { + +using session_id_t = uint32_t; +using session_key_t = std::string; + +struct CollapseDetails { + std::vector collapsedSessionNames; + std::vector collapsedSessionSizes; +}; + +class NodeSessionMetadata { + std::unordered_map> details; + std::vector sessionsLevels; + +public: + std::vector generateSubsessions(const std::string& nodeName, session_id_t subsessionSize) const; + std::string getSessionKey(const std::set& ignoredNodeNames = {}) const; + std::pair getCollapsedSessionMetadata(const std::set& ignoredNodeNames) const; + session_id_t getSubsessionSize(const std::string& subsessionName) const; + session_id_t getShardId(const std::set& collapsedNames = {}) const; +}; +} // namespace ovms diff --git a/src/ovms_lib/nodesessionresult.hpp b/src/ovms_lib/nodesessionresult.hpp new file mode 100644 index 0000000000..fdeb5f64ef --- /dev/null +++ b/src/ovms_lib/nodesessionresult.hpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include "nodesessionmetadata.hpp" +#include "tensormap.hpp" + +namespace ovms { + +using SessionResult = std::pair; +using SessionResults = std::unordered_map; + +} // namespace ovms diff --git a/src/ovms_lib/nodestreamidguard.hpp b/src/ovms_lib/nodestreamidguard.hpp new file mode 100644 index 0000000000..88b1678b75 --- /dev/null +++ b/src/ovms_lib/nodestreamidguard.hpp @@ -0,0 +1,70 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include + +#include "ovinferrequestsqueue.hpp" +#include "profiler.hpp" + +namespace ovms { + +struct NodeStreamIdGuard { + NodeStreamIdGuard(ovms::OVInferRequestsQueue& inferRequestsQueue) : + inferRequestsQueue_(inferRequestsQueue), + futureStreamId(inferRequestsQueue_.getIdleStream()) {} + + ~NodeStreamIdGuard() { + if (!disarmed) { + if (!streamId) { + SPDLOG_DEBUG("Trying to disarm stream Id that is not needed anymore..."); + streamId = futureStreamId.get(); + } + SPDLOG_DEBUG("Returning streamId: {}", streamId.value()); + inferRequestsQueue_.returnStream(streamId.value()); + } + } + + std::optional tryGetId(const uint microseconds = 1) { + OVMS_PROFILE_FUNCTION(); + if (!streamId) { + if (std::future_status::ready == futureStreamId.wait_for(std::chrono::microseconds(microseconds))) { + streamId = futureStreamId.get(); + } + } + return streamId; + } + + bool tryDisarm(const uint microseconds = 1) { + if (std::future_status::ready == futureStreamId.wait_for(std::chrono::microseconds(microseconds))) { + streamId = futureStreamId.get(); + SPDLOG_DEBUG("Returning streamId:", streamId.value()); + inferRequestsQueue_.returnStream(streamId.value()); + disarmed = true; + } + return disarmed; + } + +private: + ovms::OVInferRequestsQueue& inferRequestsQueue_; + std::future futureStreamId; + std::optional streamId = std::nullopt; + bool disarmed = false; +}; +} // namespace ovms diff --git a/src/ovms_lib/ov_utils.cpp b/src/ovms_lib/ov_utils.cpp new file mode 100644 index 0000000000..cd443c0512 --- /dev/null +++ b/src/ovms_lib/ov_utils.cpp @@ -0,0 +1,131 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "ov_utils.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include "logging.hpp" +#include "profiler.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +// This creates tensor without data ownership. +ov::Tensor createSharedTensor(ov::element::Type_t precision, const shape_t& shape, void* data) { + auto tensor = ov::Tensor(precision, shape, data); + return tensor; +} + +Status createSharedTensor(ov::Tensor& destinationTensor, ov::element::Type_t precision, const ov::Shape& shape) { + destinationTensor = ov::Tensor(precision, shape); + return StatusCode::OK; +} + +std::string getTensorMapString(const std::map>& inputsInfo) { + std::stringstream stringStream; + for (const auto& pair : inputsInfo) { + const auto& name = pair.first; + auto inputInfo = pair.second; + auto precision = inputInfo->getPrecision(); + auto layout = inputInfo->getLayout(); + auto shape = inputInfo->getShape(); + + stringStream << "\nname: " << name + << "; mapping: " << inputInfo->getMappedName() + << "; shape: " << shape.toString() + << "; precision: " << TensorInfo::getPrecisionAsString(precision) + << "; layout: " << TensorInfo::getStringFromLayout(layout); + } + return stringStream.str(); +} + +Status tensorClone(ov::Tensor& destinationTensor, const ov::Tensor& sourceTensor) { + OVMS_PROFILE_FUNCTION(); + destinationTensor = ov::Tensor(sourceTensor.get_element_type(), sourceTensor.get_shape()); + + if (destinationTensor.get_byte_size() != sourceTensor.get_byte_size()) { + SPDLOG_ERROR("tensorClone byte size mismatch destination:{}; source:{}", + destinationTensor.get_byte_size(), + sourceTensor.get_byte_size()); + return StatusCode::OV_CLONE_TENSOR_ERROR; + } + std::memcpy(destinationTensor.data(), sourceTensor.data(), sourceTensor.get_byte_size()); + return StatusCode::OK; +} + +std::optional getLayoutFromRTMap(const ov::RTMap& rtMap) { + for (const auto& [k, v] : rtMap) { + try { + return v.as().value; + } catch (ov::Exception& e) { + } + } + return std::nullopt; +} + +void insertSupportedKeys(std::set& aggregatedPluginSupportedConfigKeys, const std::string& pluginName, const ov::Core& ieCore) { + const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); + try { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validating plugin: {}; configuration", pluginName); + std::vector pluginSupportedConfigKeys = ieCore.get_property(pluginName, supportedConfigKey).as>(); + std::set pluginSupportedConfigKeysSet(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); + aggregatedPluginSupportedConfigKeys.insert(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); + } catch (std::exception& e) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", pluginName, supportedConfigKey, e.what()); + } catch (...) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", pluginName, supportedConfigKey); + } +} + +Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore) { + std::set pluginSupportedConfigKeys; + std::string pluginDelimiter = ":"; + auto pluginDelimeterPos = targetDevice.find(pluginDelimiter); + if (pluginDelimeterPos != std::string::npos) { + std::string pluginName = targetDevice.substr(0, pluginDelimeterPos); + insertSupportedKeys(pluginSupportedConfigKeys, pluginName, ieCore); + char deviceDelimiter = ','; + std::stringstream ss(targetDevice.substr(pluginDelimeterPos + 1, targetDevice.length())); + std::string deviceName; + + while (getline(ss, deviceName, deviceDelimiter)) { + insertSupportedKeys(pluginSupportedConfigKeys, deviceName, ieCore); + } + } else { + insertSupportedKeys(pluginSupportedConfigKeys, targetDevice, ieCore); + } + + for (auto& config : pluginConfig) { + if (std::find(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end(), config.first) == pluginSupportedConfigKeys.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config key: {} not found in supported config keys for device: {}.", config.first, targetDevice); + SPDLOG_LOGGER_INFO(modelmanager_logger, "List of supported keys for this device:"); + for (std::string supportedKey : pluginSupportedConfigKeys) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "{}", supportedKey); + } + return StatusCode::MODEL_CONFIG_INVALID; + } + } + + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/ov_utils.hpp b/src/ovms_lib/ov_utils.hpp new file mode 100644 index 0000000000..2f52005066 --- /dev/null +++ b/src/ovms_lib/ov_utils.hpp @@ -0,0 +1,47 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "modelconfig.hpp" +#include "status.hpp" + +namespace ovms { + +class TensorInfo; + +Status createSharedTensor(ov::Tensor& destinationTensor, ov::element::Type_t precision, const ov::Shape& shape); +/** + * Creates new tensor that copies data and owns the copy + **/ +ov::Tensor createSharedTensor(ov::element::Type_t precision, const shape_t& shape, void* data); + +std::string getTensorMapString(const std::map>& tensorMap); + +Status tensorClone(ov::Tensor& destinationTensor, const ov::Tensor& sourceTensor); + +std::optional getLayoutFromRTMap(const ov::RTMap& rtMap); + +Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore); + +} // namespace ovms diff --git a/src/ovms_lib/ovinferrequestsqueue.hpp b/src/ovms_lib/ovinferrequestsqueue.hpp new file mode 100644 index 0000000000..9421e9f133 --- /dev/null +++ b/src/ovms_lib/ovinferrequestsqueue.hpp @@ -0,0 +1,45 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "queue.hpp" + +namespace ovms { + +class OVInferRequestsQueue : public Queue { +public: + OVInferRequestsQueue(ov::CompiledModel& compiledModel, int streamsLength) : + Queue(streamsLength) { + for (int i = 0; i < streamsLength; ++i) { + streams[i] = i; + inferRequests.push_back(compiledModel.create_infer_request()); + } + } +}; + +} // namespace ovms diff --git a/src/ovms_lib/pipeline.cpp b/src/ovms_lib/pipeline.cpp new file mode 100644 index 0000000000..965bf386cb --- /dev/null +++ b/src/ovms_lib/pipeline.cpp @@ -0,0 +1,270 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pipeline.hpp" + +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "node.hpp" +#include "pipelineeventqueue.hpp" +#include "profiler.hpp" + +namespace ovms { + +using DeferredNodeSessions = std::vector, session_key_t>>; + +Pipeline::~Pipeline() = default; + +Pipeline::Pipeline(Node& entry, Node& exit, const std::string& name) : + name(name), + entry(entry), + exit(exit) {} + +void Pipeline::push(std::unique_ptr node) { + nodes.emplace_back(std::move(node)); +} +void Pipeline::connect(Node& from, Node& to, const Aliases& tensorNamesMapping) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Connecting from: {}, to: {}", from.getName(), to.getName()); + printNodeConnections(to.getName(), from.getName(), tensorNamesMapping); + from.addDependant(to); + to.addDependency(from, tensorNamesMapping); +} + +void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs) { + if (spdlog::default_logger()->level() > spdlog::level::debug) { + return; + } + std::stringstream ss; + ss << "Links from:" << sourceNode << " to:" << nodeName << ":\n"; + for (auto& pair : pairs) { + ss << "\t" << nodeName << "[" << pair.second << "]=" << sourceNode << "[" << pair.first << "]\n"; + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, ss.str()); +} + +void setFailIfNotFailEarlier(ovms::Status& earlierStatusCode, ovms::Status& newFailStatus) { + if (earlierStatusCode.ok()) { + earlierStatusCode = newFailStatus; + } +} + +#define IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE \ + if (!firstErrorStatus.ok()) { \ + if (finishedSessions.size() == startedSessions.size()) { \ + break; \ + } else { \ + continue; \ + } \ + } + +#define CHECK_AND_LOG_ERROR(NODE) \ + if (!status.ok()) { \ + setFailIfNotFailEarlier(firstErrorStatus, status); \ + SPDLOG_LOGGER_WARN(dag_executor_logger, "Executing pipeline: {} node: {} session: {} failed with ret code: {}, error message: {}", \ + getName(), NODE.getName(), sessionKey, status.getCode(), status.string()); \ + } + +Status Pipeline::execute() { + OVMS_PROFILE_FUNCTION(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Started execution of pipeline: {}", getName()); + PipelineEventQueue finishedNodeQueue; + ovms::Status firstErrorStatus{ovms::StatusCode::OK}; + std::set startedSessions; + std::set finishedSessions; + NodeSessionMetadata meta; + // entry node does not have setInputsCalled so it has no + // session created. Here is just assumption that this meta has the same key + // that the one in EntryNode::execute(); + auto entrySessionKey = meta.getSessionKey(); + startedSessions.emplace(entry.getName() + entrySessionKey); + ovms::Status status = entry.execute(entrySessionKey, finishedNodeQueue); // first node will triger first message + if (!status.ok()) { + SPDLOG_LOGGER_WARN(dag_executor_logger, "Executing pipeline: {} node: {} failed with: {}", + getName(), entry.getName(), status.string()); + return status; + } + DeferredNodeSessions deferredNodeSessions; + const uint WAIT_FOR_FINISHED_NODE_TIMEOUT_MICROSECONDS = 5000; + const uint WAIT_FOR_DEFERRED_NODE_DISARM_TIMEOUT_MICROSECONDS = 500; + // process finished session nodes and if no one is finished check if any node session with deferred execution + // has necessary resources already + while (true) { + spdlog::trace("Pipeline: {} waiting for message that node finished.", getName()); + OVMS_PROFILE_SYNC_BEGIN("PipelineEventQueue::tryPull"); + auto optionallyFinishedNode = finishedNodeQueue.tryPull(WAIT_FOR_FINISHED_NODE_TIMEOUT_MICROSECONDS); + OVMS_PROFILE_SYNC_END("PipelineEventQueue::tryPull"); + if (optionallyFinishedNode) { + OVMS_PROFILE_SCOPE_S("Processing Finished Node", "node_name", optionallyFinishedNode.value().first.get().getName().c_str()); + /* + Get results from finished node session. + */ + auto& [finishedNodeRef, sessionKey] = optionallyFinishedNode.value(); + Node& finishedNode = finishedNodeRef.get(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Pipeline: {} got message that node: {} session: {} finished.", getName(), finishedNode.getName(), sessionKey); + finishedSessions.emplace(finishedNode.getName() + sessionKey); + if (!firstErrorStatus.ok()) { + finishedNode.release(sessionKey); + } + IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE + SessionResults sessionResults; + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Fetching results of pipeline: {} node: {} session: {}", getName(), finishedNode.getName(), sessionKey); + status = finishedNode.fetchResults(sessionKey, sessionResults); + CHECK_AND_LOG_ERROR(finishedNode) + IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE + + /* + Feed next node sessions with results from currently finished node session. + */ + auto& nextNodesFromFinished = finishedNode.getNextNodes(); + for (auto& nextNode : nextNodesFromFinished) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "setting pipeline: {} node: {} session: {} outputs as inputs for node: {}", + getName(), finishedNode.getName(), sessionKey, nextNode.get().getName()); + status = nextNode.get().setInputs(finishedNode, sessionResults); + CHECK_AND_LOG_ERROR(nextNode.get()) + if (!firstErrorStatus.ok()) { + break; + } + } + + /* + Try to schedule node sessions that are following the currently finished session. + Defer next node sessions which are ready, but stream id is not ready yet. + Save defered node sessions to temporary container which will be later merged into global container. + */ + OVMS_PROFILE_SYNC_BEGIN("Try next nodes"); + DeferredNodeSessions tmpDeferredNodeSessions; + for (auto& nextNode : nextNodesFromFinished) { + auto readySessions = nextNode.get().getReadySessions(); + for (auto& sessionKey : readySessions) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Started execution of pipeline: {} node: {} session: {}", getName(), nextNode.get().getName(), sessionKey); + startedSessions.emplace(nextNode.get().getName() + sessionKey); + status = nextNode.get().execute(sessionKey, finishedNodeQueue); + if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", nextNode.get().getName(), sessionKey); + tmpDeferredNodeSessions.emplace_back(nextNode.get(), sessionKey); + status = StatusCode::OK; + } + CHECK_AND_LOG_ERROR(nextNode.get()) + if (!firstErrorStatus.ok()) { + break; + } + } + } + OVMS_PROFILE_SYNC_END("Try next nodes"); + + /* + Iterate over global container of deferred node sessions and try to schedule. + Keep in mind that newly deferred nodes are not iterated since those are in temporary container. + This is expected since newly deferred nodes were just checked for possible availability of stream ID in previous step. + */ + OVMS_PROFILE_SYNC_BEGIN("Try deferred nodes"); + for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { + // Quit trying to schedule deferred nodes since handling newly finished node has bigger priority (the node can unlock stream ID or allow scheduling next nodes) + if (finishedNodeQueue.size() > 0) { + break; + } + auto& [nodeRef, sessionKey] = *it; + auto& node = nodeRef.get(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to trigger node: {} session: {} execution", node.getName(), sessionKey); + status = node.execute(sessionKey, finishedNodeQueue); + if (status.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} is ready", node.getName(), sessionKey); + it = deferredNodeSessions.erase(it); + continue; + } + it++; + if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", node.getName(), sessionKey); + status = StatusCode::OK; + } else { + CHECK_AND_LOG_ERROR(node) + } + } + OVMS_PROFILE_SYNC_END("Try deferred nodes"); + + /* + Merge temporary and global deferred node session containers. + */ + OVMS_PROFILE_SYNC_BEGIN("Merge deferred containers"); + deferredNodeSessions.insert( + deferredNodeSessions.end(), + tmpDeferredNodeSessions.begin(), + tmpDeferredNodeSessions.end()); + OVMS_PROFILE_SYNC_END("Merge deferred containers"); + + if (startedSessions.size() == finishedSessions.size()) { + break; + } + } else { + OVMS_PROFILE_SCOPE("No new finished nodes"); + // If error occurred earlier, disarm stream id guards of all deferred nodes and exit + if (!firstErrorStatus.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will try to disarm all stream id guards of all {} deferred node sessions due to previous error in pipeline", deferredNodeSessions.size()); + if (deferredNodeSessions.size() > 0) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to disarm {} remaining deferred node sessions ...", deferredNodeSessions.size()); + for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { + auto& [nodeRef, sessionKey] = *it; + auto& node = nodeRef.get(); + if (node.tryDisarm(sessionKey, WAIT_FOR_DEFERRED_NODE_DISARM_TIMEOUT_MICROSECONDS)) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Stream id guard disarm of node {} session: {} has succeeded", node.getName(), sessionKey); + finishedSessions.emplace(node.getName() + sessionKey); + it = deferredNodeSessions.erase(it); + } else { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Cannot disarm stream id guard of node: {}, session: {} yet, will try again later", node.getName(), sessionKey); + it++; + } + } + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Disarming iteration completed, remaining deferred node sessions count: {}", deferredNodeSessions.size()); + } + // Check for deferred node queue size again to indicate if all nodes got freed + if (deferredNodeSessions.size() > 0) { + continue; + } else { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Disarming all stream id guards of deferred nodes completed, pipeline will shut down"); + IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE + } + } + // else scope could be executed always however it seems most reasonable at the time to + // free blocked inferRequests from exeuction first rather than free models for reloading + OVMS_PROFILE_SYNC_BEGIN("Try deferred nodes"); + for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { + auto& [nodeRef, sessionKey] = *it; + auto& node = nodeRef.get(); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to trigger node: {} session: {} execution", node.getName(), sessionKey); + status = node.execute(sessionKey, finishedNodeQueue); + if (status.ok()) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} is ready", node.getName(), sessionKey); + it = deferredNodeSessions.erase(it); + continue; + } + it++; + if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", node.getName(), sessionKey); + status = StatusCode::OK; + } else { + CHECK_AND_LOG_ERROR(node) + } + } + OVMS_PROFILE_SYNC_END("Try deferred nodes"); + } + } + return firstErrorStatus; +} +} // namespace ovms diff --git a/src/ovms_lib/pipeline.hpp b/src/ovms_lib/pipeline.hpp new file mode 100644 index 0000000000..71e4cb373b --- /dev/null +++ b/src/ovms_lib/pipeline.hpp @@ -0,0 +1,63 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include "aliases.hpp" +#include "status.hpp" + +namespace ovms { + +class Node; +template +class EntryNode; +template +class ExitNode; + +void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs); + +class Pipeline { + std::vector> nodes; + const std::string name; + Node& entry; + Node& exit; + +public: + Pipeline(Node& entry, Node& exit, const std::string& name = "default_name"); + + void push(std::unique_ptr node); + ~Pipeline(); + + Node& getEntry() const { return this->entry; } + Node& getExit() const { return this->exit; } + + static void connect(Node& from, Node& to, const Aliases& tensorNamesMapping); + + Status execute(); + const std::string& getName() const { + return name; + } + +private: + std::map prepareStatusMap() const; +}; + +} // namespace ovms diff --git a/src/ovms_lib/pipeline_factory.cpp b/src/ovms_lib/pipeline_factory.cpp new file mode 100644 index 0000000000..73588b2260 --- /dev/null +++ b/src/ovms_lib/pipeline_factory.cpp @@ -0,0 +1,146 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pipeline_factory.hpp" + +#include "logging.hpp" +#include "pipeline.hpp" +#include "pipelinedefinition.hpp" +#include "prediction_service_utils.hpp" + +namespace ovms { + +bool PipelineFactory::definitionExists(const std::string& name) const { + std::shared_lock lock(definitionsMtx); + return definitions.find(name) != definitions.end(); +} + +PipelineDefinition* PipelineFactory::findDefinitionByName(const std::string& name) const { + std::shared_lock lock(definitionsMtx); + auto it = definitions.find(name); + if (it == std::end(definitions)) { + return nullptr; + } else { + return it->second.get(); + } +} + +void PipelineFactory::retireOtherThan(std::set&& pipelinesInConfigFile, ModelManager& manager) { + std::for_each(definitions.begin(), + definitions.end(), + [&pipelinesInConfigFile, &manager](auto& nameDefinitionPair) { + if (pipelinesInConfigFile.find(nameDefinitionPair.second->getName()) == pipelinesInConfigFile.end() && nameDefinitionPair.second->getStateCode() != PipelineDefinitionStateCode::RETIRED) { + nameDefinitionPair.second->retire(manager); + } + }); +} + +Status PipelineFactory::createDefinition(const std::string& pipelineName, + const std::vector& nodeInfos, + const pipeline_connections_t& connections, + ModelManager& manager) { + if (definitionExists(pipelineName)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "pipeline definition: {} is already created", pipelineName); + return StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST; + } + std::unique_ptr pipelineDefinition = std::make_unique(pipelineName, nodeInfos, connections); + + pipelineDefinition->makeSubscriptions(manager); + Status validationResult = pipelineDefinition->validate(manager); + if (!validationResult.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline definition: {} failed: {}", pipelineName, validationResult.string()); + if (validationResult == StatusCode::PIPELINE_NAME_OCCUPIED) { + pipelineDefinition->resetSubscriptions(manager); + return validationResult; + } + } else { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Loading pipeline definition: {} succeeded", pipelineName); + } + + std::unique_lock lock(definitionsMtx); + definitions[pipelineName] = std::move(pipelineDefinition); + + return validationResult; +} + +template +Status PipelineFactory::createInternal(std::unique_ptr& pipeline, + const std::string& name, + const RequestType* request, + ResponseType* response, + ModelManager& manager) const { + if (!definitionExists(name)) { + SPDLOG_LOGGER_INFO(dag_executor_logger, "Pipeline with requested name: {} does not exist", name); + return StatusCode::PIPELINE_DEFINITION_NAME_MISSING; + } + std::shared_lock lock(definitionsMtx); + auto& definition = *definitions.at(name); + lock.unlock(); + return definition.create(pipeline, request, response, manager); +} +Status PipelineFactory::create(std::unique_ptr& pipeline, + const std::string& name, + const ::inference::ModelInferRequest* request, + ::inference::ModelInferResponse* response, + ModelManager& manager) const { + return this->createInternal(pipeline, name, request, response, manager); +} +Status PipelineFactory::create(std::unique_ptr& pipeline, + const std::string& name, + const tensorflow::serving::PredictRequest* request, + tensorflow::serving::PredictResponse* response, + ModelManager& manager) const { + return this->createInternal(pipeline, name, request, response, manager); +} + +Status PipelineFactory::reloadDefinition(const std::string& pipelineName, + const std::vector&& nodeInfos, + const pipeline_connections_t&& connections, + ModelManager& manager) { + auto pd = findDefinitionByName(pipelineName); + if (pd == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested to reload pipeline definition but it does not exist: {}", pipelineName); + return StatusCode::UNKNOWN_ERROR; + } + return pd->reload(manager, std::move(nodeInfos), std::move(connections)); +} + +Status PipelineFactory::revalidatePipelines(ModelManager& manager) { + Status firstErrorStatus = StatusCode::OK; + for (auto& [name, definition] : definitions) { + if (definition->getStatus().isRevalidationRequired()) { + auto validationResult = definition->validate(manager); + if (!validationResult.ok()) { + if (firstErrorStatus.ok()) { + firstErrorStatus = validationResult; + } + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Revalidation pipeline definition: {} failed: {}", name, validationResult.string()); + } else { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Revalidation of pipeline: {} succeeded", name); + } + } + } + return firstErrorStatus; +} +const std::vector PipelineFactory::getPipelinesNames() const { + std::vector names; + std::shared_lock lock(definitionsMtx); + names.reserve(definitions.size()); + for (auto& [name, definition] : definitions) { + names.push_back(definition->getName()); + } + return names; +} +} // namespace ovms diff --git a/src/ovms_lib/pipeline_factory.hpp b/src/ovms_lib/pipeline_factory.hpp new file mode 100644 index 0000000000..b3780b33a3 --- /dev/null +++ b/src/ovms_lib/pipeline_factory.hpp @@ -0,0 +1,84 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop +#include "kfs_grpc_inference_service.hpp" +#include "nodeinfo.hpp" +#include "status.hpp" + +namespace ovms { + +class ModelManager; +class Pipeline; +class PipelineDefinition; + +class PipelineFactory { + std::map> definitions; + mutable std::shared_mutex definitionsMtx; + +public: + Status createDefinition(const std::string& pipelineName, + const std::vector& nodeInfos, + const pipeline_connections_t& connections, + ModelManager& manager); + + bool definitionExists(const std::string& name) const; + +private: + template + Status createInternal(std::unique_ptr& pipeline, + const std::string& name, + const RequestType* request, + ResponseType* response, + ModelManager& manager) const; + +public: + Status create(std::unique_ptr& pipeline, + const std::string& name, + const ::inference::ModelInferRequest* request, + ::inference::ModelInferResponse* response, + ModelManager& manager) const; + Status create(std::unique_ptr& pipeline, + const std::string& name, + const tensorflow::serving::PredictRequest* request, + tensorflow::serving::PredictResponse* response, + ModelManager& manager) const; + + PipelineDefinition* findDefinitionByName(const std::string& name) const; + Status reloadDefinition(const std::string& pipelineName, + const std::vector&& nodeInfos, + const pipeline_connections_t&& connections, + ModelManager& manager); + + void retireOtherThan(std::set&& pipelinesInConfigFile, ModelManager& manager); + Status revalidatePipelines(ModelManager&); + const std::vector getPipelinesNames() const; +}; + +} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinition.cpp b/src/ovms_lib/pipelinedefinition.cpp new file mode 100644 index 0000000000..0c42bd3e83 --- /dev/null +++ b/src/ovms_lib/pipelinedefinition.cpp @@ -0,0 +1,1367 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pipelinedefinition.hpp" + +#include +#include +#include + +#include "custom_node.hpp" +#include "dl_node.hpp" +#include "entry_node.hpp" +#include "exit_node.hpp" +#include "logging.hpp" +#include "modelmanager.hpp" +#include "node_library_utils.hpp" +#include "pipeline.hpp" +#include "pipelinedefinitionunloadguard.hpp" +#include "prediction_service_utils.hpp" + +namespace ovms { + +Status toNodeKind(const std::string& str, NodeKind& nodeKind) { + if (str == DL_NODE_CONFIG_TYPE) { + nodeKind = NodeKind::DL; + return StatusCode::OK; + } + if (str == CUSTOM_NODE_CONFIG_TYPE) { + nodeKind = NodeKind::CUSTOM; + return StatusCode::OK; + } + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unsupported node type: {}", str); + return StatusCode::PIPELINE_NODE_WRONG_KIND_CONFIGURATION; +} + +Status PipelineDefinition::validate(ModelManager& manager) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of pipeline: {}", getName()); + ValidationResultNotifier notifier(status, loadedNotify); + auto& models = manager.getModels(); + if (std::find_if(models.begin(), models.end(), [this](auto pair) { return this->pipelineName == pair.first; }) != models.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline name: {} is already occupied by model.", pipelineName); + return StatusCode::PIPELINE_NAME_OCCUPIED; + } + + Status validationResult = initializeNodeResources(manager); + if (!validationResult.ok()) { + return validationResult; + } + validationResult = validateNodes(manager); + if (!validationResult.ok()) { + return validationResult; + } + validationResult = validateForCycles(); + if (!validationResult.ok()) { + return validationResult; + } + validationResult = validateDemultiplexerGatherNodesOrder(); + if (!validationResult.ok()) { + return validationResult; + } + std::unique_lock lock(metadataMtx); + validationResult = updateInputsInfo(manager); + if (!validationResult.ok()) { + return validationResult; + } + validationResult = updateOutputsInfo(manager); + if (!validationResult.ok()) { + return validationResult; + } + lock.unlock(); + notifier.passed = true; + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Finished validation of pipeline: {}", getName()); + SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} inputs: {}", getName(), getTensorMapString(inputsInfo)); + SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} outputs: {}", getName(), getTensorMapString(outputsInfo)); + return validationResult; +} + +Status PipelineDefinition::initializeNodeResources(ModelManager& manager) { + for (const auto& nodeInfo : nodeInfos) { + if (nodeInfo.kind == NodeKind::CUSTOM) { + void* customNodeLibraryInternalManager = nullptr; + auto params = createCustomNodeParamArray(nodeInfo.parameters); + int paramsLength = nodeInfo.parameters.size(); + if (!nodeInfo.library.isValid()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to invalid library", pipelineName, nodeInfo.nodeName); + return StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY; + } + auto status = nodeInfo.library.initialize(&customNodeLibraryInternalManager, params.get(), paramsLength); + if (status != 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Initialization of library with base path: {} failed", nodeInfo.library.basePath); + return StatusCode::NODE_LIBRARY_INITIALIZE_FAILED; + } + std::shared_ptr sharedCustomNodeLibraryInternalManager(new CNLIMWrapper{customNodeLibraryInternalManager, nodeInfo.library.deinitialize}); + manager.addResourceToCleaner(sharedCustomNodeLibraryInternalManager); + nodeResources.emplace(std::make_pair(nodeInfo.nodeName, std::move(sharedCustomNodeLibraryInternalManager))); + } + } + return StatusCode::OK; +} + +// returns NodeInfos that are in PipelineDefinition, but are not in nodeInfos(std::vector argument) +std::vector PipelineDefinition::calculateNodeInfosDiff(const std::vector& nodeInfos) { + std::vector diff; + for (const auto& nodeInfo : this->nodeInfos) { + auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), + [&nodeInfo](const auto& x) { return x.nodeName == nodeInfo.nodeName; }); + if (it == nodeInfos.end()) { + diff.push_back(nodeInfo); + } + } + return diff; +} + +void PipelineDefinition::deinitializeNodeResources(const std::vector& nodeInfosDiff) { + for (const auto& nodeInfo : nodeInfosDiff) { + if (nodeInfo.kind == NodeKind::CUSTOM) { + if (nodeResources.find(nodeInfo.nodeName) == nodeResources.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Library deinitialization of Node: {} failed. Couldn't find any initialized resources", nodeInfo.nodeName); + continue; + } + nodeResources.erase(nodeInfo.nodeName); + } + } +} + +Status PipelineDefinition::reload(ModelManager& manager, const std::vector&& nodeInfos, const pipeline_connections_t&& connections) { + // block creating new unloadGuards + this->status.handle(ReloadEvent()); + resetSubscriptions(manager); + while (requestsHandlesCounter > 0) { + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + // deinitialize all resources that are associated with nodes that are currently in PipelineDefinition, but not in nodeInfos + deinitializeNodeResources(calculateNodeInfosDiff(nodeInfos)); + this->nodeInfos = std::move(nodeInfos); + this->connections = std::move(connections); + makeSubscriptions(manager); + + return validate(manager); +} + +void PipelineDefinition::retire(ModelManager& manager) { + resetSubscriptions(manager); + this->status.handle(RetireEvent()); + while (requestsHandlesCounter > 0) { + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + // deinitalize all resources + deinitializeNodeResources(this->nodeInfos); + this->nodeResources.clear(); + this->nodeInfos.clear(); + this->connections.clear(); +} + +Status PipelineDefinition::waitForLoaded(std::unique_ptr& unloadGuard, const uint waitForLoadedTimeoutMicroseconds) { + unloadGuard = std::make_unique(*this); + + const uint waitLoadedTimestepMicroseconds = 100; + const uint waitCheckpoints = waitForLoadedTimeoutMicroseconds / waitLoadedTimestepMicroseconds; + uint waitCheckpointsCounter = waitCheckpoints; + std::mutex cvMtx; + std::unique_lock cvLock(cvMtx); + while (waitCheckpointsCounter-- != 0) { + if (status.isAvailable()) { + SPDLOG_DEBUG("Successfully waited for pipeline definition: {}", getName()); + return StatusCode::OK; + } + unloadGuard.reset(); + if (!status.canEndLoaded()) { + if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { + SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; + } else { + SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; + } + } + SPDLOG_DEBUG("Waiting for available state for pipeline: {}, with timestep: {}us timeout: {}us check count: {}", + getName(), waitLoadedTimestepMicroseconds, waitForLoadedTimeoutMicroseconds, waitCheckpointsCounter); + loadedNotify.wait_for(cvLock, + std::chrono::microseconds(waitLoadedTimestepMicroseconds), + [this]() { + return this->status.isAvailable() || + !this->status.canEndLoaded(); + }); + unloadGuard = std::make_unique(*this); + } + if (!status.isAvailable()) { + if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { + SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; + } else { + SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; + } + } + SPDLOG_DEBUG("Succesfully waited for pipeline definition: {}", getName()); + return StatusCode::OK; +} + +template +Status PipelineDefinition::create(std::unique_ptr& pipeline, + const RequestType* request, + ResponseType* response, + ModelManager& manager) { + std::unique_ptr unloadGuard; + Status status = waitForLoaded(unloadGuard); + if (!status.ok()) { + return status; + } + + std::unordered_map> nodes; + EntryNode* entry = nullptr; + ExitNode* exit = nullptr; + + for (const auto& info : nodeInfos) { + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Creating pipeline: {}. Adding nodeName: {}, modelName: {}", + getName(), info.nodeName, info.modelName); + switch (info.kind) { + case NodeKind::ENTRY: { + auto node = std::make_unique>(request, getInputsInfo(), info.demultiplyCount); + entry = node.get(); + nodes.emplace(info.nodeName, std::move(node)); + break; + } + case NodeKind::DL: + nodes.emplace(info.nodeName, std::make_unique( + info.nodeName, + info.modelName, + info.modelVersion, + manager, + info.outputNameAliases, + info.demultiplyCount, + info.gatherFromNode)); + break; + case NodeKind::CUSTOM: + nodes.emplace(info.nodeName, std::make_unique( + info.nodeName, + info.library, + info.parameters, + info.outputNameAliases, + info.demultiplyCount, + info.gatherFromNode, + nodeResources.at(info.nodeName))); + break; + case NodeKind::EXIT: { + auto node = std::make_unique>(response, getOutputsInfo(), info.gatherFromNode); + exit = node.get(); + nodes.emplace(info.nodeName, std::move(node)); + break; + } + default: + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Requested pipeline: {} contains unknown node kind", getName()); + throw std::invalid_argument("unknown node kind"); + } + } + for (const auto& kv : connections) { + const auto& dependantNode = nodes.at(kv.first); + for (const auto& pair : kv.second) { + const auto& dependencyNode = nodes.at(pair.first); + SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Connecting pipeline: {}, from: {}, to: {}", getName(), dependencyNode->getName(), dependantNode->getName()); + Pipeline::connect(*dependencyNode, *dependantNode, pair.second); + } + } + pipeline = std::make_unique(*entry, *exit, pipelineName); + for (auto& kv : nodes) { + pipeline->push(std::move(kv.second)); + } + return status; +} + +void PipelineDefinition::resetSubscriptions(ModelManager& manager) { + for (auto& [modelName, modelVersion] : subscriptions) { + if (modelVersion) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Unsubscribing pipeline: {} from model: {}, version: {}", + getName(), modelName, modelVersion); + manager.findModelByName(modelName)->getModelInstanceByVersion(modelVersion)->unsubscribe(*this); + } else { // using default version + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Unsubscribing pipeline: {} from model: {}", + getName(), modelName); + manager.findModelByName(modelName)->unsubscribe(*this); + } + } + subscriptions.clear(); +} + +static std::string createSubscriptionErrorMessage(const std::string& pipelineName, const NodeInfo& nodeInfo) { + std::stringstream ss; + ss << "Pipeline: " << pipelineName << " Failed to make subscription to model: " << nodeInfo.modelName; + if (nodeInfo.modelVersion) { + ss << " version: " << nodeInfo.modelVersion.value(); + } + ss << " because it was missing"; + return ss.str(); +} + +void PipelineDefinition::makeSubscriptions(ModelManager& manager) { + for (auto& node : nodeInfos) { + if (node.kind == NodeKind::DL) { + if (subscriptions.find({node.modelName, node.modelVersion.value_or(0)}) != subscriptions.end()) { + continue; + } + auto model = manager.findModelByName(node.modelName); + if (nullptr == model) { + SPDLOG_LOGGER_WARN(modelmanager_logger, createSubscriptionErrorMessage(getName(), node)); + continue; + } + if (node.modelVersion) { + auto modelInstance = model->getModelInstanceByVersion(node.modelVersion.value()); + if (nullptr == modelInstance) { + SPDLOG_LOGGER_WARN(modelmanager_logger, createSubscriptionErrorMessage(getName(), node)); + continue; + } + modelInstance->subscribe(*this); + } else { + model->subscribe(*this); + } + subscriptions.insert({node.modelName, node.modelVersion.value_or(0)}); + } + } +} + +class NodeValidator { + const std::string& pipelineName; + ModelManager& manager; + const NodeInfo& dependantNodeInfo; + const pipeline_connections_t& connections; + const std::vector& nodeInfos; + std::map>& nodeResources; + const bool isMultiBatchAllowed; + + std::unique_ptr dependantModelUnloadGuard; + std::shared_ptr dependantModelInstance; + std::set remainingUnconnectedDependantInputs; + + tensor_map_t inputsInfo, outputsInfo; + tensor_map_t dependencyInputsInfo, dependencyOutputsInfo; + +public: + NodeValidator( + const std::string& pipelineName, + ModelManager& manager, + const NodeInfo& dependantNodeInfo, + const pipeline_connections_t& connections, + const std::vector& nodeInfos, + std::map>& nodeResources, + const bool isMultiBatchAllowed = true) : + pipelineName(pipelineName), + manager(manager), + dependantNodeInfo(dependantNodeInfo), + connections(connections), + nodeInfos(nodeInfos), + nodeResources(nodeResources), + isMultiBatchAllowed(isMultiBatchAllowed) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validation of pipeline: {}; node name: {}; node kind: {}", + pipelineName, + dependantNodeInfo.nodeName, + dependantNodeInfo.kind); + } + + Status fetchUnderlyingModelInstance() { + if (!manager.getModelInstance( + dependantNodeInfo.modelName, + dependantNodeInfo.modelVersion.value_or(0), + dependantModelInstance, + dependantModelUnloadGuard) + .ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing model: {}; version: {}", + pipelineName, + dependantNodeInfo.modelName, + dependantNodeInfo.modelVersion.value_or(0)); + return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL; + } + return StatusCode::OK; + } + + Status getDependencyNodeInfo(const std::string& dependencyNodeName, std::vector::const_iterator& dependencyNodeInfo) { + // Find dependency node info object. + dependencyNodeInfo = std::find_if( + std::begin(this->nodeInfos), + std::end(this->nodeInfos), + [dependencyNodeName](const NodeInfo& nodeInfo) { return nodeInfo.nodeName == dependencyNodeName; }); + if (dependencyNodeInfo == std::end(this->nodeInfos)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node (name: {}) is connected to missing dependency node (name: {})", + pipelineName, + dependantNodeInfo.nodeName, + dependencyNodeName); + return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_NODE; + } + + if (dependencyNodeInfo->kind == NodeKind::EXIT) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Exit node used as dependency node", + pipelineName); + return StatusCode::PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY; + } + + return StatusCode::OK; + } + + Status checkForForbiddenDynamicParameters() { + const auto& config = dependantModelInstance->getModelConfig(); + if (config.getBatchingMode() == Mode::AUTO || config.anyShapeSetToAuto()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node name: {} used model name: {} with batch/shape parameter set to 'auto' which is forbidden. Use dynamic shape.", + pipelineName, + dependantNodeInfo.nodeName, + dependantNodeInfo.modelName); + return StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER; + } + return StatusCode::OK; + } + + Status validateGatherNode(const NodeInfo& dependantNodeInfo) const { + for (const auto& gather : dependantNodeInfo.gatherFromNode) { + auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), [gather](const NodeInfo& nodeInfo) { return nodeInfo.nodeName == gather; }); + if (it == nodeInfos.end()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Node name: {}, have gather_from: {} which does not exist in pipeline", + pipelineName, + dependantNodeInfo.nodeName, + gather); + return StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE; + } + if (!it->demultiplyCount) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Node name: {}, have gather_from: {} which is not demultiplexer node", + pipelineName, + dependantNodeInfo.nodeName, + gather); + return StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER; + } + } + return StatusCode::OK; + } + + Status checkConnectionMappedToExistingDataSource(const NodeInfo& dependencyNodeInfo, const std::string& dataSource) { + // Check whether dependency node is configured to have required output. + if (dependencyNodeInfo.outputNameAliases.count(dataSource) == 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing dependency node: {} data item: {} for dependant node: {}", + pipelineName, + dependencyNodeInfo.nodeName, + dataSource, + dependantNodeInfo.nodeName); + return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE; + } + + // If dependency node is of type DL model, make sure there is underlying model output present. + if (dependencyNodeInfo.kind == NodeKind::DL || dependencyNodeInfo.kind == NodeKind::CUSTOM) { + // Check whether underlying model contains required output. + const auto& modelOutputName = dependencyNodeInfo.outputNameAliases.at(dataSource); + if (this->dependencyOutputsInfo.count(modelOutputName) == 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing output: {} of dependency node: {}; data source: {}", + pipelineName, + modelOutputName, + dependencyNodeInfo.nodeName, + dataSource); + return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT; + } + } + return StatusCode::OK; + } + + Status validateShapeWithDemultiplexer(const Shape& shape, const NodeInfo& demultiplicatorNodeInfo) const { + if (!demultiplicatorNodeInfo.demultiplyCount) { + return StatusCode::OK; + } + if (shape.size() < 3) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} demultiply cannot occur due to not enough shape dimensions: {}", + this->pipelineName, + demultiplicatorNodeInfo.nodeName, + shape.size()); + return StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY; + } + if (demultiplicatorNodeInfo.demultiplyCount.value() != -1) { + if (!shape[0].isAny()) { + auto demultiplyDimension = Dimension(demultiplicatorNodeInfo.demultiplyCount.value()); + if (!shape[0].partiallyFitsInto(demultiplyDimension)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of node: {} does not match tensor first dimension value: {}", + this->pipelineName, + demultiplicatorNodeInfo.demultiplyCount.value(), + demultiplicatorNodeInfo.nodeName, + shape[0].toString()); + return StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT; + } + } else { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is fixed while first dimenson value of node library is not: {}. This pipeline may fail at execution stage.", + this->pipelineName, + demultiplicatorNodeInfo.demultiplyCount.value(), + demultiplicatorNodeInfo.nodeName, + shape[0].toString()); + } + } else if (!shape[0].isAny()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is dynamic while first dimenson value of gather node is not: {}. This pipeline may fail at execution stage.", + this->pipelineName, + demultiplicatorNodeInfo.demultiplyCount.value(), + demultiplicatorNodeInfo.nodeName, + shape[0].toString()); + } + return StatusCode::OK; + } + + Status influenceShapeWithDemultiplexer(Shape& shape, const NodeInfo& demultiplicatorNodeInfo) { + auto result = validateShapeWithDemultiplexer(shape, demultiplicatorNodeInfo); + if (!result.ok()) { + return result; + } + shape.erase(shape.begin()); + return StatusCode::OK; + } + + bool areShapesMatching(const Shape& tensorInputShape, const Shape& tensorOutputShape) { + if (tensorInputShape.size() != tensorOutputShape.size()) { + return false; + } + + for (size_t i = 0; i < tensorInputShape.size(); i++) { + if (!tensorInputShape[i].partiallyFitsInto(tensorOutputShape[i])) { + return false; + } + } + return true; + } + + Status checkConnectionMetadataCorrectness(const NodeInfo& dependencyNodeInfo, const std::string& modelInputName, const std::string& modelOutputName) { + // If validated connection pair connects two DL model/Custom nodes, + // check if both input/output exist and its metadata (shape, precision) matches. + // Affect shape by demultiplexer/gather if applies. + const auto& tensorInput = this->inputsInfo.at(modelInputName); + const auto& tensorOutput = this->dependencyOutputsInfo.at(modelOutputName); + Shape tensorInputShape = tensorInput->getShape(); + Shape tensorOutputShape = tensorOutput->getShape(); + if (dependencyNodeInfo.demultiplyCount) { + auto result = influenceShapeWithDemultiplexer(tensorOutputShape, dependencyNodeInfo); + if (!result.ok()) { + return result; + } + } + if (dependantNodeInfo.gatherFromNode.size() == 1) { + std::vector::const_iterator demultiplicatorNode; + auto result = getDependencyNodeInfo(*dependantNodeInfo.gatherFromNode.begin(), demultiplicatorNode); + if (!result.ok()) { + return result; + } + result = influenceShapeWithDemultiplexer(tensorInputShape, *demultiplicatorNode); + if (!result.ok()) { + SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of gather_from node: {} does not match tensor first dimenson value: {} of node: {}", + this->pipelineName, + demultiplicatorNode->demultiplyCount.value(), + demultiplicatorNode->nodeName, + tensorInputShape[1].toString(), + dependencyNodeInfo.nodeName); + return result; + } + } else if (dependantNodeInfo.gatherFromNode.size() > 1) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Manual gathering from multiple nodes is not supported in node name: {}", + this->pipelineName, + dependantNodeInfo.nodeName); + return StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED; + } + if (!areShapesMatching(tensorInputShape, tensorOutputShape)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Shape mismatch between: dependant node: {}; input: {}; shape: {} vs dependency node: {}; output: {}; shape: {}", + pipelineName, + dependantNodeInfo.nodeName, + modelInputName, + tensorInputShape.toString(), + dependencyNodeInfo.nodeName, + modelOutputName, + tensorOutputShape.toString()); + return StatusCode::INVALID_SHAPE; + } + if (tensorInput->getPrecision() != tensorOutput->getPrecision()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Precision mismatch between: dependant node: {}; input: {}; precision: {} vs dependency node: {}; output: {}; precision: {}", + pipelineName, + dependantNodeInfo.nodeName, + modelInputName, + tensorInput->getPrecisionAsString(), + dependencyNodeInfo.nodeName, + modelOutputName, + tensorOutput->getPrecisionAsString()); + return StatusCode::INVALID_PRECISION; + } + return StatusCode::OK; + } + + void prepareRemainingUnconnectedDependantInputsSet() { + // Save set of inputs which are required by underlying model/custom node of currently validated node. + // This is later used to make sure we feed each input exactly one data source. + std::transform( + this->inputsInfo.begin(), + this->inputsInfo.end(), + std::inserter( + remainingUnconnectedDependantInputs, + remainingUnconnectedDependantInputs.end()), + [](auto pair) { return pair.first; }); + } + + Status ensureAllModelInputsOfValidatedNodeHaveDataSource() { + // Make sure all model inputs of validated node is fed with some data source. + if (remainingUnconnectedDependantInputs.size() > 0) { + std::stringstream ss; + for (const auto& input : remainingUnconnectedDependantInputs) { + ss << input << ", "; + } + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} has inputs:: {} not connected to any source", + pipelineName, + dependantNodeInfo.nodeName, + ss.str()); + return StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED; + } + return StatusCode::OK; + } + + Status markInputAsConnected(const std::string& name) { + // If currently validated node is of type DL model or Custom, mark its input as connected + // by erasing from previously gathered input set. + // If such input cannot be found in the map, it means we refer + // to non existing model input or we already connected it to some other data source which is invalid. + if (this->inputsInfo.count(name) == 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} has no input with name: {}", + pipelineName, + dependantNodeInfo.nodeName, + name); + return StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT; + } + if (remainingUnconnectedDependantInputs.erase(name) == 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} input name: {} is connected to more than one data source", + pipelineName, + dependantNodeInfo.nodeName, + name); + return StatusCode::PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES; + } + return StatusCode::OK; + } + + Status validateConnection(const NodeInfo& dependencyNodeInfo, const Aliases& mapping) { + // At this point dependency node can only be either DL model node, Custom node or entry node. + // Take care when adding new node types. + std::unique_ptr dependencyModelUnloadGuard; + std::shared_ptr dependencyModelInstance; + if (dependencyNodeInfo.kind == NodeKind::DL) { + if (!manager.getModelInstance( + dependencyNodeInfo.modelName, + dependencyNodeInfo.modelVersion.value_or(0), + dependencyModelInstance, + dependencyModelUnloadGuard) + .ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Dependency DL model node refers to unavailable model - name: {}; version: {}", + pipelineName, + dependencyNodeInfo.modelName, + dependencyNodeInfo.modelVersion.value_or(0)); + return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL; + } + retrieveModelNodeDependencyMetadata(dependencyModelInstance); + } + + if (dependencyNodeInfo.kind == NodeKind::CUSTOM) { + auto result = retrieveCustomNodeDependencyMetadata(dependencyNodeInfo); + if (!result.ok()) { + return result; + } + } + + for (const auto& [alias, realName] : mapping) { + if (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) { + auto result = markInputAsConnected(realName); + if (!result.ok()) { + return result; + } + } + + auto result = checkConnectionMappedToExistingDataSource(dependencyNodeInfo, alias); + if (!result.ok()) { + return result; + } + + if ( + (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) && + (dependencyNodeInfo.kind == NodeKind::DL || dependencyNodeInfo.kind == NodeKind::CUSTOM)) { + result = checkConnectionMetadataCorrectness(dependencyNodeInfo, realName, dependencyNodeInfo.outputNameAliases.at(alias)); + if (!result.ok()) { + return result; + } + } + } + + return StatusCode::OK; + } + + Status retrieveDependantMetadata() { + if (dependantNodeInfo.kind == NodeKind::DL) { + this->inputsInfo = this->dependantModelInstance->getInputsInfo(); + this->outputsInfo = this->dependantModelInstance->getOutputsInfo(); + return StatusCode::OK; + } else if (dependantNodeInfo.kind == NodeKind::CUSTOM) { + auto result = PipelineDefinition::getCustomNodeMetadata( + dependantNodeInfo, + this->inputsInfo, + dependantNodeInfo.library.getInputsInfo, + this->pipelineName, + getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); + if (!result.ok()) { + return result; + } + result = PipelineDefinition::getCustomNodeMetadata( + dependantNodeInfo, + this->outputsInfo, + dependantNodeInfo.library.getOutputsInfo, + this->pipelineName, + getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); + if (!result.ok()) { + return result; + } + } + return StatusCode::OK; + } + + void retrieveModelNodeDependencyMetadata(const std::shared_ptr& dependencyModelInstance) { + this->dependencyInputsInfo = dependencyModelInstance->getInputsInfo(); + this->dependencyOutputsInfo = dependencyModelInstance->getOutputsInfo(); + } + + Status retrieveCustomNodeDependencyMetadata(const NodeInfo& dependencyNodeInfo) { + auto result = PipelineDefinition::getCustomNodeMetadata( + dependencyNodeInfo, + this->dependencyInputsInfo, + dependencyNodeInfo.library.getInputsInfo, + this->pipelineName, + getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); + if (!result.ok()) { + return result; + } + result = PipelineDefinition::getCustomNodeMetadata( + dependencyNodeInfo, + this->dependencyOutputsInfo, + dependencyNodeInfo.library.getOutputsInfo, + this->pipelineName, + getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); + if (!result.ok()) { + return result; + } + return StatusCode::OK; + } + + Status validate() { + if (dependantNodeInfo.kind == NodeKind::DL) { + auto result = fetchUnderlyingModelInstance(); + if (!result.ok()) { + return result; + } + + result = retrieveDependantMetadata(); + if (!result.ok()) { + return result; + } + + result = checkForForbiddenDynamicParameters(); + if (!result.ok()) { + return result; + } + + prepareRemainingUnconnectedDependantInputsSet(); + } + + if (dependantNodeInfo.kind == NodeKind::CUSTOM) { + if (!dependantNodeInfo.library.isValid()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to incorrect library", pipelineName, dependantNodeInfo.nodeName); + return StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY; + } + + auto result = retrieveDependantMetadata(); + if (!result.ok()) { + return result; + } + + prepareRemainingUnconnectedDependantInputsSet(); + } + + if (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) { + for (const auto& [name, tensorOutput] : outputsInfo) { + auto result = validateShapeWithDemultiplexer(tensorOutput->getShape(), dependantNodeInfo); + if (!result.ok()) { + return result; + } + } + } + + if (!dependantNodeInfo.gatherFromNode.empty()) { + auto result = validateGatherNode(dependantNodeInfo); + if (!result.ok()) { + return result; + } + } + auto it = connections.find(dependantNodeInfo.nodeName); + if (it != connections.end()) { + for (const auto& [dependencyNodeName, mapping] : it->second) { + if (mapping.size() == 0) { + return StatusCode::UNKNOWN_ERROR; + } + + this->dependencyInputsInfo.clear(); + this->dependencyOutputsInfo.clear(); + std::vector::const_iterator dependencyNodeInfo; + auto result = getDependencyNodeInfo(dependencyNodeName, dependencyNodeInfo); + if (!result.ok()) { + return result; + } + + result = validateConnection(*dependencyNodeInfo, mapping); + if (!result.ok()) { + return result; + } + } + } + + return ensureAllModelInputsOfValidatedNodeHaveDataSource(); + } +}; + +Status PipelineDefinition::validateNode(ModelManager& manager, const NodeInfo& dependantNodeInfo, const bool isMultiBatchAllowed) { + NodeValidator validator(this->pipelineName, manager, dependantNodeInfo, connections, nodeInfos, nodeResources, isMultiBatchAllowed); + return validator.validate(); +} + +// Because of the way how pipeline_connections is implemented, this function is using +// transpose of PipelineDefinition graph.(Transpose contains same cycles as original graph) +Status PipelineDefinition::validateForCycles() { + std::vector visited; + std::vector parentNodes; + visited.reserve(nodeInfos.size()); + parentNodes.reserve(nodeInfos.size()); + + auto pred = [](const NodeInfo& nodeInfo) { + return nodeInfo.kind == NodeKind::EXIT; + }; + + const auto& itr = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), pred); + if (itr == nodeInfos.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} does not contain response node.", getName()); + return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; + } + std::string nodeName = itr->nodeName; + visited.push_back(nodeName); + + bool anyUnvisitedLeft = true; + while (anyUnvisitedLeft) { + bool unvisistedFound = false; + const auto& connectedToNode = connections[nodeName]; + for (const auto& node : connectedToNode) { + if (nodeName == node.first) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Node: {} is connected to itself in pipeline: {}", nodeName, getName()); + return StatusCode::PIPELINE_CYCLE_FOUND; + } + + if (std::find(visited.begin(), visited.end(), node.first) == visited.end()) { + parentNodes.push_back(nodeName); + visited.push_back(node.first); + nodeName = node.first; + unvisistedFound = true; + break; + } else { + if (std::find(parentNodes.begin(), parentNodes.end(), node.first) != parentNodes.end()) { + std::string cycleNodes; + for (auto& cycleNode : parentNodes) { + cycleNodes += cycleNode; + if (cycleNode != parentNodes.back()) { + cycleNodes += ", "; + } + } + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {}, following nodes creates cycle: {}", getName(), cycleNodes); + return StatusCode::PIPELINE_CYCLE_FOUND; + } + } + } + + if (!unvisistedFound) { + if (parentNodes.size() == 0) { + anyUnvisitedLeft = false; + if (visited.size() != nodeInfos.size()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {}, there are not connected nodes", getName()); + return StatusCode::PIPELINE_CONTAINS_UNCONNECTED_NODES; + } + } else { + nodeName = parentNodes.back(); + parentNodes.pop_back(); + } + } + } + return StatusCode::OK; +} + +Status PipelineDefinition::validateDemultiplexerGatherNodesOrder() { + auto exitNode = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), [](const NodeInfo& nodeInfo) { return nodeInfo.kind == NodeKind::EXIT; }); + using gatherFromNode_t = std::set; + using demultiplyStack_t = std::vector; + std::vector> nodesToCheck{{exitNode->nodeName, {exitNode->gatherFromNode}}}; + if (exitNode->gatherFromNode.empty()) { + nodesToCheck.back().second.clear(); + } + std::map visitedNodes; + while (!nodesToCheck.empty()) { + auto [nodeName, demultiplyStack] = nodesToCheck.back(); + nodesToCheck.pop_back(); + for (auto& [connectedNodeName, aliasName] : connections[nodeName]) { + auto newDemultiplyStack(demultiplyStack); + auto& connectedNodeInfo = findNodeByName(connectedNodeName); + if (connectedNodeInfo.demultiplyCount) { + if (newDemultiplyStack.empty()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path that doesn't gather from demultiplexer node: {}, connection to node: {}.", getName(), connectedNodeName, nodeName); + return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; + } + auto& lastGatherSet = newDemultiplyStack.back(); + if (lastGatherSet.find(connectedNodeName) == lastGatherSet.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path where after demultiplexer node: {} there is gathering from different nodes: {}.", + getName(), + connectedNodeName, + std::accumulate(lastGatherSet.begin(), lastGatherSet.end(), std::string{}, [](const std::string& lhs, const std::string& rhs) { + if (lhs.empty()) { + return rhs; + } + return lhs + ", " + rhs; })); + return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; + } + lastGatherSet.erase(connectedNodeName); + if (lastGatherSet.empty()) { + newDemultiplyStack.pop_back(); + } + } + if (!connectedNodeInfo.gatherFromNode.empty()) { + newDemultiplyStack.emplace_back(connectedNodeInfo.gatherFromNode); + } + if (connectedNodeInfo.kind == NodeKind::ENTRY && !newDemultiplyStack.empty()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path that gathers from nodes that are not in path: {}. Consider changing inputs of the node that gathers from mentioned demultiplexer nodes", + getName(), + std::accumulate(newDemultiplyStack.back().begin(), newDemultiplyStack.back().end(), std::string{}, [](const std::string& lhs, const std::string& rhs) { + if (lhs.empty()) { + return rhs; + } + return lhs + ", " + rhs; })); + return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; + } + auto visitedNode = std::find_if(std::begin(visitedNodes), std::end(visitedNodes), + [&connectedNodeName](const auto& visitedNode) { return visitedNode.first == connectedNodeName; }); + if (visitedNode != visitedNodes.end()) { + if (visitedNode->second != newDemultiplyStack) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} after node: {} exist paths that have different demultiply levels. Consider changing output connections of node: {}", getName(), connectedNodeName, connectedNodeName); + return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; + } + } else { + nodesToCheck.emplace_back(std::pair{connectedNodeName, newDemultiplyStack}); + visitedNodes.emplace(connectedNodeName, std::move(newDemultiplyStack)); + } + } + } + return StatusCode::OK; +} + +Status PipelineDefinition::validateNodes(ModelManager& manager) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validation of pipeline definition: {} nodes started.", getName()); + + int entryNodeCount = std::count_if( + this->nodeInfos.begin(), + this->nodeInfos.end(), + [](const NodeInfo& info) { return info.kind == NodeKind::ENTRY; }); + + int exitNodeCount = std::count_if( + this->nodeInfos.begin(), + this->nodeInfos.end(), + [](const NodeInfo& info) { return info.kind == NodeKind::EXIT; }); + + if (entryNodeCount <= 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing request node", pipelineName); + return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; + } + + if (exitNodeCount <= 0) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing response node", pipelineName); + return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; + } + + if (entryNodeCount > 1) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple request nodes", pipelineName); + return StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES; + } + + if (exitNodeCount > 1) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple response nodes", pipelineName); + return StatusCode::PIPELINE_MULTIPLE_EXIT_NODES; + } + + bool isAnyNodeDynamicDemultiplexer = (std::find_if(this->nodeInfos.begin(), this->nodeInfos.end(), [](const NodeInfo& info) { + if (info.demultiplyCount) { + if (info.demultiplyCount.value() == -1) + return true; + return false; + } + return false; + }) != this->nodeInfos.end()); + int demultiplexerCount = std::count_if( + this->nodeInfos.begin(), + this->nodeInfos.end(), + [](const NodeInfo& info) { return info.demultiplyCount.has_value(); }); + if (isAnyNodeDynamicDemultiplexer && (demultiplexerCount > 1)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple demultiplexers with at least one dynamic.", pipelineName); + return StatusCode::NOT_IMPLEMENTED; + } + + const bool isMultiBatchAllowed = !std::any_of(nodeInfos.begin(), nodeInfos.end(), [](const auto& node) { return node.demultiplyCount; }); + for (const auto& node : nodeInfos) { + auto findByName = [node](const NodeInfo& nodeInfo) { + return nodeInfo.nodeName == node.nodeName; + }; + + if (std::count_if(nodeInfos.begin(), nodeInfos.end(), findByName) > 1) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple nodes with name: {}", pipelineName, node.nodeName); + return StatusCode::PIPELINE_NODE_NAME_DUPLICATE; + } + + auto result = validateNode(manager, node, isMultiBatchAllowed); + if (!result.ok()) { + return result; + } + } + return StatusCode::OK; +} + +const tensor_map_t PipelineDefinition::getInputsInfo() const { + std::shared_lock lock(metadataMtx); + tensor_map_t copy = inputsInfo; + return copy; +} + +const tensor_map_t PipelineDefinition::getOutputsInfo() const { + std::shared_lock lock(metadataMtx); + tensor_map_t copy = outputsInfo; + return copy; +} + +std::shared_ptr applyDemultiplexerShapeForTensor(const std::shared_ptr& tensorInfo, int32_t demultiplyCount) { + return tensorInfo->createCopyWithDemultiplexerDimensionPrefix(demultiplyCount ? Dimension(demultiplyCount) : Dimension::any()); +} + +std::shared_ptr createOutputTensorInfoForPipeline(const std::string& mappedName, const std::shared_ptr& tensorInfo, const Shape& gatherShape, bool isConnectionFromDemultiplexer) { + std::shared_ptr newOwnedTensorInfo; + if (gatherShape.size() == 0) { + newOwnedTensorInfo = std::make_shared(*tensorInfo); + newOwnedTensorInfo->setMappedName(mappedName); + return newOwnedTensorInfo; + } + Shape newShape = tensorInfo->getShape(); + if (isConnectionFromDemultiplexer) { + newShape.erase(newShape.begin()); + } + newShape.insert(newShape.begin(), gatherShape.begin(), gatherShape.end()); + newOwnedTensorInfo = tensorInfo->createCopyWithNewShape(newShape); + newOwnedTensorInfo->setMappedName(mappedName); + return newOwnedTensorInfo; +} + +Status updateInputsInfoWithNodeConnection(tensor_map_t& inputsInfo, const TensorInfo& tensorInfo, const std::string& alias) { + auto newTensorInfo = std::make_shared(alias, tensorInfo.getPrecision(), tensorInfo.getShape(), tensorInfo.getLayout()); + auto it = inputsInfo.find(alias); + if (it != inputsInfo.end()) { + if (!it->second->isTensorSpecEqual(*newTensorInfo)) { + auto intersectionTensorInfo = it->second->createIntersection(*newTensorInfo); + if (intersectionTensorInfo == nullptr) { + Status status = StatusCode::PIPELINE_INPUTS_AMBIGUOUS_METADATA; + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error validating pipeline: {};\n{}\n{}", + status.string(), + it->second->asString(), + newTensorInfo->asString()); + return status; + } + inputsInfo[alias] = intersectionTensorInfo; + return StatusCode::OK; + } + } + inputsInfo[alias] = newTensorInfo; + return StatusCode::OK; +} + +template +Status updateInputsInfoWithNodeConnections(tensor_map_t& inputsInfo, const Aliases& specificDependencyMapping, Extractor extractor) { + for (const auto& [alias, realName] : specificDependencyMapping) { + auto status = updateInputsInfoWithNodeConnection(inputsInfo, extractor(realName), alias); + if (!status.ok()) { + return status; + } + } + return StatusCode::OK; +} + +Status PipelineDefinition::updateInputsInfo(const ModelManager& manager) { + // Assumptions: this can only be called on available pipeline definition. + // Add check if available when pipeline status will be implemented. + inputsInfo.clear(); + static const auto byName = [](const std::string& name) { + return [name](const NodeInfo& nodeInfo) { + return nodeInfo.nodeName == name; + }; + }; + for (const auto& [dependantNodeName, allMappings] : connections) { + const auto& dependantNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependantNodeName)); + for (const auto& [dependencyNodeName, specificDependencyMapping] : allMappings) { + const auto& dependencyNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependencyNodeName)); + if (dependencyNodeInfo->kind != NodeKind::ENTRY) { + continue; + } + + switch (dependantNodeInfo->kind) { + case NodeKind::EXIT: { + for (const auto& [alias, realName] : specificDependencyMapping) { + inputsInfo.insert({alias, TensorInfo::getUnspecifiedTensorInfo()}); + } + break; + } + case NodeKind::DL: { + auto instance = manager.findModelInstance(dependantNodeInfo->modelName, dependantNodeInfo->modelVersion.value_or(0)); + if (!instance) { + SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} inputs info fetching", dependantNodeInfo->modelName, this->getName()); + return StatusCode::MODEL_MISSING; + } + std::unique_ptr unloadGuard; + auto status = instance->waitForLoaded(0, unloadGuard); + if (!status.ok()) { + SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} inputs info fetching", instance->getName(), this->getName()); + return status; + } + status = updateInputsInfoWithNodeConnections(inputsInfo, + specificDependencyMapping, + [&instance](const std::string& realName) { + return *instance->getInputsInfo().at(realName); + }); + if (!status.ok()) { + return status; + } + break; + } + case NodeKind::CUSTOM: { + if (!dependantNodeInfo->library.isValid()) { + return StatusCode::NODE_LIBRARY_MISSING; + } + + tensor_map_t info; + auto status = getCustomNodeMetadata(*dependantNodeInfo, info, dependantNodeInfo->library.getInputsInfo, this->getName(), + getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo->nodeName))); + if (!status.ok()) { + return status; + } + + status = updateInputsInfoWithNodeConnections(inputsInfo, + specificDependencyMapping, + [&info](const std::string& realName) { + return *info.at(realName); + }); + if (!status.ok()) { + return status; + } + break; + } + default: { + // Pipeline validation does not allow connections into entry node. + SPDLOG_ERROR("Unexpected dependant node kind (name: {})", this->getName()); + return StatusCode::UNKNOWN_ERROR; + } + } + } + } + auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), [](const NodeInfo& info) { return info.kind == NodeKind::ENTRY && info.demultiplyCount; }); + if (it != nodeInfos.end()) { + int32_t demultiplyCount = it->demultiplyCount.value(); + for (auto& [inputName, inputTensorInfo] : inputsInfo) { + inputTensorInfo = applyDemultiplexerShapeForTensor(inputTensorInfo, demultiplyCount); + } + } + return StatusCode::OK; +} + +Status PipelineDefinition::populateOutputsInfoWithDLModelOutputs(const NodeInfo& dependencyNodeInfo, const ModelManager& manager, tensor_map_t& outputsInfo, const Aliases& specificDependencyMapping, const Shape& gatherShape) const { + auto instance = manager.findModelInstance(dependencyNodeInfo.modelName, dependencyNodeInfo.modelVersion.value_or(0)); + if (!instance) { + SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} outputs info fetching", dependencyNodeInfo.modelName, this->getName()); + return StatusCode::MODEL_MISSING; + } + std::unique_ptr unloadGuard; + auto status = instance->waitForLoaded(0, unloadGuard); + if (!status.ok()) { + SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} outputs info fetching", instance->getName(), this->getName()); + return status; + } + for (const auto& [alias, realName] : specificDependencyMapping) { + const auto& finalName = dependencyNodeInfo.outputNameAliases.count(alias) > 0 ? dependencyNodeInfo.outputNameAliases.at(alias) : alias; + outputsInfo[realName] = createOutputTensorInfoForPipeline(realName, instance->getOutputsInfo().at(finalName), gatherShape, dependencyNodeInfo.demultiplyCount.has_value()); + } + return StatusCode::OK; +} + +Status PipelineDefinition::populateOutputsInfoWithCustomNodeOutputs(const NodeInfo& dependencyNodeInfo, const ModelManager& manager, tensor_map_t& outputsInfo, const Aliases& specificDependencyMapping, const Shape& gatherShape) const { + if (!dependencyNodeInfo.library.isValid()) { + return StatusCode::NODE_LIBRARY_MISSING; + } + tensor_map_t info; + auto status = getCustomNodeMetadata(dependencyNodeInfo, info, dependencyNodeInfo.library.getOutputsInfo, this->getName(), + getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); + if (!status.ok()) { + return status; + } + for (const auto& [alias, realName] : specificDependencyMapping) { + const auto& finalName = dependencyNodeInfo.outputNameAliases.count(alias) > 0 ? dependencyNodeInfo.outputNameAliases.at(alias) : alias; + outputsInfo[realName] = createOutputTensorInfoForPipeline(realName, info.at(finalName), gatherShape, dependencyNodeInfo.demultiplyCount.has_value()); + } + return StatusCode::OK; +} + +Status PipelineDefinition::updateOutputsInfo(const ModelManager& manager) { + // Assumptions: this can only be called on available pipeline definition. + // Add check if available when pipeline status will be implemented. + outputsInfo.clear(); + static const auto byName = [](const std::string& name) { + return [name](const NodeInfo& nodeInfo) { + return nodeInfo.nodeName == name; + }; + }; + for (const auto& [dependantNodeName, allMappings] : connections) { + const auto& dependantNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependantNodeName)); + if (dependantNodeInfo->kind != NodeKind::EXIT) { + continue; + } + + auto gatherShape = this->getNodeGatherShape(*dependantNodeInfo); + + for (const auto& [dependencyNodeName, specificDependencyMapping] : allMappings) { + const auto& dependencyNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependencyNodeName)); + + switch (dependencyNodeInfo->kind) { + case NodeKind::ENTRY: { + for (const auto& [alias, realName] : specificDependencyMapping) { + outputsInfo.insert({realName, TensorInfo::getUnspecifiedTensorInfo()}); + } + break; + } + case NodeKind::DL: { + auto status = populateOutputsInfoWithDLModelOutputs( + *dependencyNodeInfo, manager, outputsInfo, specificDependencyMapping, gatherShape); + if (!status.ok()) { + return status; + } + break; + } + case NodeKind::CUSTOM: { + auto status = populateOutputsInfoWithCustomNodeOutputs( + *dependencyNodeInfo, manager, outputsInfo, specificDependencyMapping, gatherShape); + if (!status.ok()) { + return status; + } + break; + } + default: { + // Pipeline validation does not allow connections from exit node. + SPDLOG_ERROR("Unexpected dependency node kind (name: {})", this->getName()); + return StatusCode::UNKNOWN_ERROR; + } + } + } + } + return StatusCode::OK; +} + +Status PipelineDefinition::getCustomNodeMetadata(const NodeInfo& customNodeInfo, tensor_map_t& inputsInfo, metadata_fn callback, const std::string& pipelineName, void* customNodeLibraryInternalManager) { + struct CustomNodeTensorInfo* info = nullptr; + int infoCount = 0; + auto paramArray = createCustomNodeParamArray(customNodeInfo.parameters); + int paramArrayLength = customNodeInfo.parameters.size(); + int result = callback(&info, &infoCount, paramArray.get(), paramArrayLength, customNodeLibraryInternalManager); + if (result != 0) { + SPDLOG_ERROR("Metadata call to custom node: {} in pipeline: {} returned error code: {}", + customNodeInfo.nodeName, pipelineName, result); + return StatusCode::NODE_LIBRARY_METADATA_FAILED; + } + return createTensorInfoMap(info, infoCount, inputsInfo, customNodeInfo.library.release, customNodeLibraryInternalManager); +} + +const NodeInfo& PipelineDefinition::findNodeByName(const std::string& name) const { + return *std::find_if(std::begin(this->nodeInfos), std::end(this->nodeInfos), [&name](const NodeInfo& nodeInfo) { + return nodeInfo.nodeName == name; + }); +} + +Shape PipelineDefinition::getNodeGatherShape(const NodeInfo& info) const { + if (info.gatherFromNode.size() == 0) { + return {}; + } + Shape shape; + shape.reserve(info.gatherFromNode.size()); + + std::function search; + search = [this, &info, &search, &shape](const std::string& nodeName) { + if (this->connections.count(nodeName) == 0) { + return; + } + if (info.gatherFromNode.count(nodeName) > 0) { + auto someNodeInfo = this->findNodeByName(nodeName); + dimension_value_t demultiplyCount = static_cast(someNodeInfo.demultiplyCount.value_or(0)); + Dimension dim = demultiplyCount == 0 ? Dimension::any() : Dimension(demultiplyCount); + if (dim.isAny()) { + tensor_map_t nodeOutputsInfo; + if (someNodeInfo.kind == NodeKind::CUSTOM) { + auto result = PipelineDefinition::getCustomNodeMetadata( + someNodeInfo, + nodeOutputsInfo, + someNodeInfo.library.getOutputsInfo, + this->pipelineName, + getCNLIMWrapperPtr(nodeResources.at(someNodeInfo.nodeName))); + if (!result.ok()) { + SPDLOG_ERROR("Failed to read node: {} library metadata with error: {}", nodeName, result.string()); + return; + } + if (nodeOutputsInfo.size() == 0) { + SPDLOG_ERROR("Node: {} library metadata reports no outputs", nodeName); + return; + } else if (nodeOutputsInfo.begin()->second->getShape().size() < 3) { + SPDLOG_ERROR("Node: {} library metadata reports output with too small number of dimensions", nodeName); + return; + } + dim = nodeOutputsInfo.begin()->second->getShape()[0]; + } else if (someNodeInfo.kind == NodeKind::ENTRY) { + dim = Dimension::any(); + } + } + shape.emplace_back(dim); + } + + if (this->connections.at(nodeName).size() > 0) { + search(this->connections.at(nodeName).begin()->first); + } + }; + search(info.nodeName); + + if (info.gatherFromNode.size() != shape.size()) { + SPDLOG_ERROR("Pipeline: {} node: {} is misconfigured, gather shape has different number of dimensions that gather from node elements: {} vs {}", + this->getName(), info.nodeName, shape.size(), info.gatherFromNode.size()); + throw std::invalid_argument("Gather shape has different number of dimensions that gather from node elements"); + } + + std::reverse(shape.begin(), shape.end()); + return shape; +} +template Status PipelineDefinition::create( + std::unique_ptr& pipeline, + const tensorflow::serving::PredictRequest* request, + tensorflow::serving::PredictResponse* response, + ModelManager& manager); +template Status PipelineDefinition::create<::inference::ModelInferRequest, ::inference::ModelInferResponse>( + std::unique_ptr& pipeline, + const ::inference::ModelInferRequest* request, + ::inference::ModelInferResponse* response, + ModelManager& manager); + +} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinition.hpp b/src/ovms_lib/pipelinedefinition.hpp new file mode 100644 index 0000000000..4d21ca8992 --- /dev/null +++ b/src/ovms_lib/pipelinedefinition.hpp @@ -0,0 +1,189 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop +#include "aliases.hpp" +#include "custom_node_library_internal_manager_wrapper.hpp" +#include "kfs_grpc_inference_service.hpp" +#include "modelversion.hpp" +#include "nodeinfo.hpp" +#include "pipelinedefinitionstatus.hpp" +#include "pipelinedefinitionunloadguard.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +class ModelManager; +class Pipeline; +class NodeValidator; + +class PipelineDefinition { + friend NodeValidator; + friend PipelineDefinitionUnloadGuard; + struct ValidationResultNotifier { + ValidationResultNotifier(PipelineDefinitionStatus& status, std::condition_variable& loadedNotify) : + status(status), + loadedNotify(loadedNotify) {} + ~ValidationResultNotifier() { + if (passed) { + status.handle(ValidationPassedEvent()); + loadedNotify.notify_all(); + } else { + status.handle(ValidationFailedEvent()); + } + } + bool passed = false; + + private: + PipelineDefinitionStatus& status; + std::condition_variable& loadedNotify; + }; + + const std::string pipelineName; + std::vector nodeInfos; + std::map> nodeResources = {}; + pipeline_connections_t connections; + +protected: + tensor_map_t inputsInfo; + tensor_map_t outputsInfo; + +private: + mutable std::shared_mutex metadataMtx; + std::atomic requestsHandlesCounter = 0; + std::shared_mutex loadMtx; + + std::condition_variable loadedNotify; + + // Pipelines are not versioned and any available definition has constant version equal 1. + static constexpr model_version_t VERSION = 1; + +protected: + PipelineDefinitionStatus status; + +private: + std::set> subscriptions; + + Status validateNode(ModelManager& manager, const NodeInfo& node, const bool isMultiBatchAllowed); + + const NodeInfo& findNodeByName(const std::string& name) const; + Shape getNodeGatherShape(const NodeInfo& info) const; + +public: + static constexpr uint64_t WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS = 10000; + PipelineDefinition(const std::string& pipelineName, + const std::vector& nodeInfos, + const pipeline_connections_t& connections) : + pipelineName(pipelineName), + nodeInfos(nodeInfos), + connections(connections), + status(this->pipelineName) {} + template + Status create(std::unique_ptr& pipeline, + const RequestType* request, + ResponseType* response, + ModelManager& manager); + +private: + template + Status createPrivate(std::unique_ptr& pipeline, + const RequestType* request, + ResponseType* response, + ModelManager& manager); + +public: + Status reload(ModelManager& manager, const std::vector&& nodeInfos, const pipeline_connections_t&& connections); + void retire(ModelManager& manager); + Status validate(ModelManager& manager); + Status validateNodes(ModelManager& manager); + Status validateForCycles(); + Status validateDemultiplexerGatherNodesOrder(); + Status initializeNodeResources(ModelManager& manager); + std::vector calculateNodeInfosDiff(const std::vector& nodeInfos); + void deinitializeNodeResources(const std::vector& nodeInfosDiff); + + const std::string& getName() const { return pipelineName; } + const PipelineDefinitionStateCode getStateCode() const { return status.getStateCode(); } + const model_version_t getVersion() const { return VERSION; } + + void notifyUsedModelChanged(const std::string& ownerDetails) { + this->status.handle(UsedModelChangedEvent(ownerDetails)); + } + + const PipelineDefinitionStatus& getStatus() const { + return this->status; + } + + const std::vector& getNodeInfos() { + return this->nodeInfos; + } + + void makeSubscriptions(ModelManager& manager); + void resetSubscriptions(ModelManager& manager); + +protected: + virtual Status updateInputsInfo(const ModelManager& manager); + virtual Status updateOutputsInfo(const ModelManager& manager); + +public: + const tensor_map_t getInputsInfo() const; + const tensor_map_t getOutputsInfo() const; + +private: + static Status getCustomNodeMetadata(const NodeInfo& customNodeInfo, tensor_map_t& inputsInfo, metadata_fn callback, const std::string& pipelineName, void* customNodeLibraryInternalManager); + + Status populateOutputsInfoWithDLModelOutputs( + const NodeInfo& dependencyNodeInfo, + const ModelManager& manager, + tensor_map_t& outputsInfo, + const Aliases& aliases, + const Shape& gatherShape) const; + + Status populateOutputsInfoWithCustomNodeOutputs( + const NodeInfo& dependencyNodeInfo, + const ModelManager& manager, + tensor_map_t& outputsInfo, + const Aliases& aliases, + const Shape& gatherShape) const; + + void increaseRequestsHandlesCount() { + ++requestsHandlesCounter; + } + + void decreaseRequestsHandlesCount() { + --requestsHandlesCounter; + } + +public: + Status waitForLoaded(std::unique_ptr& unloadGuard, const uint waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); +}; +} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionstatus.hpp b/src/ovms_lib/pipelinedefinitionstatus.hpp new file mode 100644 index 0000000000..20421422d1 --- /dev/null +++ b/src/ovms_lib/pipelinedefinitionstatus.hpp @@ -0,0 +1,407 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include "logging.hpp" +#include "modelversionstatus.hpp" +#include "status.hpp" + +namespace ovms { + +enum class PipelineDefinitionStateCode { + BEGIN, + RELOADING, + LOADING_PRECONDITION_FAILED, + LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, + AVAILABLE_REQUIRED_REVALIDATION, + AVAILABLE, + RETIRED +}; + +inline const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code) { + static const std::unordered_map names{ + {PipelineDefinitionStateCode::BEGIN, "BEGIN"}, + {PipelineDefinitionStateCode::RELOADING, "RELOADING"}, + {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED, "LOADING_PRECONDITION_FAILED"}, + {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, + {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, + {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, + {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; + return names.at(code); +} + +template +class MachineState { +public: + MachineState(const std::string& name) : + name(name) {} + template + void handle(const Event& event) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} state: {} handling: {}: {}", + name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, event.getDetails()); + try { + std::visit([this, &event](auto state) { state->handle(event).execute(*this); }, currentState); + } catch (std::logic_error& le) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} state: {} handling: {} error: {}", name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, le.what()); + throw; + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} state changed to: {} after handling: {}: {}", + name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, event.getDetails()); + } + + template + void changeStateTo() { + currentState = &std::get(allPossibleStates); + } + void printState() const { + std::visit([](const auto state) { state->print(); }, currentState); + } + PipelineDefinitionStateCode getStateCode() const { + while (true) { + try { + return std::visit([](const auto state) { return state->getStateCode(); }, currentState); + } catch (const std::bad_variant_access&) { + continue; + } + } + } + +private: + const std::string& name; + std::tuple allPossibleStates; + std::variant currentState{&std::get<0>(allPossibleStates)}; +}; +/** + * State in which pipeline is only defined + */ +struct BeginState; +/** + * State in which pipeline is available + */ +struct AvailableState; +/** + * State in which pipeline is available + * but there is revalidation pending since we know that one of used + * models changed + */ +struct AvailableRequiredRevalidation; +/** + * State in which pipeline is reloading + */ +struct ReloadState; +/** + * State in which pipeline is defined in config and failed validation. + */ +struct LoadingPreconditionFailedState; +/** + * State in which pipeline is defined in config, failed validation, + * but there is revalidation pending since we know that one of used + * models changed + */ +struct LoadingFailedLastValidationRequiredRevalidation; +/** + * State in which pipeline is retired - removed from config + */ +struct RetiredState; + +#define EVENT_STRUCT_WITH_NAME(x) \ + struct x { \ + static constexpr const char* name = #x; \ + x(const std::string& details = "") : \ + details(details) {} \ + const std::string& getDetails() const { \ + return details; \ + } \ + \ + private: \ + const std::string details; \ + }; + +EVENT_STRUCT_WITH_NAME(ReloadEvent); +EVENT_STRUCT_WITH_NAME(ValidationFailedEvent); +EVENT_STRUCT_WITH_NAME(ValidationPassedEvent); +EVENT_STRUCT_WITH_NAME(UsedModelChangedEvent); +EVENT_STRUCT_WITH_NAME(RetireEvent); + +template +struct StateChanger { + template + void execute(MachineState& pds) { + pds.template changeStateTo(); + } +}; + +struct StateKeeper { + template + void execute(MachineState& machine) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Keeping state"); + } +}; + +constexpr const char* INVALID_TRANSITION_MESSAGE = "Tried to conduct invalid transition."; + +struct BeginState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::BEGIN; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateChanger handle(const ValidationPassedEvent& e) const { + return {}; + } + StateChanger handle(const ValidationFailedEvent& e) const { + return {}; + } + StateKeeper handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } +}; + +struct ReloadState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RELOADING; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateKeeper handle(const ReloadEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateChanger handle(const ValidationPassedEvent& e) const { + return {}; + } + StateChanger handle(const ValidationFailedEvent& e) const { + return {}; + } + StateKeeper handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } +}; + +struct AvailableState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + return {}; + } + StateKeeper handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateChanger handle(const UsedModelChangedEvent& e) const { + return {}; + } + StateChanger handle(const RetireEvent& e) const { + return {}; + } +}; + +struct AvailableRequiredRevalidation { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + return {}; + } + StateChanger handle(const ValidationPassedEvent& e) const { + return {}; + } + StateChanger handle(const ValidationFailedEvent& e) const { + return {}; + } + StateKeeper handle(const UsedModelChangedEvent& e) const { + return {}; + } + StateChanger handle(const RetireEvent& e) const { + return {}; + } +}; + +struct LoadingPreconditionFailedState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + return {}; + } + StateKeeper handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateChanger handle(const UsedModelChangedEvent& e) const { + return {}; + } + StateChanger handle(const RetireEvent& e) const { + return {}; + } +}; + +struct LoadingFailedLastValidationRequiredRevalidation { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + return {}; + } + StateChanger handle(const ValidationPassedEvent& e) const { + return {}; + } + StateChanger handle(const ValidationFailedEvent& e) const { + return {}; + } + StateKeeper handle(const UsedModelChangedEvent& e) const { + return {}; + } + StateChanger handle(const RetireEvent& e) const { + return {}; + } +}; + +struct RetiredState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RETIRED; + PipelineDefinitionStateCode getStateCode() const { + return code; + } + void print() const { + SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); + } + StateChanger handle(const ReloadEvent& e) const { + return {}; + } + StateChanger handle(const ValidationPassedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateChanger handle(const ValidationFailedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const UsedModelChangedEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } + StateKeeper handle(const RetireEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; + } +}; + +class PipelineDefinitionStatus : public MachineState { +public: + PipelineDefinitionStatus(const std::string& name) : + MachineState(name) {} + bool isAvailable() const { + auto state = getStateCode(); + return (state == PipelineDefinitionStateCode::AVAILABLE) || + (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); + } + bool canEndLoaded() const { + auto state = getStateCode(); + return isAvailable() || + (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || + (state == PipelineDefinitionStateCode::BEGIN) || + (state == PipelineDefinitionStateCode::RELOADING); + } + bool isRevalidationRequired() const { + auto state = getStateCode(); + return (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || + (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); + } + + std::tuple convertToModelStatus() const { + switch (getStateCode()) { + case PipelineDefinitionStateCode::BEGIN: + case PipelineDefinitionStateCode::RELOADING: + case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: + return { + ModelVersionState::LOADING, + ModelVersionStatusErrorCode::OK}; + + case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: + return { + ModelVersionState::LOADING, + ModelVersionStatusErrorCode::FAILED_PRECONDITION}; + + case PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION: + case PipelineDefinitionStateCode::AVAILABLE: + return { + ModelVersionState::AVAILABLE, + ModelVersionStatusErrorCode::OK}; + + case PipelineDefinitionStateCode::RETIRED: + return { + ModelVersionState::END, + ModelVersionStatusErrorCode::OK}; + + default: + return {}; + } + } +}; + +} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionunloadguard.cpp b/src/ovms_lib/pipelinedefinitionunloadguard.cpp new file mode 100644 index 0000000000..fc4c6964fd --- /dev/null +++ b/src/ovms_lib/pipelinedefinitionunloadguard.cpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pipelinedefinitionunloadguard.hpp" + +#include "pipelinedefinition.hpp" + +namespace ovms { +PipelineDefinitionUnloadGuard::PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition) : + pipelineDefinition(pipelineDefinition) { + pipelineDefinition.increaseRequestsHandlesCount(); +} + +PipelineDefinitionUnloadGuard::~PipelineDefinitionUnloadGuard() { + pipelineDefinition.decreaseRequestsHandlesCount(); +} +} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionunloadguard.hpp b/src/ovms_lib/pipelinedefinitionunloadguard.hpp new file mode 100644 index 0000000000..c36466f21b --- /dev/null +++ b/src/ovms_lib/pipelinedefinitionunloadguard.hpp @@ -0,0 +1,30 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +namespace ovms { +class PipelineDefinition; + +class PipelineDefinitionUnloadGuard { +public: + PipelineDefinitionUnloadGuard() = delete; + PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition); + ~PipelineDefinitionUnloadGuard(); + +private: + PipelineDefinition& pipelineDefinition; +}; +} // namespace ovms diff --git a/src/ovms_lib/pipelineeventqueue.hpp b/src/ovms_lib/pipelineeventqueue.hpp new file mode 100644 index 0000000000..85ab741d65 --- /dev/null +++ b/src/ovms_lib/pipelineeventqueue.hpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "threadsafequeue.hpp" + +namespace ovms { + +class Node; + +using NodeSessionKeyPair = std::pair, session_key_t>; +using PipelineEventQueue = ThreadSafeQueue; +} // namespace ovms diff --git a/src/ovms_lib/precision.cpp b/src/ovms_lib/precision.cpp new file mode 100644 index 0000000000..46d28463f2 --- /dev/null +++ b/src/ovms_lib/precision.cpp @@ -0,0 +1,249 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "precision.hpp" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include + +namespace ovms { + +std::string toString(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::BF16, "BF16"}, + {Precision::FP64, "FP64"}, + {Precision::FP32, "FP32"}, + {Precision::FP16, "FP16"}, + {Precision::I64, "I64"}, + {Precision::I32, "I32"}, + {Precision::I16, "I16"}, + {Precision::I8, "I8"}, + {Precision::I4, "I4"}, + {Precision::U64, "U64"}, + {Precision::U32, "U32"}, + {Precision::U16, "U16"}, + {Precision::U8, "U8"}, + {Precision::U4, "U4"}, + {Precision::U1, "U1"}, + {Precision::MIXED, "MIXED"}, + {Precision::Q78, "Q78"}, + {Precision::BIN, "BIN"}, + {Precision::BOOL, "BOOL"}, + {Precision::UNDEFINED, "UNDEFINED"}, + {Precision::CUSTOM, "CUSTOM"}}; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return "UNKNOWN"; + } + return it->second; +} + +Precision fromString(const std::string& s) { + static std::unordered_map precisionMap{ + {"BF16", Precision::BF16}, + {"FP64", Precision::FP64}, + {"FP32", Precision::FP32}, + {"FP16", Precision::FP16}, + {"I64", Precision::I64}, + {"I32", Precision::I32}, + {"I16", Precision::I16}, + {"I8", Precision::I8}, + {"I4", Precision::I4}, + {"U64", Precision::U64}, + {"U32", Precision::U32}, + {"U16", Precision::U16}, + {"U8", Precision::U8}, + {"U4", Precision::U4}, + {"U1", Precision::U1}, + {"MIXED", Precision::MIXED}, + {"Q78", Precision::Q78}, + {"BIN", Precision::BIN}, + {"BOOL", Precision::BOOL}, + {"UNDEFINED", Precision::UNDEFINED}, + {"CUSTOM", Precision::CUSTOM}}; + auto it = precisionMap.find(s); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +Precision KFSPrecisionToOvmsPrecision(const KFSDataType& datatype) { + static std::unordered_map precisionMap{ + {"BOOL", Precision::BOOL}, + {"FP64", Precision::FP64}, + {"FP32", Precision::FP32}, + {"FP16", Precision::FP16}, + {"INT64", Precision::I64}, + {"INT32", Precision::I32}, + {"INT16", Precision::I16}, + {"INT8", Precision::I8}, + {"UINT64", Precision::U64}, + {"UINT32", Precision::U32}, + {"UINT16", Precision::U16}, + // {"BYTES", Precision::??}, + {"UINT8", Precision::U8}}; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { + static std::unordered_map precisionMap{ + {TFSDataType::DT_FLOAT, Precision::FP32}, + {TFSDataType::DT_DOUBLE, Precision::FP64}, + {TFSDataType::DT_HALF, Precision::FP16}, + {TFSDataType::DT_INT64, Precision::I64}, + {TFSDataType::DT_INT32, Precision::I32}, + {TFSDataType::DT_INT16, Precision::I16}, + {TFSDataType::DT_INT8, Precision::I8}, + {TFSDataType::DT_UINT64, Precision::U64}, + {TFSDataType::DT_UINT16, Precision::U16}, + {TFSDataType::DT_UINT8, Precision::U8}, + {TFSDataType::DT_BOOL, Precision::BOOL}}; + auto it = precisionMap.find(datatype); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} + +size_t KFSDataTypeSize(const KFSDataType& datatype) { + static std::unordered_map datatypeSizeMap{ + {"BOOL", 1}, + {"UINT8", 1}, + {"UINT16", 2}, + {"UINT32", 4}, + {"UINT64", 8}, + {"INT8", 1}, + {"INT16", 2}, + {"INT32", 4}, + {"INT64", 8}, + {"FP16", 2}, + {"FP32", 4}, + {"FP64", 8} + // {"BYTES", }, + }; + auto it = datatypeSizeMap.find(datatype); + if (it == datatypeSizeMap.end()) { + return 0; + } + return it->second; +} + +KFSDataType ovmsPrecisionToKFSPrecision(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP64, "FP64"}, + {Precision::FP32, "FP32"}, + {Precision::FP16, "FP16"}, + {Precision::I64, "INT64"}, + {Precision::I32, "INT32"}, + {Precision::I16, "INT16"}, + {Precision::I8, "INT8"}, + {Precision::U64, "UINT64"}, + {Precision::U32, "UINT32"}, + {Precision::U16, "UINT16"}, + {Precision::U8, "UINT8"}, + {Precision::BOOL, "BOOL"}}; + // {Precision::BF16, ""}, + // {Precision::U4, ""}, + // {Precision::U1, ""}, + // {Precision::CUSTOM, ""}, + // {Precision::DYNAMIC, ""}, + // {Precision::MIXED, ""}, + // {Precision::Q78, ""}, + // {Precision::BIN, ""}, + // {Precision::I4, ""}, + // {Precision::UNDEFINED, "UNDEFINED"}}; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return "INVALID"; + } + return it->second; +} + +ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP64, ov::element::Type_t::f64}, + {Precision::FP32, ov::element::Type_t::f32}, + {Precision::FP16, ov::element::Type_t::f16}, + {Precision::I64, ov::element::Type_t::i64}, + {Precision::I32, ov::element::Type_t::i32}, + {Precision::I16, ov::element::Type_t::i16}, + {Precision::I8, ov::element::Type_t::i8}, + {Precision::I4, ov::element::Type_t::i4}, + {Precision::U64, ov::element::Type_t::u64}, + {Precision::U32, ov::element::Type_t::u32}, + {Precision::U16, ov::element::Type_t::u16}, + {Precision::U8, ov::element::Type_t::u8}, + {Precision::U4, ov::element::Type_t::u4}, + {Precision::U1, ov::element::Type_t::u1}, + {Precision::BOOL, ov::element::Type_t::boolean}, + {Precision::UNDEFINED, ov::element::Type_t::undefined}, + {Precision::DYNAMIC, ov::element::Type_t::dynamic} + // {Precision::MIXED, ov::element::Type_t::MIXED}, + // {Precision::Q78, ov::element::Type_t::Q78}, + // {Precision::BIN, ov::element::Type_t::BIN}, + // {Precision::CUSTOM, ov::element::Type_t::CUSTOM + }; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return ov::element::Type_t::undefined; + } + return it->second; +} + +Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type) { + static std::unordered_map precisionMap{ + {ov::element::Type_t::f64, Precision::FP64}, + {ov::element::Type_t::f32, Precision::FP32}, + {ov::element::Type_t::f16, Precision::FP16}, + {ov::element::Type_t::bf16, Precision::BF16}, + {ov::element::Type_t::i64, Precision::I64}, + {ov::element::Type_t::i32, Precision::I32}, + {ov::element::Type_t::i16, Precision::I16}, + {ov::element::Type_t::i8, Precision::I8}, + {ov::element::Type_t::i4, Precision::I4}, + {ov::element::Type_t::u64, Precision::U64}, + {ov::element::Type_t::u32, Precision::U32}, + {ov::element::Type_t::u16, Precision::U16}, + {ov::element::Type_t::u8, Precision::U8}, + {ov::element::Type_t::u4, Precision::U4}, + {ov::element::Type_t::u1, Precision::U1}, + {ov::element::Type_t::undefined, Precision::UNDEFINED}, + {ov::element::Type_t::dynamic, Precision::DYNAMIC}, + // {ov::element::Type_t::???, Precision::MIXED}, + // {ov::element::Type_t::???, Precision::Q78}, + // {ov::element::Type_t::???, Precision::BIN}, + {ov::element::Type_t::boolean, Precision::BOOL} + // {ov::element::Type_t::CUSTOM, Precision::CUSTOM} + /* + undefined, + dynamic, +*/ + }; + auto it = precisionMap.find(type); + if (it == precisionMap.end()) { + return Precision::UNDEFINED; + } + return it->second; +} +} // namespace ovms diff --git a/src/ovms_lib/precision.hpp b/src/ovms_lib/precision.hpp new file mode 100644 index 0000000000..9f50f7ae7d --- /dev/null +++ b/src/ovms_lib/precision.hpp @@ -0,0 +1,70 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include + +using KFSDataType = std::string; +namespace tensorflow { +enum DataType : int; +} + +namespace ovms { + +using TFSDataType = tensorflow::DataType; + +enum class Precision { + BF16, + FP64, + FP32, + FP16, + I64, + I32, + I16, + I8, + I4, + U64, + U32, + U16, + U8, + U4, + U1, + BOOL, + CUSTOM, + UNDEFINED, + DYNAMIC, + MIXED, + Q78, + BIN +}; + +std::string toString(Precision precision); + +Precision fromString(const std::string& s); + +Precision KFSPrecisionToOvmsPrecision(const KFSDataType& s); +Precision TFSPrecisionToOvmsPrecision(const TFSDataType& s); + +size_t KFSDataTypeSize(const KFSDataType& datatype); + +KFSDataType ovmsPrecisionToKFSPrecision(Precision precision); + +ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision); + +Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type); +} // namespace ovms diff --git a/src/ovms_lib/predict_request_validation_utils.cpp b/src/ovms_lib/predict_request_validation_utils.cpp new file mode 100644 index 0000000000..ee2d58a6e6 --- /dev/null +++ b/src/ovms_lib/predict_request_validation_utils.cpp @@ -0,0 +1,696 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "predict_request_validation_utils.hpp" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop +#include +#include + +#include + +#include "kfs_grpc_inference_service.hpp" +#include "modelconfig.hpp" +#include "profiler.hpp" +#include "tfs_frontend/tfs_utils.hpp" + +namespace ovms { +namespace request_validation_utils { +template +struct RequestShapeInfo { + const RequestTensorType& tensor; + RequestShapeInfo(const RequestTensorType& tensor) : + tensor(tensor) {} + dimension_value_t getDim(size_t i); + size_t getShapeSize(); + const RequestTensorShapeType& getShape(); +}; + +using KFSRequestType = ::inference::ModelInferRequest; +using KFSInputTensorType = ::inference::ModelInferRequest_InferInputTensor; +using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; +using KFSShapeType = google::protobuf::RepeatedField; +using TFSRequestType = tensorflow::serving::PredictRequest; +using TFSInputTensorType = tensorflow::TensorProto; +using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; +using TFSShapeType = tensorflow::TensorShapeProto; + +template <> +dimension_value_t RequestShapeInfo::getDim(size_t i) { + return tensor.shape()[i]; +} +template <> +dimension_value_t RequestShapeInfo::getDim(size_t i) { + return tensor.tensor_shape().dim(i).size(); +} +template <> +size_t RequestShapeInfo::getShapeSize() { + return tensor.shape().size(); +} +template <> +size_t RequestShapeInfo::getShapeSize() { + return tensor.tensor_shape().dim_size(); +} +template <> +const TFSShapeType& RequestShapeInfo::getShape() { + return tensor.tensor_shape(); +} +template <> +const KFSShapeType& RequestShapeInfo::getShape() { + return tensor.shape(); +} + +template +class RequestValidator { + const RequestType& request; + const tensor_map_t& inputsInfo; + const std::string& servableName; + const model_version_t servableVersion; + const std::set& optionalAllowedInputNames; + const Mode batchingMode; + const shapes_info_map_t& shapeInfo; + + InputIterator it; + + RequestValidator() = delete; + + const std::string& getCurrentlyValidatedInputName() const; + const InputTensorType& getInputFromIt(const InputIterator& it) const; + +public: + RequestValidator( + const RequestType& request, const tensor_map_t& inputsInfo, + const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, + const Mode batchingMode, const shapes_info_map_t& shapeInfo) : + request(request), + inputsInfo(inputsInfo), + servableName(servableName), + servableVersion(servableVersion), + optionalAllowedInputNames(optionalAllowedInputNames), + batchingMode(batchingMode), + shapeInfo(shapeInfo) {} + + Status validateNumberOfInputs() const; + Status validateAndGetInput(const RequestType& request, const std::string& name, InputIterator& it, size_t& bufferId); + Status checkIfShapeValuesNegative(const InputTensorType& proto) const; + Status validateNumberOfBinaryInputShapeDimensions(const InputTensorType& proto) const; + Status checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; + Status checkBinaryBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; + Status checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; + Status validateTensorContentSize(const InputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const; + Status validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; + Status validatePrecision(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; + bool checkIfBinaryInputUsed(const InputTensorType& proto, const std::string inputName) const; + Status validateRequestCoherency() const; + Status validate(); +}; + +template <> +Status RequestValidator::validateRequestCoherency() const { + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateRequestCoherency() const { + if (!request.raw_input_contents().empty()) { + for (auto& input : request.inputs()) { + if (input.has_contents()) { + std::stringstream ss; + ss << "Passing buffers both in InferInputTensor contents and in raw_input_contents is not allowed. Detected buffer in InferInputTensor contents for input: " << input.name(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid request message - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_MESSAGE_STRUCTURE, details); + } + } + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateNumberOfInputs() const { + size_t expectedNumberOfInputs = inputsInfo.size(); + + if (optionalAllowedInputNames.size() > 0) { + auto it = request.inputs().begin(); + while (it != request.inputs().end()) { + if (optionalAllowedInputNames.find(it->name()) != optionalAllowedInputNames.end()) { + ++expectedNumberOfInputs; + } + ++it; + } + } + if (request.inputs_size() > 0 && expectedNumberOfInputs == static_cast(request.inputs_size())) { + return StatusCode::OK; + } + std::stringstream ss; + ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.inputs_size(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_INPUTS, details); +} + +template <> +const std::string& RequestValidator::getCurrentlyValidatedInputName() const { + return it->name(); +} + +template <> +const std::string& RequestValidator::getCurrentlyValidatedInputName() const { + return it->first; +} +template <> +const KFSInputTensorType& RequestValidator::getInputFromIt(const KFSInputTensorIteratorType& it) const { + return *it; +} + +template <> +const TFSInputTensorType& RequestValidator::getInputFromIt(const TFSInputTensorIteratorType& it) const { + return it->second; +} + +template <> +Status RequestValidator::validateNumberOfInputs() const { + size_t expectedNumberOfInputs = inputsInfo.size(); + for (auto optionalAllowedInputName : optionalAllowedInputNames) { + if (request.inputs().count(optionalAllowedInputName)) + expectedNumberOfInputs++; + } + if (request.inputs_size() > 0 && expectedNumberOfInputs == static_cast(request.inputs_size())) { + return StatusCode::OK; + } + std::stringstream ss; + ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.inputs_size(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_INPUTS, details); +} + +template <> +Status RequestValidator::validateAndGetInput(const TFSRequestType& request, const std::string& name, TFSInputTensorIteratorType& it, size_t& bufferId) { + it = request.inputs().find(name); + if (it != request.inputs().end()) { + return StatusCode::OK; + } + std::stringstream ss; + ss << "Required input: " << name; + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Missing input with specific name - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_MISSING_INPUT, details); +} + +template <> +Status RequestValidator::validateAndGetInput(const KFSRequestType& request, const std::string& name, KFSInputTensorIteratorType& it, size_t& bufferId) { + it = request.inputs().begin(); + bufferId = 0; + while (it != request.inputs().end()) { + if (it->name() == name) { + break; + } + ++it; + ++bufferId; + } + if (it != request.inputs().end()) { + return StatusCode::OK; + } + std::stringstream ss; + ss << "Required input: " << name; + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Missing input with specific name - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_MISSING_INPUT, details); +} + +template <> +template +Status RequestValidator::checkIfShapeValuesNegative(const InputTensorType& proto) const { + RequestShapeInfo rsi(proto); + for (size_t i = 0; i < rsi.getShapeSize(); i++) { + if (rsi.getDim(i) <= 0) { + std::stringstream ss; + ss << "Negative or zero dimension size is not acceptable: " << tensorShapeToString(rsi.getShape()) << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_SHAPE, details); + } + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const TFSInputTensorType& proto) const { + RequestShapeInfo rsi(proto); + if (rsi.getShapeSize() != 1) { + std::stringstream ss; + ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const KFSInputTensorType& proto) const { + RequestShapeInfo rsi(proto); + if (rsi.getShapeSize() != 1) { + std::stringstream ss; + ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} + +template +Status RequestValidator::checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + RequestShapeInfo rsi(proto); + if (servableBatchSize.match(rsi.getDim(batchSizeIndex))) { + return StatusCode::OK; + } + if (batchingMode == AUTO) { + finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; + return StatusCode::OK; + } else if (shapeMode != AUTO) { + std::stringstream ss; + ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << rsi.getDim(batchSizeIndex) << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::checkBinaryBatchSizeMismatch(const TFSInputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + RequestShapeInfo rsi(proto); + if (proto.string_val_size() <= 0) { + std::stringstream ss; + ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + if (servableBatchSize.match(rsi.getDim(0))) { + return StatusCode::OK; + } + if (batchingMode == AUTO) { + finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; + return StatusCode::OK; + } else if (shapeMode != AUTO) { + std::stringstream ss; + ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << proto.string_val_size() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + return StatusCode::OK; +} +template <> +Status RequestValidator::checkBinaryBatchSizeMismatch(const KFSInputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + RequestShapeInfo rsi(proto); + if (proto.contents().bytes_contents_size() <= 0) { + std::stringstream ss; + ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + if (servableBatchSize.match(rsi.getDim(0))) { + return StatusCode::OK; + } + if (batchingMode == AUTO) { + finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; + return StatusCode::OK; + } else if (shapeMode != AUTO) { + std::stringstream ss; + ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << proto.contents().bytes_contents_size() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + return StatusCode::OK; +} + +template +Status RequestValidator::checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + const auto& shape = inputInfo.getShape(); + bool mismatch = false; + RequestShapeInfo rsi(proto); + if (batchingMode == AUTO) { // Skip batch dimension + for (int i = 0; i < batchSizeIndex; i++) { + if (!shape[i].match(static_cast(rsi.getDim(i)))) { + mismatch = true; + break; + } + } + for (int i = batchSizeIndex + 1; i < rsi.getShapeSize(); i++) { + if (!shape[i].match(static_cast(rsi.getDim(i)))) { + mismatch = true; + break; + } + } + } else { // Do not skip batch dimension + for (int i = 0; i < rsi.getShapeSize(); i++) { + if (!shape[i].match(static_cast(rsi.getDim(i)))) { + mismatch = true; + break; + } + } + } + if (!mismatch) { + return StatusCode::OK; + } + if (shapeMode == AUTO) { + finalStatus = StatusCode::RESHAPE_REQUIRED; + return StatusCode::OK; + } else { + std::stringstream ss; + ss << "Expected: " << inputInfo.getShape().toString() + << "; Actual: " << tensorShapeToString(rsi.getShape()) + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_SHAPE, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateTensorContentSize(const TFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { + /* + int8 data in request.tensor_content + uint8 data in request.tensor_content + int16 data in request.tensor_content + uint16 request.tensor_content is empty, data located in request.int_val + int32 data in request.tensor_content + uint32 data in request.tensor_content + int64 data in request.tensor_content + uint64 data in request.tensor_content + float16 request.tensor_content is empty, data located in request.half_val + float32 data in request.tensor_content + double data in request.tensor_content + + _TENSOR_CONTENT_TYPES + https://github.com/tensorflow/tensorflow/blob/903a6399aab19b549fefd0ead836af644f3d00f8/tensorflow/python/framework/tensor_util.py#L237 +*/ + + size_t expectedValueCount = 1; + for (int i = 0; i < proto.tensor_shape().dim_size(); i++) { + expectedValueCount *= proto.tensor_shape().dim(i).size(); + } + + // Network expects tensor content size or value count + if (proto.dtype() == tensorflow::DataType::DT_UINT16) { + if (proto.int_val_size() < 0 || + expectedValueCount != static_cast(proto.int_val_size())) { + std::stringstream ss; + ss << "Expected: " << expectedValueCount << "; Actual: " << proto.int_val_size() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_VALUE_COUNT, details); + } + } else if (proto.dtype() == tensorflow::DataType::DT_HALF) { + if (proto.half_val_size() < 0 || + expectedValueCount != static_cast(proto.half_val_size())) { + std::stringstream ss; + ss << "Expected: " << expectedValueCount << "; Actual: " << proto.half_val_size() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_VALUE_COUNT, details); + } + } else { + size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); + if (expectedContentSize != proto.tensor_content().size()) { + std::stringstream ss; + ss << "Expected: " << expectedContentSize << " bytes; Actual: " << proto.tensor_content().size() << " bytes; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_CONTENT_SIZE, details); + } + } + return StatusCode::OK; +} + +size_t getElementsCount(const KFSInputTensorType& proto, ovms::Precision expectedPrecision) { + switch (expectedPrecision) { + case ovms::Precision::BOOL: { + return proto.contents().bool_contents().size(); + } + /// int_contents + case ovms::Precision::I8: + case ovms::Precision::I16: + case ovms::Precision::I32: { + return proto.contents().int_contents().size(); + } + /// int64_contents + case ovms::Precision::I64: { + return proto.contents().int64_contents().size(); + } + // uint_contents + case ovms::Precision::U8: + case ovms::Precision::U16: + case ovms::Precision::U32: { + return proto.contents().uint_contents().size(); + } + // uint64_contents + case ovms::Precision::U64: { + return proto.contents().uint64_contents().size(); + } + // fp32_contents + case ovms::Precision::FP32: { + return proto.contents().fp32_contents().size(); + } + // fp64_contentes + case ovms::Precision::FP64: { + return proto.contents().fp64_contents().size(); + } + case ovms::Precision::FP16: + case ovms::Precision::U1: + case ovms::Precision::CUSTOM: + case ovms::Precision::UNDEFINED: + case ovms::Precision::DYNAMIC: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + default: + return 0; + } +} + +template <> +Status RequestValidator::validateTensorContentSize(const KFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { + size_t expectedValueCount = 1; + for (int i = 0; i < proto.shape().size(); i++) { + expectedValueCount *= proto.shape()[i]; + } + if (request.raw_input_contents().size()) { + size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); + if (expectedContentSize != request.raw_input_contents()[bufferId].size()) { + std::stringstream ss; + ss << "Expected: " << expectedContentSize << " bytes; Actual: " << request.raw_input_contents()[bufferId].size() << " bytes; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_CONTENT_SIZE, details); + } + } else { // buffers placed in InputTensor content + // here we should check that the elements count is equal since for some precisions there is padding + // we need to decide first which exact datatype_contents we extract that information from + size_t elementsCount = getElementsCount(proto, expectedPrecision); + if (expectedValueCount != elementsCount) { + std::stringstream ss; + ss << "Expected: " << expectedValueCount << " values; Actual: " << elementsCount << " values; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid value count of tensor proto - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_VALUE_COUNT, details); + } + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { + // Network and request must have the same number of shape dimensions, higher than 0 + const auto& shape = inputInfo.getShape(); + if (proto.tensor_shape().dim_size() <= 0 || + shape.size() != static_cast(proto.tensor_shape().dim_size())) { + std::stringstream ss; + ss << "Expected: " << shape.toString() + << "; Actual: " << tensorShapeToString(proto.tensor_shape()) + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { + // Network and request must have the same number of shape dimensions, higher than 0 + const auto& shape = inputInfo.getShape(); + if (proto.shape().size() <= 0 || + shape.size() != static_cast(proto.shape().size())) { + std::stringstream ss; + ss << "Expected: " << shape.toString() + << "; Actual: " << tensorShapeToString(proto.shape()) + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { + if (proto.dtype() != getPrecisionAsDataType(inputInfo.getPrecision())) { + std::stringstream ss; + ss << "Expected: " << inputInfo.getPrecisionAsString() + << "; Actual: " << getDataTypeAsString(proto.dtype()) + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_PRECISION, details); + } + return StatusCode::OK; +} + +template <> +Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { + if (proto.datatype() != inputInfo.getPrecisionAsKFSPrecision()) { + std::stringstream ss; + ss << "Expected: " << inputInfo.getPrecisionAsString() + << "; Actual: " << proto.datatype() + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_PRECISION, details); + } + return StatusCode::OK; +} + +Mode getShapeMode(const shapes_info_map_t& shapeInfo, const std::string& name) { + if (shapeInfo.size() == 0) { + return Mode::FIXED; + } + auto it = shapeInfo.find(name); + if (it == shapeInfo.end()) { + it = shapeInfo.find(ANONYMOUS_INPUT_NAME); + } + if (it == shapeInfo.end()) { + return Mode::FIXED; + } + return it->second.shapeMode; +} + +template <> +bool RequestValidator::checkIfBinaryInputUsed(const TFSInputTensorType& proto, const std::string inputName) const { + if (proto.dtype() == tensorflow::DataType::DT_STRING) { + SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.string_val_size()); + return true; + } + return false; +} + +template <> +bool RequestValidator::checkIfBinaryInputUsed(const KFSInputTensorType& proto, const std::string inputName) const { + if (proto.datatype() == "BYTES") { + SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.contents().bytes_contents_size()); + return true; + } + return false; +} + +template +Status RequestValidator::validate() { + Status finalStatus = StatusCode::OK; + + auto status = validateNumberOfInputs(); + if (!status.ok()) + return status; + status = validateRequestCoherency(); + if (!status.ok()) + return status; + + size_t bufferId = 0; + for (const auto& [name, inputInfo] : inputsInfo) { + status = validateAndGetInput(request, name, it, bufferId); + if (!status.ok()) + return status; + + const auto& proto = getInputFromIt(it); + + status = checkIfShapeValuesNegative(proto); + if (!status.ok()) + return status; + auto batchIndex = inputInfo->getLayout().getBatchIndex(); + if (!batchIndex.has_value()) { + SPDLOG_DEBUG("[servable name: {} version: {}] Missing batch index in input: {} layout: {}", + servableName, servableVersion, name, inputInfo->getLayout()); + return StatusCode::INTERNAL_ERROR; + } + if (inputInfo->getShape().size() < batchIndex.value() + 1) { + SPDLOG_DEBUG("[servable name: {} version: {}] Batch index out of shape range for input: {} layout: {} shape: {}", + servableName, servableVersion, name, inputInfo->getLayout(), inputInfo->getShape().toString()); + return StatusCode::INTERNAL_ERROR; + } + const Dimension& batchSize = inputInfo->getShape()[batchIndex.value()]; + Mode shapeMode = getShapeMode(shapeInfo, name); + if (checkIfBinaryInputUsed(proto, name)) { + status = validateNumberOfBinaryInputShapeDimensions(proto); + if (!status.ok()) + return status; + + status = checkBinaryBatchSizeMismatch(proto, batchSize, finalStatus, batchingMode, shapeMode); + if (!status.ok()) + return status; + + continue; + } + + status = validatePrecision(*inputInfo, proto); + if (!status.ok()) + return status; + status = validateNumberOfShapeDimensions(*inputInfo, proto); + if (!status.ok()) + return status; + status = checkBatchSizeMismatch(proto, batchSize, batchIndex.value(), finalStatus, batchingMode, shapeMode); + if (!status.ok()) + return status; + status = checkShapeMismatch(proto, *inputInfo, batchIndex.value(), finalStatus, batchingMode, shapeMode); + if (!status.ok()) + return status; + status = validateTensorContentSize(proto, inputInfo->getPrecision(), bufferId); + if (!status.ok()) + return status; + } + return finalStatus; +} + +template <> +Status validate(const TFSRequestType& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { + OVMS_PROFILE_FUNCTION(); + return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); +} + +template <> +Status validate(const KFSRequestType& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { + OVMS_PROFILE_FUNCTION(); + return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); +} +} // namespace request_validation_utils +} // namespace ovms diff --git a/src/ovms_lib/predict_request_validation_utils.hpp b/src/ovms_lib/predict_request_validation_utils.hpp new file mode 100644 index 0000000000..67931e3666 --- /dev/null +++ b/src/ovms_lib/predict_request_validation_utils.hpp @@ -0,0 +1,42 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include + +#include "modelversion.hpp" +#include "shape.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { +namespace request_validation_utils { + +template +Status validate( + const RequestType& request, + const tensor_map_t& inputsInfo, + const std::string& servableName, + const model_version_t servableVersion, + const std::set& optionalAllowedInputNames = {}, + const Mode batchingMode = Mode::FIXED, + const shapes_info_map_t& shapeInfo = shapes_info_map_t()); + +} // namespace request_validation_utils +} // namespace ovms diff --git a/src/ovms_lib/prediction_service.cpp b/src/ovms_lib/prediction_service.cpp new file mode 100644 index 0000000000..0594c317df --- /dev/null +++ b/src/ovms_lib/prediction_service.cpp @@ -0,0 +1,137 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "prediction_service.hpp" + +#include +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#pragma GCC diagnostic pop + +#include "get_model_metadata_impl.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "ovinferrequestsqueue.hpp" +#include "prediction_service_utils.hpp" +#include "profiler.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" +#include "status.hpp" +#include "timer.hpp" + +using grpc::ServerContext; + +using tensorflow::TensorProto; + +using tensorflow::serving::PredictionService; +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; + +namespace ovms { + +PredictionServiceImpl::PredictionServiceImpl(ovms::Server& ovmsServer) : + ovmsServer(ovmsServer), + getModelMetadataImpl(ovmsServer) {} + +Status PredictionServiceImpl::getModelInstance(const PredictRequest* request, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return manager.getModelInstance(request->model_spec().name(), request->model_spec().version().value(), modelInstance, modelInstanceUnloadGuardPtr); +} + +Status PredictionServiceImpl::getPipeline(const PredictRequest* request, + PredictResponse* response, + std::unique_ptr& pipelinePtr) { + OVMS_PROFILE_FUNCTION(); + auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return StatusCode::MODEL_NOT_LOADED; + } + auto servableManagerModule = dynamic_cast(module); + // TODO if not succeed then return error + auto& manager = servableManagerModule->getServableManager(); + return manager.createPipeline(pipelinePtr, request->model_spec().name(), request, response); +} + +grpc::Status ovms::PredictionServiceImpl::Predict( + ServerContext* context, + const PredictRequest* request, + PredictResponse* response) { + OVMS_PROFILE_FUNCTION(); + Timer timer; + timer.start("total"); + using std::chrono::microseconds; + SPDLOG_DEBUG("Processing gRPC request for model: {}; version: {}", + request->model_spec().name(), + request->model_spec().version().value()); + + std::shared_ptr modelInstance; + std::unique_ptr pipelinePtr; + + std::unique_ptr modelInstanceUnloadGuard; + auto status = getModelInstance(request, modelInstance, modelInstanceUnloadGuard); + + if (status == StatusCode::MODEL_NAME_MISSING) { + SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", request->model_spec().name()); + status = getPipeline(request, response, pipelinePtr); + } + if (!status.ok()) { + SPDLOG_INFO("Getting modelInstance or pipeline failed. {}", status.string()); + return status.grpc(); + } + + if (pipelinePtr) { + status = pipelinePtr->execute(); + } else { + status = modelInstance->infer(request, response, modelInstanceUnloadGuard); + } + + if (!status.ok()) { + return status.grpc(); + } + + timer.stop("total"); + SPDLOG_DEBUG("Total gRPC request processing time: {} ms", timer.elapsed("total") / 1000); + return grpc::Status::OK; +} + +grpc::Status PredictionServiceImpl::GetModelMetadata( + grpc::ServerContext* context, + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response) { + OVMS_PROFILE_FUNCTION(); + return getModelMetadataImpl.getModelStatus(request, response).grpc(); +} + +const GetModelMetadataImpl& PredictionServiceImpl::getTFSModelMetadataImpl() const { + return this->getModelMetadataImpl; +} + +} // namespace ovms diff --git a/src/ovms_lib/prediction_service.hpp b/src/ovms_lib/prediction_service.hpp new file mode 100644 index 0000000000..4483b79088 --- /dev/null +++ b/src/ovms_lib/prediction_service.hpp @@ -0,0 +1,64 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "get_model_metadata_impl.hpp" + +namespace ovms { +class ModelInstance; +class ModelInstanceUnloadGuard; +class ModelManager; +class Pipeline; +class Server; +class Status; + +class PredictionServiceImpl final : public tensorflow::serving::PredictionService::Service { + ovms::Server& ovmsServer; + GetModelMetadataImpl getModelMetadataImpl; + +public: + PredictionServiceImpl(ovms::Server& ovmsServer); + grpc::Status Predict( + grpc::ServerContext* context, + const tensorflow::serving::PredictRequest* request, + tensorflow::serving::PredictResponse* response) override; + + grpc::Status GetModelMetadata( + grpc::ServerContext* context, + const tensorflow::serving::GetModelMetadataRequest* request, + tensorflow::serving::GetModelMetadataResponse* response) override; + + const GetModelMetadataImpl& getTFSModelMetadataImpl() const; + +protected: + Status getModelInstance(const tensorflow::serving::PredictRequest* request, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr); + Status getPipeline(const tensorflow::serving::PredictRequest* request, + tensorflow::serving::PredictResponse* response, + std::unique_ptr& pipelinePtr); +}; + +} // namespace ovms diff --git a/src/ovms_lib/prediction_service_utils.cpp b/src/ovms_lib/prediction_service_utils.cpp new file mode 100644 index 0000000000..2f56ca39bc --- /dev/null +++ b/src/ovms_lib/prediction_service_utils.cpp @@ -0,0 +1,90 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "prediction_service_utils.hpp" + +#include + +#include "deserialization.hpp" +#include "executingstreamidguard.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "serialization.hpp" +#include "stringutils.hpp" +#include "timer.hpp" + +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; + +namespace ovms { + +std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex) { + auto requestInputItr = request->inputs().begin(); + if (requestInputItr == request->inputs().end()) { + SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); + return std::nullopt; + } + auto& requestInput = requestInputItr; // assuming same batch size for all inputs + if (requestInput->shape().size() < batchSizeIndex + 1) { + SPDLOG_DEBUG("Failed to get batch size of a request. Batch size index out of shape range. Validation of request failed"); + return std::nullopt; + } + return Dimension(requestInput->shape()[batchSizeIndex]); +} + +std::map getRequestShapes(const ::inference::ModelInferRequest* request) { + std::map requestShapes; + for (auto& it : request->inputs()) { + shape_t requestShape; + std::string name = it.name(); + auto& requestInput = it; + for (int i = 0; i < requestInput.shape().size(); i++) { + requestShape.push_back(requestInput.shape()[i]); + } + requestShapes[name] = std::move(requestShape); + } + return requestShapes; +} + +std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex) { + auto requestInputItr = request->inputs().begin(); + if (requestInputItr == request->inputs().end()) { + SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); + return std::nullopt; + } + auto& requestInput = requestInputItr->second; // assuming same batch size for all inputs + if (requestInput.tensor_shape().dim_size() < batchSizeIndex + 1) { + SPDLOG_DEBUG("Failed to get batch size of a request. Batch size index out of shape range. Validation of request failed"); + return std::nullopt; + } + return Dimension(requestInput.tensor_shape().dim(batchSizeIndex).size()); +} + +std::map getRequestShapes(const tensorflow::serving::PredictRequest* request) { + std::map requestShapes; + for (auto& it : request->inputs()) { + shape_t requestShape; + std::string name = it.first; + auto& requestInput = it.second; + for (int i = 0; i < requestInput.tensor_shape().dim_size(); i++) { + requestShape.push_back(requestInput.tensor_shape().dim(i).size()); + } + requestShapes[name] = std::move(requestShape); + } + return requestShapes; +} + +} // namespace ovms diff --git a/src/ovms_lib/prediction_service_utils.hpp b/src/ovms_lib/prediction_service_utils.hpp new file mode 100644 index 0000000000..03d598f5da --- /dev/null +++ b/src/ovms_lib/prediction_service_utils.hpp @@ -0,0 +1,38 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop +#include "kfs_grpc_inference_service.hpp" +#include "shape.hpp" + +namespace ovms { + +std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex); + +std::map getRequestShapes(const ::inference::ModelInferRequest* request); + +std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); +std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); + +} // namespace ovms diff --git a/src/ovms_lib/profiler.cpp b/src/ovms_lib/profiler.cpp new file mode 100644 index 0000000000..da6607db4a --- /dev/null +++ b/src/ovms_lib/profiler.cpp @@ -0,0 +1,56 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "profiler.hpp" + +#include + +namespace ovms { + +bool profiler_init(const char* file_path) { +#ifndef MTR_ENABLED + return true; +#endif + FILE* f = fopen(file_path, "wb"); + if (f == nullptr) { + return false; + } + mtr_init_from_stream(f); + return true; +} + +void profiler_shutdown() { +#ifndef MTR_ENABLED + return; +#endif + mtr_flush(); + mtr_shutdown(); +} + +Profiler::Profiler(const std::string& file_path) { + this->initialized = profiler_init(file_path.c_str()); +} + +Profiler::~Profiler() { + if (this->isInitialized()) { + profiler_shutdown(); + } +} + +bool Profiler::isInitialized() const { + return this->initialized; +} + +} // namespace ovms diff --git a/src/ovms_lib/profiler.hpp b/src/ovms_lib/profiler.hpp new file mode 100644 index 0000000000..b323c11351 --- /dev/null +++ b/src/ovms_lib/profiler.hpp @@ -0,0 +1,49 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "minitrace.h" // NOLINT + +#define OVMS_PROFILE_SCOPE(name) MTR_SCOPE("OVMS", name); +#define OVMS_PROFILE_SCOPE_S(name, vname, cstr) MTR_SCOPE_S("OVMS", name, vname, cstr); +#define OVMS_PROFILE_FUNCTION() OVMS_PROFILE_SCOPE(__PRETTY_FUNCTION__) + +#define OVMS_PROFILE_SYNC_BEGIN(name) MTR_BEGIN("OVMS", name); +#define OVMS_PROFILE_SYNC_END(name) MTR_END("OVMS", name); +#define OVMS_PROFILE_SYNC_BEGIN_S(name, vname, cstr) MTR_BEGIN_S("OVMS", name, vname, cstr); +#define OVMS_PROFILE_SYNC_END_S(name, vname, cstr) MTR_END_S("OVMS", name, vname, cstr); + +#define OVMS_PROFILE_ASYNC_BEGIN(name, id) MTR_START("OVMS", name, id); +#define OVMS_PROFILE_ASYNC_END(name, id) MTR_FINISH("OVMS", name, id); + +namespace ovms { + +bool profiler_init(const char* file_path); +void profiler_shutdown(); + +class Profiler { +public: + Profiler(const std::string& file_path); + ~Profiler(); + bool isInitialized() const; + +private: + bool initialized = false; +}; + +} // namespace ovms diff --git a/src/ovms_lib/queue.hpp b/src/ovms_lib/queue.hpp new file mode 100644 index 0000000000..037b3ebc94 --- /dev/null +++ b/src/ovms_lib/queue.hpp @@ -0,0 +1,136 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ovms { + +template +class Queue { +public: + /** + * @brief Allocating idle stream for execution + */ + std::future getIdleStream() { + int value; + std::promise idleStreamPromise; + std::future idleStreamFuture = idleStreamPromise.get_future(); + std::unique_lock lk(front_mut); + if (streams[front_idx] < 0) { // we need to wait for any idle stream to be returned + std::unique_lock queueLock(queue_mutex); + promises.push(std::move(idleStreamPromise)); + } else { // we can give idle stream right away + value = streams[front_idx]; + streams[front_idx] = -1; // negative value indicate consumed vector index + front_idx = (front_idx + 1) % streams.size(); + lk.unlock(); + idleStreamPromise.set_value(value); + } + return idleStreamFuture; + } + + std::optional tryToGetIdleStream() { + int value; + std::unique_lock lk(front_mut); + if (streams[front_idx] < 0) { // we need to wait for any idle stream to be returned + return std::nullopt; + } else { // we can give idle stream right away + value = streams[front_idx]; + streams[front_idx] = -1; // negative value indicate consumed vector index + front_idx = (front_idx + 1) % streams.size(); + lk.unlock(); + return value; + } + } + + /** + * @brief Release stream after execution + */ + void returnStream(int streamID) { + std::unique_lock lk(queue_mutex); + if (promises.size()) { + std::promise promise = std::move(promises.front()); + promises.pop(); + lk.unlock(); + promise.set_value(streamID); + return; + } + std::uint32_t old_back = back_idx.load(); + while (!back_idx.compare_exchange_weak( + old_back, + (old_back + 1) % streams.size(), + std::memory_order_relaxed)) { + } + streams[old_back] = streamID; + } + + /** + * @brief Constructor with initialization + */ + Queue(int streamsLength) : + streams(streamsLength), + front_idx{0}, + back_idx{0} { + for (int i = 0; i < streamsLength; ++i) { + streams[i] = i; + } + } + + /** + * @brief Give InferRequest + */ + T& getInferRequest(int streamID) { + return inferRequests[streamID]; + } + +protected: + /** + * @brief Vector representing circular buffer for infer queue + */ + std::vector streams; + + /** + * @brief Index of the front of the idle streams list + */ + std::uint32_t front_idx; + + /** + * @brief Index of the back of the idle streams list + */ + std::atomic back_idx; + + /** + * @brief Vector representing OV streams and used for notification about completed inference operations + */ + std::mutex front_mut; + std::mutex queue_mutex; + /** + * + */ + std::vector inferRequests; + std::queue> promises; +}; +} // namespace ovms diff --git a/src/ovms_lib/rest_parser.cpp b/src/ovms_lib/rest_parser.cpp new file mode 100644 index 0000000000..08a2e5149e --- /dev/null +++ b/src/ovms_lib/rest_parser.cpp @@ -0,0 +1,468 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "rest_parser.hpp" + +#include +#include + +#include "rest_utils.hpp" +#include "tfs_frontend/tfs_utils.hpp" + +namespace ovms { + +RestParser::RestParser(const tensor_map_t& tensors) { + for (const auto& kv : tensors) { + const auto& name = kv.first; + const auto& tensor = kv.second; + tensorPrecisionMap[name] = tensor->getPrecision(); + auto& input = (*requestProto.mutable_inputs())[name]; + input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); + + auto fold = [](size_t a, const Dimension& b) { + if (b.isDynamic()) { + if (b.isAny()) { + return static_cast(0); + } else { + return static_cast(b.getMaxValue()); + } + } else { + return a * static_cast(b.getStaticValue()); + } + }; + input.mutable_tensor_content()->reserve(std::accumulate( + tensor->getShape().cbegin(), + tensor->getShape().cend(), + 1, + fold) * + DataTypeSize(getPrecisionAsDataType(tensor->getPrecision()))); + } +} + +void RestParser::removeUnusedInputs() { + auto& inputs = (*requestProto.mutable_inputs()); + auto it = inputs.begin(); + while (it != inputs.end()) { + if (!it->second.tensor_shape().dim_size()) { + SPDLOG_DEBUG("Removing {} input from proto since it's not included in the request", it->first); + it = inputs.erase(it); + } else { + it++; + } + } +} + +bool RestParser::parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { + proto.set_dtype(tensorflow::DataType::DT_UINT64); + for (auto& value : doc.GetArray()) { + if (value.IsUint64()) + proto.add_uint64_val(value.GetUint64()); + else + return false; + } + return true; +} + +bool RestParser::parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { + proto.set_dtype(tensorflow::DataType::DT_UINT32); + for (auto& value : doc.GetArray()) { + if (value.IsUint()) + proto.add_uint32_val(value.GetUint()); + else + return false; + } + return true; +} + +bool RestParser::parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { + // Special tensors are given in 1 dimentional array + if (doc.GetArray()[0].IsArray()) + return false; + + if (tensorName == "sequence_id") + return parseSequenceIdInput(doc, proto, tensorName); + else if (tensorName == "sequence_control_input") + return parseSequenceControlInput(doc, proto, tensorName); + + return false; +} + +static bool isBinary(const rapidjson::Value& value) { + if (!value.IsObject()) { + return false; + } + + if (!(value.HasMember("b64") && ((value.MemberEnd() - value.MemberBegin()) == 1))) { + return false; + } + + if (!value["b64"].IsString()) { + return false; + } + + return true; +} + +bool RestParser::parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName) { + if (isBinary(doc)) { + if (!addValue(proto, doc)) { + return false; + } + return true; + } + if (!doc.IsArray()) { + return false; + } + if (doc.GetArray().Size() == 0) { + return false; + } + if (!setDimOrValidate(proto, dim, doc.GetArray().Size())) { + return false; + } + if (tensorName == "sequence_id" || tensorName == "sequence_control_input") { + if (!parseSpecialInput(doc, proto, tensorName)) { + return false; + } + return true; + } + if (doc.GetArray()[0].IsArray()) { + for (auto& itr : doc.GetArray()) { + if (!parseArray(itr, dim + 1, proto, tensorName)) { + return false; + } + } + return true; + } else { + if (!setDTypeIfNotSet(doc.GetArray()[0], proto, tensorName)) { + return false; + } + for (auto& value : doc.GetArray()) { + if (!addValue(proto, value)) { + return false; + } + } + return true; + } + return false; +} + +bool RestParser::parseInstance(rapidjson::Value& doc) { + if (doc.GetObject().MemberCount() == 0) { + return false; + } + for (auto& itr : doc.GetObject()) { + std::string tensorName = itr.name.GetString(); + auto& proto = (*requestProto.mutable_inputs())[tensorName]; + increaseBatchSize(proto); + if (!parseArray(itr.value, 1, proto, tensorName)) { + return false; + } + } + return true; +} + +bool RestParser::isBatchSizeEqualForAllInputs() const { + int64_t size = 0; + for (const auto& kv : requestProto.inputs()) { + if (size == 0) { + size = kv.second.tensor_shape().dim(0).size(); + } else if (kv.second.tensor_shape().dim(0).size() != size) { + return false; + } + } + return true; +} + +Status RestParser::parseRowFormat(rapidjson::Value& node) { + order = Order::ROW; + if (!node.IsArray()) { + return StatusCode::REST_INSTANCES_NOT_AN_ARRAY; + } + if (node.GetArray().Size() == 0) { + return StatusCode::REST_NO_INSTANCES_FOUND; + } + if (node.GetArray()[0].IsObject() && !isBinary(node.GetArray()[0])) { + // named format + for (auto& instance : node.GetArray()) { + if (!instance.IsObject()) { + return StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT; + } + + if (!this->parseInstance(instance)) { + return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; + } + } + } else if (node.GetArray()[0].IsArray() || node.GetArray()[0].IsNumber() || isBinary(node.GetArray()[0])) { + // no named format + if (requestProto.inputs_size() != 1) { + return StatusCode::REST_INPUT_NOT_PREALLOCATED; + } + auto inputsIterator = requestProto.mutable_inputs()->begin(); + if (inputsIterator == requestProto.mutable_inputs()->end()) { + const std::string details = "Failed to parse row formatted request."; + SPDLOG_ERROR("Internal error occured: {}", details); + return Status(StatusCode::INTERNAL_ERROR, details); + } + if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { + return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; + } else { + format = Format::NONAMED; + return StatusCode::OK; + } + } else { + return StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED; + } + removeUnusedInputs(); + if (!isBatchSizeEqualForAllInputs()) { + return StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER; + } + format = Format::NAMED; + return StatusCode::OK; +} + +Status RestParser::parseColumnFormat(rapidjson::Value& node) { + order = Order::COLUMN; + // no named format + if (node.IsArray()) { + if (requestProto.inputs_size() != 1) { + return StatusCode::REST_INPUT_NOT_PREALLOCATED; + } + auto inputsIterator = requestProto.mutable_inputs()->begin(); + if (inputsIterator == requestProto.mutable_inputs()->end()) { + const std::string details = "Failed to parse column formatted request."; + SPDLOG_ERROR("Internal error occured: {}", details); + return Status(StatusCode::INTERNAL_ERROR, details); + } + if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { + return StatusCode::REST_COULD_NOT_PARSE_INPUT; + } + format = Format::NONAMED; + return StatusCode::OK; + } + // named format + if (!node.IsObject()) { + return StatusCode::REST_INPUTS_NOT_AN_OBJECT; + } + if (node.GetObject().MemberCount() == 0) { + return StatusCode::REST_NO_INPUTS_FOUND; + } + for (auto& kv : node.GetObject()) { + std::string tensorName = kv.name.GetString(); + auto& proto = (*requestProto.mutable_inputs())[tensorName]; + if (!parseArray(kv.value, 0, proto, tensorName)) { + return StatusCode::REST_COULD_NOT_PARSE_INPUT; + } + } + removeUnusedInputs(); + format = Format::NAMED; + return StatusCode::OK; +} + +Status RestParser::parse(const char* json) { + rapidjson::Document doc; + if (doc.Parse(json).HasParseError()) { + return StatusCode::JSON_INVALID; + } + if (!doc.IsObject()) { + return StatusCode::REST_BODY_IS_NOT_AN_OBJECT; + } + auto instancesItr = doc.FindMember("instances"); + auto inputsItr = doc.FindMember("inputs"); + if (instancesItr != doc.MemberEnd() && inputsItr != doc.MemberEnd()) { + return StatusCode::REST_PREDICT_UNKNOWN_ORDER; + } + if (instancesItr != doc.MemberEnd()) { + return parseRowFormat(instancesItr->value); + } + if (inputsItr != doc.MemberEnd()) { + return parseColumnFormat(inputsItr->value); + } + return StatusCode::REST_PREDICT_UNKNOWN_ORDER; +} + +void RestParser::increaseBatchSize(tensorflow::TensorProto& proto) { + if (proto.tensor_shape().dim_size() < 1) { + proto.mutable_tensor_shape()->add_dim()->set_size(0); + } + proto.mutable_tensor_shape()->mutable_dim(0)->set_size(proto.tensor_shape().dim(0).size() + 1); +} + +bool RestParser::setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size) { + if (proto.tensor_shape().dim_size() > dim) { + return proto.tensor_shape().dim(dim).size() == size; + } else { + while (proto.tensor_shape().dim_size() <= dim) { + proto.mutable_tensor_shape()->add_dim()->set_size(0); + } + proto.mutable_tensor_shape()->mutable_dim(dim)->set_size(size); + return true; + } +} + +bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { + if (!isBinary(value)) { + return false; + } + + b64Val = value["b64"].GetString(); + return true; +} + +template +bool addToTensorContent(tensorflow::TensorProto& proto, T value) { + if (sizeof(T) != DataTypeSize(proto.dtype())) { + return false; + } + proto.mutable_tensor_content()->append(reinterpret_cast(&value), sizeof(T)); + return true; +} + +template +bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& value) { + if (value.IsDouble()) { + return addToTensorContent(proto, static_cast(value.GetDouble())); + } + if (value.IsInt64()) { + return addToTensorContent(proto, static_cast(value.GetInt64())); + } + if (value.IsUint64()) { + return addToTensorContent(proto, static_cast(value.GetUint64())); + } + if (value.IsInt()) { + return addToTensorContent(proto, static_cast(value.GetInt())); + } + if (value.IsUint()) { + return addToTensorContent(proto, static_cast(value.GetUint())); + } + + return false; +} + +bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { + if (value.IsDouble()) { + proto.add_half_val(value.GetDouble()); + return true; + } + if (value.IsInt64()) { + proto.add_half_val(value.GetInt64()); + return true; + } + if (value.IsUint64()) { + proto.add_half_val(value.GetUint64()); + return true; + } + if (value.IsInt()) { + proto.add_half_val(value.GetInt()); + return true; + } + if (value.IsUint()) { + proto.add_half_val(value.GetUint()); + return true; + } + + return false; +} + +bool addToIntVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { + if (value.IsDouble()) { + proto.add_int_val(value.GetDouble()); + return true; + } + if (value.IsInt64()) { + proto.add_int_val(value.GetInt64()); + return true; + } + if (value.IsUint64()) { + proto.add_int_val(value.GetUint64()); + return true; + } + if (value.IsInt()) { + proto.add_int_val(value.GetInt()); + return true; + } + if (value.IsUint()) { + proto.add_int_val(value.GetUint()); + return true; + } + + return false; +} + +bool RestParser::addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value) { + if (isBinary(value)) { + std::string b64Val; + if (!getB64FromValue(value, b64Val)) + return false; + std::string decodedBytes; + if (decodeBase64(b64Val, decodedBytes) == StatusCode::OK) { + proto.add_string_val(decodedBytes.c_str(), decodedBytes.length()); + proto.set_dtype(tensorflow::DataType::DT_STRING); + return true; + } else { + return false; + } + } + + if (!value.IsNumber()) { + return false; + } + + switch (proto.dtype()) { + case tensorflow::DataType::DT_FLOAT: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_INT32: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_INT8: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_UINT8: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_DOUBLE: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_HALF: + return addToHalfVal(proto, value); + case tensorflow::DataType::DT_INT16: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_UINT16: + return addToIntVal(proto, value); + case tensorflow::DataType::DT_INT64: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_UINT32: + return addToTensorContent(proto, value); + case tensorflow::DataType::DT_UINT64: + return addToTensorContent(proto, value); + default: + return false; + } + return false; +} + +// This is still required for parsing inputs which are not present in model/DAG. +// Such inputs are then removed from proto at the end of parsing phase. +bool RestParser::setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName) { + if (tensorPrecisionMap.count(tensorName)) + return true; + + if (value.IsInt()) + tensorPrecisionMap[tensorName] = ovms::Precision::I32; + else if (value.IsDouble()) + tensorPrecisionMap[tensorName] = ovms::Precision::FP32; + else + return false; + + proto.set_dtype(getPrecisionAsDataType(tensorPrecisionMap[tensorName])); + return true; +} + +} // namespace ovms diff --git a/src/ovms_lib/rest_parser.hpp b/src/ovms_lib/rest_parser.hpp new file mode 100644 index 0000000000..b8a0745a7c --- /dev/null +++ b/src/ovms_lib/rest_parser.hpp @@ -0,0 +1,218 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +/** + * @brief Request order types + */ +enum class Order { + UNKNOWN, + ROW, + COLUMN +}; + +/** + * @brief Request format types + */ +enum class Format { + UNKNOWN, + NAMED, + NONAMED +}; + +/** + * @brief This class encapsulates http request body string parsing to request proto. + */ +class RestParser { + /** + * @brief Request order + */ + Order order = Order::UNKNOWN; + + /** + * @brief Request format + */ + Format format = Format::UNKNOWN; + + /** + * @brief Request proto + */ + tensorflow::serving::PredictRequest requestProto; + + /** + * @brief Request content precision + */ + std::map tensorPrecisionMap; + + void removeUnusedInputs(); + + /** + * @brief Increases batch size (0th-dimension) of tensor + */ + static void increaseBatchSize(tensorflow::TensorProto& proto); + + /** + * @brief Sets specific dimension to given size + * + * @return returns false if dimension already existed and did not match requested size, true otherwise + */ + static bool setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size); + + /** + * Parses and adds rapidjson value to tensor proto depending on underlying tensor data type + */ + static bool addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value); + + bool parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); + bool parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); + bool parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); + + /** + * @brief Parses rapidjson Node for arrays or numeric values on certain level of nesting. + * + * @param doc rapidjson Node + * @param dim level of nesting + * @param input destination for parsing numeric values + * + * @return false if processing failed, true when succeeded + * + * Rapid json node expected to be passed in following structure: + * [ + * [...], + * [...], + * ... + * ] + */ + bool parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName); + + /** + * @brief Parses rapidjson Node for inputs in a string(name)=>array(data) format + * + * @param doc rapidjson Node + * + * @return false if processing failed, true when succeeded + * + * Rapid json node expected to be passed in following structure: + * { + * "input1": [[...], [...], ...], + * "input2": [[...], [...], ...], + * ... + * } + */ + bool parseInstance(rapidjson::Value& doc); + + /** + * @brief Checks whether all inputs have equal batch size, 0th-dimension + * + * @return true or false + */ + bool isBatchSizeEqualForAllInputs() const; + + /** + * @brief Parses row format: list of objects, each object corresponding to one batch with one or multiple inputs. + * When no named format is detected, instance is treated as array of single input batches with no name. + * + * @param node rapidjson Node + * + * @return Status indicating if processing succeeded, error code otherwise + * + * Rapid json node expected to be passed in following structure: + * [{inputs...}, {inputs...}, {inputs...}, ...] + * or: + * [no named input data batches...] + */ + Status parseRowFormat(rapidjson::Value& node); + + /** + * @brief Parses column format: object of input:batches key value pairs. + * When no named format is detected, instance is treated as array of single input batches with no name. + * + * @param node rapidjson Node + * + * @return Status indicating if processing succeeded, error code otherwise + * + * Rapid json node expected to be passed in following structure: + * {"inputA": [...], "inputB": [...], ...} + * or: + * [no named input data batches...] + */ + Status parseColumnFormat(rapidjson::Value& node); + +public: + bool setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName); + /** + * @brief Constructor for preallocating memory for inputs beforehand. Size is calculated from tensor shape required by backend. + * + * @param tensors Tensor map with model input parameters + */ + RestParser(const tensor_map_t& tensors); + + /** + * @brief Gets parsed request proto + * + * @return proto + */ + tensorflow::serving::PredictRequest& getProto() { return requestProto; } + + /** + * @brief Gets request order + */ + Order getOrder() const { + return order; + } + + /** + * @brief Gets request format + */ + Format getFormat() const { + return format; + } + + /** + * @brief Parses http request body string + * + * @param json request string + * + * @return Status indicating error code or success + * + * JSON expected to be passed in following structure: + * { + * "signature_name": "serving_default", + * "instances": [ + * {...}, {...}, {...}, ... + * ] + * } + */ + Status parse(const char* json); +}; + +} // namespace ovms diff --git a/src/ovms_lib/rest_utils.cpp b/src/ovms_lib/rest_utils.cpp new file mode 100644 index 0000000000..d3a33fbcbc --- /dev/null +++ b/src/ovms_lib/rest_utils.cpp @@ -0,0 +1,198 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "rest_utils.hpp" + +#include + +#include "absl/strings/escaping.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/util/json_tensor.h" +#pragma GCC diagnostic pop + +#include "tfs_frontend/tfs_utils.hpp" +#include "timer.hpp" + +using tensorflow::DataType; +using tensorflow::DataTypeSize; +using tensorflow::serving::JsonPredictRequestFormat; +using tensorflow::serving::MakeJsonFromTensors; +using tensorflow::serving::PredictResponse; + +namespace ovms { + +Status checkValField(const size_t& fieldSize, const size_t& expectedElementsNumber) { + if (fieldSize == 0) + return StatusCode::REST_SERIALIZE_NO_DATA; + if (fieldSize != expectedElementsNumber) + return StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE; + return StatusCode::OK; +} + +Status makeJsonFromPredictResponse( + PredictResponse& response_proto, + std::string* response_json, + Order order) { + if (order == Order::UNKNOWN) { + return StatusCode::REST_PREDICT_UNKNOWN_ORDER; + } + + Timer timer; + using std::chrono::microseconds; + + timer.start("convert"); + + for (auto& kv : *response_proto.mutable_outputs()) { + auto& tensor = kv.second; + + size_t expectedContentSize = DataTypeSize(tensor.dtype()); + for (int i = 0; i < tensor.tensor_shape().dim_size(); i++) { + expectedContentSize *= tensor.tensor_shape().dim(i).size(); + } + size_t dataTypeSize = DataTypeSize(tensor.dtype()); + size_t expectedElementsNumber = dataTypeSize > 0 ? expectedContentSize / dataTypeSize : 0; + bool seekDataInValField = false; + + if (tensor.tensor_content().size() == 0) + seekDataInValField = true; + else if (tensor.tensor_content().size() != expectedContentSize) + return StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE; + + switch (tensor.dtype()) { + case DataType::DT_FLOAT: + if (seekDataInValField) { + auto status = checkValField(tensor.float_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(float)) + tensor.add_float_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_INT32: + if (seekDataInValField) { + auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int32_t)) + tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_INT8: + if (seekDataInValField) { + auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int8_t)) + tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_UINT8: + if (seekDataInValField) { + auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint8_t)) + tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_DOUBLE: + if (seekDataInValField) { + auto status = checkValField(tensor.double_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(double)) + tensor.add_double_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_INT16: + if (seekDataInValField) { + auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int16_t)) + tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_INT64: + if (seekDataInValField) { + auto status = checkValField(tensor.int64_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int64_t)) + tensor.add_int64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_UINT32: + if (seekDataInValField) { + auto status = checkValField(tensor.uint32_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint32_t)) + tensor.add_uint32_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + case DataType::DT_UINT64: + if (seekDataInValField) { + auto status = checkValField(tensor.uint64_val_size(), expectedElementsNumber); + if (!status.ok()) + return status; + } else { + for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint64_t)) + tensor.add_uint64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); + } + break; + default: + return StatusCode::REST_UNSUPPORTED_PRECISION; + } + } + + timer.stop("convert"); + timer.start("MakeJsonFromTensors"); + + const auto& tf_status = MakeJsonFromTensors( + response_proto.outputs(), + order == Order::ROW ? JsonPredictRequestFormat::kRow : JsonPredictRequestFormat::kColumnar, + response_json); + + timer.stop("MakeJsonFromTensors"); + SPDLOG_DEBUG("tensor_content to *_val container conversion: {:.3f} ms", timer.elapsed("convert") / 1000); + SPDLOG_DEBUG("MakeJsonFromTensors call: {:.3f} ms", timer.elapsed("MakeJsonFromTensors") / 1000); + + if (!tf_status.ok()) { + SPDLOG_ERROR("Creating json from tensors failed: {}", tf_status.error_message()); + return StatusCode::REST_PROTO_TO_STRING_ERROR; + } + + return StatusCode::OK; +} + +Status decodeBase64(std::string& bytes, std::string& decodedBytes) { + auto status = Status(absl::Base64Unescape(bytes, &decodedBytes) ? StatusCode::OK : StatusCode::REST_BASE64_DECODE_ERROR); + if (!status.ok()) { + return status; + } + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/rest_utils.hpp b/src/ovms_lib/rest_utils.hpp new file mode 100644 index 0000000000..fbc928f9e3 --- /dev/null +++ b/src/ovms_lib/rest_utils.hpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "rest_parser.hpp" +#include "status.hpp" + +namespace ovms { +Status makeJsonFromPredictResponse( + tensorflow::serving::PredictResponse& response_proto, + std::string* response_json, + Order order); + +Status decodeBase64(std::string& bytes, std::string& decodedBytes); + +} // namespace ovms diff --git a/src/ovms_lib/s3filesystem.cpp b/src/ovms_lib/s3filesystem.cpp new file mode 100644 index 0000000000..b4fc5f47e3 --- /dev/null +++ b/src/ovms_lib/s3filesystem.cpp @@ -0,0 +1,552 @@ +// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "s3filesystem.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "stringutils.hpp" + +namespace ovms { + +namespace s3 = Aws::S3; +namespace fs = std::filesystem; + +const std::string S3FileSystem::S3_URL_PREFIX = "s3://"; + +StatusCode S3FileSystem::parsePath(const std::string& path, std::string* bucket, std::string* object) { + std::smatch sm; + + if (isPathEscaped(path)) { + SPDLOG_LOGGER_ERROR(s3_logger, "Path {} escape with .. is forbidden.", path); + return StatusCode::PATH_INVALID; + } + + if (!std::regex_match(path, sm, s3_regex_)) { + int bucket_start = path.find(S3_URL_PREFIX) + S3_URL_PREFIX.size(); + int bucket_end = path.find("/", bucket_start); + + if (bucket_end > bucket_start) { + *bucket = path.substr(bucket_start, bucket_end - bucket_start); + *object = path.substr(bucket_end + 1); + } else { + *bucket = path.substr(bucket_start); + *object = ""; + } + } else { + *bucket = sm[3]; + *object = sm[4]; + } + + if (bucket->empty()) { + SPDLOG_LOGGER_ERROR(s3_logger, "No bucket name found in path {}", path); + + return StatusCode::S3_BUCKET_NOT_FOUND; + } + + return StatusCode::OK; +} + +S3FileSystem::S3FileSystem(const Aws::SDKOptions& options, const std::string& s3_path) : + options_(options), + s3_regex_(S3_URL_PREFIX + "([0-9a-zA-Z-.]+):([0-9]+)/([0-9a-z.-]+)(((/" + "[0-9a-zA-Z.-_]+)*)?)"), + proxy_regex_("^(https?)://(([^:]{1,128}):([^@]{1,256})@)?([^:/]{1,255})(:([0-9]{1,5}))?/?") { + Aws::Client::ClientConfiguration config; + Aws::Auth::AWSCredentials credentials; + + const char* profile_name = std::getenv("AWS_PROFILE"); + const char* secret_key = std::getenv("AWS_SECRET_ACCESS_KEY"); + const char* key_id = std::getenv("AWS_ACCESS_KEY_ID"); + const char* region = std::getenv("AWS_REGION"); + const char* s3_endpoint = std::getenv("S3_ENDPOINT"); + const char* http_proxy = std::getenv("http_proxy") != nullptr ? std::getenv("http_proxy") : std::getenv("HTTP_PROXY"); + const char* https_proxy = std::getenv("https_proxy") != nullptr ? std::getenv("https_proxy") : std::getenv("HTTPS_PROXY"); + const std::string default_proxy = https_proxy != nullptr ? std::string(https_proxy) : http_proxy != nullptr ? std::string(http_proxy) : ""; + + if ((secret_key != NULL) && (key_id != NULL)) { + credentials.SetAWSAccessKeyId(key_id); + credentials.SetAWSSecretKey(secret_key); + config = Aws::Client::ClientConfiguration(); + if (region != NULL) { + config.region = region; + } + } else if (profile_name) { + config = Aws::Client::ClientConfiguration(profile_name); + } else { + config = Aws::Client::ClientConfiguration("default"); + } + + std::string host_name, host_port, bucket, object; + std::smatch sm; + if (std::regex_match(s3_path, sm, s3_regex_)) { + host_name = sm[1]; + host_port = sm[2]; + bucket = sm[3]; + object = sm[4]; + + config.endpointOverride = Aws::String(host_name + ":" + host_port); + config.scheme = Aws::Http::Scheme::HTTP; + } + if (s3_endpoint != NULL) { + std::string endpoint(s3_endpoint); + if (endpoint.rfind("http://") != std::string::npos) { + endpoint = endpoint.substr(7); + } + config.endpointOverride = Aws::String(endpoint.c_str()); + config.scheme = Aws::Http::Scheme::HTTP; + } + + if (!default_proxy.empty()) { + if (std::regex_match(default_proxy, sm, proxy_regex_)) { + config.proxyHost = sm[5].str(); + config.proxyPort = std::stoi(sm[7].str()); + config.proxyScheme = sm[1].str().size() == 4 ? Aws::Http::Scheme::HTTP : Aws::Http::Scheme::HTTPS; + if (!sm[3].str().empty()) { + config.proxyUserName = sm[3].str(); + } + if (!sm[4].str().empty()) { + config.proxyPassword = sm[4].str(); + } + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Couldn't parse proxy: {}", default_proxy); + } + } + + if (profile_name) { + client_ = s3::S3Client( + config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false); + } else if ((secret_key != NULL) && (key_id != NULL)) { + client_ = s3::S3Client( + credentials, + config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false); + } else { + std::shared_ptr provider = std::make_shared(); + client_ = s3::S3Client( + provider, + config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + true); + } +} + +S3FileSystem::~S3FileSystem() { + Aws::ShutdownAPI(options_); +} + +StatusCode S3FileSystem::fileExists(const std::string& path, bool* exists) { + *exists = false; + std::string bucket, object; + + auto status = parsePath(path, &bucket, &object); + if (status != StatusCode::OK) { + return status; + } + + s3::Model::HeadObjectRequest head_request; + head_request.SetBucket(bucket.c_str()); + head_request.SetKey(object.c_str()); + + auto head_object_outcome = client_.HeadObject(head_request); + if (head_object_outcome.IsSuccess()) { + *exists = true; + return StatusCode::OK; + } + + bool is_dir; + status = isDirectory(path, &is_dir); + if (status != StatusCode::OK) { + return status; + } + *exists = is_dir; + + return StatusCode::OK; +} + +StatusCode S3FileSystem::isDirectory(const std::string& path, bool* is_dir) { + *is_dir = false; + std::string bucket, object_path; + auto status = parsePath(path, &bucket, &object_path); + if (status != StatusCode::OK) { + return status; + } + + // Check if the bucket exists + s3::Model::HeadBucketRequest head_request; + head_request.WithBucket(bucket.c_str()); + + auto head_bucket_outcome = client_.HeadBucket(head_request); + if (!head_bucket_outcome.IsSuccess()) { + SPDLOG_LOGGER_ERROR(s3_logger, "Invalid or missing S3 credentials, or bucket does not exist - {}. {}", bucket, head_bucket_outcome.GetError().GetMessage()); + return StatusCode::S3_METADATA_FAIL; + } + + // Root case - bucket exists and object path is empty + if (object_path.empty()) { + *is_dir = true; + return StatusCode::OK; + } + + // List the objects in the bucket + s3::Model::ListObjectsRequest list_objects_request; + list_objects_request.SetBucket(bucket.c_str()); + list_objects_request.SetPrefix(appendSlash(object_path).c_str()); + auto list_objects_outcome = client_.ListObjects(list_objects_request); + + if (list_objects_outcome.IsSuccess()) { + *is_dir = !list_objects_outcome.GetResult().GetContents().empty(); + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to list objects with prefix {}", path); + return StatusCode::S3_FAILED_LIST_OBJECTS; + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::getDirectoryContents(const std::string& path, std::set* contents) { + // Parse bucket and dir_path + std::string bucket, dir_path, full_dir; + auto status = parsePath(path, &bucket, &dir_path); + if (status != StatusCode::OK) { + return status; + } + std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; + + // Capture the full path to facilitate content listing + full_dir = appendSlash(dir_path); + + // Issue request for objects with prefix + s3::Model::ListObjectsRequest objects_request; + objects_request.SetBucket(bucket.c_str()); + objects_request.SetPrefix(full_dir.c_str()); + auto list_objects_outcome = client_.ListObjects(objects_request); + + if (list_objects_outcome.IsSuccess()) { + Aws::Vector object_list = list_objects_outcome.GetResult().GetContents(); + for (auto const& s3_object : object_list) { + // In the case of empty directories, the directory itself will appear here + if (s3_object.GetKey().c_str() == full_dir) { + continue; + } + + // We have to make sure that subdirectory contents do not appear here + std::string name(s3_object.GetKey().c_str()); + int item_start = name.find(full_dir) + full_dir.size(); + int item_end = name.find("/", item_start); + + // Let set take care of subdirectory contents + std::string item = name.substr(item_start, item_end - item_start); + contents->insert(item); + } + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Could not list contents of directory {}", true_path); + return StatusCode::S3_INVALID_ACCESS; + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::getDirectorySubdirs(const std::string& path, std::set* subdirs) { + // Parse bucket and dir_path + std::string bucket, dir_path; + auto status = parsePath(path, &bucket, &dir_path); + if (status != StatusCode::OK) { + return status; + } + std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; + status = getDirectoryContents(true_path, subdirs); + if (status != StatusCode::OK) { + return status; + } + + // Erase non-directory entries... + for (auto iter = subdirs->begin(); iter != subdirs->end();) { + bool is_dir; + status = isDirectory(joinPath({true_path, *iter}), &is_dir); + if (status != StatusCode::OK) { + return status; + } + if (!is_dir) { + iter = subdirs->erase(iter); + } else { + ++iter; + } + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::getDirectoryFiles(const std::string& path, std::set* files) { + // Parse bucket and dir_path + std::string bucket, dir_path; + auto status = parsePath(path, &bucket, &dir_path); + if (status != StatusCode::OK) { + return status; + } + + std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; + status = getDirectoryContents(true_path, files); + if (status != StatusCode::OK) { + return status; + } + + // Erase directory entries... + for (auto iter = files->begin(); iter != files->end();) { + bool is_dir; + status = isDirectory(joinPath({true_path, *iter}), &is_dir); + if (status != StatusCode::OK) { + return status; + } + if (is_dir) { + iter = files->erase(iter); + } else { + ++iter; + } + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::readTextFile(const std::string& path, std::string* contents) { + bool exists; + auto status = fileExists(path, &exists); + if (status != StatusCode::OK) { + return status; + } + + if (!exists) { + SPDLOG_LOGGER_ERROR(s3_logger, "File does not exist at {}", path); + return StatusCode::S3_FILE_NOT_FOUND; + } + + std::string bucket, object; + status = parsePath(path, &bucket, &object); + if (status != StatusCode::OK) { + return status; + } + + // Send a request for the objects metadata + s3::Model::GetObjectRequest object_request; + object_request.SetBucket(bucket.c_str()); + object_request.SetKey(object.c_str()); + + auto get_object_outcome = client_.GetObject(object_request); + if (get_object_outcome.IsSuccess()) { + auto& object_result = get_object_outcome.GetResultWithOwnership().GetBody(); + + std::string data = ""; + char c = 0; + while (object_result.get(c)) { + data += c; + } + + *contents = data; + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", path); + return StatusCode::S3_FILE_INVALID; + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { + bool exists; + auto status = fileExists(path, &exists); + if (status != StatusCode::OK) { + return status; + } + if (!exists) { + SPDLOG_LOGGER_ERROR(s3_logger, "File/folder does not exist at {}", path); + return StatusCode::S3_FILE_NOT_FOUND; + } + + std::string effective_path, host_name, host_port, bucket, object; + + std::smatch sm; + if (std::regex_match(path, sm, s3_regex_)) { + host_name = sm[1]; + host_port = sm[2]; + bucket = sm[3]; + object = sm[4]; + effective_path = S3_URL_PREFIX + bucket + object; + } else { + effective_path = path; + } + + bool is_dir = false; + std::set contents, files; + status = isDirectory(effective_path, &is_dir); + if (status != StatusCode::OK) { + return status; + } + if (is_dir) { + status = getDirectoryContents(effective_path, &contents); + if (status != StatusCode::OK) { + return status; + } + + for (auto iter = contents.begin(); iter != contents.end(); ++iter) { + bool is_subdir; + std::string s3_fpath = joinPath({effective_path, *iter}); + std::string local_fpath = joinPath({local_path, *iter}); + status = isDirectory(s3_fpath, &is_subdir); + if (status != StatusCode::OK) { + return status; + } + if (is_subdir) { + // Create local mirror of sub-directories + int status = mkdir(const_cast(local_fpath.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); + if (status == -1) { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to create local folder: {} {} ", local_fpath, strerror(errno)); + return StatusCode::PATH_INVALID; + } + + // Add with s3 path + std::set subdir_files; + auto s = getDirectoryFiles(s3_fpath, &subdir_files); + if (s != StatusCode::OK) { + return s; + } + for (auto itr = subdir_files.begin(); itr != subdir_files.end(); ++itr) { + files.insert(joinPath({s3_fpath, *itr})); + } + } else { + files.insert(s3_fpath); + } + } + + for (auto iter = files.begin(); iter != files.end(); ++iter) { + if (std::any_of(acceptedFiles.begin(), acceptedFiles.end(), [&iter](const std::string& x) { + return iter->size() > 0 && endsWith(*iter, x); + })) { + std::string bucket, object; + status = parsePath(*iter, &bucket, &object); + if (status != StatusCode::OK) { + return status; + } + + // Send a request for the objects metadata + s3::Model::GetObjectRequest object_request; + object_request.SetBucket(bucket.c_str()); + object_request.SetKey(object.c_str()); + + auto get_object_outcome = client_.GetObject(object_request); + if (get_object_outcome.IsSuccess()) { + auto& retrieved_file = + get_object_outcome.GetResultWithOwnership().GetBody(); + std::string s3_removed_path = (*iter).substr(effective_path.size()); + std::string local_file_path = joinPath({local_path, s3_removed_path}); + std::ofstream output_file(local_file_path.c_str(), std::ios::binary); + output_file << retrieved_file.rdbuf(); + output_file.close(); + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", *iter); + return StatusCode::S3_FAILED_GET_OBJECT; + } + } + } + } else { + std::string bucket, object; + auto s = parsePath(effective_path, &bucket, &object); + if (s != StatusCode::OK) { + return s; + } + + // Send a request for the objects metadata + s3::Model::GetObjectRequest object_request; + object_request.SetBucket(bucket.c_str()); + object_request.SetKey(object.c_str()); + + auto get_object_outcome = client_.GetObject(object_request); + if (get_object_outcome.IsSuccess()) { + auto& retrieved_file = get_object_outcome.GetResultWithOwnership().GetBody(); + std::ofstream output_file(local_path.c_str(), std::ios::binary); + output_file << retrieved_file.rdbuf(); + output_file.close(); + } else { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", effective_path); + return StatusCode::S3_FAILED_GET_OBJECT; + } + } + + return StatusCode::OK; +} + +StatusCode S3FileSystem::downloadModelVersions(const std::string& path, + std::string* local_path, + const std::vector& versions) { + auto sc = createTempPath(local_path); + if (sc != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to create a temporary path {}", sc); + return sc; + } + + StatusCode result = StatusCode::OK; + for (auto& ver : versions) { + std::string versionpath = path; + if (!endsWith(versionpath, "/")) { + versionpath.append("/"); + } + versionpath.append(std::to_string(ver)); + std::string lpath = *local_path; + if (!endsWith(lpath, "/")) { + lpath.append("/"); + } + lpath.append(std::to_string(ver)); + fs::create_directory(lpath); + auto status = downloadFileFolder(versionpath, lpath); + if (status != StatusCode::OK) { + result = status; + SPDLOG_LOGGER_ERROR(s3_logger, "Failed to download model version {}", versionpath); + } + } + + return result; +} + +StatusCode S3FileSystem::deleteFileFolder(const std::string& path) { + remove(path.c_str()); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/s3filesystem.hpp b/src/ovms_lib/s3filesystem.hpp new file mode 100644 index 0000000000..56757c3ddf --- /dev/null +++ b/src/ovms_lib/s3filesystem.hpp @@ -0,0 +1,155 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include +#include + +#include "filesystem.hpp" +#include "status.hpp" + +namespace ovms { + +class S3FileSystem : public FileSystem { +public: + /** + * @brief Construct a new S3FileSystem object + * + * @param options + * @param s3_path + */ + S3FileSystem(const Aws::SDKOptions& options, const std::string& s3_path); + + /** + * @brief Destroy the S3FileSystem object + * + */ + ~S3FileSystem(); + + /** + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ + StatusCode fileExists(const std::string& path, bool* exists) override; + + /** + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ + StatusCode isDirectory(const std::string& path, bool* is_dir) override; + + /** + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; + + /** + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ + StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; + + /** + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ + StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; + + /** + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ + StatusCode readTextFile(const std::string& path, std::string* contents) override; + + /** + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ + StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; + + /** + * @brief Download selected model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ + StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; + + /** + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ + StatusCode deleteFileFolder(const std::string& path) override; + + static const std::string S3_URL_PREFIX; + +private: + /** + * @brief + * + * @param path + * @param bucket + * @param object + * @return StatusCode + */ + StatusCode parsePath(const std::string& path, std::string* bucket, std::string* object); + + /** + * @brief + * + */ + Aws::SDKOptions options_; + + /** + * @brief + * + */ + Aws::S3::S3Client client_; + std::regex s3_regex_; + std::regex proxy_regex_; +}; + +} // namespace ovms diff --git a/src/ovms_lib/schema.cpp b/src/ovms_lib/schema.cpp new file mode 100644 index 0000000000..d35b8ea947 --- /dev/null +++ b/src/ovms_lib/schema.cpp @@ -0,0 +1,351 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "schema.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace ovms { +const char* MODELS_CONFIG_SCHEMA = R"({ + "definitions": { + "custom_loader_config": { + "type": "object", + "required": ["config"], + "properties": { + "config": { + "type": "object", + "required": ["loader_name", "library_path"], + "properties": { + "loader_name": { + "type": "string" + }, + "library_path": { + "type": "string" + }, + "loader_config_file": { + "type": "string" + } + }, + "additionalProperties": false + }, + "additionalProperties": false + } + }, + "model_config": { + "type": "object", + "required": ["config"], + "properties": { + "config": { + "type": "object", + "required": ["name", "base_path"], + "properties": { + "name": { + "type": "string" + }, + "base_path": { + "type": "string" + }, + "batch_size": { + "type": ["integer", "string"], + "minimum": 0 + }, + "model_version_policy": { + "type": "object" + }, + "shape": { + "type": ["object", "string"] + }, + "layout": { + "type": ["object", "string"] + }, + "nireq": { + "type": "integer", + "minimum": 0 + }, + "target_device": { + "type": "string" + }, + "allow_cache": { + "type": "boolean" + }, + "plugin_config": { + "type": "object" + }, + "stateful": { + "type": "boolean" + }, + "idle_sequence_cleanup": { + "type": "boolean" + }, + "low_latency_transformation": { + "type": "boolean" + }, + "max_sequence_number": { + "type": "integer", + "minimum": 0 + }, + "custom_loader_options": { + "type": "object", + "required": ["loader_name"], + "properties": { + "loader_name": { + "type": "string" + } + }, + "minProperties": 1 + } + }, + "additionalProperties": false + }, + "additionalProperties": false + } + }, + "source_node_names": { + "type": "object", + "required": ["node_name", "data_item"], + "properties": { + "node_name": { + "type": "string" + }, + "data_item": { + "type": "string" + } + }, + "additionalProperties": false + }, + "source_node": { + "type": "object", + "additionalProperties" : { + "$ref": "#/definitions/source_node_names" + }, + "minProperties": 1, + "maxProperties": 1 + }, + "output_alias": { + "type": "object", + "required": ["data_item", "alias"], + "properties": { + "data_item": { + "type": "string" + }, + "alias": { + "type": "string" + } + }, + "additionalProperties": false + }, + "node_config": { + "type": "object", + "required": ["name", "type", "inputs", "outputs"], + "oneOf": [ + { + "properties": { "type": { "enum": ["custom"] } }, + "required": ["library_name"], + "not": { "required": ["model_name"] } + }, + { + "properties": { "type": { "enum": ["DL model"] } }, + "not": { "required": ["library_name"] }, + "required": ["model_name"] + } + ], + "properties": { + "name": { + "type": "string" + }, + "model_name": { + "type": "string" + }, + "library_name": { + "type": "string" + }, + "type": { + "type": "string", + "enum": ["DL model", "custom"] + }, + "version": { + "type": "integer", + "minimum": 1 + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/definitions/source_node" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/definitions/output_alias" + } + }, + "params": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "demultiply_count": { + "type": "integer", + "minimum": -1, + "maximum": 10000 + }, + "gather_from_node": { + "type": "string" + } + }, + "additionalProperties": false + }, + "pipeline_config": { + "type": "object", + "required": ["name", "nodes", "inputs", "outputs"], + "properties": { + "name": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/definitions/node_config" + } + }, + "inputs": { + "type": "array", + "items": { + "type": "string" + } + }, + "outputs": { + "type": "array", + "items": { + "$ref": "#/definitions/source_node" + } + }, + "demultiply_count" : { + "type": "integer", + "minimum": -1, + "maximum": 10000 + } + }, + "additionalProperties": false + }, + "custom_node_library_config": { + "type": "object", + "required": ["name", "base_path"], + "properties": { + "name": { + "type": "string" + }, + "base_path": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "type": "object", + "required": ["model_config_list"], + "properties": { + "custom_loader_config_list": { + "type": "array", + "items": { + "$ref": "#/definitions/custom_loader_config" + } + }, + "model_config_list": { + "type": "array", + "items": { + "$ref": "#/definitions/model_config" + } + }, + "pipeline_config_list": { + "type": "array", + "items": { + "$ref": "#/definitions/pipeline_config" + } + }, + "custom_node_library_config_list": { + "type": "array", + "items": { + "$ref": "#/definitions/custom_node_library_config" + } + } + }, + "additionalProperties": false +})"; + +const char* MODELS_MAPPING_INPUTS_SCHEMA = R"({ + "type": "object", + "required": [ + "inputs" + ], + "properties": { + "outputs":{ + "type": "object" + }, + "inputs":{ + "type": "object" + } + }, + "additionalProperties": false + })"; + +const char* MODELS_MAPPING_OUTPUTS_SCHEMA = R"({ + "type": "object", + "required": [ + "outputs" + ], + "properties": { + "outputs":{ + "type": "object" + }, + "inputs":{ + "type": "object" + } + }, + "additionalProperties": false + })"; + +StatusCode validateJsonAgainstSchema(rapidjson::Document& json, const char* schema) { + rapidjson::Document schemaJson; + rapidjson::ParseResult parsingSucceeded = schemaJson.Parse(schema); + if (!parsingSucceeded) { + SPDLOG_ERROR("JSON schema parse error: {}, at: {}", rapidjson::GetParseError_En(parsingSucceeded.Code()), parsingSucceeded.Offset()); + return StatusCode::JSON_INVALID; + } + rapidjson::SchemaDocument parsedSchema(schemaJson); + rapidjson::SchemaValidator validator(parsedSchema); + if (!json.Accept(validator)) { + rapidjson::StringBuffer sb; + validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); + std::string schema = sb.GetString(); + std::string keyword = validator.GetInvalidSchemaKeyword(); + sb.Clear(); + validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); + std::string key = sb.GetString(); + + SPDLOG_ERROR("Given config is invalid according to schema: {}. Keyword: {} Key: {}", schema, keyword, key); + return StatusCode::JSON_INVALID; + } + + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/ovms_lib/schema.hpp b/src/ovms_lib/schema.hpp new file mode 100644 index 0000000000..3753314619 --- /dev/null +++ b/src/ovms_lib/schema.hpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include "status.hpp" + +namespace ovms { +extern const char* MODELS_CONFIG_SCHEMA; +extern const char* MODELS_MAPPING_INPUTS_SCHEMA; +extern const char* MODELS_MAPPING_OUTPUTS_SCHEMA; + +StatusCode validateJsonAgainstSchema(rapidjson::Document& json, const char* schema); +} // namespace ovms diff --git a/src/ovms_lib/sequence.cpp b/src/ovms_lib/sequence.cpp new file mode 100644 index 0000000000..3b3cabaf5c --- /dev/null +++ b/src/ovms_lib/sequence.cpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "sequence.hpp" + +#include + +using namespace InferenceEngine; + +namespace ovms { + +const uint64_t Sequence::getId() const { + return sequenceId; +} + +const sequence_memory_state_t& Sequence::getMemoryState() const { + return memoryState; +} + +const bool Sequence::isIdle() const { + return idle; +} + +void Sequence::setIdle(bool idle) { + this->idle = idle; +} + +Status Sequence::updateMemoryState(model_memory_state_t& newState) { + for (auto&& state : newState) { + auto stateName = state.get_name(); + ov::Tensor tensor = state.get_state(); + ov::Tensor copyTensor; + auto status = tensorClone(copyTensor, tensor); + if (!status.ok()) { + return status; + } + memoryState[stateName] = copyTensor; + } + setIdle(false); + return StatusCode::OK; +} + +std::mutex& Sequence::getMutex() { + return mutex; +} + +bool Sequence::isTerminated() const { + return terminated; +} + +void Sequence::setTerminated() { + this->terminated = true; +} + +} // namespace ovms diff --git a/src/ovms_lib/sequence.hpp b/src/ovms_lib/sequence.hpp new file mode 100644 index 0000000000..2c6157e5ed --- /dev/null +++ b/src/ovms_lib/sequence.hpp @@ -0,0 +1,60 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ov_utils.hpp" +#include "status.hpp" + +namespace ovms { + +using sequence_memory_state_t = std::unordered_map; +using model_memory_state_t = std::vector; + +class Sequence { +private: + uint64_t sequenceId; + sequence_memory_state_t memoryState; + std::mutex mutex; + bool terminated; + bool idle; + +public: + Sequence(uint64_t sequenceId) : + sequenceId(sequenceId), + terminated(false), + idle(false) {} + const sequence_memory_state_t& getMemoryState() const; + const uint64_t getId() const; + const bool isIdle() const; + void setIdle(bool idle = true); + // In case updateMemoryState returns non-OK status code the sequence should be dropped + Status updateMemoryState(model_memory_state_t& newState); + std::mutex& getMutex(); + bool isTerminated() const; + void setTerminated(); +}; + +} // namespace ovms diff --git a/src/ovms_lib/sequence_manager.cpp b/src/ovms_lib/sequence_manager.cpp new file mode 100644 index 0000000000..bf61396fb5 --- /dev/null +++ b/src/ovms_lib/sequence_manager.cpp @@ -0,0 +1,157 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "sequence_manager.hpp" + +#include + +#include "logging.hpp" + +namespace ovms { + +uint64_t SequenceManager::getUniqueSequenceId() { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "No sequence id has been provided on SEQUENCE_START. Seeking unique sequence id..."); + bool uniqueIdFound = false; + while (!uniqueIdFound) { + if (sequenceExists(this->sequenceIdCounter) || this->sequenceIdCounter == 0) + this->sequenceIdCounter++; + else + uniqueIdFound = true; + } + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Found unique sequence id: {}", this->sequenceIdCounter); + return this->sequenceIdCounter; +} + +const uint32_t SequenceManager::getMaxSequenceNumber() const { + return maxSequenceNumber; +} + +void SequenceManager::setMaxSequenceNumber(uint32_t maxSequenceNumber) { + this->maxSequenceNumber = maxSequenceNumber; +} + +std::mutex& SequenceManager::getMutex() { + return mutex; +} + +bool SequenceManager::sequenceExists(const uint64_t sequenceId) const { + return sequences.find(sequenceId) != sequences.end(); +} + +Status SequenceManager::removeIdleSequences() { + std::unique_lock sequenceManagerLock(mutex); + for (auto it = sequences.begin(); it != sequences.end();) { + Sequence& sequence = it->second; + // Non blocking try to get mutex + std::unique_lock sequenceLock(sequence.getMutex(), std::try_to_lock); + if (!sequence.isTerminated() && sequenceLock.owns_lock()) { + sequenceLock.unlock(); + // We hold sequence manager lock before lock and after unlock so no other thread even attempts accessing that sequence at that moment + if (sequence.isIdle()) { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "[Idle sequence cleanup] Removing sequence with id: {} on model {}, version: {}", sequence.getId(), modelName, modelVersion); + it = sequences.erase(it); + continue; + } else { + sequence.setIdle(); + } + } + ++it; + } + + return StatusCode::OK; +} + +Status SequenceManager::hasSequence(const uint64_t sequenceId) { + if (!sequenceExists(sequenceId)) + return StatusCode::SEQUENCE_MISSING; + + if (getSequence(sequenceId).isTerminated()) + return StatusCode::SEQUENCE_MISSING; + + return StatusCode::OK; +} + +Status SequenceManager::createSequence(SequenceProcessingSpec& sequenceProcessingSpec) { + if (sequences.size() >= this->maxSequenceNumber) { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Max sequence number has been reached. Could not create new sequence.", modelName, modelVersion); + return StatusCode::MAX_SEQUENCE_NUMBER_REACHED; + } + + uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); + + if (sequenceId == 0) { + uint64_t uniqueSequenceId = getUniqueSequenceId(); + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Adding new sequence with ID: {}", modelName, modelVersion, uniqueSequenceId); + sequences.emplace(uniqueSequenceId, uniqueSequenceId); + sequenceProcessingSpec.setSequenceId(uniqueSequenceId); + return StatusCode::OK; + } + + if (sequenceExists(sequenceId)) { + if (getSequence(sequenceId).isTerminated()) { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID is currently being removed", modelName, modelVersion); + return StatusCode::SEQUENCE_TERMINATED; + } + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID already exists", modelName, modelVersion); + return StatusCode::SEQUENCE_ALREADY_EXISTS; + } else { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Adding new sequence with ID: {}", modelName, modelVersion, sequenceId); + sequences.emplace(sequenceId, sequenceId); + } + return StatusCode::OK; +} + +Status SequenceManager::terminateSequence(const uint64_t sequenceId) { + auto status = hasSequence(sequenceId); + if (!status.ok()) + return status; + + getSequence(sequenceId).setTerminated(); + return StatusCode::OK; +} + +Sequence& SequenceManager::getSequence(const uint64_t sequenceId) { + return sequences.at(sequenceId); +} + +Status SequenceManager::removeSequence(const uint64_t sequenceId) { + auto it = sequences.find(sequenceId); + if (it != sequences.end()) { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} versions {} Removing sequence with ID: {}", modelName, modelVersion, sequenceId); + sequences.erase(it); + } else { + SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID does not exists", modelName, modelVersion); + return StatusCode::SEQUENCE_MISSING; + } + return StatusCode::OK; +} + +Status SequenceManager::processRequestedSpec(SequenceProcessingSpec& sequenceProcessingSpec) { + const uint32_t sequenceControlInput = sequenceProcessingSpec.getSequenceControlInput(); + const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); + Status status; + + if (sequenceControlInput == SEQUENCE_START) { + status = createSequence(sequenceProcessingSpec); + } else if (sequenceControlInput == NO_CONTROL_INPUT) { + status = hasSequence(sequenceId); + } else { // sequenceControlInput == SEQUENCE_END + status = terminateSequence(sequenceId); + } + return status; +} + +} // namespace ovms diff --git a/src/ovms_lib/sequence_manager.hpp b/src/ovms_lib/sequence_manager.hpp new file mode 100644 index 0000000000..a72460c5b3 --- /dev/null +++ b/src/ovms_lib/sequence_manager.hpp @@ -0,0 +1,83 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#pragma once + +#include +#include +#include +#include + +#include "modelversion.hpp" +#include "sequence.hpp" +#include "sequence_processing_spec.hpp" +#include "status.hpp" + +namespace ovms { + +const uint32_t NO_CONTROL_INPUT = 0; +const uint32_t SEQUENCE_START = 1; +const uint32_t SEQUENCE_END = 2; + +class SequenceManager { +private: + uint32_t maxSequenceNumber; + std::string modelName; + model_version_t modelVersion; + std::mutex mutex; + +protected: + std::unordered_map sequences; + + uint64_t sequenceIdCounter; + + uint64_t getUniqueSequenceId(); + + Status hasSequence(const uint64_t sequenceId); + + Status createSequence(SequenceProcessingSpec& sequenceProcessingSpec); + + Status terminateSequence(const uint64_t sequenceId); + +public: + SequenceManager() = default; + SequenceManager(uint32_t maxSequenceNumber, std::string modelName, model_version_t modelVersion) : + maxSequenceNumber(maxSequenceNumber), + modelName(modelName), + modelVersion(modelVersion), + sequenceIdCounter(1) {} + + uint64_t getSequencesCount() { + return sequences.size(); + } + + const uint32_t getMaxSequenceNumber() const; + + void setMaxSequenceNumber(uint32_t maxSequenceNumber); + + std::mutex& getMutex(); + + bool sequenceExists(const uint64_t sequenceId) const; + + Sequence& getSequence(const uint64_t sequenceId); + + Status removeSequence(const uint64_t sequenceId); + + Status removeIdleSequences(); + + Status processRequestedSpec(SequenceProcessingSpec& sequenceProcessingSpec); +}; +} // namespace ovms diff --git a/src/ovms_lib/sequence_processing_spec.hpp b/src/ovms_lib/sequence_processing_spec.hpp new file mode 100644 index 0000000000..782087df42 --- /dev/null +++ b/src/ovms_lib/sequence_processing_spec.hpp @@ -0,0 +1,36 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { + +class SequenceProcessingSpec { + uint32_t sequenceControlInput = 0; + uint64_t sequenceId = 0; + +public: + SequenceProcessingSpec() = default; + SequenceProcessingSpec(uint32_t sequenceControlInput, uint64_t sequenceId) : + sequenceControlInput(sequenceControlInput), + sequenceId(sequenceId) {} + const uint32_t getSequenceControlInput() const { return sequenceControlInput; } + void setSequenceControlInput(uint32_t sequenceControlInput) { this->sequenceControlInput = sequenceControlInput; } + const uint64_t getSequenceId() const { return sequenceId; } + void setSequenceId(uint64_t sequenceId) { this->sequenceId = sequenceId; } +}; +} // namespace ovms diff --git a/src/ovms_lib/serialization.cpp b/src/ovms_lib/serialization.cpp new file mode 100644 index 0000000000..686fa2e2ef --- /dev/null +++ b/src/ovms_lib/serialization.cpp @@ -0,0 +1,245 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "serialization.hpp" + +#include "ov_utils.hpp" +#include "tfs_frontend/tfs_utils.hpp" + +namespace ovms { + +Status serializePrecision( + tensorflow::TensorProto& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + if (servableOutput->getOvPrecision() != tensor.get_element_type()) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in precision expected:{} vs actual:{}", + servableOutput->getName(), + TensorInfo::getPrecisionAsString(servableOutput->getPrecision()), + tensor.get_element_type().get_type_name()); + return StatusCode::INTERNAL_ERROR; + } + switch (servableOutput->getPrecision()) { + case ovms::Precision::FP32: + case ovms::Precision::I32: + case ovms::Precision::FP64: + case ovms::Precision::I8: + case ovms::Precision::U8: + case ovms::Precision::I16: // 2 byte padding [v1, v0, 0, 0, u1, u0, 0, 0, ...] + case ovms::Precision::U16: + case ovms::Precision::FP16: + case ovms::Precision::I64: + responseOutput.set_dtype(getPrecisionAsDataType(servableOutput->getPrecision())); + break; + + case ovms::Precision::Q78: + case ovms::Precision::BIN: + case ovms::Precision::BOOL: + case ovms::Precision::MIXED: + case ovms::Precision::CUSTOM: + default: { + Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; + SPDLOG_ERROR(status.string()); + return status; + } + } + return StatusCode::OK; +} + +Status serializePrecision( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + if (servableOutput->getOvPrecision() != tensor.get_element_type()) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in precision expected:{} vs actual:{}", + servableOutput->getName(), + TensorInfo::getPrecisionAsString(servableOutput->getPrecision()), + tensor.get_element_type().get_type_name()); + return StatusCode::INTERNAL_ERROR; + } + switch (servableOutput->getPrecision()) { + case ovms::Precision::FP64: + case ovms::Precision::FP32: + case ovms::Precision::FP16: + case ovms::Precision::I64: + case ovms::Precision::I32: + case ovms::Precision::I16: + case ovms::Precision::I8: + case ovms::Precision::U64: + case ovms::Precision::U32: + case ovms::Precision::U16: + case ovms::Precision::U8: + case ovms::Precision::BOOL: + responseOutput.set_datatype(servableOutput->getPrecisionAsKFSPrecision()); + break; + case ovms::Precision::UNDEFINED: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + default: { + Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; + SPDLOG_ERROR(status.string()); + return status; + } + } + return StatusCode::OK; +} + +Status serializeShape( + tensorflow::TensorProto& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + responseOutput.mutable_tensor_shape()->Clear(); + auto& effectiveNetworkOutputShape = servableOutput->getShape(); + ov::Shape actualTensorShape = tensor.get_shape(); + if (effectiveNetworkOutputShape.size() != actualTensorShape.size()) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in number of dimensions expected:{} vs actual:{}", + servableOutput->getName(), effectiveNetworkOutputShape.size(), actualTensorShape.size()); + return StatusCode::INTERNAL_ERROR; + } + for (size_t i = 0; i < effectiveNetworkOutputShape.size(); ++i) { + dimension_value_t dim = actualTensorShape[i]; + if (!effectiveNetworkOutputShape[i].match(dim)) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in dimension:{} expected:{} vs actual:{}", + servableOutput->getName(), i, effectiveNetworkOutputShape[i].toString(), dim); + return StatusCode::INTERNAL_ERROR; + } + responseOutput.mutable_tensor_shape()->add_dim()->set_size(dim); + } + return StatusCode::OK; +} + +Status serializeShape( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + responseOutput.clear_shape(); + auto& effectiveNetworkOutputShape = servableOutput->getShape(); + ov::Shape actualTensorShape = tensor.get_shape(); + if (effectiveNetworkOutputShape.size() != actualTensorShape.size()) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in number of dimensions expected:{} vs actual:{}", + servableOutput->getName(), effectiveNetworkOutputShape.size(), actualTensorShape.size()); + return StatusCode::INTERNAL_ERROR; + } + for (size_t i = 0; i < effectiveNetworkOutputShape.size(); ++i) { + dimension_value_t dim = actualTensorShape[i]; + if (!effectiveNetworkOutputShape[i].match(dim)) { + SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in dimension:{} expected:{} vs actual:{}", + servableOutput->getName(), i, effectiveNetworkOutputShape[i].toString(), dim); + return StatusCode::INTERNAL_ERROR; + } + responseOutput.add_shape(dim); + } + return StatusCode::OK; +} + +void serializeContent(std::string* content, ov::Tensor& tensor) { + // We only fill if the content is not already filled. + // It can be filled in gather exit node handler. + if (content->size() == 0) { + content->assign((char*)tensor.data(), tensor.get_byte_size()); + } +} + +Status serializeTensorToTensorProto( + tensorflow::TensorProto& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + auto status = serializePrecision(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + status = serializeShape(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + serializeContent(responseOutput.mutable_tensor_content(), tensor); + return StatusCode::OK; +} + +Status serializeTensorToTensorProto( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + std::string* rawOutputContents, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor) { + auto status = serializePrecision(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + status = serializeShape(responseOutput, servableOutput, tensor); + if (!status.ok()) { + return status; + } + serializeContent(rawOutputContents, tensor); + return StatusCode::OK; +} + +template <> +Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { + try { + tensor = outputSource.get_tensor(name); + } catch (const ov::Exception& e) { + Status status = StatusCode::OV_INTERNAL_SERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } + return StatusCode::OK; +} + +template <> +tensorflow::TensorProto& ProtoGetter::createOutput(const std::string& name) { + return (*protoStorage->mutable_outputs())[name]; +} + +template <> +std::string* ProtoGetter::createContent(const std::string& name) { + return nullptr; +} + +template <> +::inference::ModelInferResponse::InferOutputTensor& ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createOutput(const std::string& name) { + for (int i = 0; i < protoStorage->outputs_size(); i++) { + auto& tensor = *protoStorage->mutable_outputs(i); + if (tensor.name() == name) { + return tensor; + } + } + auto* output = protoStorage->add_outputs(); + output->set_name(name); + return *output; +} + +template <> +std::string* ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createContent(const std::string& name) { + for (int i = 0; i < protoStorage->outputs_size(); i++) { + auto& tensor = *protoStorage->mutable_outputs(i); + if (tensor.name() == name) { + if (protoStorage->raw_output_contents_size() <= i) { + return protoStorage->add_raw_output_contents(); + } + return protoStorage->mutable_raw_output_contents(i); + } + } + return protoStorage->add_raw_output_contents(); +} + +const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo) { + return tensorInfo.getName(); +} + +const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo) { + return first; +} +} // namespace ovms diff --git a/src/ovms_lib/serialization.hpp b/src/ovms_lib/serialization.hpp new file mode 100644 index 0000000000..2eb5f56456 --- /dev/null +++ b/src/ovms_lib/serialization.hpp @@ -0,0 +1,126 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "kfs_grpc_inference_service.hpp" +#include "profiler.hpp" +#include "status.hpp" +#include "tensorinfo.hpp" + +namespace ovms { + +template +class OutputGetter { +public: + OutputGetter(T t) : + outputSource(t) {} + Status get(const std::string& name, ov::Tensor& tensor); + +private: + T outputSource; +}; + +template +class ProtoGetter { + ProtoStorage protoStorage; + +public: + ProtoGetter(ProtoStorage protoStorage) : + protoStorage(protoStorage) {} + ProtoType createOutput(const std::string& name); + std::string* createContent(const std::string& name); +}; + +Status serializeTensorToTensorProto( + tensorflow::TensorProto& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor); + +Status serializeTensorToTensorProto( + ::inference::ModelInferResponse::InferOutputTensor& responseOutput, + std::string* rawOutputContents, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor); + +typedef const std::string& (*outputNameChooser_t)(const std::string&, const TensorInfo&); +const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo); +const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo); + +template +Status serializePredictResponse( + OutputGetter& outputGetter, + const tensor_map_t& outputMap, + ResponseType* response, + outputNameChooser_t outputNameChooser); +// partial template specialization +template +Status serializePredictResponse( + OutputGetter& outputGetter, + const tensor_map_t& outputMap, + tensorflow::serving::PredictResponse* response, + outputNameChooser_t outputNameChooser) { + OVMS_PROFILE_FUNCTION(); + Status status; + ProtoGetter protoGetter(response); + for (const auto& [outputName, outputInfo] : outputMap) { + ov::Tensor tensor; + status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); + if (!status.ok()) { + return status; + } + auto& tensorProto = protoGetter.createOutput(outputInfo->getMappedName()); + status = serializeTensorToTensorProto(tensorProto, outputInfo, tensor); + if (!status.ok()) { + return status; + } + } + return status; +} +template +Status serializePredictResponse( + OutputGetter& outputGetter, + const tensor_map_t& outputMap, + ::inference::ModelInferResponse* response, + outputNameChooser_t outputNameChooser) { + Status status; + ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(response); + for (const auto& [outputName, outputInfo] : outputMap) { + ov::Tensor tensor; + status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); + if (!status.ok()) { + return status; + } + auto& inferOutputTensor = protoGetter.createOutput(outputInfo->getMappedName()); + status = serializeTensorToTensorProto(inferOutputTensor, protoGetter.createContent(outputInfo->getMappedName()), outputInfo, tensor); + if (!status.ok()) { + return status; + } + } + return status; +} +} // namespace ovms diff --git a/src/ovms_lib/servablemanagermodule.cpp b/src/ovms_lib/servablemanagermodule.cpp new file mode 100644 index 0000000000..9a7bc9f0e9 --- /dev/null +++ b/src/ovms_lib/servablemanagermodule.cpp @@ -0,0 +1,56 @@ +//*************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "servablemanagermodule.hpp" + +#include +#include + +#include "config.hpp" +#include "logging.hpp" +#include "modelmanager.hpp" + +namespace ovms { + +ServableManagerModule::ServableManagerModule() : + servableManager(std::make_unique()) {} + +int ServableManagerModule::start(const ovms::Config& config) { + state = ModuleState::STARTED_INITIALIZE; + SPDLOG_INFO("{} starting", SERVABLE_MANAGER_MODULE_NAME); + auto status = servableManager->start(config); + if (status.ok()) { + state = ModuleState::INITIALIZED; + SPDLOG_INFO("{} started", SERVABLE_MANAGER_MODULE_NAME); + // FIXME this should be reenabled in grpcserver module when functional tests are switched to wait + // for servablemanager module start log + // #KFS_CLEANUP + SPDLOG_INFO("Server started on port {}", config.port()); + return EXIT_SUCCESS; + } + SPDLOG_ERROR("ovms::ModelManager::Start() Error: {}", status.string()); + return EXIT_FAILURE; +} +void ServableManagerModule::shutdown() { + state = ModuleState::STARTED_SHUTDOWN; + SPDLOG_INFO("{} shutting down", SERVABLE_MANAGER_MODULE_NAME); + servableManager->join(); + state = ModuleState::SHUTDOWN; + SPDLOG_INFO("{} shutdown", SERVABLE_MANAGER_MODULE_NAME); +} +ModelManager& ServableManagerModule::getServableManager() const { + return *servableManager; +} +} // namespace ovms diff --git a/src/ovms_lib/servablemanagermodule.hpp b/src/ovms_lib/servablemanagermodule.hpp new file mode 100644 index 0000000000..77b1b2d75c --- /dev/null +++ b/src/ovms_lib/servablemanagermodule.hpp @@ -0,0 +1,35 @@ +//**************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include + +#include "server.hpp" + +namespace ovms { +class Config; +class ModelManager; + +class ServableManagerModule : public Module { +protected: + mutable std::unique_ptr servableManager; + +public: + ServableManagerModule(); + int start(const ovms::Config& config) override; + void shutdown() override; + ModelManager& getServableManager() const; +}; +} // namespace ovms diff --git a/src/ovms_lib/server.cpp b/src/ovms_lib/server.cpp new file mode 100644 index 0000000000..5765cd9817 --- /dev/null +++ b/src/ovms_lib/server.cpp @@ -0,0 +1,348 @@ +//**************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "server.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.hpp" +#include "grpcservermodule.hpp" +#include "http_server.hpp" +#include "kfs_grpc_inference_service.hpp" +#include "logging.hpp" +#include "model_service.hpp" +#include "modelmanager.hpp" +#include "prediction_service.hpp" +#include "profiler.hpp" +#include "servablemanagermodule.hpp" +#include "stringutils.hpp" +#include "version.hpp" + +using grpc::ServerBuilder; + +namespace ovms { +const std::string PROFILER_MODULE_NAME = "ProfilerModule"; +const std::string GRPC_SERVER_MODULE_NAME = "GRPCServerModule"; +const std::string HTTP_SERVER_MODULE_NAME = "HTTPServerModule"; +const std::string SERVABLE_MANAGER_MODULE_NAME = "ServableManagerModule"; +} // namespace ovms +using namespace ovms; + +namespace { +volatile sig_atomic_t shutdown_request = 0; +} + +Server& Server::instance() { + static Server global; + return global; +} + +void logConfig(const Config& config) { + std::string project_name(PROJECT_NAME); + std::string project_version(PROJECT_VERSION); + SPDLOG_INFO(project_name + " " + project_version); + SPDLOG_INFO("OpenVINO backend {}", OPENVINO_NAME); + SPDLOG_DEBUG("CLI parameters passed to ovms server"); + if (config.configPath().empty()) { + SPDLOG_DEBUG("model_path: {}", config.modelPath()); + SPDLOG_DEBUG("model_name: {}", config.modelName()); + SPDLOG_DEBUG("batch_size: {}", config.batchSize()); + SPDLOG_DEBUG("shape: {}", config.shape()); + SPDLOG_DEBUG("model_version_policy: {}", config.modelVersionPolicy()); + SPDLOG_DEBUG("nireq: {}", config.nireq()); + SPDLOG_DEBUG("target_device: {}", config.targetDevice()); + SPDLOG_DEBUG("plugin_config: {}", config.pluginConfig()); + SPDLOG_DEBUG("stateful: {}", config.stateful()); + SPDLOG_DEBUG("idle_sequence_cleanup: {}", config.idleSequenceCleanup()); + SPDLOG_DEBUG("max_sequence_number: {}", config.maxSequenceNumber()); + SPDLOG_DEBUG("low_latency_transformation: {}", config.lowLatencyTransformation()); + } else { + SPDLOG_DEBUG("config_path: {}", config.configPath()); + } + SPDLOG_DEBUG("gRPC port: {}", config.port()); + SPDLOG_DEBUG("REST port: {}", config.restPort()); + SPDLOG_DEBUG("gRPC bind address: {}", config.grpcBindAddress()); + SPDLOG_DEBUG("REST bind address: {}", config.restBindAddress()); + SPDLOG_DEBUG("REST workers: {}", config.restWorkers()); + SPDLOG_DEBUG("gRPC workers: {}", config.grpcWorkers()); + SPDLOG_DEBUG("gRPC channel arguments: {}", config.grpcChannelArguments()); + SPDLOG_DEBUG("log level: {}", config.logLevel()); + SPDLOG_DEBUG("log path: {}", config.logPath()); + SPDLOG_DEBUG("file system poll wait seconds: {}", config.filesystemPollWaitSeconds()); + SPDLOG_DEBUG("sequence cleaner poll wait minutes: {}", config.sequenceCleanerPollWaitMinutes()); +} + +void onInterrupt(int status) { + shutdown_request = 1; +} + +void onTerminate(int status) { + shutdown_request = 1; +} + +void onIllegal(int status) { + shutdown_request = 2; +} + +void installSignalHandlers(ovms::Server& server) { + static struct sigaction sigIntHandler; + sigIntHandler.sa_handler = onInterrupt; + sigemptyset(&sigIntHandler.sa_mask); + sigIntHandler.sa_flags = 0; + sigaction(SIGINT, &sigIntHandler, NULL); + + static struct sigaction sigTermHandler; + sigTermHandler.sa_handler = onTerminate; + sigemptyset(&sigTermHandler.sa_mask); + sigTermHandler.sa_flags = 0; + sigaction(SIGTERM, &sigTermHandler, NULL); + + static struct sigaction sigIllHandler; + sigIllHandler.sa_handler = onIllegal; + sigemptyset(&sigIllHandler.sa_mask); + sigIllHandler.sa_flags = 0; + sigaction(SIGILL, &sigIllHandler, NULL); +} + +static const int GIGABYTE = 1024 * 1024 * 1024; + +ModuleState Module::getState() const { + return state; +} + +// TODO should replace all messages like +// start REST Server with start HTTP Server +// start Server with start gRPC server +// this should be synchronized with validation tests changes + +class HTTPServerModule : public Module { + std::unique_ptr server; + Server& ovmsServer; + +public: + HTTPServerModule(ovms::Server& ovmsServer) : + ovmsServer(ovmsServer) {} + int start(const ovms::Config& config) override { + state = ModuleState::STARTED_INITIALIZE; + const std::string server_address = config.restBindAddress() + ":" + std::to_string(config.restPort()); + int workers = config.restWorkers() ? config.restWorkers() : 10; + + SPDLOG_INFO("Will start {} REST workers", workers); + server = ovms::createAndStartHttpServer(config.restBindAddress(), config.restPort(), workers, this->ovmsServer); + if (server != nullptr) { + SPDLOG_INFO("Started REST server at {}", server_address); + } else { + SPDLOG_ERROR("Failed to start REST server at " + server_address); + return EXIT_FAILURE; + } + state = ModuleState::INITIALIZED; + return EXIT_SUCCESS; + } + void shutdown() override { + if (server == nullptr) + return; + state = ModuleState::STARTED_SHUTDOWN; + server->Terminate(); + server->WaitForTermination(); + SPDLOG_INFO("Shutdown HTTP server"); + state = ModuleState::SHUTDOWN; + } +}; + +bool Server::isReady() const { + std::shared_lock lock(modulesMtx); + auto it = modules.find(SERVABLE_MANAGER_MODULE_NAME); + if (it == modules.end()) + return false; + if (ModuleState::INITIALIZED != it->second->getState()) + return false; + return true; +} + +bool Server::isLive() const { + // TODO we might want at some time start REST only/ or respond with true only if both servers started if both are requested to start. This is to be resolved especially if we implement REST API for Kserver & potentially switch to check for starting specific module + std::shared_lock lock(modulesMtx); + auto it = modules.find(GRPC_SERVER_MODULE_NAME); + if (it == modules.end()) + return false; + if (ModuleState::INITIALIZED != it->second->getState()) + return false; + return true; +} + +ModuleState Server::getModuleState(const std::string& name) const { + std::shared_lock lock(modulesMtx); + auto it = modules.find(name); + if (it == modules.end()) + return ModuleState::NOT_INITIALIZED; + return it->second->getState(); +} + +const Module* Server::getModule(const std::string& name) const { + std::shared_lock lock(modulesMtx); + auto it = modules.find(name); + if (it == modules.end()) + return nullptr; + return it->second.get(); +} + +#ifdef MTR_ENABLED +class ProfilerModule : public Module { + std::unique_ptr profiler; + +public: + ProfilerModule() = default; + int start(const Config& config) override { + state = ModuleState::STARTED_INITIALIZE; + auto profiler = std::make_unique(config.tracePath()); + if (!profiler.isInitialized()) { + SPDLOG_ERROR("Cannot open file for profiler, --trace_path: {}", config.tracePath()); + return EXIT_FAILURE; + } + state = ModuleState::INITIALIZED; + return EXIT_SUCCESS; + } + void shutdown() override { +#ifdef MTR_ENABLED + state = ModuleState::STARTED_SHUTDOWN; + profiler.reset(); + state = ModuleState::SHUTDOWN; +#endif + } +}; +#endif + +void Server::setShutdownRequest(int i) { + shutdown_request = i; +} + +Server::~Server() = default; + +std::unique_ptr Server::createModule(const std::string& name) { +#ifdef MTR_ENABLED + if (name == PROFILER_MODULE_NAME) + return std::make_unique(); +#endif + if (name == GRPC_SERVER_MODULE_NAME) + return std::make_unique(*this); + if (name == HTTP_SERVER_MODULE_NAME) + return std::make_unique(*this); + if (name == SERVABLE_MANAGER_MODULE_NAME) + return std::make_unique(); + return nullptr; +} + +int Server::startModules(ovms::Config& config) { + auto retCode = EXIT_SUCCESS; + bool inserted = false; + auto it = modules.end(); +#if MTR_ENABLED + { + auto module = this->createModule(PROFILER_MODULE_NAME); + std::unique_lock lock(modulesMtx); + std::tie(it, inserted) = this->modules.emplace(PROFILER_MODULE_NAME, std::move(module)); + } + retCode = modules.at(PROFILER_MODULE_NAME)->start(config); + if (retCode) + return retCode; +#endif + { + auto module = this->createModule(GRPC_SERVER_MODULE_NAME); + std::unique_lock lock(modulesMtx); + std::tie(it, inserted) = this->modules.emplace(GRPC_SERVER_MODULE_NAME, std::move(module)); + } + + if (!inserted) + return EXIT_FAILURE; + // if we ever decide not to start GRPC module then we need to implement HTTP responses without using grpc implementations + retCode = it->second->start(config); + if (retCode) + return retCode; + if (config.restPort() != 0) { + { + auto module = this->createModule(HTTP_SERVER_MODULE_NAME); + std::unique_lock lock(modulesMtx); + std::tie(it, inserted) = this->modules.emplace(HTTP_SERVER_MODULE_NAME, std::move(module)); + } + retCode = it->second->start(config); + if (retCode) + return retCode; + } + { + auto module = this->createModule(SERVABLE_MANAGER_MODULE_NAME); + std::unique_lock lock(modulesMtx); + std::tie(it, inserted) = this->modules.emplace(SERVABLE_MANAGER_MODULE_NAME, std::move(module)); + } + retCode = it->second->start(config); + return retCode; +} + +void Server::shutdownModules(ovms::Config& config) { + modules.at(GRPC_SERVER_MODULE_NAME)->shutdown(); + if (config.restPort() != 0) + modules.at(HTTP_SERVER_MODULE_NAME)->shutdown(); + modules.at(SERVABLE_MANAGER_MODULE_NAME)->shutdown(); +#ifdef MTR_ENABLED + modules.at(PROFILER_MODULE_NAME)->shutdown(); +#endif + // FIXME we need to be able to quickly start grpc or start it without port + // this is because the OS can have a delay between freeing up port before it can be requested and used again + modules.clear(); +} + +int Server::start(int argc, char** argv) { + ovms::Server& server = ovms::Server::instance(); + installSignalHandlers(server); + try { + auto& config = ovms::Config::instance().parse(argc, argv); + configure_logger(config.logLevel(), config.logPath()); + logConfig(config); + auto retCode = this->startModules(config); + if (retCode) + return retCode; + + while (!shutdown_request) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + if (shutdown_request == 2) { + SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); + } + SPDLOG_INFO("Shutting down"); + this->shutdownModules(config); + } catch (std::exception& e) { + SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); + return EXIT_FAILURE; + } catch (...) { + SPDLOG_ERROR("Unknown exception catch - will now terminate."); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/src/ovms_lib/server.hpp b/src/ovms_lib/server.hpp new file mode 100644 index 0000000000..8b1cfb54f1 --- /dev/null +++ b/src/ovms_lib/server.hpp @@ -0,0 +1,75 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include +#include +#include +#include + +namespace ovms { +class Config; +enum class ModuleState { + NOT_INITIALIZED, + STARTED_INITIALIZE, + INITIALIZED, + RELOADING, + STARTED_SHUTDOWN, + SHUTDOWN +}; + +class Module { +protected: + ModuleState state = ModuleState::NOT_INITIALIZED; + +public: + virtual int start(const ovms::Config& config) = 0; + virtual void shutdown() = 0; + virtual ~Module() = default; + ModuleState getState() const; +}; + +extern const std::string PROFILER_MODULE_NAME; +extern const std::string GRPC_SERVER_MODULE_NAME; +extern const std::string HTTP_SERVER_MODULE_NAME; +extern const std::string SERVABLE_MANAGER_MODULE_NAME; + +class Server { + mutable std::shared_mutex modulesMtx; + +protected: + std::unordered_map> modules; + Server() = default; + virtual std::unique_ptr createModule(const std::string& name); + +public: + static Server& instance(); + int start(int argc, char** argv); + ModuleState getModuleState(const std::string& name) const; + const Module* getModule(const std::string& name) const; + bool isReady() const; + bool isLive() const; + + void setShutdownRequest(int i); + virtual ~Server(); + + // TODO potentially to be hiden under protected and exposed only in tests by inheritance + // #KFS_CLEANUP + int startModules(ovms::Config& config); + void shutdownModules(ovms::Config& config); +}; +} // namespace ovms diff --git a/src/ovms_lib/session_id.hpp b/src/ovms_lib/session_id.hpp new file mode 100644 index 0000000000..e7d6282b22 --- /dev/null +++ b/src/ovms_lib/session_id.hpp @@ -0,0 +1,24 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { + +using session_id_t = uint32_t; +using session_key_t = std::string; +} // namespace ovms diff --git a/src/ovms_lib/shape.cpp b/src/ovms_lib/shape.cpp new file mode 100644 index 0000000000..b4d557d141 --- /dev/null +++ b/src/ovms_lib/shape.cpp @@ -0,0 +1,505 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "shape.hpp" + +#include +#include +#include +#include +#include + +#include "logging.hpp" +#include "stringutils.hpp" + +namespace ovms { + +Dimension::Dimension() : + Dimension(DYNAMIC_DIMENSION) { +} + +Dimension::Dimension(const ov::Dimension& dim) { + if (dim.is_static()) { + this->minimum = dim.get_length(); + this->maximum = dim.get_length(); + } else if (!dim.get_interval().has_upper_bound()) { + this->minimum = DYNAMIC_DIMENSION; + this->maximum = DYNAMIC_DIMENSION; + } else { + this->minimum = dim.get_min_length(); + this->maximum = dim.get_max_length(); + } +} + +Dimension::Dimension(dimension_value_t dim) : + Dimension(dim, dim) { +} + +Dimension::Dimension(dimension_value_t minimum, dimension_value_t maximum) { + if (minimum == -1 && maximum != -1) { + throw std::invalid_argument("Invalid range"); + } + if (minimum < -1 || maximum < -1) { + throw std::invalid_argument("Range must not be lower than -1"); + } + if (minimum > maximum) { + throw std::invalid_argument("Range maximum must be higher or equal to minimum"); + } + + this->minimum = minimum; + this->maximum = maximum; +} + +bool Dimension::isDynamic() const { + if (this->minimum != this->maximum) + return true; + if (this->minimum == DYNAMIC_DIMENSION) + return true; + return false; +} + +bool Dimension::isStatic() const { + return !this->isDynamic(); +} + +ov::Dimension Dimension::createPartialDimension() const { + if (this->isStatic()) { + return ov::Dimension(this->getStaticValue()); + } + if (this->minimum == DYNAMIC_DIMENSION) { + return ov::Dimension::dynamic(); + } + return ov::Dimension(this->minimum, this->maximum); +} + +dimension_value_t Dimension::getStaticValue() const { + if (this->isDynamic()) + throw std::invalid_argument("getStaticValue but dimension dynamic"); + return this->maximum; +} + +dimension_value_t Dimension::getMinValue() const { + if (this->isStatic()) + throw std::invalid_argument("getMinValue but dimension static"); + if (this->isAny()) + throw std::invalid_argument("getMinValue but dimension any"); + return this->minimum; +} + +dimension_value_t Dimension::getMaxValue() const { + if (this->isStatic()) + throw std::invalid_argument("getMaxValue but dimension static"); + if (this->isAny()) + throw std::invalid_argument("getMinValue but dimension any"); + return this->maximum; +} + +bool Dimension::operator==(const Dimension& rhs) const { + return this->minimum == rhs.minimum && this->maximum == rhs.maximum; +} + +bool Dimension::operator!=(const Dimension& rhs) const { + return !(this->operator==(rhs)); +} + +Status Dimension::fromString(const std::string& str, Dimension& dimOut) { + Dimension dim; + + std::string strCopy = str; + erase_spaces(strCopy); + if (strCopy.find(DIMENSION_RANGE_DELIMETER) != std::string::npos) { + // Range + if (strCopy.find_first_not_of("0123456789:") != std::string::npos) { + SPDLOG_ERROR("Parsing dimension string not a range: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + size_t delimCount = std::count(strCopy.begin(), strCopy.end(), DIMENSION_RANGE_DELIMETER); + if (delimCount != 1) { + SPDLOG_ERROR("Parsing dimension string, wrong amount of '{}' - {}; {}", DIMENSION_RANGE_DELIMETER, delimCount, strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } else { + std::vector tokens = tokenize(strCopy, DIMENSION_RANGE_DELIMETER); + if (tokens.size() == 2) { + try { + int dimNumberMin = std::stoi(tokens[0]); + int dimNumberMax = std::stoi(tokens[1]); + if (dimNumberMin > 0 && dimNumberMax > 0) { + dim = Dimension(dimNumberMin, dimNumberMax); + } else if (dimNumberMin >= dimNumberMax) { + SPDLOG_ERROR("Parsing dimension string range max must be higher than min: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } else { + SPDLOG_ERROR("Parsing dimension string range must be lager than 0: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + } catch (const std::out_of_range& e) { + SPDLOG_ERROR("Parsing dimension string out of range: {}, error: {}", strCopy, e.what()); + return StatusCode::DIM_WRONG_FORMAT; + } catch (...) { + SPDLOG_ERROR("Parsing dimension string: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + } else { + SPDLOG_ERROR("Parsing dimension string, not a number between '{}' - {}", DIMENSION_RANGE_DELIMETER, strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + } + } else { + size_t count = std::count(strCopy.begin(), strCopy.end(), '-'); + if (count > 1) { + SPDLOG_ERROR("Parsing dimension string: {}; too many '-' characters", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } else if (count == 1 && *strCopy.begin() != '-') { + SPDLOG_ERROR("Parsing dimension string: {}; invalid '-' position", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + // Number + if (strCopy.find_first_not_of("0123456789-") != std::string::npos) { + SPDLOG_ERROR("Parsing dimension string not a number: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + try { + int dimNumber = std::stoi(strCopy); + if (dimNumber == DYNAMIC_DIMENSION) { + dim = Dimension::any(); + } else if (dimNumber > 0) { + dim = Dimension(dimNumber); + } else { + SPDLOG_ERROR("Parsing dimension string out of range: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + } catch (const std::out_of_range& e) { + SPDLOG_ERROR("Parsing dimension string out of range: {}, error: {}", strCopy, e.what()); + return StatusCode::DIM_WRONG_FORMAT; + } catch (...) { + SPDLOG_ERROR("Parsing dimension string: {}", strCopy); + return StatusCode::DIM_WRONG_FORMAT; + } + } + + dimOut = dim; + return StatusCode::OK; +} + +bool Dimension::isAny() const { + return (this->maximum == DYNAMIC_DIMENSION) && + (this->minimum == DYNAMIC_DIMENSION); +} + +bool Dimension::partiallyFitsInto(const Dimension& next) const { + if (next.isAny() || isAny()) { + return true; + } + if (isStatic()) { + return next.match(getStaticValue()); + } + if (next.isStatic()) { + return this->match(next.getStaticValue()); + } + // both are dynamic + if (next.getMinValue() > getMaxValue()) { + return false; + } + if (next.getMaxValue() < getMinValue()) { + return false; + } + return true; +} + +bool Dimension::match(dimension_value_t value) const { + if (value < -1) + return false; + if (isAny()) { + return true; + } + if (isStatic()) { + return getStaticValue() == value; + } + if (value < getMinValue()) { + return false; + } + if (value > getMaxValue()) { + return false; + } + return true; +} + +std::string Dimension::toString() const { + std::stringstream ss; + + if (this->isStatic()) { + ss << this->minimum; + } else { + if (this->maximum == DYNAMIC_DIMENSION) { + ss << DYNAMIC_DIMENSION; + } else { + ss << "[" << this->minimum << "~" << this->maximum << "]"; + } + } + + return ss.str(); +} + +Dimension Dimension::any() { + return Dimension(); +} + +dimension_value_t Dimension::getLowerBound() const { + return isStatic() ? getStaticValue() : getMinValue(); +} + +dimension_value_t Dimension::getUpperBound() const { + return isStatic() ? getStaticValue() : getMaxValue(); +} + +std::optional Dimension::createIntersection(const Dimension& other) const { + if (*this == Dimension::any()) + return other; + if (other == Dimension::any()) + return *this; + auto start = std::max(this->getLowerBound(), other.getLowerBound()); + auto end = std::min(this->getUpperBound(), other.getUpperBound()); + if (end < start) + return std::nullopt; + return Dimension{start, end}; +} + +Shape::Shape() { +} + +Shape::Shape(std::initializer_list list) : + std::vector(list) {} + +Shape::Shape(const shape_t& s) { + auto status = fromFlatShape(s, *this); + if (!status.ok()) { + throw std::invalid_argument("Could not convert from flat shape"); + } +} + +Status Shape::fromFlatShape(const shape_t& shapeIn, Shape& shapeOut) { + Shape shape; + for (size_t dim : shapeIn) { + if (dim > std::numeric_limits::max()) { + return StatusCode::CANNOT_CONVERT_FLAT_SHAPE; + } else { + shape.add(Dimension{static_cast(dim)}); + } + } + shapeOut = shape; + return StatusCode::OK; +} + +Shape::Shape(const ov::PartialShape& shape) { + this->reserve(shape.size()); + for (const ov::Dimension& dim : shape) { + if (dim.is_static()) { + this->emplace_back(Dimension{dim.get_length()}); + } else if (!dim.get_interval().has_upper_bound()) { + this->emplace_back(Dimension::any()); + } else { + this->emplace_back(Dimension{dim.get_min_length(), dim.get_max_length()}); + } + } +} + +Shape& Shape::add(const Dimension& dim) { + return this->add(dim, this->size()); +} +Shape& Shape::add(const Dimension& dim, size_t pos) { + this->emplace(this->begin() + pos, dim); + return *this; +} + +ov::PartialShape Shape::createPartialShape() const { + ov::PartialShape shape; + + shape.reserve(this->size()); + for (const Dimension& dim : *this) { + if (dim.isStatic()) { + shape.push_back(ov::Dimension(dim.getStaticValue())); + } else if (dim.isAny()) { + shape.push_back(ov::Dimension::dynamic()); + } else { + shape.push_back(ov::Dimension{dim.getMinValue(), dim.getMaxValue()}); + } + } + + return shape; +} + +bool Shape::operator==(const Shape& rhs) const { + if (this->size() != rhs.size()) + return false; + + for (size_t i = 0; i < this->size(); i++) { + if ((*this)[i] != rhs[i]) { + return false; + } + } + return true; +} + +bool Shape::operator!=(const Shape& rhs) const { + return !(this->operator==(rhs)); +} + +bool Shape::match(const ov::Shape& ovShape) const { + for (size_t i = 0; i < this->size(); i++) { + if (!(*this)[i].match(ovShape[i])) { + return false; + } + } + return true; +} + +bool Shape::match(const ov::Shape& ovShape, const size_t skipPosition) const { + if (this->size() != ovShape.size()) { + return false; + } + for (size_t i = 0; i < skipPosition; i++) { + if (!(*this)[i].match(ovShape[i])) { + return false; + } + } + for (size_t i = skipPosition + 1; i < this->size(); i++) { + if (!(*this)[i].match(ovShape[i])) { + return false; + } + } + return true; +} + +std::optional Shape::createIntersection(const Shape& other) const { + if (this->size() != other.size()) + return std::nullopt; + Shape intersected; + intersected.reserve(this->size()); + for (size_t i = 0; i < this->size(); ++i) { + auto intersectedDim = (*this)[i].createIntersection(other[i]); + if (!intersectedDim.has_value()) { + return std::nullopt; + } + intersected.emplace_back(std::move(intersectedDim.value())); + } + return intersected; +} + +std::string Shape::toString() const { + std::stringstream ss; + + ss << "("; + + size_t dimensionCount = this->size(); + if (dimensionCount > 0) { + for (size_t i = 0; i < dimensionCount - 1; i++) { + ss << (*this)[i].toString() << ","; + } + ss << (*this)[dimensionCount - 1].toString(); + } + + ss << ")"; + + return ss.str(); +} + +Status Shape::fromString(const std::string& strIn, Shape& shapeOut) { + Shape shape; + std::string str = strIn; + + erase_spaces(str); + if (str.find_first_not_of("0123456789(),-:") != std::string::npos) + return StatusCode::SHAPE_WRONG_FORMAT; + + if (std::count(str.begin(), str.end(), '(') != 1) + return StatusCode::SHAPE_WRONG_FORMAT; + + if (std::count(str.begin(), str.end(), ')') != 1) + return StatusCode::SHAPE_WRONG_FORMAT; + + if (str.size() <= 2) + return StatusCode::SHAPE_WRONG_FORMAT; + + if (str.front() != '(' || str.back() != ')') + return StatusCode::SHAPE_WRONG_FORMAT; + + str.pop_back(); + str.erase(str.begin()); + + std::vector tokens = tokenize(str, ','); + + for (const std::string& token : tokens) { + size_t count = std::count(token.begin(), token.end(), '-'); + if (count > 1) { + SPDLOG_ERROR("Parsing model shape string: {}; too many '-' characters", token); + return StatusCode::SHAPE_WRONG_FORMAT; + } else if (count == 1 && !token.empty() && *token.begin() != '-') { + SPDLOG_ERROR("Parsing model shape string: {}; invalid '-' position", token); + return StatusCode::SHAPE_WRONG_FORMAT; + } + + count = std::count(token.begin(), token.end(), DIMENSION_RANGE_DELIMETER); + if (count > 1) { + SPDLOG_ERROR("Parsing model shape string: {}; too many '{}' characters", DIMENSION_RANGE_DELIMETER, token); + return StatusCode::SHAPE_WRONG_FORMAT; + } + try { + if (count == 0) { + int dimValue = std::stoi(token); + if (dimValue == DYNAMIC_DIMENSION || dimValue > 0) { + shape.add(Dimension(dimValue)); + } else { + SPDLOG_ERROR("Parsing model shape string: {}; must be {} (any) or higher than 0", token, DYNAMIC_DIMENSION); + return StatusCode::SHAPE_WRONG_FORMAT; + } + } else { + std::vector subTokens = tokenize(token, DIMENSION_RANGE_DELIMETER); + if (subTokens.size() != 2 || subTokens[0].empty() || subTokens[1].empty()) { + SPDLOG_ERROR("Parsing model shape string: {}; range must have min and max", strIn); + return StatusCode::SHAPE_WRONG_FORMAT; + } + int dimMin = std::stoi(subTokens[0]); + int dimMax = std::stoi(subTokens[1]); + if (dimMin <= 0 || dimMax <= 0) { + SPDLOG_ERROR("Parsing model shape string: {}; range must be higher than 0", token); + return StatusCode::SHAPE_WRONG_FORMAT; + } + if (dimMin >= dimMax) { + SPDLOG_ERROR("Parsing model shape string: {}; range max must be higher than min", token); + return StatusCode::SHAPE_WRONG_FORMAT; + } + shape.add(Dimension(dimMin, dimMax)); + } + } catch (const std::out_of_range& e) { + SPDLOG_ERROR("Parsing model shape string out of range: {}, error: {}", str, e.what()); + return StatusCode::SHAPE_WRONG_FORMAT; + } catch (...) { + SPDLOG_ERROR("Parsing model shape string: {}", strIn); + return StatusCode::SHAPE_WRONG_FORMAT; + } + } + + shapeOut = shape; + return StatusCode::OK; +} + +ShapeInfo::operator std::string() const { + std::stringstream ss; + ss << this->shape.toString() << " (" << (this->shapeMode == Mode::FIXED ? "fixed" : "auto") << ")"; + return ss.str(); +} + +} // namespace ovms diff --git a/src/ovms_lib/shape.hpp b/src/ovms_lib/shape.hpp new file mode 100644 index 0000000000..e8b504815a --- /dev/null +++ b/src/ovms_lib/shape.hpp @@ -0,0 +1,128 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include + +#include "status.hpp" + +namespace ovms { + +using dimension_value_t = std::int64_t; + +constexpr dimension_value_t DYNAMIC_DIMENSION = -1; + +constexpr char DIMENSION_RANGE_DELIMETER = ':'; + +enum Mode { FIXED, + AUTO }; +using shape_t = std::vector; + +class Dimension { + dimension_value_t minimum, maximum; + +public: + Dimension(); + + Dimension(const ov::Dimension& dim); + + Dimension(dimension_value_t dim); + + Dimension(dimension_value_t minimum, dimension_value_t maximum); + + bool isStatic() const; + bool isDynamic() const; + + ov::Dimension createPartialDimension() const; + + dimension_value_t getStaticValue() const; + dimension_value_t getMinValue() const; + dimension_value_t getMaxValue() const; + + bool operator==(const Dimension& rhs) const; + bool operator!=(const Dimension& rhs) const; + + static Status fromString(const std::string& str, Dimension& dimOut); + std::string toString() const; + + static Dimension any(); + + bool match(dimension_value_t value) const; + bool partiallyFitsInto(const Dimension& value) const; + bool isAny() const; + std::optional createIntersection(const Dimension& other) const; + +private: + dimension_value_t getLowerBound() const; + dimension_value_t getUpperBound() const; +}; + +class Shape : public std::vector { +public: + Shape(); + Shape(const shape_t& s); + // Create shape out of ovms::Shape{1, 5, 100, 100} + Shape(std::initializer_list list); + + // Create ovms::Shape out of oridnary vector of dimensions. + static Status fromFlatShape(const shape_t& shapeIn, Shape& shapeOut); + + // Create ovms::Shape out of ov::PartialShape. + Shape(const ov::PartialShape& shape); + + Shape& add(const Dimension& dim, size_t pos); + Shape& add(const Dimension& dim); + + bool isStatic() const; + bool isDynamic() const; + + ov::PartialShape createPartialShape() const; + + bool operator==(const Shape& rhs) const; + bool operator!=(const Shape& rhs) const; + + bool match(const ov::Shape& rhs) const; + bool match(const ov::Shape& rhs, const size_t skipPosition) const; + std::optional createIntersection(const Shape& other) const; + + std::string toString() const; + static Status fromString(const std::string& strIn, Shape& shapeOut); +}; + +using shapes_map_t = std::unordered_map; + +struct ShapeInfo { + Mode shapeMode = FIXED; + Shape shape; + + operator std::string() const; + + bool operator==(const ShapeInfo& rhs) const { + return this->shapeMode == rhs.shapeMode && this->shape == rhs.shape; + } + + bool operator!=(const ShapeInfo& rhs) const { + return !(*this == rhs); + } +}; + +using shapes_info_map_t = std::unordered_map; + +} // namespace ovms diff --git a/src/ovms_lib/statefulmodelinstance.cpp b/src/ovms_lib/statefulmodelinstance.cpp new file mode 100644 index 0000000000..6a71c0cb34 --- /dev/null +++ b/src/ovms_lib/statefulmodelinstance.cpp @@ -0,0 +1,324 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "statefulmodelinstance.hpp" + +#include +#include + +#include "deserialization.hpp" +#include "executingstreamidguard.hpp" +#include "logging.hpp" +#include "predict_request_validation_utils.hpp" +#include "profiler.hpp" +#include "serialization.hpp" +#include "timer.hpp" + +namespace ovms { + +const std::set StatefulModelInstance::SPECIAL_INPUT_NAMES{"sequence_id", "sequence_control_input"}; + +const Status StatefulModelInstance::extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId) { + if (!proto.tensor_shape().dim_size()) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto does not contain tensor shape information", getName(), getVersion()); + return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; + } else if (proto.tensor_shape().dim_size() != 1) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid number of dimensions. Expecting shape with one dimension", getName(), getVersion()); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_id is: (1)"); + } + + if (proto.tensor_shape().dim(0).size() != 1) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); + return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_id is: (1)"); + } + + if (proto.uint64_val_size() == 1) { + sequenceId = proto.uint64_val(0); + return StatusCode::OK; + } + return StatusCode::SEQUENCE_ID_BAD_TYPE; +} + +const Status StatefulModelInstance::extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput) { + if (proto.tensor_shape().dim_size() == 0) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto does not contain tensor shape information", getName(), getVersion()); + return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; + } else if (proto.tensor_shape().dim_size() != 1) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid number of dimensions. Expecting shape with one dimension.", getName(), getVersion()); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_control_input is: (1)"); + } + + if (proto.tensor_shape().dim(0).size() != 1) { + SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); + return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_control_input is: (1)"); + } + + if (proto.uint32_val_size() == 1) { + sequenceControlInput = proto.uint32_val(0); + return StatusCode::OK; + } + return StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE; +} + +Status StatefulModelInstance::loadModel(const ModelConfig& config) { + std::lock_guard loadingLock(loadingMutex); + autoCleanupEnabled = config.getIdleSequenceCleanup(); + + Status status = ModelInstance::loadModel(config); + if (!status.ok()) + return status; + + if (autoCleanupEnabled) { + status = globalSequencesViewer->registerForCleanup(getName(), getVersion(), sequenceManager); + if (!status.ok()) + return status; + } + return StatusCode::OK; +} + +Status StatefulModelInstance::reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter) { + std::lock_guard loadingLock(loadingMutex); + Status status; + if (autoCleanupEnabled && this->status.getState() == ModelVersionState::AVAILABLE) { + status = globalSequencesViewer->unregisterFromCleanup(getName(), getVersion()); + if (!status.ok()) + return status; + } + status = ModelInstance::reloadModel(config, parameter); + if (!status.ok()) + return status; + autoCleanupEnabled = config.getIdleSequenceCleanup(); + + if (autoCleanupEnabled) { + status = globalSequencesViewer->registerForCleanup(getName(), getVersion(), sequenceManager); + if (!status.ok()) + return status; + } + return StatusCode::OK; +} + +void StatefulModelInstance::retireModel(bool isPermanent) { + std::lock_guard loadingLock(loadingMutex); + if (isPermanent && autoCleanupEnabled) { + globalSequencesViewer->unregisterFromCleanup(getName(), getVersion()); + } + ModelInstance::retireModel(isPermanent); + sequenceManager.reset(); +} + +void StatefulModelInstance::cleanupFailedLoad() { + std::lock_guard loadingLock(loadingMutex); + ModelInstance::cleanupFailedLoad(); + sequenceManager.reset(); +} + +Status StatefulModelInstance::loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter) { + performLowLatencyTransformation = config.isLowLatencyTransformationUsed(); + sequenceManager = std::make_shared(config.getMaxSequenceNumber(), config.getName(), config.getVersion()); + return ModelInstance::loadModelImpl(config, parameter); +} + +Status StatefulModelInstance::loadOVCompiledModel(const ModelConfig& config) { + if (performLowLatencyTransformation) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "[Model: {} version: {}] Performing Low Latency Transformation on the model", getName(), getVersion()); + try { + ov::pass::LowLatency2().run_on_model(model); + } catch (ov::Exception& ex) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error: {}; occurred during low latency transformation on model: {} version: {}", ex.what(), getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } catch (std::exception& ex) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error: {}; occurred during low latency transformation on model: {} version: {}", ex.what(), getName(), getVersion()); + return StatusCode::INTERNAL_ERROR; + } + } + return ModelInstance::loadOVCompiledModel(config); +} + +template <> +const Status StatefulModelInstance::validateSpecialKeys(const tensorflow::serving::PredictRequest* request, SequenceProcessingSpec& sequenceProcessingSpec) { + uint64_t sequenceId = 0; + uint32_t sequenceControlInput = 0; + Status status; + auto it = request->inputs().find("sequence_id"); + if (it != request->inputs().end()) { + status = extractSequenceId(it->second, sequenceId); + if (!status.ok()) + return status; + } + it = request->inputs().find("sequence_control_input"); + if (it != request->inputs().end()) { + status = extractSequenceControlInput(it->second, sequenceControlInput); + if (!status.ok()) + return status; + } + + if (sequenceControlInput != SEQUENCE_END && sequenceControlInput != NO_CONTROL_INPUT && sequenceControlInput != SEQUENCE_START) { + return StatusCode::INVALID_SEQUENCE_CONTROL_INPUT; + } + if ((sequenceControlInput == SEQUENCE_END || sequenceControlInput == NO_CONTROL_INPUT) && sequenceId == 0) { + return StatusCode::SEQUENCE_ID_NOT_PROVIDED; + } + + sequenceProcessingSpec.setSequenceId(sequenceId); + sequenceProcessingSpec.setSequenceControlInput(sequenceControlInput); + + return StatusCode::OK; +} + +template +const Status StatefulModelInstance::validate(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec) { + OVMS_PROFILE_FUNCTION(); + auto status = validateSpecialKeys(request, sequenceProcessingSpec); + if (!status.ok()) + return status; + + return request_validation_utils::validate( + *request, + getInputsInfo(), + getName(), + getVersion(), + SPECIAL_INPUT_NAMES, + getModelConfig().getBatchingMode(), + getModelConfig().getShapes()); +} + +Status StatefulModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, + tensorflow::serving::PredictResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + Timer timer; + using std::chrono::microseconds; + SequenceProcessingSpec sequenceProcessingSpec; + auto status = validate(requestProto, sequenceProcessingSpec); + if (!status.ok()) + return status; + + std::unique_lock sequenceManagerLock(sequenceManager->getMutex()); + status = sequenceManager->processRequestedSpec(sequenceProcessingSpec); + if (!status.ok()) + return status; + const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); + if (!sequenceManager->sequenceExists(sequenceId)) + return StatusCode::INTERNAL_ERROR; + Sequence& sequence = sequenceManager->getSequence(sequenceId); + + std::unique_lock sequenceLock(sequence.getMutex()); + sequenceManagerLock.unlock(); + + timer.start("get infer request"); + ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); + int executingInferId = executingStreamIdGuard.getId(); + (void)executingInferId; + ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); + timer.stop("get infer request"); + SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); + + timer.start("preprocess"); + status = preInferenceProcessing(inferRequest, sequence, sequenceProcessingSpec); + timer.stop("preprocess"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Preprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("preprocess") / 1000); + + timer.start("deserialize"); + InputSink inputSink(inferRequest); + bool isPipeline = false; + status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); + timer.stop("deserialize"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); + + timer.start("prediction"); + status = performInference(inferRequest); + timer.stop("prediction"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); + + timer.start("serialize"); + OutputGetter outputGetter(inferRequest); + status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); + timer.stop("serialize"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); + + timer.start("postprocess"); + status = postInferenceProcessing(responseProto, inferRequest, sequence, sequenceProcessingSpec); + timer.stop("postprocess"); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Postprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", + requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("postprocess") / 1000); + + sequenceLock.unlock(); + if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { + sequenceManagerLock.lock(); + status = sequenceManager->removeSequence(sequenceId); + if (!status.ok()) + return status; + } + + return StatusCode::OK; +} + +const Status StatefulModelInstance::preInferenceProcessing(ov::InferRequest& inferRequest, Sequence& sequence, + SequenceProcessingSpec& sequenceProcessingSpec) { + if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_START) { + // On SEQUENCE_START reset memory state of infer request to default + for (auto&& state : inferRequest.query_state()) { + state.reset(); + } + } else { + // For next requests in the sequence set infer request memory state to the last state saved by the sequence + const sequence_memory_state_t& sequenceMemoryState = sequence.getMemoryState(); + for (auto&& state : inferRequest.query_state()) { + auto stateName = state.get_name(); + if (!sequenceMemoryState.count(stateName)) + return StatusCode::INTERNAL_ERROR; + state.set_state(sequenceMemoryState.at(stateName)); + } + } + return StatusCode::OK; +} + +const Status StatefulModelInstance::postInferenceProcessing(tensorflow::serving::PredictResponse* response, + ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec) { + // Reset inferRequest states on SEQUENCE_END + if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { + spdlog::debug("Received SEQUENCE_END signal. Reseting model state and removing sequence"); + for (auto&& state : inferRequest.query_state()) { + state.reset(); + } + } else { + auto modelState = inferRequest.query_state(); + sequence.updateMemoryState(modelState); + } + + // Include sequence_id in server response + auto& tensorProto = (*response->mutable_outputs())["sequence_id"]; + tensorProto.mutable_tensor_shape()->add_dim()->set_size(1); + tensorProto.set_dtype(tensorflow::DataType::DT_UINT64); + tensorProto.add_uint64_val(sequenceProcessingSpec.getSequenceId()); + + return StatusCode::OK; +} +} // namespace ovms diff --git a/src/ovms_lib/statefulmodelinstance.hpp b/src/ovms_lib/statefulmodelinstance.hpp new file mode 100644 index 0000000000..ecb230fc41 --- /dev/null +++ b/src/ovms_lib/statefulmodelinstance.hpp @@ -0,0 +1,101 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "global_sequences_viewer.hpp" +#include "modelconfig.hpp" +#include "modelinstance.hpp" +#include "sequence_manager.hpp" + +namespace ovms { + +class StatefulModelInstance : public ModelInstance { + static const std::set SPECIAL_INPUT_NAMES; + +public: + /** + * @brief A default constructor + */ + StatefulModelInstance(const std::string& name, model_version_t version, ov::Core& ieCore, GlobalSequencesViewer* globalSequencesViewer) : + ModelInstance(name, version, ieCore), + globalSequencesViewer(globalSequencesViewer) { + sequenceManager = std::make_shared(config.getMaxSequenceNumber(), name, version); + } + + const std::shared_ptr& getSequenceManager() const { + return this->sequenceManager; + } + + const Status extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId); + + const Status extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput); + /* + Performs pre inference operations: + - for SEQUENCE_START control input - reset InferRequest memory state + - for SEQUENCE_END control input or for no control input - load sequence memory state into InferRequest + + Always returns StatusCode::OK + */ + const Status preInferenceProcessing(ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec); + + /* + Performs pre inference operations: + - for SEQUENCE_START or for no control input - save InferRequest memory state in sequence memory state + - for SEQUENCE_END control input - reset InferRequest memory state + - for all requests - append sequence id to the response + + Always returns StatusCode::OK + */ + const Status postInferenceProcessing(tensorflow::serving::PredictResponse* response, + ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec); + + Status infer(const tensorflow::serving::PredictRequest* requestProto, + tensorflow::serving::PredictResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr) override; + + Status loadModel(const ModelConfig& config) override; + + Status reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; + + void retireModel(bool isPermanent = true) override; + + void cleanupFailedLoad() override; + +protected: + std::shared_ptr sequenceManager; + + bool performLowLatencyTransformation; + + bool autoCleanupEnabled; + + GlobalSequencesViewer* globalSequencesViewer; + + template + const Status validate(const RequestType* request, SequenceProcessingSpec& processingSpec); + + Status loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; + + Status loadOVCompiledModel(const ModelConfig& config) override; + +private: + template + const Status validateSpecialKeys(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec); +}; +} // namespace ovms diff --git a/src/ovms_lib/status.cpp b/src/ovms_lib/status.cpp new file mode 100644 index 0000000000..78cfa7279c --- /dev/null +++ b/src/ovms_lib/status.cpp @@ -0,0 +1,426 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "status.hpp" + +namespace ovms { + +const std::unordered_map Status::statusMessageMap = { + {StatusCode::OK, ""}, + + {StatusCode::PATH_INVALID, "The provided base path is invalid or doesn't exists"}, + {StatusCode::FILE_INVALID, "File not found or cannot open"}, + {StatusCode::CONFIG_FILE_INVALID, "Configuration file not found or cannot open"}, + {StatusCode::FILESYSTEM_ERROR, "Error during filesystem operation"}, + {StatusCode::NOT_IMPLEMENTED, "Functionality not implemented"}, + {StatusCode::NO_MODEL_VERSION_AVAILABLE, "Not a single model version directory has valid numeric name"}, + {StatusCode::MODEL_NOT_LOADED, "Error while loading a model"}, + {StatusCode::JSON_INVALID, "The file is not valid json"}, + {StatusCode::JSON_SERIALIZATION_ERROR, "Data serialization to json format failed"}, + {StatusCode::MODELINSTANCE_NOT_FOUND, "ModelInstance not found"}, + {StatusCode::SHAPE_WRONG_FORMAT, "The provided shape is in wrong format"}, + {StatusCode::LAYOUT_WRONG_FORMAT, "The provided layout is in wrong format"}, + {StatusCode::DIM_WRONG_FORMAT, "The provided dimension is in wrong format"}, + {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, "Plugin config is in wrong format"}, + {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, "Model version policy is in wrong format"}, + {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, "Model version policy contains unsupported key"}, + {StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT, "Grpc channel arguments passed in wrong format"}, + {StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED, "Error during config file timestamp reading"}, + {StatusCode::RESHAPE_ERROR, "Model could not be reshaped with requested shape"}, + {StatusCode::RESHAPE_REQUIRED, "Model needs to be reloaded with new shape"}, + {StatusCode::BATCHSIZE_CHANGE_REQUIRED, "Model needs to be reloaded with new batchsize"}, + {StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER, "Value of provided parameter is forbidden"}, + {StatusCode::ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED, "Anonymous fixed shape is invalid for models with multiple inputs"}, + {StatusCode::ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED, "Anonymous fixed layout is invalid for models with multiple inputs"}, + {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, "Cannot compile model into target device"}, + {StatusCode::MODEL_MISSING, "Model with requested name and/or version is not found"}, + {StatusCode::MODEL_CONFIG_INVALID, "Model config is invalid"}, + {StatusCode::MODEL_NAME_MISSING, "Model with requested name is not found"}, + {StatusCode::MODEL_NAME_OCCUPIED, "Given model name is already occupied"}, + {StatusCode::MODEL_VERSION_MISSING, "Model with requested version is not found"}, + {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, "Model with requested version is retired"}, + {StatusCode::MODEL_VERSION_NOT_LOADED_YET, "Model with requested version is not loaded yet"}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, "Pipeline is retired"}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, "Pipeline is not loaded yet"}, + {StatusCode::MODEL_SPEC_MISSING, "model_spec missing in request"}, + {StatusCode::MODEL_VERSION_INVALID_FORMAT, "invalid model version format in request"}, + {StatusCode::INVALID_SIGNATURE_DEF, "Invalid signature name"}, + {StatusCode::CONFIG_SHAPE_IS_NOT_IN_MODEL, "Shape from config not found in model"}, + {StatusCode::CONFIG_LAYOUT_IS_NOT_IN_MODEL, "Layout from config not found in model"}, + {StatusCode::CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME, "Shape from config has real name. Use mapped name instead"}, + {StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME, "Layout from config has real name. Use mapped name instead"}, + {StatusCode::INVALID_NIREQ, "Nireq parameter too high"}, + {StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL, "Requested dynamic parameters but model is used in pipeline"}, + {StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET, "Node is not ready for execution"}, + {StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL, "Dynamic shape and dynamic batch size are not supported for stateful models"}, + {StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL, "Stateful model cannot be subscribed to pipeline"}, + {StatusCode::REQUESTED_MODEL_TYPE_CHANGE, "Model type cannot be changed after it is loaded"}, + {StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER, "Stateful model config parameter used for non stateful model"}, + {StatusCode::INVALID_MAX_SEQUENCE_NUMBER, "Sequence max number parameter too high"}, + {StatusCode::CANNOT_CONVERT_FLAT_SHAPE, "Cannot convert flat shape to Shape object"}, + {StatusCode::INVALID_BATCH_DIMENSION, "Invalid batch dimension in shape"}, + {StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE, "Layout incompatible with given shape"}, + {StatusCode::ALLOW_CACHE_WITH_CUSTOM_LOADER, "allow_cache is set to true with custom loader usage"}, + {StatusCode::UNKNOWN_ERROR, "Unknown error"}, + + // Sequence management + {StatusCode::SEQUENCE_MISSING, "Sequence with provided ID does not exist"}, + {StatusCode::SEQUENCE_ALREADY_EXISTS, "Sequence with provided ID already exists"}, + {StatusCode::SEQUENCE_ID_NOT_PROVIDED, "Sequence ID has not been provided in request inputs"}, + {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, "Unexpected value of sequence control input"}, + {StatusCode::SEQUENCE_ID_BAD_TYPE, "Could not find sequence id in expected tensor proto field uint64_val"}, + {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, "Could not find sequence control input in expected tensor proto field uint32_val"}, + {StatusCode::SEQUENCE_TERMINATED, "Sequence last request is being processed and it's not available anymore"}, + {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, "Special input proto does not contain tensor shape information"}, + {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, "Max sequence number has been reached. Could not create new sequence."}, + + // Predict request validation + {StatusCode::INVALID_NO_OF_INPUTS, "Invalid number of inputs"}, + {StatusCode::INVALID_MISSING_INPUT, "Missing input with specific name"}, + {StatusCode::INVALID_MISSING_OUTPUT, "Missing output with specific name"}, + {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Invalid number of shape dimensions"}, + {StatusCode::INVALID_BATCH_SIZE, "Invalid input batch size"}, + {StatusCode::INVALID_SHAPE, "Invalid input shape"}, + {StatusCode::INVALID_PRECISION, "Invalid input precision"}, + {StatusCode::INVALID_VALUE_COUNT, "Invalid number of values in tensor proto container"}, + {StatusCode::INVALID_CONTENT_SIZE, "Invalid content size of tensor proto"}, + {StatusCode::INVALID_MESSAGE_STRUCTURE, "Passing buffers both in ModelInferRequest::InferInputTensor::contents and in ModelInferRequest::raw_input_contents is not allowed"}, + {StatusCode::UNSUPPORTED_LAYOUT, "Received binary image input but resource not configured to accept NHWC layout"}, + + // Deserialization + {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, "Unsupported deserialization precision"}, + {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, "Internal deserialization error"}, + + // Inference + {StatusCode::OV_INTERNAL_INFERENCE_ERROR, "Internal inference error"}, + + // Serialization + {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, "Unsupported serialization precision"}, + {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, "Internal serialization error"}, + {StatusCode::OV_CLONE_TENSOR_ERROR, "Error during tensor clone"}, + + // GetModelStatus + {StatusCode::INTERNAL_ERROR, "Internal server error"}, + + // Rest handler failure + {StatusCode::REST_NOT_FOUND, "Requested REST resource not found"}, + {StatusCode::REST_COULD_NOT_PARSE_VERSION, "Could not parse model version in request"}, + {StatusCode::REST_INVALID_URL, "Invalid request URL"}, + {StatusCode::REST_UNSUPPORTED_METHOD, "Unsupported method"}, + {StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE, "Request components type not recognized"}, + + // Rest parser failure + {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, "Request body should be JSON object"}, + {StatusCode::REST_PREDICT_UNKNOWN_ORDER, "Invalid JSON structure. Could not detect row or column format"}, + {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, "Invalid JSON structure. Nonamed instance is not an array."}, + {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, "Invalid JSON structure. One of named instances is not a JSON object."}, + {StatusCode::REST_INPUT_NOT_PREALLOCATED, "Internal allocation error"}, + {StatusCode::REST_NO_INSTANCES_FOUND, "Invalid JSON structure. Missing instances in row format"}, + {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, "Could not detect neither named or nonamed format"}, + {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, "Could not parse instance content. Not valid ndarray detected"}, + {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, "Invalid JSON structure. Request inputs have different batch sizes"}, + {StatusCode::REST_INPUTS_NOT_AN_OBJECT, "Invalid JSON structure. One of inputs is not a JSON object."}, + {StatusCode::REST_NO_INPUTS_FOUND, "Invalid JSON structure. Missing inputs in column format"}, + {StatusCode::REST_COULD_NOT_PARSE_INPUT, "Could not parse input content. Not valid ndarray detected"}, + {StatusCode::REST_PROTO_TO_STRING_ERROR, "Response parsing to JSON error"}, + {StatusCode::REST_BASE64_DECODE_ERROR, "Decode Base64 to string error"}, + {StatusCode::REST_UNSUPPORTED_PRECISION, "Could not parse input content. Unsupported data precision detected"}, + {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, "Size of data in tensor_content does not match declared tensor shape"}, + {StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE, "Number of elements in xxx_val field does not match declared tensor shape"}, + {StatusCode::REST_SERIALIZE_NO_DATA, "No data found in tensor_content or xxx_val field matching tensor dtype"}, + + // Pipeline validation errors + {StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST, "Pipeline definition with the same name already exists"}, + {StatusCode::PIPELINE_NODE_WRONG_KIND_CONFIGURATION, "Unsupported node type"}, + {StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES, "Pipeline definition has multiple request nodes"}, + {StatusCode::PIPELINE_MULTIPLE_EXIT_NODES, "Pipeline definition has multiple response nodes"}, + {StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT, "Pipeline definition is missing request or response node"}, + {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, "Model with requested name is not found"}, + {StatusCode::PIPELINE_NODE_NAME_DUPLICATE, "Pipeline definition has multiple nodes with the same name"}, + {StatusCode::PIPELINE_CYCLE_FOUND, "Pipeline definition contains a cycle"}, + {StatusCode::PIPELINE_CONTAINS_UNCONNECTED_NODES, "Pipeline definition has unconnected nodes"}, + {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_NODE, "Pipeline definition has reference to missing node"}, + {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL, "Pipeline definition has reference to missing model"}, + {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE, "Pipeline definition has reference to missing data source"}, + {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT, "Pipeline definition has reference to missing model output"}, + {StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT, "Pipeline definition has connection to non existing model input"}, + {StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED, "Pipeline definition does not have connections for all inputs of underlying models"}, + {StatusCode::PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES, "Pipeline definition has multiple connections to the same input of underlying model"}, + {StatusCode::PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY, "Pipeline definition has response node used as dependency node"}, + {StatusCode::PIPELINE_NAME_OCCUPIED, "Pipeline has the same name as model"}, + {StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY, "Pipeline refers to incorrect library"}, + {StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS, "Gathered tensor shards dimensions are different"}, + {StatusCode::PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY, "Wrong number of dimensions in a tensor to be sharded"}, + {StatusCode::PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY, "Wrong dimension size. Should match demultiply count"}, + {StatusCode::PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE, "Tried to set the same input twice for node input handler"}, + {StatusCode::PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER, "Tried to set input with shard id > 0 for ordinary input handler"}, + {StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE, "Gather node refers to not existing node"}, + {StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER, "Gather node refers to node that isn't demultiplexer"}, + {StatusCode::PIPELINE_NODE_GATHER_FROM_ENTRY_NODE, "Gathering from entry node is not allowed"}, + {StatusCode::PIPELINE_DEMULTIPLY_ENTRY_NODE, "Demultiplication at entry node is not allowed"}, + {StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT, "Demultiplication count does not match tensor first dimension"}, + {StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED, "Manual gathering from multiple nodes is not supported"}, + {StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY, "Pipeline has not enough shape dimensions to demultiply"}, + {StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY, "Too large dynamic demultiplication requested."}, + {StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER, "Demultiplexer and gather nodes are not in LIFO order"}, + {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, "Pipeline execution aborted due to no content from custom node"}, + {StatusCode::PIPELINE_INPUTS_AMBIGUOUS_METADATA, "Multiple nodes connected to the same pipeline input require different tensor metadata"}, + + // Storage errors + // S3 + {StatusCode::S3_BUCKET_NOT_FOUND, "S3 Bucket not found"}, + {StatusCode::S3_METADATA_FAIL, "S3 metadata failure"}, + {StatusCode::S3_FAILED_LIST_OBJECTS, "S3 Failed to list objects"}, + {StatusCode::S3_FAILED_GET_TIME, "S3 Failed to get modification time"}, + {StatusCode::S3_INVALID_ACCESS, "S3 Invalid access rights"}, + {StatusCode::S3_FILE_NOT_FOUND, "S3 File or directory not found"}, + {StatusCode::S3_FILE_INVALID, "S3 File path is invalid"}, + {StatusCode::S3_FAILED_GET_OBJECT, "S3 Failed to get object from path"}, + + // GCS + {StatusCode::GCS_BUCKET_NOT_FOUND, "GCS Bucket not found"}, + {StatusCode::GCS_METADATA_FAIL, "GCS metadata failure"}, + {StatusCode::GCS_FAILED_LIST_OBJECTS, "GCS Failed to list objects"}, + {StatusCode::GCS_FAILED_GET_TIME, "GCS Failed to list objects"}, + {StatusCode::GCS_INVALID_ACCESS, "GCS Invalid access rights"}, + {StatusCode::GCS_FILE_NOT_FOUND, "GCS File or directory not found"}, + {StatusCode::GCS_FILE_INVALID, "GCS File path is invalid"}, + {StatusCode::GCS_FAILED_GET_OBJECT, "GCS Failed to get object from path"}, + {StatusCode::GCS_INCORRECT_REQUESTED_OBJECT_TYPE, "GCS invalid object type in path"}, + + // AS + {StatusCode::AS_INVALID_PATH, "AS Invalid path"}, + {StatusCode::AS_CONTAINER_NOT_FOUND, "AS Container not found"}, + {StatusCode::AS_SHARE_NOT_FOUND, "AS Share not found"}, + {StatusCode::AS_METADATA_FAIL, "AS metadata failure"}, + {StatusCode::AS_FAILED_LIST_OBJECTS, "AS Failed to list objects"}, + {StatusCode::AS_FAILED_GET_TIME, "AS Failed to list objects"}, + {StatusCode::AS_INVALID_ACCESS, "AS Invalid access rights"}, + {StatusCode::AS_FILE_NOT_FOUND, "AS File or directory not found"}, + {StatusCode::AS_FILE_INVALID, "AS File path is invalid"}, + {StatusCode::AS_FAILED_GET_OBJECT, "AS Failed to get object from path"}, + {StatusCode::AS_INCORRECT_REQUESTED_OBJECT_TYPE, "AS invalid object type in path"}, + + // Custom Loader + {StatusCode::CUSTOM_LOADER_LIBRARY_INVALID, "Custom Loader library not found or cannot open"}, + {StatusCode::CUSTOM_LOADER_LIBRARY_LOAD_FAILED, "Cannot load the custom library"}, + {StatusCode::CUSTOM_LOADER_EXISTS, "The custom loader is already present in loaders list"}, + {StatusCode::CUSTOM_LOADER_NOT_PRESENT, "The custom loader is not present in loaders list"}, + {StatusCode::CUSTOM_LOADER_INIT_FAILED, "Custom Loader LoadInit failed"}, + {StatusCode::CUSTOM_LOADER_ERROR, "Custom Loader Generic / Unknown Error"}, + + // Custom Node + {StatusCode::NODE_LIBRARY_ALREADY_LOADED, "Custom node library is already loaded"}, + {StatusCode::NODE_LIBRARY_LOAD_FAILED_OPEN, "Custom node library failed to open"}, + {StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM, "Custom node library failed to load symbol"}, + {StatusCode::NODE_LIBRARY_MISSING, "Custom node library not found"}, + {StatusCode::NODE_LIBRARY_MISSING_OUTPUT, "Custom node output is missing"}, + {StatusCode::NODE_LIBRARY_EXECUTION_FAILED, "Custom node failed during execution"}, + {StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED, "Custom node library has returned corrupted outputs handle"}, + {StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT, "Custom node library has produced corrupted number of outputs"}, + {StatusCode::NODE_LIBRARY_INVALID_PRECISION, "Custom node has produced tensor with unspecified precision"}, + {StatusCode::NODE_LIBRARY_INVALID_SHAPE, "Custom node has produced tensor with not matching shape"}, + {StatusCode::NODE_LIBRARY_INVALID_CONTENT_SIZE, "Custom node output has invalid content size"}, + {StatusCode::NODE_LIBRARY_METADATA_FAILED, "Custom node failed on metadata call"}, + {StatusCode::NODE_LIBRARY_OUTPUT_MISSING_NAME, "Custom node output is missing name"}, + + // Binary inputs + {StatusCode::IMAGE_PARSING_FAILED, "Image parsing failed"}, + {StatusCode::INVALID_NO_OF_CHANNELS, "Invalid number of channels in binary input"}, + {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, "Binary input images for this endpoint are required to have the same resolution"}, + {StatusCode::STRING_VAL_EMPTY, "String val is empty"}, + {StatusCode::BYTES_CONTENTS_EMPTY, "Bytes contents is empty"}, + {StatusCode::NODE_LIBRARY_INITIALIZE_FAILED, "Failure during custom node library initialization"}, + + // Model control API + {StatusCode::OK_NOT_RELOADED, "Config reload was not needed"}, + {StatusCode::OK_RELOADED, "Config reload successful"}, +}; + +const std::unordered_map Status::grpcStatusMap = { + {StatusCode::OK, grpc::StatusCode::OK}, + + {StatusCode::PATH_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::FILE_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::NO_MODEL_VERSION_AVAILABLE, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_NOT_LOADED, grpc::StatusCode::INTERNAL}, + {StatusCode::JSON_INVALID, grpc::StatusCode::INTERNAL}, + {StatusCode::MODELINSTANCE_NOT_FOUND, grpc::StatusCode::INTERNAL}, + {StatusCode::SHAPE_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, + {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, grpc::StatusCode::INTERNAL}, + {StatusCode::RESHAPE_ERROR, grpc::StatusCode::FAILED_PRECONDITION}, + {StatusCode::MODEL_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, + {StatusCode::MODEL_SPEC_MISSING, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::MODEL_VERSION_INVALID_FORMAT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SIGNATURE_DEF, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, grpc::StatusCode::ABORTED}, + {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, grpc::StatusCode::FAILED_PRECONDITION}, + + // Sequence management + {StatusCode::SEQUENCE_MISSING, grpc::StatusCode::NOT_FOUND}, + {StatusCode::SEQUENCE_ALREADY_EXISTS, grpc::StatusCode::ALREADY_EXISTS}, + {StatusCode::SEQUENCE_ID_NOT_PROVIDED, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_ID_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::SEQUENCE_TERMINATED, grpc::StatusCode::FAILED_PRECONDITION}, + {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, grpc::StatusCode::UNAVAILABLE}, + + // Predict request validation + {StatusCode::INVALID_NO_OF_INPUTS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_MISSING_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_BATCH_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_PRECISION, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_VALUE_COUNT, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_CONTENT_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_MESSAGE_STRUCTURE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::UNSUPPORTED_LAYOUT, grpc::StatusCode::INVALID_ARGUMENT}, + + // Deserialization + + // Should never occur - ModelInstance::validate takes care of that + {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, + {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, + + // Inference + {StatusCode::OV_INTERNAL_INFERENCE_ERROR, grpc::StatusCode::INTERNAL}, + + // Serialization + + // Should never occur - it should be validated during model loading + {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, + {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, + + // GetModelStatus + {StatusCode::INTERNAL_ERROR, grpc::StatusCode::INTERNAL}, + + // Binary input + {StatusCode::INVALID_NO_OF_CHANNELS, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::STRING_VAL_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::BYTES_CONTENTS_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, +}; + +const std::unordered_map Status::httpStatusMap = { + {StatusCode::OK, net_http::HTTPStatusCode::OK}, + {StatusCode::OK_RELOADED, net_http::HTTPStatusCode::CREATED}, + {StatusCode::OK_NOT_RELOADED, net_http::HTTPStatusCode::OK}, + + // REST handler failure + {StatusCode::REST_INVALID_URL, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_UNSUPPORTED_METHOD, net_http::HTTPStatusCode::NONE_ACC}, + + // REST parser failure + {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_PREDICT_UNKNOWN_ORDER, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INPUT_NOT_PREALLOCATED, net_http::HTTPStatusCode::ERROR}, + {StatusCode::REST_NO_INSTANCES_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_INPUTS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_NO_INPUTS_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_COULD_NOT_PARSE_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_PROTO_TO_STRING_ERROR, net_http::HTTPStatusCode::ERROR}, + {StatusCode::REST_UNSUPPORTED_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, net_http::HTTPStatusCode::ERROR}, + + {StatusCode::PATH_INVALID, net_http::HTTPStatusCode::ERROR}, + {StatusCode::FILE_INVALID, net_http::HTTPStatusCode::ERROR}, + {StatusCode::NO_MODEL_VERSION_AVAILABLE, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_NOT_LOADED, net_http::HTTPStatusCode::ERROR}, + {StatusCode::JSON_INVALID, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MODELINSTANCE_NOT_FOUND, net_http::HTTPStatusCode::ERROR}, + {StatusCode::SHAPE_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, + {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, net_http::HTTPStatusCode::ERROR}, + {StatusCode::RESHAPE_ERROR, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MODEL_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_VERSION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::MODEL_SPEC_MISSING, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SIGNATURE_DEF, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, net_http::HTTPStatusCode::NO_CONTENT}, + {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, net_http::HTTPStatusCode::PRECOND_FAILED}, + + // Sequence management + {StatusCode::SEQUENCE_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, + {StatusCode::SEQUENCE_ALREADY_EXISTS, net_http::HTTPStatusCode::CONFLICT}, + {StatusCode::SEQUENCE_ID_NOT_PROVIDED, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_ID_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::SEQUENCE_TERMINATED, net_http::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, net_http::HTTPStatusCode::SERVICE_UNAV}, + + // Predict request validation + {StatusCode::INVALID_NO_OF_INPUTS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_MISSING_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_BATCH_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_VALUE_COUNT, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_CONTENT_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_MESSAGE_STRUCTURE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::UNSUPPORTED_LAYOUT, net_http::HTTPStatusCode::BAD_REQUEST}, + + // Deserialization + + // Should never occur - ModelInstance::validate takes care of that + {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, + {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Inference + {StatusCode::OV_INTERNAL_INFERENCE_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Serialization + + // Should never occur - it should be validated during model loading + {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, + {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, + + // GetModelStatus + {StatusCode::INTERNAL_ERROR, net_http::HTTPStatusCode::ERROR}, + + // Binary input + {StatusCode::INVALID_NO_OF_CHANNELS, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::STRING_VAL_EMPTY, net_http::HTTPStatusCode::BAD_REQUEST}, +}; + +} // namespace ovms diff --git a/src/ovms_lib/status.hpp b/src/ovms_lib/status.hpp new file mode 100644 index 0000000000..a49b873340 --- /dev/null +++ b/src/ovms_lib/status.hpp @@ -0,0 +1,381 @@ +//***************************************************************************** +// Copyright 2020-2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/util/net_http/server/public/response_code_enum.h" +#pragma GCC diagnostic pop + +namespace ovms { + +namespace net_http = tensorflow::serving::net_http; + +enum class StatusCode { + OK, /*!< Success */ + + PATH_INVALID, /*!< The provided path is invalid or doesn't exists */ + FILE_INVALID, /*!< File not found or cannot open */ + CONFIG_FILE_INVALID, /*!< Config file not found or cannot open */ + FILESYSTEM_ERROR, /*!< Underlaying filesystem error */ + MODEL_NOT_LOADED, + JSON_INVALID, /*!< The file/content is not valid json */ + JSON_SERIALIZATION_ERROR, /*!< Data serialization to json format failed */ + MODELINSTANCE_NOT_FOUND, + SHAPE_WRONG_FORMAT, /*!< The provided shape param is in wrong format */ + LAYOUT_WRONG_FORMAT, /*!< The provided layout param is in wrong format */ + DIM_WRONG_FORMAT, /*!< The provided dimension param is in wrong format */ + PLUGIN_CONFIG_WRONG_FORMAT, /*!< Plugin config is in wrong format */ + MODEL_VERSION_POLICY_WRONG_FORMAT, /*!< Model version policy is in wrong format */ + MODEL_VERSION_POLICY_UNSUPPORTED_KEY, /*!< Model version policy contains invalid key */ + GRPC_CHANNEL_ARG_WRONG_FORMAT, + CONFIG_FILE_TIMESTAMP_READING_FAILED, /*!< Reading config file timestamp failed */ + NO_MODEL_VERSION_AVAILABLE, /*!< No model version found in path */ + RESHAPE_ERROR, /*!< Impossible to perform reshape */ + RESHAPE_REQUIRED, /*!< Model instance needs to be reloaded with new shape */ + BATCHSIZE_CHANGE_REQUIRED, /*!< Model instance needs to be reloaded with new batch size */ + FORBIDDEN_MODEL_DYNAMIC_PARAMETER, /*!< Value of the provided param is forbidden */ + ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED, /*!< Anonymous fixed shape is invalid for models with multiple inputs */ + ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED, /*!< Anonymous fixed layout is invalid for models with multiple inputs */ + CONFIG_SHAPE_IS_NOT_IN_MODEL, + CONFIG_LAYOUT_IS_NOT_IN_MODEL, + CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME, /*!< Using old name of input/output in config shape when mapped in mapping_config.json*/ + CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME, /*!< Using old name of input/output in config layout when mapped in mapping_config.json*/ + CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, + REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL, + CANNOT_CONVERT_FLAT_SHAPE, + INVALID_BATCH_DIMENSION, /*!< Invalid batch dimension in shape */ + ALLOW_CACHE_WITH_CUSTOM_LOADER, + LAYOUT_INCOMPATIBLE_WITH_SHAPE, + + // Model management + MODEL_MISSING, /*!< Model with such name and/or version does not exist */ + MODEL_CONFIG_INVALID, /*!< Model config is invalid */ + MODEL_NAME_MISSING, /*!< Model with requested name is not found */ + MODEL_NAME_OCCUPIED, /*!< Given model name is already occupied */ + MODEL_VERSION_MISSING, /*!< Model with requested version is not found */ + MODEL_VERSION_NOT_LOADED_ANYMORE, /*!< Model with requested version is retired */ + MODEL_VERSION_NOT_LOADED_YET, /*!< Model with requested version is not loaded yet */ + INVALID_NIREQ, /*!< Invalid NIREQ requested */ + REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL, /*!< Dynamic shape and dynamic batch size not supported for stateful models */ + REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL, /*!< Stateful model cannot be subscribed to pipeline */ + REQUESTED_MODEL_TYPE_CHANGE, /*!< Model type cannot be changed after it's loaded */ + INVALID_NON_STATEFUL_MODEL_PARAMETER, /*!< Stateful model config parameter used for non stateful model */ + INVALID_MAX_SEQUENCE_NUMBER, /*!< Sequence max number parameter too high */ + + // Sequence management + SEQUENCE_MISSING, /*!< Sequence with provided ID does not exist */ + SEQUENCE_ALREADY_EXISTS, /*!< Sequence with provided ID already exists */ + SEQUENCE_ID_NOT_PROVIDED, /*!< Sequence ID has not been provided in request inputs */ + SEQUENCE_ID_BAD_TYPE, /*!< Wrong sequence ID type */ + INVALID_SEQUENCE_CONTROL_INPUT, /*!< Unexpected value of sequence control input */ + SEQUENCE_CONTROL_INPUT_BAD_TYPE, /*!< Sequence control input in bad type */ + SEQUENCE_TERMINATED, /*!< Sequence last request is being processed and it's not available anymore */ + SPECIAL_INPUT_NO_TENSOR_SHAPE, /*!< Special input proto does not contain tensor shape information */ + MAX_SEQUENCE_NUMBER_REACHED, /*!< Model handles maximum number of sequences and will not accept new ones */ + + // Predict request validation + INVALID_NO_OF_INPUTS, /*!< Invalid number of inputs */ + INVALID_MISSING_INPUT, /*!< Missing one or more of inputs */ + INVALID_MISSING_OUTPUT, /*!< Missing one or more of outputs */ + INVALID_NO_OF_SHAPE_DIMENSIONS, /*!< Invalid number of shape dimensions */ + INVALID_BATCH_SIZE, /*!< Input batch size other than required */ + INVALID_SHAPE, /*!< Invalid shape dimension number or dimension value */ + INVALID_PRECISION, /*!< Invalid precision */ + INVALID_VALUE_COUNT, /*!< Invalid value count error status for uint16 and half float data types */ + INVALID_CONTENT_SIZE, /*!< Invalid content size error status for types using tensor_content() */ + INVALID_MESSAGE_STRUCTURE, /*!< Buffers can't be both in raw_input_content & input tensor content */ + + // Deserialization + OV_UNSUPPORTED_DESERIALIZATION_PRECISION, /*!< Unsupported deserialization precision, theoretically should never be returned since ModelInstance::validation checks against model precision */ + OV_INTERNAL_DESERIALIZATION_ERROR, /*!< Error occured during deserialization */ + + // Inference + OV_INTERNAL_INFERENCE_ERROR, /*!< Error occured during inference */ + + // Serialization + OV_UNSUPPORTED_SERIALIZATION_PRECISION, /*!< Unsupported serializaton precision */ + OV_INTERNAL_SERIALIZATION_ERROR, /*!< Error occurred during serialization */ + OV_CLONE_TENSOR_ERROR, /*!< Error during tensor clone */ + + // GetModelStatus + INVALID_SIGNATURE_DEF, /*!< Requested signature is not supported */ + + // Common request validation errors + MODEL_SPEC_MISSING, /*!< Request lacks model_spec */ + MODEL_VERSION_INVALID_FORMAT, + + INTERNAL_ERROR, + + UNKNOWN_ERROR, + + NOT_IMPLEMENTED, + + // S3 + S3_BUCKET_NOT_FOUND, /*!< S3 Bucket not found */ + S3_METADATA_FAIL, + S3_FAILED_LIST_OBJECTS, + S3_FAILED_GET_TIME, + S3_INVALID_ACCESS, + S3_FILE_NOT_FOUND, + S3_FILE_INVALID, + S3_FAILED_GET_OBJECT, + + // GCS + GCS_BUCKET_NOT_FOUND, + GCS_METADATA_FAIL, + GCS_FAILED_LIST_OBJECTS, + GCS_FAILED_GET_TIME, + GCS_INVALID_ACCESS, + GCS_FILE_NOT_FOUND, + GCS_FILE_INVALID, + GCS_FAILED_GET_OBJECT, + GCS_INCORRECT_REQUESTED_OBJECT_TYPE, + + // AS + AS_INVALID_PATH, + AS_CONTAINER_NOT_FOUND, + AS_SHARE_NOT_FOUND, + AS_METADATA_FAIL, + AS_FAILED_LIST_OBJECTS, + AS_FAILED_GET_TIME, + AS_INVALID_ACCESS, + AS_FILE_NOT_FOUND, + AS_FILE_INVALID, + AS_FAILED_GET_OBJECT, + AS_INCORRECT_REQUESTED_OBJECT_TYPE, + + // REST handler + REST_NOT_FOUND, /*!< Requested REST resource not found */ + REST_COULD_NOT_PARSE_VERSION, /*!< Could not parse model version in request */ + REST_INVALID_URL, /*!< Malformed REST request url */ + REST_UNSUPPORTED_METHOD, /*!< Request sent with unsupported method */ + UNKNOWN_REQUEST_COMPONENTS_TYPE, /*!< Components type not recognized */ + + // REST Parse + REST_BODY_IS_NOT_AN_OBJECT, /*!< REST body should be JSON object */ + REST_PREDICT_UNKNOWN_ORDER, /*!< Could not detect order (row/column) */ + REST_INSTANCES_NOT_AN_ARRAY, /*!< When parsing row order, instances must be an array */ + REST_NAMED_INSTANCE_NOT_AN_OBJECT, /*!< When parsing named instance it needs to be an object */ + REST_INPUT_NOT_PREALLOCATED, /*!< When parsing no named instance, exactly one input need to be preallocated */ + REST_NO_INSTANCES_FOUND, /*!< Missing instances in row order */ + REST_INSTANCES_NOT_NAMED_OR_NONAMED, /*!< Unknown instance format, neither named or nonamed */ + REST_COULD_NOT_PARSE_INSTANCE, /*!< Error while parsing instance content, not valid ndarray */ + REST_INSTANCES_BATCH_SIZE_DIFFER, /*!< In row order 0-th dimension (batch size) must be equal for all inputs */ + REST_INPUTS_NOT_AN_OBJECT, /*!< When parsing column order, inputs must be an object */ + REST_NO_INPUTS_FOUND, /*!< Missing inputs in column order */ + REST_COULD_NOT_PARSE_INPUT, /*!< Error while parsing input content, not valid ndarray */ + REST_PROTO_TO_STRING_ERROR, /*!< Error while parsing ResponseProto to JSON string */ + REST_BASE64_DECODE_ERROR, /*!< Error while decoding base64 REST binary input */ + REST_UNSUPPORTED_PRECISION, /*!< Unsupported conversion from tensor_content to _val container */ + REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, /*!< Size of data in tensor_content does not match declared tensor shape */ + REST_SERIALIZE_VAL_FIELD_INVALID_SIZE, /*!< Number of elements in xxx_val field does not match declared tensor shape */ + REST_SERIALIZE_NO_DATA, /*!< No data found in tensor_content or xxx_val field matching tensor dtype */ + + // Pipeline validation errors + PIPELINE_DEFINITION_ALREADY_EXIST, + PIPELINE_NODE_WRONG_KIND_CONFIGURATION, + PIPELINE_MULTIPLE_ENTRY_NODES, + PIPELINE_MULTIPLE_EXIT_NODES, + PIPELINE_MISSING_ENTRY_OR_EXIT, + PIPELINE_DEFINITION_NAME_MISSING, + PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, + PIPELINE_DEFINITION_NOT_LOADED_YET, + PIPELINE_NODE_NAME_DUPLICATE, + PIPELINE_STREAM_ID_NOT_READY_YET, + PIPELINE_CYCLE_FOUND, + PIPELINE_CONTAINS_UNCONNECTED_NODES, + PIPELINE_NODE_REFERING_TO_MISSING_NODE, + PIPELINE_NODE_REFERING_TO_MISSING_MODEL, + PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE, + PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT, + PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT, + PIPELINE_NOT_ALL_INPUTS_CONNECTED, + PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES, + PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY, + PIPELINE_NAME_OCCUPIED, + PIPELINE_DEFINITION_INVALID_NODE_LIBRARY, + PIPELINE_INCONSISTENT_SHARD_DIMENSIONS, + PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY, + PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY, + PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE, + PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER, + PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE, + PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER, + PIPELINE_NODE_GATHER_FROM_ENTRY_NODE, + PIPELINE_DEMULTIPLY_ENTRY_NODE, + PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT, + PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED, + PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY, + PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY, + PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER, + PIPELINE_DEMULTIPLEXER_NO_RESULTS, + PIPELINE_INPUTS_AMBIGUOUS_METADATA, + + // Custom Loader + CUSTOM_LOADER_LIBRARY_INVALID, + CUSTOM_LOADER_LIBRARY_LOAD_FAILED, + CUSTOM_LOADER_EXISTS, + CUSTOM_LOADER_NOT_PRESENT, + CUSTOM_LOADER_INIT_FAILED, + CUSTOM_LOADER_ERROR, + + // Custom Node + NODE_LIBRARY_ALREADY_LOADED, + NODE_LIBRARY_LOAD_FAILED_OPEN, + NODE_LIBRARY_LOAD_FAILED_SYM, + NODE_LIBRARY_MISSING, + NODE_LIBRARY_MISSING_OUTPUT, + NODE_LIBRARY_EXECUTION_FAILED, + NODE_LIBRARY_OUTPUTS_CORRUPTED, + NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT, + NODE_LIBRARY_INVALID_PRECISION, + NODE_LIBRARY_INVALID_SHAPE, + NODE_LIBRARY_INVALID_CONTENT_SIZE, + NODE_LIBRARY_METADATA_FAILED, + NODE_LIBRARY_OUTPUT_MISSING_NAME, + NODE_LIBRARY_INITIALIZE_FAILED, + + // Binary inputs + IMAGE_PARSING_FAILED, + UNSUPPORTED_LAYOUT, + INVALID_NO_OF_CHANNELS, + BINARY_IMAGES_RESOLUTION_MISMATCH, + STRING_VAL_EMPTY, + BYTES_CONTENTS_EMPTY, + + // Model control API + OK_NOT_RELOADED, /*!< Operation succeeded but no config reload was needed */ + OK_RELOADED, /*!< Operation succeeded and config reload was needed */ + + STATUS_CODE_END +}; + +class Status { + StatusCode code; + std::unique_ptr message; + + static const std::unordered_map statusMessageMap; + static const std::unordered_map grpcStatusMap; + static const std::unordered_map httpStatusMap; + + void appendDetails(const std::string& details) { + ensureMessageAllocated(); + *this->message += " - " + details; + } + void ensureMessageAllocated() { + if (nullptr == message) { + message = std::make_unique(); + } + } + +public: + Status(StatusCode code = StatusCode::OK) : + code(code) { + if (code == StatusCode::OK) { + return; + } + auto it = statusMessageMap.find(code); + if (it != statusMessageMap.end()) + this->message = std::make_unique(it->second); + else + this->message = std::make_unique("Undefined error"); + } + + Status(StatusCode code, const std::string& details) : + Status(code) { + appendDetails(details); + } + + Status(const Status& rhs) : + code(rhs.code), + message(rhs.message != nullptr ? std::make_unique(*(rhs.message)) : nullptr) {} + + Status(Status&& rhs) = default; + + Status operator=(const Status& rhs) { + this->code = rhs.code; + this->message = (rhs.message != nullptr ? std::make_unique(*rhs.message) : nullptr); + return *this; + } + + Status& operator=(Status&&) = default; + + bool ok() const { + return (code == StatusCode::OK || code == StatusCode::OK_RELOADED || code == StatusCode::OK_NOT_RELOADED); + } + + const StatusCode getCode() const { + return this->code; + } + + bool batchSizeChangeRequired() const { + return code == StatusCode::BATCHSIZE_CHANGE_REQUIRED; + } + + bool reshapeRequired() const { + return code == StatusCode::RESHAPE_REQUIRED; + } + + bool operator==(const Status& status) const { + return this->code == status.code; + } + + bool operator!=(const Status& status) const { + return this->code != status.code; + } + + const grpc::Status grpc() const { + auto it = grpcStatusMap.find(code); + if (it != grpcStatusMap.end()) { + return grpc::Status(it->second, + this->message ? *this->message : ""); + } else { + return grpc::Status(grpc::StatusCode::UNKNOWN, "Unknown error"); + } + } + + operator grpc::Status() const { + return this->grpc(); + } + + const std::string& string() const { + return this->message ? *this->message : statusMessageMap.at(code); + } + + operator const std::string&() const { + return this->string(); + } + + const net_http::HTTPStatusCode http() const { + auto it = httpStatusMap.find(code); + if (it != httpStatusMap.end()) { + return it->second; + } else { + return net_http::HTTPStatusCode::ERROR; + } + } +}; + +} // namespace ovms diff --git a/src/ovms_lib/stringutils.cpp b/src/ovms_lib/stringutils.cpp new file mode 100644 index 0000000000..7d7407357c --- /dev/null +++ b/src/ovms_lib/stringutils.cpp @@ -0,0 +1,139 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include "stringutils.hpp" + +#include +#include +#include +#include +#include +#include +#include +namespace ovms { + +std::string joins(const std::vector& listOfStrings, const std::string delimiter) { + std::stringstream ss; + auto it = listOfStrings.cbegin(); + if (it == listOfStrings.end()) { + return ""; + } + for (; it != (listOfStrings.end() - 1); ++it) { + ss << *it << delimiter; + } + if (it != listOfStrings.end()) { + ss << *it; + } + return ss.str(); +} + +void ltrim(std::string& str) { + str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int c) { + return !std::isspace(c); + })); +} + +void rtrim(std::string& str) { + str.erase(std::find_if(str.rbegin(), str.rend(), [](int c) { + return !std::isspace(c); + }) + .base(), + str.end()); +} + +void trim(std::string& str) { + ltrim(str); + rtrim(str); +} + +void erase_spaces(std::string& str) { + str.erase(std::remove_if(str.begin(), str.end(), + [](char c) -> bool { + return std::isspace(c, std::locale::classic()); + }), + str.end()); +} + +std::vector tokenize(const std::string& str, const char delimiter) { + std::vector tokens; + std::string token; + std::istringstream iss(str); + while (std::getline(iss, token, delimiter)) { + tokens.push_back(token); + } + + return tokens; +} + +bool endsWith(const std::string& str, const std::string& match) { + auto it = match.begin(); + return str.size() >= match.size() && + std::all_of(std::next(str.begin(), str.size() - match.size()), str.end(), [&it](const char& c) { + return ::tolower(c) == ::tolower(*(it++)); + }); +} + +std::optional stou32(const std::string& input) { + std::string str = input; + ovms::erase_spaces(str); + + if (str.size() > 0 && str[0] == '-') { + return std::nullopt; + } + + try { + uint64_t val = std::stoul(str); + if (val > std::numeric_limits::max()) { + return std::nullopt; + } + return {static_cast(val)}; + } catch (...) { + return std::nullopt; + } +} + +std::optional stoi32(const std::string& str) { + try { + return {static_cast(std::stoi(str))}; + } catch (...) { + return std::nullopt; + } +} + +std::optional stoi64(const std::string& str) { + if (!str.size()) { + return std::nullopt; + } + bool isMinus = (str[0] == '-'); + size_t i = 0; + if (isMinus) { + i = 1; + } + for (; i < str.size(); ++i) { + if (!std::isdigit(str[i])) { + return std::nullopt; + } + } + if (str[isMinus] == '0') { + return std::nullopt; + } + try { + return std::stoll(str); + } catch (...) { + return std::nullopt; + } +} +} // namespace ovms diff --git a/src/ovms_lib/stringutils.hpp b/src/ovms_lib/stringutils.hpp new file mode 100644 index 0000000000..cc0f14a634 --- /dev/null +++ b/src/ovms_lib/stringutils.hpp @@ -0,0 +1,91 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +namespace ovms { + +std::string joins(const std::vector& listOfStrings, const std::string delimiter); + +/** + * @brief Trims the string on the left side + * + * @param std::string& + */ +void ltrim(std::string& str); + +/** + * @brief Trims the string on the right side + * + * @param str + */ +void rtrim(std::string& str); + +/** + * @brief Trims the string + * + * @param str + */ +void trim(std::string& str); + +/** + * @brief Erases all whitespace characters from string + * + * @param str + */ +void erase_spaces(std::string& str); + +/** + * @brief Tokenizes a string into a vector of tokens + * + * @param str + * @param delimiter + * @return std::vector + */ +std::vector tokenize(const std::string& str, const char delimiter); + +/** + * @brief Checks if given string ends with another one + * + * @param str + * @param match + * @return true + * @return false + */ +bool endsWith(const std::string& str, const std::string& match); + +/** + * @brief Converts string to uint32, returns 0 or specified default value if conversion failed, fails if negative number is provided + * + * @param string input + * @param default value + * @return converted value and result indicating if conversion succeeded + */ +std::optional stou32(const std::string& input); + +/** + * @brief Converts string to int32, returns 0 or specified default value if conversion failed + * + * @param string input + * @param default value + * @return converted value and result indicating if conversion succeeded + */ +std::optional stoi32(const std::string& str); + +std::optional stoi64(const std::string& str); +} // namespace ovms diff --git a/src/ovms_lib/tensor_utils.hpp b/src/ovms_lib/tensor_utils.hpp new file mode 100644 index 0000000000..f17d3edd21 --- /dev/null +++ b/src/ovms_lib/tensor_utils.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { + +class TensorWithSource { +public: + TensorWithSource(const ov::Tensor& actual) : + actual(actual) {} + TensorWithSource(const ov::Tensor& actual, const ov::Tensor& source) : + actual(actual), + source(source) {} + + ov::Tensor& getActualTensor() { + return this->actual; + } + + ov::Tensor& getSourceTensor() { + return this->source; + } + + bool hasSource() const { + return (bool)this->source; + } + +private: + ov::Tensor actual, source; +}; + +} // namespace ovms diff --git a/src/ovms_lib/tensorinfo.cpp b/src/ovms_lib/tensorinfo.cpp new file mode 100644 index 0000000000..79bafdc6a2 --- /dev/null +++ b/src/ovms_lib/tensorinfo.cpp @@ -0,0 +1,289 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "tensorinfo.hpp" + +#include +#include +#include +#include +#include + +#include "kfs_grpc_inference_service.hpp" +#include "logging.hpp" + +namespace ovms { + +// in case we change behaviour for this constructor we may need to write additional tests for TensorInfo intersection / DAGs +TensorInfo::TensorInfo(const std::string& name, + const Precision& precision, + const Shape& shape) : + name(name), + mapping(""), + precision(precision), + shape(shape), + layout(Layout::getDefaultLayout()) {} + +TensorInfo::TensorInfo(const std::string& name, + const Precision& precision, + const shape_t& shape) : + name(name), + mapping(""), + precision(precision), + shape(shape), + layout(Layout::getDefaultLayout()) { +} + +TensorInfo::TensorInfo(const std::string& name, + const ovms::Precision& precision, + const shape_t& shape, + const Layout& layout) : + name(name), + mapping(""), + precision(precision), + shape(shape), + layout(layout) { +} + +TensorInfo::TensorInfo(const std::string& name, + const ovms::Precision& precision, + const Shape& shape, + const Layout& layout) : + name(name), + mapping(""), + precision(precision), + shape(shape), + layout(layout) { +} + +TensorInfo::TensorInfo(const std::string& name, + const std::string& mapping, + const ovms::Precision& precision, + const shape_t& shape, + const Layout& layout) : + name(name), + mapping(mapping), + precision(precision), + shape(shape), + layout(layout) { +} +TensorInfo::TensorInfo(const std::string& name, + const std::string& mapping, + const ovms::Precision& precision, + const Shape& shape, + const Layout& layout) : + name(name), + mapping(mapping), + precision(precision), + shape(shape), + layout(layout) { +} +TensorInfo::TensorInfo(const std::string& name, + const std::string& mapping, + const Precision& precision, + const shape_t& shape) : + name(name), + mapping(mapping), + precision(precision), + shape(shape), + layout(Layout::getDefaultLayout()) { +} + +const std::string& TensorInfo::getName() const { + return name; +} + +const std::string& TensorInfo::getMappedName() const { + return mapping.size() == 0 ? name : mapping; +} + +void TensorInfo::setMappedName(const std::string& mappedName) { + mapping = mappedName; +} + +const Precision TensorInfo::getPrecision() const { + return precision; +} + +void TensorInfo::setPrecision(const ovms::Precision& requestedPrecision) { + precision = requestedPrecision; +} + +std::string TensorInfo::getPrecisionAsString(Precision precision) { + return toString(precision); +} + +ov::element::Type TensorInfo::getOvPrecision() const { + return ovmsPrecisionToIE2Precision(precision); +} + +std::string TensorInfo::getPrecisionAsString() const { + return getPrecisionAsString(precision); +} + +std::string TensorInfo::getPrecisionAsKFSPrecision(Precision precision) { + return ovmsPrecisionToKFSPrecision(precision); +} + +std::string TensorInfo::getPrecisionAsKFSPrecision() const { + return getPrecisionAsKFSPrecision(precision); +} + +std::string TensorInfo::getStringFromLayout(const Layout& layout) { + return layout; +} + +const Layout& TensorInfo::getLayout() const { + return layout; +} + +bool TensorInfo::isInfluencedByDemultiplexer() const { + return influencedByDemultiplexer; +} + +void TensorInfo::setShape(const Shape& shape) { + this->shape = shape; +} + +const Shape& TensorInfo::getShape() const { + return this->shape; +} + +void TensorInfo::setLayout(const Layout& layout) { + this->layout = layout; +} + +std::shared_ptr TensorInfo::createCopyWithNewShape(const Shape& shape) const { + auto copy = std::make_shared(*this); + copy->shape = shape; + copy->layout = Layout::getUnspecifiedLayout(); + return copy; +} + +std::shared_ptr TensorInfo::createCopyWithDemultiplexerDimensionPrefix(const Dimension& dim) const { + auto copy = std::make_shared(*this); + copy->influencedByDemultiplexer = true; + copy->shape.emplace(copy->shape.begin(), dim); + copy->layout = this->getLayout(); + auto batchPosition = copy->layout.find(BATCH_DIMENSION_LETTER); + if (batchPosition != std::string::npos) { + copy->layout.replace(batchPosition, 1, std::string(1, UNDEFINED_DIMENSION_CHAR)); + } + copy->layout = std::string(1, BATCH_DIMENSION_LETTER[0]) + copy->layout; + return copy; +} + +std::shared_ptr TensorInfo::createIntersection(const TensorInfo& other) { + if (this->isTensorUnspecified()) + return std::make_shared(other); + if (other.isTensorUnspecified()) + return std::make_shared(*this); + if ((this->getName() != other.getName()) || + (this->getMappedName() != other.getMappedName())) { + return nullptr; + } + Precision precision; + if (this->getPrecision() != other.getPrecision()) { + if (this->getPrecision() == Precision::UNDEFINED) { + precision = other.getPrecision(); + } else if (other.getPrecision() == Precision::UNDEFINED) { + precision = this->getPrecision(); + } else { + return nullptr; + } + } else { + precision = this->getPrecision(); + } + auto newShape = this->getShape().createIntersection(other.getShape()); + if (!newShape.has_value()) + return nullptr; + auto layout = this->getLayout().createIntersection(other.getLayout(), newShape.value().size()); + if (!layout.has_value()) + return nullptr; + return std::make_shared(this->getName(), + this->getMappedName(), + precision, + std::move(newShape.value()), + layout.value()); +} + +bool TensorInfo::isTensorSpecEqual(const TensorInfo& other) const { + return (this->getShape() == other.getShape()) && + (this->getPrecision() == other.getPrecision()) && + (this->getLayout() == other.getLayout()); +} + +bool TensorInfo::isTensorUnspecified() const { + return (this->getPrecision() == Precision::UNDEFINED) && + (this->getName() == "") && + (this->getShape() == Shape()); +} + +std::string TensorInfo::shapeToString(const shape_t& shape) { + std::ostringstream oss; + oss << "("; + size_t i = 0; + if (shape.size() > 0) { + for (; i < shape.size() - 1; i++) { + oss << shape[i] << ","; + } + oss << shape[i]; + } + oss << ")"; + + return oss.str(); +} + +std::string tensorShapeToString(const google::protobuf::RepeatedField& shape) { + std::ostringstream oss; + oss << "("; + size_t i = 0; + if (shape.size() > 0) { + for (; i < shape.size() - 1; i++) { + oss << shape[i] << ","; + } + oss << shape[i]; + } + oss << ")"; + + return oss.str(); +} + +std::shared_ptr TensorInfo::getUnspecifiedTensorInfo() { + return std::make_shared("", Precision::UNDEFINED, Shape{}); +} + +const std::optional TensorInfo::getBatchSize() const { + const auto batchIndex = this->layout.getBatchIndex(); + if (!batchIndex.has_value()) { + return std::nullopt; + } + if (getShape().size() < batchIndex.value() + 1) { + throw std::logic_error("batch outside of shape range"); + } + return getShape()[batchIndex.value()]; +} + +std::string TensorInfo::asString() const { + std::stringstream ss; + ss + << "name: " << getName() << "; " + << "mapping_name: " << getMappedName() << "; " + << "shape: " << getShape().toString() << "; " + << "precision: " << getPrecisionAsString() << "; " + << "layout: " << getStringFromLayout(getLayout()); + return ss.str(); +} +} // namespace ovms diff --git a/src/ovms_lib/tensorinfo.hpp b/src/ovms_lib/tensorinfo.hpp new file mode 100644 index 0000000000..a785d05fc9 --- /dev/null +++ b/src/ovms_lib/tensorinfo.hpp @@ -0,0 +1,238 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "layout.hpp" +#include "precision.hpp" +#include "shape.hpp" + +namespace google::protobuf { +template +class RepeatedField; +} + +namespace ovms { + +class TensorInfo; + +using tensor_map_t = std::map>; + +/** + * @brief Class containing information about the tensor + */ +class TensorInfo { +protected: + /** + * @brief Input name + */ + std::string name; + + /** + * @brief Mapping name + */ + std::string mapping; + + Precision precision; + + /** + * @brief Model input shape + */ + Shape shape; + + /** + * @brief Tensor layout + */ + Layout layout; + + /** + * @brief Information if influenced by demultiplexer + */ + bool influencedByDemultiplexer = false; + +public: + /** + * @brief Construct a new Tensor Info object + * + */ + TensorInfo() = default; + TensorInfo(const TensorInfo&) = default; + + /** + * @brief Construct a new Tensor Info object + * + * @param name + * @param precision + * @param shape + */ + TensorInfo(const std::string& name, + const Precision& precision, + const shape_t& shape); + TensorInfo(const std::string& name, + const Precision& precision, + const Shape& shape); + + /** + * @brief Construct a new Tensor Info object + * + * @param name + * @param precision + * @param shape + * @param layout + */ + TensorInfo(const std::string& name, + const Precision& precision, + const shape_t& shape, + const Layout& layout); + TensorInfo(const std::string& name, + const Precision& precision, + const Shape& shape, + const Layout& layout); + + /** + * @brief Construct a new Tensor Info object + * + * @param name + * @param precision + * @param shape + * @param layout + */ + TensorInfo(const std::string& name, + const std::string& mapping, + const Precision& precision, + const shape_t& shape, + const Layout& layout); + TensorInfo(const std::string& name, + const std::string& mapping, + const Precision& precision, + const Shape& shape, + const Layout& layout); + TensorInfo(const std::string& name, + const std::string& mapping, + const Precision& precision, + const shape_t& shape); + + /** + * @brief Get the Name object + * + * @return const std::string& + */ + const std::string& getName() const; + + /** + * @brief Get the tensor name - as in model or mapped name + * + * @return const std::string& + */ + const std::string& getMappedName() const; + void setMappedName(const std::string& mappedName); + + /** + * @brief Get the Precision object + * + * @return const InferenceEngine::Precision + */ + const Precision getPrecision() const; + + /** + * @brief Set the Precision object + * + * @return const InferenceEngine::Precision + */ + void setPrecision(const ovms::Precision& requestedPrecision); + + /** + * @brief Set the Layout object + */ + void setLayout(const Layout& layout); + + ov::element::Type getOvPrecision() const; + + /** + * @brief Get the Precision As String object + * + * @return const std::string + */ + std::string getPrecisionAsString() const; + + /** + * @brief Get the Precision As String object representing KFS precision + * + * @return const std::string + */ + std::string getPrecisionAsKFSPrecision() const; + + /** + * @brief Get the string representation of TensorInfo object + * + * @return String representation + */ + std::string asString() const; + + static std::string getPrecisionAsString(Precision precision); + + static std::string getPrecisionAsKFSPrecision(Precision precision); + + /** + * @brief Get the layout name from Layout + * + * @param Layout + * @return std::string + */ + static std::string getStringFromLayout(const Layout& layout); + + /** + * @brief Get the Layout string + * + * @return const Layout& + */ + const Layout& getLayout() const; + + /** + * @brief Gets input shape + * + * @return shape + */ + const Shape& getShape() const; + void setShape(const Shape& shape); + + bool isInfluencedByDemultiplexer() const; + + std::shared_ptr createCopyWithNewShape(const Shape& shape) const; + + std::shared_ptr createCopyWithDemultiplexerDimensionPrefix(const Dimension& dim) const; + std::shared_ptr createIntersection(const TensorInfo& other); + + bool isTensorUnspecified() const; + + bool isTensorSpecEqual(const TensorInfo& other) const; + + static std::string shapeToString(const shape_t& shape); + + static std::shared_ptr getUnspecifiedTensorInfo(); + + const std::optional getBatchSize() const; +}; +std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); + +} // namespace ovms diff --git a/src/ovms_lib/tensormap.hpp b/src/ovms_lib/tensormap.hpp new file mode 100644 index 0000000000..830bf24e4b --- /dev/null +++ b/src/ovms_lib/tensormap.hpp @@ -0,0 +1,33 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include + +#include "tensor_utils.hpp" + +namespace ovms { + +using TensorMap = std::unordered_map; +using TensorVector = std::vector; +using TensorWithSourceMap = std::unordered_map; + +} // namespace ovms diff --git a/src/ovms_lib/tfs_frontend/tfs_utils.cpp b/src/ovms_lib/tfs_frontend/tfs_utils.cpp new file mode 100644 index 0000000000..31361c8416 --- /dev/null +++ b/src/ovms_lib/tfs_frontend/tfs_utils.cpp @@ -0,0 +1,98 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "tfs_utils.hpp" + +#include +#include +#include +#include +#include + +#include "../logging.hpp" + +namespace ovms { + +tensorflow::DataType getPrecisionAsDataType(Precision precision) { + static std::unordered_map precisionMap{ + {Precision::FP32, tensorflow::DataType::DT_FLOAT}, + {Precision::FP64, tensorflow::DataType::DT_DOUBLE}, + {Precision::FP16, tensorflow::DataType::DT_HALF}, + {Precision::I64, tensorflow::DataType::DT_INT64}, + {Precision::I32, tensorflow::DataType::DT_INT32}, + {Precision::I16, tensorflow::DataType::DT_INT16}, + {Precision::I8, tensorflow::DataType::DT_INT8}, + {Precision::U64, tensorflow::DataType::DT_UINT64}, + {Precision::U16, tensorflow::DataType::DT_UINT16}, + {Precision::U8, tensorflow::DataType::DT_UINT8}, + // {Precision::MIXED, tensorflow::DataType::DT_INVALID}, + // {Precision::Q78, tensorflow::DataType::DT_INVALID}, + // {Precision::BIN, tensorflow::DataType::DT_INVALID}, + {Precision::BOOL, tensorflow::DataType::DT_BOOL} + // {Precision::CUSTOM, tensorflow::DataType::DT_INVALID} + }; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return tensorflow::DataType::DT_INVALID; + } + return it->second; +} + +std::string getDataTypeAsString(tensorflow::DataType dataType) { + switch (dataType) { + case tensorflow::DataType::DT_FLOAT: + return "FP32"; + case tensorflow::DataType::DT_DOUBLE: + return "FP64"; + case tensorflow::DataType::DT_INT32: + return "I32"; + case tensorflow::DataType::DT_INT8: + return "I8"; + case tensorflow::DataType::DT_UINT8: + return "U8"; + case tensorflow::DataType::DT_HALF: + return "FP16"; + case tensorflow::DataType::DT_INT16: + return "I16"; + case tensorflow::DataType::DT_UINT16: + return "U16"; + case tensorflow::DataType::DT_UINT64: + return "U64"; + case tensorflow::DataType::DT_INT64: + return "I64"; + case tensorflow::DataType::DT_BOOL: + return "BOOL"; + case tensorflow::DataType::DT_STRING: + return "STRING"; + default: + return "INVALID"; + } +} + +std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape) { + std::ostringstream oss; + oss << "("; + int i = 0; + if (tensorShape.dim_size() > 0) { + for (; i < tensorShape.dim_size() - 1; i++) { + oss << tensorShape.dim(i).size() << ","; + } + oss << tensorShape.dim(i).size(); + } + oss << ")"; + + return oss.str(); +} +} // namespace ovms diff --git a/src/ovms_lib/tfs_frontend/tfs_utils.hpp b/src/ovms_lib/tfs_frontend/tfs_utils.hpp new file mode 100644 index 0000000000..578aa73892 --- /dev/null +++ b/src/ovms_lib/tfs_frontend/tfs_utils.hpp @@ -0,0 +1,33 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop + +#include "../precision.hpp" + +namespace ovms { +tensorflow::DataType getPrecisionAsDataType(Precision precision); +std::string getDataTypeAsString(tensorflow::DataType dataType); +std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape); +// static std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); +} // namespace ovms diff --git a/src/ovms_lib/threadsafequeue.hpp b/src/ovms_lib/threadsafequeue.hpp new file mode 100644 index 0000000000..2f514da08c --- /dev/null +++ b/src/ovms_lib/threadsafequeue.hpp @@ -0,0 +1,69 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace ovms { + +template +class ThreadSafeQueue { +public: + ThreadSafeQueue() {} + ~ThreadSafeQueue() {} + void push(const T& element) { + std::unique_lock lock(mtx); + queue.push(std::move(element)); + lock.unlock(); + signal.notify_one(); + } + + void push(T&& element) { + std::unique_lock lock(mtx); + queue.push(std::move(element)); + lock.unlock(); + signal.notify_one(); + } + + std::optional tryPull(const uint waitDurationMicroseconds) { + std::unique_lock lock(mtx); + if (signal.wait_for(lock, + std::chrono::microseconds(waitDurationMicroseconds), + [this]() { return queue.size() > 0; })) { + T element = std::move(queue.front()); + queue.pop(); + return std::optional{std::move(element)}; + } else { + return std::nullopt; + } + } + + size_t size() { + return queue.size(); + } + +private: + std::mutex mtx; + std::queue queue; + std::condition_variable signal; +}; +} // namespace ovms diff --git a/src/ovms_lib/timer.hpp b/src/ovms_lib/timer.hpp new file mode 100644 index 0000000000..9a761f4b12 --- /dev/null +++ b/src/ovms_lib/timer.hpp @@ -0,0 +1,50 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +namespace ovms { + +template +struct is_chrono_duration_type : std::false_type {}; + +template +struct is_chrono_duration_type> : std::true_type {}; + +class Timer { + std::unordered_map startTimestamps; + std::unordered_map stopTimestamps; + +public: + void start(const std::string& name) { + startTimestamps[name] = std::chrono::high_resolution_clock::now(); + } + + void stop(const std::string& name) { + stopTimestamps[name] = std::chrono::high_resolution_clock::now(); + } + + template + double elapsed(const std::string& name) { + static_assert(is_chrono_duration_type::value, "Non supported type."); + return std::chrono::duration_cast(stopTimestamps[name] - startTimestamps[name]).count(); + } +}; + +} // namespace ovms diff --git a/src/ovms_lib/version.hpp b/src/ovms_lib/version.hpp new file mode 100644 index 0000000000..e9a682cdd1 --- /dev/null +++ b/src/ovms_lib/version.hpp @@ -0,0 +1,21 @@ +//***************************************************************************** +// Copyright 2020 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#ifndef SRC_VERSION_HPP_ +#define SRC_VERSION_HPP_ +#define PROJECT_NAME "REPLACE_PROJECT_NAME" +#define PROJECT_VERSION "REPLACE_PROJECT_VERSION" +#define OPENVINO_NAME "REPLACE_OPENVINO_NAME" +#endif // SRC_VERSION_HPP_" diff --git a/src/server.cpp b/src/server.cpp index df763fb470..6afe332bd7 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -62,6 +62,11 @@ namespace { volatile sig_atomic_t shutdown_request = 0; } +Server& Server::instance() { + static Server global; + return global; +} + static void logConfig(const Config& config) { std::string project_name(PROJECT_NAME); std::string project_version(PROJECT_VERSION); diff --git a/src/server.hpp b/src/server.hpp index 87f2c5bec0..b9f3fb61d8 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -40,10 +40,7 @@ class Server { virtual std::unique_ptr createModule(const std::string& name); public: - static Server& instance() { - static Server global; - return global; - } + static Server& instance(); int start(int argc, char** argv); ModuleState getModuleState(const std::string& name) const; const Module* getModule(const std::string& name) const; From 1bb74744fed94b596d731270549bb78a6e8e9d1a Mon Sep 17 00:00:00 2001 From: atobisze Date: Mon, 17 Oct 2022 11:16:10 +0200 Subject: [PATCH 083/130] poc app without OVMS classes exposed in header --- src/BUILD | 7 +- src/main2.cpp | 60 +- src/modelinstance.cpp | 35 +- src/ovms_lib/BUILD | 630 -------- src/ovms_lib/aliases.hpp | 25 - src/ovms_lib/azurefilesystem.cpp | 320 ---- src/ovms_lib/azurefilesystem.hpp | 166 -- src/ovms_lib/azurestorage.cpp | 1217 --------------- src/ovms_lib/azurestorage.hpp | 195 --- src/ovms_lib/binaryutils.cpp | 447 ------ src/ovms_lib/binaryutils.hpp | 26 - src/ovms_lib/cleaner_utils.hpp | 43 - src/ovms_lib/config.cpp | 303 ---- src/ovms_lib/config.hpp | 394 ----- src/ovms_lib/custom_node.cpp | 103 -- src/ovms_lib/custom_node.hpp | 65 - src/ovms_lib/custom_node_interface.h | 79 - ..._node_library_internal_manager_wrapper.cpp | 27 - ..._node_library_internal_manager_wrapper.hpp | 40 - src/ovms_lib/custom_node_library_manager.cpp | 135 -- src/ovms_lib/custom_node_library_manager.hpp | 36 - src/ovms_lib/custom_node_output_allocator.cpp | 54 - src/ovms_lib/custom_node_output_allocator.hpp | 39 - src/ovms_lib/customloaderconfig.hpp | 145 -- src/ovms_lib/customloaderinterface.hpp | 117 -- src/ovms_lib/customloaders.cpp | 90 -- src/ovms_lib/customloaders.hpp | 92 -- src/ovms_lib/customnodesession.cpp | 250 --- src/ovms_lib/customnodesession.hpp | 59 - src/ovms_lib/deserialization.cpp | 73 - src/ovms_lib/deserialization.hpp | 405 ----- src/ovms_lib/dl_node.cpp | 134 -- src/ovms_lib/dl_node.hpp | 98 -- src/ovms_lib/dlnodesession.cpp | 345 ----- src/ovms_lib/dlnodesession.hpp | 73 - src/ovms_lib/entry_node.cpp | 177 --- src/ovms_lib/entry_node.hpp | 63 - src/ovms_lib/executingstreamidguard.hpp | 39 - src/ovms_lib/exit_node.cpp | 77 - src/ovms_lib/exit_node.hpp | 63 - src/ovms_lib/exitnodesession.cpp | 46 - src/ovms_lib/exitnodesession.hpp | 37 - src/ovms_lib/filesystem.hpp | 196 --- src/ovms_lib/gatherexitnodeinputhandler.cpp | 53 - src/ovms_lib/gatherexitnodeinputhandler.hpp | 68 - src/ovms_lib/gathernodeinputhandler.cpp | 116 -- src/ovms_lib/gathernodeinputhandler.hpp | 48 - src/ovms_lib/gcsfilesystem.cpp | 407 ----- src/ovms_lib/gcsfilesystem.hpp | 165 -- src/ovms_lib/get_model_metadata_impl.cpp | 202 --- src/ovms_lib/get_model_metadata_impl.hpp | 71 - src/ovms_lib/global_sequences_viewer.cpp | 73 - src/ovms_lib/global_sequences_viewer.hpp | 46 - src/ovms_lib/grpcservermodule.cpp | 183 --- src/ovms_lib/grpcservermodule.hpp | 46 - src/ovms_lib/http_rest_api_handler.cpp | 606 -------- src/ovms_lib/http_rest_api_handler.hpp | 190 --- src/ovms_lib/http_server.cpp | 131 -- src/ovms_lib/http_server.hpp | 41 - src/ovms_lib/kfs_grpc_inference_service.cpp | 343 ----- src/ovms_lib/kfs_grpc_inference_service.hpp | 68 - src/ovms_lib/layout.cpp | 169 -- src/ovms_lib/layout.hpp | 59 - src/ovms_lib/layout_configuration.cpp | 98 -- src/ovms_lib/layout_configuration.hpp | 52 - src/ovms_lib/localfilesystem.cpp | 173 --- src/ovms_lib/localfilesystem.hpp | 116 -- src/ovms_lib/log | 0 src/ovms_lib/logging.cpp | 87 -- src/ovms_lib/logging.hpp | 36 - src/ovms_lib/main.cpp | 24 - src/ovms_lib/main2.cpp | 15 - src/ovms_lib/model.cpp | 305 ---- src/ovms_lib/model.hpp | 246 --- src/ovms_lib/model_service.cpp | 220 --- src/ovms_lib/model_service.hpp | 65 - src/ovms_lib/model_version_policy.cpp | 67 - src/ovms_lib/model_version_policy.hpp | 135 -- src/ovms_lib/modelchangesubscription.cpp | 57 - src/ovms_lib/modelchangesubscription.hpp | 42 - src/ovms_lib/modelconfig.cpp | 669 -------- src/ovms_lib/modelconfig.hpp | 992 ------------ src/ovms_lib/modelinstance.cpp | 1225 --------------- src/ovms_lib/modelinstance.hpp | 556 ------- src/ovms_lib/modelinstanceunloadguard.cpp | 29 - src/ovms_lib/modelinstanceunloadguard.hpp | 30 - src/ovms_lib/modelmanager.cpp | 1316 ---------------- src/ovms_lib/modelmanager.hpp | 427 ----- src/ovms_lib/modelversion.hpp | 24 - src/ovms_lib/modelversionstatus.cpp | 122 -- src/ovms_lib/modelversionstatus.hpp | 105 -- src/ovms_lib/my_cc_combine.bzl | 70 - src/ovms_lib/node.cpp | 272 ---- src/ovms_lib/node.hpp | 102 -- src/ovms_lib/node_library.cpp | 29 - src/ovms_lib/node_library.hpp | 52 - src/ovms_lib/node_library_utils.cpp | 143 -- src/ovms_lib/node_library_utils.hpp | 40 - src/ovms_lib/nodeinfo.hpp | 92 -- src/ovms_lib/nodeinputhandler.cpp | 60 - src/ovms_lib/nodeinputhandler.hpp | 54 - src/ovms_lib/nodeoutputhandler.cpp | 20 - src/ovms_lib/nodeoutputhandler.hpp | 28 - src/ovms_lib/nodesession.cpp | 72 - src/ovms_lib/nodesession.hpp | 62 - src/ovms_lib/nodesessionmetadata.cpp | 197 --- src/ovms_lib/nodesessionmetadata.hpp | 48 - src/ovms_lib/nodesessionresult.hpp | 29 - src/ovms_lib/nodestreamidguard.hpp | 70 - src/ovms_lib/ov_utils.cpp | 131 -- src/ovms_lib/ov_utils.hpp | 47 - src/ovms_lib/ovinferrequestsqueue.hpp | 45 - src/ovms_lib/pipeline.cpp | 270 ---- src/ovms_lib/pipeline.hpp | 63 - src/ovms_lib/pipeline_factory.cpp | 146 -- src/ovms_lib/pipeline_factory.hpp | 84 - src/ovms_lib/pipelinedefinition.cpp | 1367 ----------------- src/ovms_lib/pipelinedefinition.hpp | 189 --- src/ovms_lib/pipelinedefinitionstatus.hpp | 407 ----- .../pipelinedefinitionunloadguard.cpp | 29 - .../pipelinedefinitionunloadguard.hpp | 30 - src/ovms_lib/pipelineeventqueue.hpp | 28 - src/ovms_lib/precision.cpp | 249 --- src/ovms_lib/precision.hpp | 70 - .../predict_request_validation_utils.cpp | 696 --------- .../predict_request_validation_utils.hpp | 42 - src/ovms_lib/prediction_service.cpp | 137 -- src/ovms_lib/prediction_service.hpp | 64 - src/ovms_lib/prediction_service_utils.cpp | 90 -- src/ovms_lib/prediction_service_utils.hpp | 38 - src/ovms_lib/profiler.cpp | 56 - src/ovms_lib/profiler.hpp | 49 - src/ovms_lib/queue.hpp | 136 -- src/ovms_lib/rest_parser.cpp | 468 ------ src/ovms_lib/rest_parser.hpp | 218 --- src/ovms_lib/rest_utils.cpp | 198 --- src/ovms_lib/rest_utils.hpp | 36 - src/ovms_lib/s3filesystem.cpp | 552 ------- src/ovms_lib/s3filesystem.hpp | 155 -- src/ovms_lib/schema.cpp | 351 ----- src/ovms_lib/schema.hpp | 28 - src/ovms_lib/sequence.cpp | 67 - src/ovms_lib/sequence.hpp | 60 - src/ovms_lib/sequence_manager.cpp | 157 -- src/ovms_lib/sequence_manager.hpp | 83 - src/ovms_lib/sequence_processing_spec.hpp | 36 - src/ovms_lib/serialization.cpp | 245 --- src/ovms_lib/serialization.hpp | 126 -- src/ovms_lib/servablemanagermodule.cpp | 56 - src/ovms_lib/servablemanagermodule.hpp | 35 - src/ovms_lib/server.cpp | 348 ----- src/ovms_lib/server.hpp | 75 - src/ovms_lib/shape.cpp | 505 ------ src/ovms_lib/shape.hpp | 128 -- src/ovms_lib/statefulmodelinstance.cpp | 324 ---- src/ovms_lib/statefulmodelinstance.hpp | 101 -- src/ovms_lib/status.cpp | 426 ----- src/ovms_lib/status.hpp | 381 ----- src/ovms_lib/stringutils.cpp | 139 -- src/ovms_lib/stringutils.hpp | 91 -- src/ovms_lib/tensor_utils.hpp | 46 - src/ovms_lib/tensorinfo.cpp | 289 ---- src/ovms_lib/tensorinfo.hpp | 238 --- src/ovms_lib/tensormap.hpp | 33 - src/ovms_lib/tfs_frontend/tfs_utils.cpp | 98 -- src/ovms_lib/tfs_frontend/tfs_utils.hpp | 33 - src/ovms_lib/threadsafequeue.hpp | 69 - src/ovms_lib/timer.hpp | 50 - src/ovms_lib/version.hpp | 21 - src/pocapi.cpp | 46 + src/{ovms_lib/session_id.hpp => pocapi.hpp} | 14 +- 171 files changed, 97 insertions(+), 29475 deletions(-) delete mode 100644 src/ovms_lib/BUILD delete mode 100644 src/ovms_lib/aliases.hpp delete mode 100644 src/ovms_lib/azurefilesystem.cpp delete mode 100644 src/ovms_lib/azurefilesystem.hpp delete mode 100644 src/ovms_lib/azurestorage.cpp delete mode 100644 src/ovms_lib/azurestorage.hpp delete mode 100644 src/ovms_lib/binaryutils.cpp delete mode 100644 src/ovms_lib/binaryutils.hpp delete mode 100644 src/ovms_lib/cleaner_utils.hpp delete mode 100644 src/ovms_lib/config.cpp delete mode 100644 src/ovms_lib/config.hpp delete mode 100644 src/ovms_lib/custom_node.cpp delete mode 100644 src/ovms_lib/custom_node.hpp delete mode 100644 src/ovms_lib/custom_node_interface.h delete mode 100644 src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp delete mode 100644 src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp delete mode 100644 src/ovms_lib/custom_node_library_manager.cpp delete mode 100644 src/ovms_lib/custom_node_library_manager.hpp delete mode 100644 src/ovms_lib/custom_node_output_allocator.cpp delete mode 100644 src/ovms_lib/custom_node_output_allocator.hpp delete mode 100644 src/ovms_lib/customloaderconfig.hpp delete mode 100644 src/ovms_lib/customloaderinterface.hpp delete mode 100644 src/ovms_lib/customloaders.cpp delete mode 100644 src/ovms_lib/customloaders.hpp delete mode 100644 src/ovms_lib/customnodesession.cpp delete mode 100644 src/ovms_lib/customnodesession.hpp delete mode 100644 src/ovms_lib/deserialization.cpp delete mode 100644 src/ovms_lib/deserialization.hpp delete mode 100644 src/ovms_lib/dl_node.cpp delete mode 100644 src/ovms_lib/dl_node.hpp delete mode 100644 src/ovms_lib/dlnodesession.cpp delete mode 100644 src/ovms_lib/dlnodesession.hpp delete mode 100644 src/ovms_lib/entry_node.cpp delete mode 100644 src/ovms_lib/entry_node.hpp delete mode 100644 src/ovms_lib/executingstreamidguard.hpp delete mode 100644 src/ovms_lib/exit_node.cpp delete mode 100644 src/ovms_lib/exit_node.hpp delete mode 100644 src/ovms_lib/exitnodesession.cpp delete mode 100644 src/ovms_lib/exitnodesession.hpp delete mode 100644 src/ovms_lib/filesystem.hpp delete mode 100644 src/ovms_lib/gatherexitnodeinputhandler.cpp delete mode 100644 src/ovms_lib/gatherexitnodeinputhandler.hpp delete mode 100644 src/ovms_lib/gathernodeinputhandler.cpp delete mode 100644 src/ovms_lib/gathernodeinputhandler.hpp delete mode 100644 src/ovms_lib/gcsfilesystem.cpp delete mode 100644 src/ovms_lib/gcsfilesystem.hpp delete mode 100644 src/ovms_lib/get_model_metadata_impl.cpp delete mode 100644 src/ovms_lib/get_model_metadata_impl.hpp delete mode 100644 src/ovms_lib/global_sequences_viewer.cpp delete mode 100644 src/ovms_lib/global_sequences_viewer.hpp delete mode 100644 src/ovms_lib/grpcservermodule.cpp delete mode 100644 src/ovms_lib/grpcservermodule.hpp delete mode 100644 src/ovms_lib/http_rest_api_handler.cpp delete mode 100644 src/ovms_lib/http_rest_api_handler.hpp delete mode 100644 src/ovms_lib/http_server.cpp delete mode 100644 src/ovms_lib/http_server.hpp delete mode 100644 src/ovms_lib/kfs_grpc_inference_service.cpp delete mode 100644 src/ovms_lib/kfs_grpc_inference_service.hpp delete mode 100644 src/ovms_lib/layout.cpp delete mode 100644 src/ovms_lib/layout.hpp delete mode 100644 src/ovms_lib/layout_configuration.cpp delete mode 100644 src/ovms_lib/layout_configuration.hpp delete mode 100644 src/ovms_lib/localfilesystem.cpp delete mode 100644 src/ovms_lib/localfilesystem.hpp delete mode 100644 src/ovms_lib/log delete mode 100644 src/ovms_lib/logging.cpp delete mode 100644 src/ovms_lib/logging.hpp delete mode 100644 src/ovms_lib/main.cpp delete mode 100644 src/ovms_lib/main2.cpp delete mode 100644 src/ovms_lib/model.cpp delete mode 100644 src/ovms_lib/model.hpp delete mode 100644 src/ovms_lib/model_service.cpp delete mode 100644 src/ovms_lib/model_service.hpp delete mode 100644 src/ovms_lib/model_version_policy.cpp delete mode 100644 src/ovms_lib/model_version_policy.hpp delete mode 100644 src/ovms_lib/modelchangesubscription.cpp delete mode 100644 src/ovms_lib/modelchangesubscription.hpp delete mode 100644 src/ovms_lib/modelconfig.cpp delete mode 100644 src/ovms_lib/modelconfig.hpp delete mode 100644 src/ovms_lib/modelinstance.cpp delete mode 100644 src/ovms_lib/modelinstance.hpp delete mode 100644 src/ovms_lib/modelinstanceunloadguard.cpp delete mode 100644 src/ovms_lib/modelinstanceunloadguard.hpp delete mode 100644 src/ovms_lib/modelmanager.cpp delete mode 100644 src/ovms_lib/modelmanager.hpp delete mode 100644 src/ovms_lib/modelversion.hpp delete mode 100644 src/ovms_lib/modelversionstatus.cpp delete mode 100644 src/ovms_lib/modelversionstatus.hpp delete mode 100644 src/ovms_lib/my_cc_combine.bzl delete mode 100644 src/ovms_lib/node.cpp delete mode 100644 src/ovms_lib/node.hpp delete mode 100644 src/ovms_lib/node_library.cpp delete mode 100644 src/ovms_lib/node_library.hpp delete mode 100644 src/ovms_lib/node_library_utils.cpp delete mode 100644 src/ovms_lib/node_library_utils.hpp delete mode 100644 src/ovms_lib/nodeinfo.hpp delete mode 100644 src/ovms_lib/nodeinputhandler.cpp delete mode 100644 src/ovms_lib/nodeinputhandler.hpp delete mode 100644 src/ovms_lib/nodeoutputhandler.cpp delete mode 100644 src/ovms_lib/nodeoutputhandler.hpp delete mode 100644 src/ovms_lib/nodesession.cpp delete mode 100644 src/ovms_lib/nodesession.hpp delete mode 100644 src/ovms_lib/nodesessionmetadata.cpp delete mode 100644 src/ovms_lib/nodesessionmetadata.hpp delete mode 100644 src/ovms_lib/nodesessionresult.hpp delete mode 100644 src/ovms_lib/nodestreamidguard.hpp delete mode 100644 src/ovms_lib/ov_utils.cpp delete mode 100644 src/ovms_lib/ov_utils.hpp delete mode 100644 src/ovms_lib/ovinferrequestsqueue.hpp delete mode 100644 src/ovms_lib/pipeline.cpp delete mode 100644 src/ovms_lib/pipeline.hpp delete mode 100644 src/ovms_lib/pipeline_factory.cpp delete mode 100644 src/ovms_lib/pipeline_factory.hpp delete mode 100644 src/ovms_lib/pipelinedefinition.cpp delete mode 100644 src/ovms_lib/pipelinedefinition.hpp delete mode 100644 src/ovms_lib/pipelinedefinitionstatus.hpp delete mode 100644 src/ovms_lib/pipelinedefinitionunloadguard.cpp delete mode 100644 src/ovms_lib/pipelinedefinitionunloadguard.hpp delete mode 100644 src/ovms_lib/pipelineeventqueue.hpp delete mode 100644 src/ovms_lib/precision.cpp delete mode 100644 src/ovms_lib/precision.hpp delete mode 100644 src/ovms_lib/predict_request_validation_utils.cpp delete mode 100644 src/ovms_lib/predict_request_validation_utils.hpp delete mode 100644 src/ovms_lib/prediction_service.cpp delete mode 100644 src/ovms_lib/prediction_service.hpp delete mode 100644 src/ovms_lib/prediction_service_utils.cpp delete mode 100644 src/ovms_lib/prediction_service_utils.hpp delete mode 100644 src/ovms_lib/profiler.cpp delete mode 100644 src/ovms_lib/profiler.hpp delete mode 100644 src/ovms_lib/queue.hpp delete mode 100644 src/ovms_lib/rest_parser.cpp delete mode 100644 src/ovms_lib/rest_parser.hpp delete mode 100644 src/ovms_lib/rest_utils.cpp delete mode 100644 src/ovms_lib/rest_utils.hpp delete mode 100644 src/ovms_lib/s3filesystem.cpp delete mode 100644 src/ovms_lib/s3filesystem.hpp delete mode 100644 src/ovms_lib/schema.cpp delete mode 100644 src/ovms_lib/schema.hpp delete mode 100644 src/ovms_lib/sequence.cpp delete mode 100644 src/ovms_lib/sequence.hpp delete mode 100644 src/ovms_lib/sequence_manager.cpp delete mode 100644 src/ovms_lib/sequence_manager.hpp delete mode 100644 src/ovms_lib/sequence_processing_spec.hpp delete mode 100644 src/ovms_lib/serialization.cpp delete mode 100644 src/ovms_lib/serialization.hpp delete mode 100644 src/ovms_lib/servablemanagermodule.cpp delete mode 100644 src/ovms_lib/servablemanagermodule.hpp delete mode 100644 src/ovms_lib/server.cpp delete mode 100644 src/ovms_lib/server.hpp delete mode 100644 src/ovms_lib/shape.cpp delete mode 100644 src/ovms_lib/shape.hpp delete mode 100644 src/ovms_lib/statefulmodelinstance.cpp delete mode 100644 src/ovms_lib/statefulmodelinstance.hpp delete mode 100644 src/ovms_lib/status.cpp delete mode 100644 src/ovms_lib/status.hpp delete mode 100644 src/ovms_lib/stringutils.cpp delete mode 100644 src/ovms_lib/stringutils.hpp delete mode 100644 src/ovms_lib/tensor_utils.hpp delete mode 100644 src/ovms_lib/tensorinfo.cpp delete mode 100644 src/ovms_lib/tensorinfo.hpp delete mode 100644 src/ovms_lib/tensormap.hpp delete mode 100644 src/ovms_lib/tfs_frontend/tfs_utils.cpp delete mode 100644 src/ovms_lib/tfs_frontend/tfs_utils.hpp delete mode 100644 src/ovms_lib/threadsafequeue.hpp delete mode 100644 src/ovms_lib/timer.hpp delete mode 100644 src/ovms_lib/version.hpp create mode 100644 src/pocapi.cpp rename src/{ovms_lib/session_id.hpp => pocapi.hpp} (81%) diff --git a/src/BUILD b/src/BUILD index 24672ff7a9..25e369b732 100644 --- a/src/BUILD +++ b/src/BUILD @@ -60,7 +60,7 @@ cc_library( name = "ovms_lib", linkstatic = True, # linkshared = True, - hdrs = ["server.hpp"], + hdrs = ["pocapi.hpp"], srcs = [ "aliases.hpp", "azurestorage.hpp", @@ -196,6 +196,8 @@ cc_library( "pipelineeventqueue.hpp", "pipeline_factory.cpp", "pipeline_factory.hpp", + "pocapi.hpp", + "pocapi.cpp", "precision.cpp", "precision.hpp", "prediction_service.cpp", @@ -544,7 +546,8 @@ cc_binary( "-Werror", ], deps = [ - "//src/ovms_lib:ovms_lib", + "//src:ovms_lib", + #"//src/ovms_lib:ovms_lib", ], linkstatic = True, linkshared = True, diff --git a/src/main2.cpp b/src/main2.cpp index f075c5c18b..cda8af6930 100644 --- a/src/main2.cpp +++ b/src/main2.cpp @@ -1,48 +1,38 @@ -#include -#include +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** #include -#include +#include -#include "server.hpp" - -#include "modelmanager.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "servablemanagermodule.hpp" - -using ovms::Server; +#include "pocapi.hpp" int main(int argc, char** argv) { - Server& server = Server::instance(); - std::thread t([&server, &argv, &argc](){ - std::cout << server.start(argc, argv) << std::endl; - }); + std::thread t([&argv, &argc]() { + OVMS_Start(argc, argv); + }); std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - std::cout << __LINE__ << "AERO" << std::endl; // get model instance and have a lock on reload - std::shared_ptr instance; - std::unique_ptr modelInstanceUnloadGuardPtr; - auto module = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); - std::cout << __LINE__ << "AERO" << std::endl; - if (nullptr == module) { - return 0; - } - std::cout << __LINE__ << "AERO" << std::endl; - auto servableManagerModule = dynamic_cast(module); - auto& manager = servableManagerModule->getServableManager(); - manager.getModelInstance("dummy", 0, instance, modelInstanceUnloadGuardPtr); // status code - float a[10] = {1,2,3,4,5,6,7,8,9,10}; - float b[10] = {1,2,3,4,5,6,7,8,9,10}; - std::cout << __LINE__ << "AERO" << std::endl; - instance->infer(a,b); - std::cout << __LINE__ << "AERO" << std::endl; + float a[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 11}; + float b[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + OVMS_Infer((char*)"dummy", a, b); for (int i = 0; i < 10; ++i) { std::cout << b[i] << " "; } std::cout << std::endl; - std::cout << __LINE__ << "AERO" << std::endl; + std::cout << __LINE__ << "FINISHED, press ctrl+c to stop " << std::endl; t.join(); - std::cout << __LINE__ << "AERO" << std::endl; - std::cout << __LINE__ << "AERO" << std::endl; + std::cout << __LINE__ << "FINISHED" << std::endl; return 0; } diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 7744c3952f..6a434c5667 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1147,39 +1147,38 @@ Status ModelInstance::performInference(ov::InferRequest& inferRequest) { #include Status ModelInstance::infer(float* data, float* output) { OVMS_PROFILE_FUNCTION(); - Timer timer; + Timer timer; using std::chrono::microseconds; - timer.start("get infer request"); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); + timer.start(GET_INFER_REQUEST); + ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); int executingInferId = executingStreamIdGuard.getId(); ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop("get infer request"); + timer.stop(GET_INFER_REQUEST); SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); - timer.start("deserialize"); - static ov::Shape shape{1,10}; + getName(), getVersion(), executingInferId, timer.elapsed(GET_INFER_REQUEST) / 1000); + timer.start(DESERIALIZE); + static ov::Shape shape{1, 10}; ov::element::Type precision = ov::element::Type_t::f32; ov::Tensor tensor(precision, - shape, - (void*)data); + shape, + (void*)data); inferRequest.set_tensor("b", tensor); - timer.stop("deserialize"); + timer.stop(DESERIALIZE); SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); - timer.start("prediction"); + getName(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); + timer.start(PREDICTION); auto status = performInference(inferRequest); - timer.stop("prediction"); + timer.stop(PREDICTION); if (!status.ok()) return status; SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); - timer.start("serialize"); + getName(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); + timer.start(SERIALIZE); auto otensor = inferRequest.get_tensor("a"); std::memcpy((void*)output, otensor.data(), otensor.get_byte_size()); - timer.stop("serialize"); + timer.stop(SERIALIZE); SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); - SPDLOG_ERROR("ER"); + getName(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); return StatusCode::OK; } diff --git a/src/ovms_lib/BUILD b/src/ovms_lib/BUILD deleted file mode 100644 index 624258cc33..0000000000 --- a/src/ovms_lib/BUILD +++ /dev/null @@ -1,630 +0,0 @@ -# -# Copyright (c) 2020,2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -#config_setting( -# name = "linux_distribution_family", -# constraint_values = [ -# ":debian", # like Ubuntu -# ":fedora", # like RHEL/CentOS -# ], -#) - -constraint_setting(name = "linux_distribution_family") -constraint_value(constraint_setting = "linux_distribution_family", name = "fedora") # like RHEL/CentOS -constraint_value(constraint_setting = "linux_distribution_family", name = "debian") # like Ubuntu - -cc_library( - name = "custom_nodes_common_lib", - linkstatic = 1, - hdrs = ["custom_nodes/common/buffersqueue.hpp"], - srcs = [ - "queue.hpp", - "custom_nodes/common/buffersqueue.hpp", - "custom_nodes/common/buffersqueue.cpp", - ], - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror", - "-Wno-sign-compare", - ], -) - -load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") -load(":my_cc_combine.bzl", "my_cc_combine") - -my_cc_combine( - name = "hello_combined", - # Here merge all the static libraries into one static library - genstatic = True, - output = "libcombined.so", - deps = [ - "//src:libovms_shared.so", - # "//src:ovms_lib", - ], -) - -cc_library( - name = "ovms_lib", - linkstatic = True, - # linkshared = True, - hdrs = ["server.hpp"], - srcs = [ - "aliases.hpp", - "azurestorage.hpp", - "azurestorage.cpp", - "azurefilesystem.cpp", - "azurefilesystem.hpp", - "cleaner_utils.hpp", - "config.cpp", - "config.hpp", - "custom_node.cpp", - "custom_node.hpp", - "custom_node_interface.h", - "custom_node_library_manager.cpp", - "custom_node_library_manager.hpp", - "custom_node_output_allocator.cpp", - "custom_node_output_allocator.hpp", - "custom_node_library_internal_manager_wrapper.hpp", - "custom_node_library_internal_manager_wrapper.cpp", - "customnodesession.cpp", - "customnodesession.hpp", - "customloaderconfig.hpp", - "customloaders.hpp", - "customloaders.cpp", - "customloaderinterface.hpp", - "deserialization.cpp", - "deserialization.hpp", - "dl_node.cpp", - "dl_node.hpp", - "dlnodesession.cpp", - "dlnodesession.hpp", - "entry_node.cpp", - "entry_node.hpp", - "executingstreamidguard.hpp", - "exit_node.cpp", - "exit_node.hpp", - "exitnodesession.cpp", - "exitnodesession.hpp", - "filesystem.hpp", - "get_model_metadata_impl.cpp", - "get_model_metadata_impl.hpp", - "global_sequences_viewer.hpp", - "global_sequences_viewer.cpp", - "http_rest_api_handler.cpp", - "http_rest_api_handler.hpp", - "http_server.cpp", - "http_server.hpp", - "layout.cpp", - "layout.hpp", - "layout_configuration.cpp", - "layout_configuration.hpp", - "localfilesystem.cpp", - "localfilesystem.hpp", - "gathernodeinputhandler.cpp", - "gathernodeinputhandler.hpp", - "gatherexitnodeinputhandler.cpp", - "gatherexitnodeinputhandler.hpp", - "gcsfilesystem.cpp", - "gcsfilesystem.hpp", - "grpcservermodule.cpp", - "grpcservermodule.hpp", - "kfs_grpc_inference_service.cpp", - "kfs_grpc_inference_service.hpp", - "model.cpp", - "model.hpp", - "model_version_policy.cpp", - "model_version_policy.hpp", - "modelchangesubscription.cpp", - "modelchangesubscription.hpp", - "modelconfig.cpp", - "modelconfig.hpp", - "modelmanager.cpp", - "modelmanager.hpp", - "modelinstance.cpp", - "modelinstance.hpp", - "modelinstanceunloadguard.cpp", - "modelinstanceunloadguard.hpp", - "modelversion.hpp", - "modelversionstatus.cpp", - "modelversionstatus.hpp", - "model_service.hpp", - "model_service.cpp", - "node.cpp", - "node.hpp", - "nodeinfo.hpp", - "node_library.cpp", - "node_library.hpp", - "node_library_utils.cpp", - "node_library_utils.hpp", - "nodesession.cpp", - "nodesession.hpp", - "nodesessionresult.hpp", - "nodeinputhandler.cpp", - "nodeinputhandler.hpp", - "nodeoutputhandler.cpp", - "nodeoutputhandler.hpp", - "nodesessionmetadata.hpp", - "nodesessionmetadata.cpp", - "nodestreamidguard.hpp", - "ovinferrequestsqueue.hpp", - "ov_utils.cpp", - "ov_utils.hpp", - "pipeline.cpp", - "pipeline.hpp", - "pipelinedefinition.cpp", - "pipelinedefinition.hpp", - "pipelinedefinitionstatus.hpp", - "pipelinedefinitionunloadguard.cpp", - "pipelinedefinitionunloadguard.hpp", - "pipelineeventqueue.hpp", - "pipeline_factory.cpp", - "pipeline_factory.hpp", - "precision.cpp", - "precision.hpp", - "prediction_service.cpp", - "prediction_service.hpp", - "prediction_service_utils.hpp", - "prediction_service_utils.cpp", - "predict_request_validation_utils.hpp", - "predict_request_validation_utils.cpp", - "profiler.cpp", - "profiler.hpp", - "rest_parser.cpp", - "rest_parser.hpp", - "rest_utils.cpp", - "rest_utils.hpp", - "s3filesystem.cpp", - "s3filesystem.hpp", - "schema.hpp", - "schema.cpp", - "serialization.cpp", - "serialization.hpp", - "servablemanagermodule.cpp", - "servablemanagermodule.hpp", - "server.cpp", - "server.hpp", - "session_id.hpp", - "sequence.cpp", - "sequence.hpp", - "sequence_manager.cpp", - "sequence_manager.hpp", - "sequence_processing_spec.hpp", - "shape.cpp", - "shape.hpp", - "statefulmodelinstance.cpp", - "statefulmodelinstance.hpp", - "status.cpp", - "status.hpp", - "stringutils.cpp", - "stringutils.hpp", - "queue.hpp", - "tensorinfo.cpp", - "tensorinfo.hpp", - "tfs_frontend/tfs_utils.cpp", - "tfs_frontend/tfs_utils.hpp", - "tensormap.hpp", - "tensor_utils.hpp", - "threadsafequeue.hpp", - "timer.hpp", - "version.hpp", - "logging.hpp", - "logging.cpp", - "binaryutils.hpp", - "binaryutils.cpp", - ], - deps = [ - "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", - "@tensorflow_serving//tensorflow_serving/apis:model_service_cc_proto", - "@minitrace//:trace", - "@com_github_grpc_grpc//:grpc++", - "@org_tensorflow//tensorflow/core:framework", - "@rapidjson//:rapidjson", - "@spdlog//:spdlog", - "@cxxopts//:cxxopts", - "@awssdk//:s3", - "@awssdk//:core", - "@awssdk//:deps", - "@azure//:storage", - "@cpprest//:sdk", - "@boost//:lib", - "@com_github_googleapis_google_cloud_cpp//google/cloud/storage:storage_client", - "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server", - "@tensorflow_serving//tensorflow_serving/util/net_http/server/public:http_server_api", - "@tensorflow_serving//tensorflow_serving/util:threadpool_executor", - "@tensorflow_serving//tensorflow_serving/util:json_tensor", - "@openvino//:openvino", - "@opencv//:opencv", - #":kfserving_api_cpp", - "//src/kfserving_api:kfserving_api_cpp", - ], - local_defines = [ - "SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_TRACE", - ], - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror", - "-Wno-sign-compare", - ], - visibility = [ - "//visibility:public", - ], -) - -cc_binary( - name = "libsampleloader.so", - srcs = [ - "example/SampleCustomLoader/sampleCustLoader.cpp", - "customloaderinterface.hpp", - ], - linkshared = 1, - deps = [ - "@rapidjson//:rapidjson", - ], -) - -cc_binary( - name = "lib_node_mock.so", - srcs = [ - "test/custom_nodes/node_mock.c", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "lib_node_missing_implementation.so", - srcs = [ - "test/custom_nodes/node_missing_implementation.c", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "lib_node_add_sub.so", - srcs = [ - "test/custom_nodes/node_add_sub.c", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "lib_node_dynamic_image.so", - srcs = [ - "test/custom_nodes/node_dynamic_image.c", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "lib_node_choose_maximum.so", - srcs = [ - "test/custom_nodes/node_choose_maximum.cpp", - "custom_node_interface.h", - ], - linkshared = 1, -) -cc_binary( - name = "lib_node_perform_different_operations.so", - srcs = [ - "test/custom_nodes/node_perform_different_operations.cpp", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "lib_node_dynamic_demultiplex.so", - srcs = [ - "test/custom_nodes/node_dynamic_demultiplex.cpp", - "custom_node_interface.h", - ], - linkshared = 1, -) - -cc_binary( - name = "libcustom_node_east_ocr.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/east_ocr/east_ocr.cpp", - "custom_nodes/east_ocr/nms.hpp", - "custom_node_interface.h", - ], - deps = [ - "@opencv//:opencv" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "libcustom_node_face_blur.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/face_blur/face_blur.cpp", - "custom_node_interface.h", - ], - deps = [ - "@opencv//:opencv" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "libcustom_node_add_one.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/buffersqueue.hpp", - "custom_nodes/common/buffersqueue.cpp", - "custom_nodes/common/custom_node_library_internal_manager.hpp", - "custom_nodes/common/custom_node_library_internal_manager.cpp", - "queue.hpp", - "custom_nodes/add_one/add_one.cpp", - "custom_node_interface.h", - "custom_nodes/add_one/add_one_internal_manager.hpp" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "libcustom_node_model_zoo_intel_object_detection.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/common/buffersqueue.hpp", - "custom_nodes/common/buffersqueue.cpp", - "custom_nodes/common/custom_node_library_internal_manager.hpp", - "custom_nodes/common/custom_node_library_internal_manager.cpp", - "queue.hpp", - "custom_nodes/model_zoo_intel_object_detection/model_zoo_intel_object_detection.cpp", - "custom_node_interface.h", - ], - deps = [ - "@opencv//:opencv" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "libcustom_node_image_transformation.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/image_transformation/image_transformation.cpp", - "custom_node_interface.h", - ], - deps = [ - "@opencv//:opencv" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "libcustom_node_horizontal_ocr.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/horizontal_ocr/horizontal_ocr.cpp", - "custom_node_interface.h", - ], - deps = [ - "@opencv//:opencv" - ], - linkshared = 1, - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror" - ] -) - -cc_binary( - name = "ovms", - srcs = [ - "main.cpp", - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - ], - copts = [ - "-Wconversion", - "-Werror", - ], - deps = [ - "//src:ovms_lib", - # "//src:libovms_shared.so", - ], - linkstatic = False, -) - -cc_binary( - name = "libovms_shared.so", - srcs = [ - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - ], - copts = [ - "-Wconversion", - "-Werror", - ], - deps = [ - "//src/ovms_lib:ovms_lib", - ], - linkstatic = True, - linkshared = True, -) - -cc_test( - name = "ovms_test", - linkstatic = 1, - srcs = [ - "test/azurefilesystem_test.cpp", - "test/binaryutils_test.cpp", - "test/custom_loader_test.cpp", - "test/custom_node_output_allocator_test.cpp", - "test/custom_node_buffersqueue_test.cpp", - "test/demultiplexer_node_test.cpp", - "test/deserialization_tests.cpp", - "test/ensemble_tests.cpp", - "test/ensemble_flow_custom_node_tests.cpp", - "test/ensemble_mapping_config_tests.cpp", - "test/ensemble_metadata_test.cpp", - "test/ensemble_config_change_stress.cpp", - "test/environment.hpp", - "test/gather_node_test.cpp", - "test/gcsfilesystem_test.cpp", - "test/get_model_metadata_response_test.cpp", - "test/get_pipeline_metadata_response_test.cpp", - "test/get_model_metadata_signature_test.cpp", - "test/get_model_metadata_validation_test.cpp", - "test/http_rest_api_handler_test.cpp", - "test/kfs_metadata_test.cpp", - "test/kfs_rest_test.cpp", - "test/layout_test.cpp", - "test/localfilesystem_test.cpp", - "test/mockmodelinstancechangingstates.hpp", - "test/model_cache_test.cpp", - "test/model_service_test.cpp", - "test/model_version_policy_test.cpp", - "test/model_test.cpp", - "test/modelinstance_test.cpp", - "test/modelconfig_test.cpp", - "test/node_library_manager_test.cpp", - "test/modelmanager_test.cpp", - "test/modelversionstatus_test.cpp", - "test/nodesessionmetadata_test.cpp", - "test/ovmsconfig_test.cpp", - "test/ovinferrequestqueue_test.cpp", - "test/ov_utils_test.cpp", - "test/pipelinedefinitionstatus_test.cpp", - "test/predict_validation_test.cpp", - "test/prediction_service_test.cpp", - "test/rest_parser_row_test.cpp", - "test/rest_parser_column_test.cpp", - "test/rest_parser_binary_inputs_test.cpp", - "test/rest_parser_nonamed_test.cpp", - "test/rest_utils_test.cpp", - "test/schema_test.cpp", - "test/sequence_test.cpp", - "test/serialization_tests.cpp", - "test/server_test.cpp", - "test/sequence_manager_test.cpp", - "test/shape_test.cpp", - "test/stateful_config_test.cpp", - "test/stateful_modelinstance_test.cpp", - "test/stateful_test_utils.hpp", - "test/status_test.cpp", - "test/stringutils_test.cpp", - "test/tensorinfo_test.cpp", - "test/tensorutils_test.cpp", - "test/test_utils.cpp", - "test/test_utils.hpp", - "test/threadsafequeue_test.cpp", - "test/unit_tests.cpp", - ], - data = [ - "test/dummy/1/dummy.xml", - "test/dummy/1/dummy.bin", - "test/dummy_fp64/1/saved_model.xml", - "test/dummy_fp64/1/saved_model.bin", - "test/add_two_inputs_model/1/add.xml", - "test/add_two_inputs_model/1/add.bin", - "test/increment_1x3x4x5/1/increment_1x3x4x5.xml", - "test/increment_1x3x4x5/1/increment_1x3x4x5.bin", - "test/summator/1/summator.xml", - "test/summator/1/summator.bin", - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - "-lssl", - ], - deps = [ - "//src:ovms_lib", - "//src:custom_nodes_common_lib", - "//src:libsampleloader.so", - "//src:lib_node_mock.so", - "//src:lib_node_missing_implementation.so", - "//src:lib_node_add_sub.so", - "//src:lib_node_dynamic_image.so", - "//src:lib_node_dynamic_demultiplex.so", - "//src:lib_node_perform_different_operations.so", - "//src:lib_node_choose_maximum.so", - "//src:libcustom_node_east_ocr.so", - "//src:libcustom_node_face_blur.so", - "//src:libcustom_node_model_zoo_intel_object_detection.so", - "//src:libcustom_node_image_transformation.so", - "//src:libcustom_node_add_one.so", - "//src:libcustom_node_horizontal_ocr.so", - "@com_google_googletest//:gtest", - ], - copts = [ - "-Wall", - "-Wno-unknown-pragmas", - "-Werror", - ], -) - -filegroup( - name = "static_analysis", - srcs = [ - "//src:ovms", - "//src:libcustom_node_add_one.so", - "//src:libcustom_node_east_ocr.so", - "//src:libcustom_node_face_blur.so", - "//src:libcustom_node_horizontal_ocr.so", - "//src:libcustom_node_model_zoo_intel_object_detection.so", - "//src:libcustom_node_image_transformation.so", - ] -) diff --git a/src/ovms_lib/aliases.hpp b/src/ovms_lib/aliases.hpp deleted file mode 100644 index b5a11ee16c..0000000000 --- a/src/ovms_lib/aliases.hpp +++ /dev/null @@ -1,25 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -namespace ovms { - -using Aliases = std::vector>; -} // namespace ovms diff --git a/src/ovms_lib/azurefilesystem.cpp b/src/ovms_lib/azurefilesystem.cpp deleted file mode 100644 index 6b92031f45..0000000000 --- a/src/ovms_lib/azurefilesystem.cpp +++ /dev/null @@ -1,320 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "azurefilesystem.hpp" - -#include -#include -#include - -#include "logging.hpp" -#include "stringutils.hpp" - -namespace ovms { - -const std::string AzureFileSystem::AZURE_URL_FILE_PREFIX = "azfs://"; -const std::string AzureFileSystem::AZURE_URL_BLOB_PREFIX = "az://"; - -as::cloud_storage_account createDefaultOrAnonymousAccount() { - try { - const char* env_cred = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); - - std::string credentials = std::string(_XPLATSTR("DefaultEndpointsProtocol = https;")); - - if (!env_cred) { - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Creating AzureFileSystem anonymous connection string."); - } else { - credentials = std::string(_XPLATSTR(env_cred)); - } - - as::cloud_storage_account storage_account = as::cloud_storage_account::parse(credentials); - if (!storage_account.is_initialized()) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account"); - throw std::runtime_error("Unable to create default azure storage account"); - } - - const char* use_http = std::getenv("AZURE_STORAGE_USE_HTTP_PROXY"); - - const char* proxy_env; - - std::string https_proxy = std::string(""); - if (!use_http) { - proxy_env = std::getenv("https_proxy"); - } else { - proxy_env = std::getenv("http_proxy"); - } - - if (!proxy_env) { - SPDLOG_LOGGER_DEBUG(azurestorage_logger, "No proxy detected."); - } else { - https_proxy = std::string(proxy_env); - web::web_proxy wproxy(https_proxy); - as::operation_context::set_default_proxy(wproxy); - - SPDLOG_LOGGER_DEBUG(azurestorage_logger, "Proxy detected: {}" + https_proxy); - } - - return storage_account; - } catch (const as::storage_exception& e) { - as::request_result result = e.result(); - as::storage_extended_error extended_error = result.extended_error(); - if (!extended_error.message().empty()) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", extended_error.message()); - } else { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); - } - throw e; - } catch (const std::exception& e) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); - throw e; - } -} - -AzureFileSystem::AzureFileSystem() : - account_{createDefaultOrAnonymousAccount()} { - SPDLOG_LOGGER_TRACE(azurestorage_logger, "AzureFileSystem default ctor"); -} - -AzureFileSystem::~AzureFileSystem() { SPDLOG_LOGGER_TRACE(azurestorage_logger, "AzureFileSystem dtor"); } - -StatusCode AzureFileSystem::fileExists(const std::string& path, bool* exists) { - *exists = false; - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->fileExists(exists); - - return status; -} - -StatusCode AzureFileSystem::isDirectory(const std::string& path, - bool* is_directory) { - *is_directory = false; - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->isDirectory(is_directory); - - return status; -} - -StatusCode AzureFileSystem::fileModificationTime(const std::string& path, - int64_t* mtime_ns) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->fileModificationTime(mtime_ns); - - return status; -} - -StatusCode -AzureFileSystem::getDirectoryContents(const std::string& path, - std::set* contents) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->getDirectoryContents(contents); - - return status; -} - -StatusCode AzureFileSystem::getDirectorySubdirs(const std::string& path, - std::set* subdirs) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->getDirectorySubdirs(subdirs); - - return status; -} - -StatusCode AzureFileSystem::getDirectoryFiles(const std::string& path, - std::set* files) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->getDirectoryFiles(files); - - return status; -} - -StatusCode AzureFileSystem::readTextFile(const std::string& path, - std::string* contents) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->readTextFile(contents); - - return status; -} - -StatusCode AzureFileSystem::downloadModelVersions(const std::string& path, - std::string* local_path, - const std::vector& versions) { - - auto sc = createTempPath(local_path); - if (sc != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to create a temporary path {}", sc); - return sc; - } - - for (auto& ver : versions) { - std::string versionpath = path; - if (!endsWith(versionpath, "/")) { - versionpath.append("/"); - } - versionpath.append(std::to_string(ver)); - std::string lpath = *local_path; - if (!endsWith(lpath, "/")) { - lpath.append("/"); - } - lpath.append(std::to_string(ver)); - fs::create_directory(lpath); - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(versionpath, account_); - auto status = azureStorageObj->checkPath(versionpath); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", versionpath, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->downloadFileFolderTo(lpath); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to download model version {}", versionpath); - return status; - } - } - - return StatusCode::OK; -} - -StatusCode AzureFileSystem::downloadFile(const std::string& remote_path, - const std::string& local_path) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(remote_path, account_); - auto status = azureStorageObj->checkPath(remote_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->downloadFile(local_path); - - return status; -} - -StatusCode AzureFileSystem::downloadFileFolder(const std::string& path, - const std::string& local_path) { - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->downloadFileFolder(local_path); - - return status; -} - -StatusCode AzureFileSystem::downloadFileFolderTo(const std::string& path, - const std::string& local_path) { - - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->downloadFileFolderTo(local_path); - - return status; -} - -StatusCode AzureFileSystem::deleteFileFolder(const std::string& path) { - auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); - auto status = azureStorageObj->checkPath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - status = azureStorageObj->deleteFileFolder(); - - return status; -} - -} // namespace ovms diff --git a/src/ovms_lib/azurefilesystem.hpp b/src/ovms_lib/azurefilesystem.hpp deleted file mode 100644 index ed7dd1096e..0000000000 --- a/src/ovms_lib/azurefilesystem.hpp +++ /dev/null @@ -1,166 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#define _TURN_OFF_PLATFORM_STRING - -#include -#include -#include - -#include "azurestorage.hpp" -#include "filesystem.hpp" -#include "status.hpp" - -namespace ovms { - -namespace as = azure::storage; - -class AzureFileSystem : public FileSystem { -public: - /** - * @brief Construct a new AzureFileSystem object - * - */ - AzureFileSystem(); - - /** - * @brief Destroy the AzureFileSystem object - * - */ - virtual ~AzureFileSystem(); - - /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ - StatusCode fileExists(const std::string& path, bool* exists) override; - - /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ - StatusCode isDirectory(const std::string& path, bool* is_dir) override; - - /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode getDirectoryContents(const std::string& path, - files_list_t* contents) override; - - /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ - StatusCode getDirectorySubdirs(const std::string& path, - files_list_t* subdirs) override; - - /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ - StatusCode getDirectoryFiles(const std::string& path, - files_list_t* files) override; - - /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode readTextFile(const std::string& path, - std::string* contents) override; - - /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ - StatusCode downloadFileFolder(const std::string& path, - const std::string& local_path) override; - - /** - * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ - StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; - - StatusCode fileModificationTime(const std::string& path, int64_t* mtime_ns); - - /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ - StatusCode deleteFileFolder(const std::string& path) override; - - static const std::string AZURE_URL_FILE_PREFIX; - - static const std::string AZURE_URL_BLOB_PREFIX; - -private: - /** - * - * @brief - * - * @param remote_path - * @param local_path - */ - StatusCode downloadFile(const std::string& remote_path, - const std::string& local_path); - - /** - * @brief Download a remote directory to a callee-provided local_path. - * - * @param path - * @param local_path - * @return StatusCode - */ - StatusCode downloadFileFolderTo(const std::string& path, - const std::string& local_path); - - /** - * @brief - * - */ - as::cloud_storage_account account_; -}; - -} // namespace ovms diff --git a/src/ovms_lib/azurestorage.cpp b/src/ovms_lib/azurestorage.cpp deleted file mode 100644 index 5abd65ee80..0000000000 --- a/src/ovms_lib/azurestorage.cpp +++ /dev/null @@ -1,1217 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "azurestorage.hpp" - -#include - -#include "azurefilesystem.hpp" -#include "logging.hpp" - -namespace ovms { - -using namespace utility; - -const std::string UNAVAILABLE_PATH_ERROR = "Unable to access path: {}"; - -const std::string AzureStorageAdapter::extractAzureStorageExceptionMessage(const as::storage_exception& e) { - as::request_result result = e.result(); - as::storage_extended_error extended_error = result.extended_error(); - if (!extended_error.message().empty()) { - return extended_error.message(); - } else { - return e.what(); - } -} - -std::string AzureStorageAdapter::joinPath(std::initializer_list segments) { - std::string joined; - - for (const auto& seg : segments) { - if (joined.empty()) { - joined = seg; - } else if (isAbsolutePath(seg)) { - if (joined[joined.size() - 1] == '/') { - joined.append(seg.substr(1)); - } else { - joined.append(seg); - } - } else { - if (joined[joined.size() - 1] != '/') { - joined.append("/"); - } - joined.append(seg); - } - } - - return joined; -} - -StatusCode AzureStorageAdapter::CreateLocalDir(const std::string& path) { - int status = - mkdir(const_cast(path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); - if (status == -1) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Failed to create local folder: {} {} ", path, - strerror(errno)); - return StatusCode::PATH_INVALID; - } - return StatusCode::OK; -} - -bool AzureStorageAdapter::isAbsolutePath(const std::string& path) { - return !path.empty() && (path[0] == '/'); -} - -AzureStorageBlob::AzureStorageBlob(const std::string& path, as::cloud_storage_account account) { - account_ = account; - as_blob_client_ = account_.create_cloud_blob_client(); - isPathValidationOk_ = false; -} - -StatusCode AzureStorageBlob::checkPath(const std::string& path) { - try { - if (FileSystem::isPathEscaped(path)) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - - auto status = this->parseFilePath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to parse path: {} -> {}", fullPath_, - ovms::Status(status).string()); - return status; - } - - as_container_ = as_blob_client_.get_container_reference(container_); - - if (!as_container_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Container does not exist: {} -> {}", fullPath_, container_); - return StatusCode::AS_CONTAINER_NOT_FOUND; - } - - isPathValidationOk_ = true; - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::fileExists(bool* exists) { - try { - *exists = false; - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - *exists = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::isDirectory(bool* is_directory) { - try { - *is_directory = false; - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_blob_directory temp_directory = as_container_.get_directory_reference(blockpath_); - as::cloud_blob_directory parent_directory = temp_directory.get_parent_reference(); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (!item.is_blob()) { - std::string prefix = item.as_directory().prefix(); - if (prefix.back() == '/') - prefix.pop_back(); - if (prefix == blockpath_) { - *is_directory = true; - } - } - } - - token = result.continuation_token(); - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::fileModificationTime(int64_t* mtime_ns) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as::cloud_blob_properties& prop = as_blob_.properties(); - utility::datetime time = prop.last_modified(); - std::string date = time.to_string(); - - auto nanoseconds = time.to_interval(); - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); - *mtime_ns = nanoseconds; - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::getDirectoryContents(files_list_t* contents) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (item.is_blob()) { - contents->insert(getLastPathPart(item.as_blob().name())); - } else { - contents->insert(getLastPathPart(item.as_directory().prefix())); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::getDirectorySubdirs(files_list_t* subdirs) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (!item.is_blob()) { - subdirs->insert(getLastPathPart(item.as_directory().prefix())); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::getDirectoryFiles(files_list_t* files) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (item.is_blob()) { - files->insert(getLastPathPart(item.as_blob().name())); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::readTextFile(std::string* contents) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Nlock blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_block_blob_ = as_container_.get_block_blob_reference(blockpath_); - - concurrency::streams::container_buffer> buffer; - concurrency::streams::ostream output_stream(buffer); - as_block_blob_.download_to_stream(output_stream); - *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::downloadFileFolder(const std::string& local_path) { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - SPDLOG_LOGGER_DEBUG(azurestorage_logger, - "Downloading dir {} (recursive) and saving a new local path: {}", - fullUri_, local_path); - return downloadFileFolderTo(local_path); -} - -StatusCode AzureStorageBlob::deleteFileFolder() { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_blob_.delete_blob(); - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -std::string AzureStorageBlob::getLastPathPart(const std::string& path) { - std::string proper_path = path; - if (path.back() == '/') - proper_path = path.substr(0, path.size() - 1); - - int part_start = proper_path.find_last_of("/"); - int part_end = proper_path.length(); - - return proper_path.substr(part_start + 1, part_end - part_start - 1); -} - -StatusCode AzureStorageBlob::downloadFile(const std::string& local_path) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_blob_.download_to_file(local_path); - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Downloading dir {} and saving to {}", fullPath_, local_path); - bool is_dir; - auto status = this->isDirectory(&is_dir); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File/folder does not exist at {}", fullPath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - if (!is_dir) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Path is not a directory: {}", fullPath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - std::set dirs; - status = getDirectorySubdirs(&dirs); - if (status != StatusCode::OK) { - return status; - } - - std::set files; - status = getDirectoryFiles(&files); - if (status != StatusCode::OK) { - return status; - } - - for (auto&& d : dirs) { - std::string remote_dir_path = joinPath({fullUri_, d}); - std::string local_dir_path = joinPath({local_path, d}); - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, - local_dir_path); - - auto factory = std::make_shared(); - auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); - status = azureSubdirStorageObj->checkPath(remote_dir_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, - ovms::Status(status).string()); - return status; - } - - auto mkdir_status = CreateLocalDir(local_dir_path); - if (mkdir_status != StatusCode::OK) { - return status; - } - auto download_dir_status = - azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); - if (download_dir_status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", - remote_dir_path, local_dir_path); - return download_dir_status; - } - } - - for (auto&& f : files) { - std::string remote_file_path = joinPath({fullUri_, f}); - std::string local_file_path = joinPath({local_path, f}); - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing file {} from {} -> {}", f, remote_file_path, - local_file_path); - - auto factory = std::make_shared(); - auto azureFiledirStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); - status = azureFiledirStorageObj->checkPath(remote_file_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", - remote_file_path, local_file_path); - return status; - } - - auto download_status = - azureFiledirStorageObj->downloadFile(local_file_path); - if (download_status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, - local_file_path); - return download_status; - } - } - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -std::string AzureStorageBlob::getNameFromPath(std::string& path) { - int name_start = path.find_last_of("/"); - int name_end = path.length(); - - if (name_start == name_end) - path = path.substr(0, path.size() - 1); - - name_start = path.find_last_of("/"); - name_end = path.length(); - - return path.substr(name_start, name_end - name_start); -} - -StatusCode AzureStorageBlob::parseFilePath(const std::string& path) { - // az://share/blockpath/file - // az://share/blockpath - // az://share/ - if (path.back() == '/') { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Path can not end with '/'", path); - return StatusCode::AS_INVALID_PATH; - } - - fullUri_ = path; - int share_start = 0; - // Blob path - if (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos) { - share_start = path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) + AzureFileSystem::AZURE_URL_BLOB_PREFIX.size(); - } else if (path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) != std::string::npos) { - // File path - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Wrong object type - az:// prefix in path required, azure:// found:", path); - return StatusCode::AS_INVALID_PATH; - } else { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Missing az:// prefix in path:", path); - return StatusCode::AS_INVALID_PATH; - } - - int share_end = path.find_first_of("/", share_start); - int file_end = path.length(); - - fullPath_ = path.substr(share_end + 1, file_end - share_end - 1); - - subdirs_ = FindSubdirectories(fullPath_); - - std::set subdirs; - - if (share_end > share_start) { - container_ = path.substr(share_start, share_end - share_start); - - blockpath_ = path.substr(share_end + 1, file_end - share_end - 1); - - } else { - // No directory and no file - container_ = path.substr(share_start); - blockpath_ = ""; - } - - // No container - if (container_.empty()) { - return StatusCode::AS_INVALID_PATH; - } - - return StatusCode::OK; -} - -AzureStorageFile::AzureStorageFile(const std::string& path, as::cloud_storage_account account) { - account_ = account; - as_file_client_ = account_.create_cloud_file_client(); - isPathValidationOk_ = false; -} - -StatusCode AzureStorageFile::checkPath(const std::string& path) { - try { - if (FileSystem::isPathEscaped(path)) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - - auto status = this->parseFilePath(path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to parse path: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - as_file_client_ = account_.create_cloud_file_client(); - as_share_ = as_file_client_.get_share_reference(share_); - - if (!as_share_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Share does not exist: {} -> {}", path, share_); - return StatusCode::AS_SHARE_NOT_FOUND; - } - - if (directory_.empty()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory required in path: {} -> {}", path, directory_); - return StatusCode::AS_INVALID_PATH; - } - - isPathValidationOk_ = true; - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::fileExists(bool* exists) { - try { - *exists = false; - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(file_); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - *exists = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::isDirectory(bool* is_directory) { - try { - *is_directory = false; - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - return StatusCode::OK; - } - } - - *is_directory = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { - } - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::fileModificationTime(int64_t* mtime_ns) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as_directory_ = as_share_.get_directory_reference(_XPLATSTR(directory_)); - if (!as_directory_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory does not exist: {} -> {}", fullPath_, directory_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_file1_ = as_directory_.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as::cloud_file_properties& prop = as_file1_.properties(); - utility::datetime time = prop.last_modified(); - std::string date = time.to_string(); - - auto nanoseconds = time.to_interval(); - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); - *mtime_ns = nanoseconds; - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::getDirectoryContents(files_list_t* contents) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_file()) { - contents->insert(item.as_file().name()); - } - if (item.is_directory()) { - contents->insert(item.as_directory().name()); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::getDirectorySubdirs(files_list_t* subdirs) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_directory()) { - subdirs->insert(item.as_directory().name()); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::getDirectoryFiles(files_list_t* files) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_file()) { - files->insert(item.as_file().name()); - } - } - } while (!token.empty()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::readTextFile(std::string* contents) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - concurrency::streams::container_buffer> buffer; - concurrency::streams::ostream output_stream(buffer); - as_file1_.download_to_stream(output_stream); - *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::downloadFileFolder(const std::string& local_path) { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - SPDLOG_LOGGER_DEBUG(azurestorage_logger, - "Downloading dir {} (recursive) and saving a new local path: {}", - fullPath_, local_path); - return downloadFileFolderTo(local_path); -} - -StatusCode AzureStorageFile::deleteFileFolder() { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_file1_.delete_file(); - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::downloadFile(const std::string& local_path) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; - } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_file1_.download_to_file(local_path); - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) { - try { - if (!isPathValidationOk_) { - auto status = checkPath(fullUri_); - if (status != StatusCode::OK) - return status; - } - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Downloading dir {} and saving to {}", fullPath_, local_path); - bool is_dir; - auto status = this->isDirectory(&is_dir); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Folder does not exist at {}", fullPath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - if (!is_dir) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Path is not a directory: {}", fullPath_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - std::set dirs; - status = getDirectorySubdirs(&dirs); - if (status != StatusCode::OK) { - return status; - } - - std::set files; - status = getDirectoryFiles(&files); - if (status != StatusCode::OK) { - return status; - } - - for (auto&& d : dirs) { - std::string remote_dir_path = joinPath({fullUri_, d}); - std::string local_dir_path = joinPath({local_path, d}); - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, - local_dir_path); - - auto factory = std::make_shared(); - auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); - auto status = azureSubdirStorageObj->checkPath(remote_dir_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, - ovms::Status(status).string()); - return status; - } - - auto mkdir_status = CreateLocalDir(local_dir_path); - if (mkdir_status != StatusCode::OK) { - return status; - } - auto download_dir_status = - azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); - if (download_dir_status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", - remote_dir_path, local_dir_path); - return download_dir_status; - } - } - - for (auto&& f : files) { - std::string remote_file_path = joinPath({fullUri_, f}); - std::string local_file_path = joinPath({local_path, f}); - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Processing file {} from {} -> {}", f, remote_file_path, - local_file_path); - - auto factory = std::make_shared(); - auto azureFileStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); - auto status = azureFileStorageObj->checkPath(remote_file_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_file_path, - ovms::Status(status).string()); - return status; - } - - auto download_status = - azureFileStorageObj->downloadFile(local_file_path); - if (download_status != StatusCode::OK) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, - local_file_path); - return download_status; - } - } - return StatusCode::OK; - } catch (const as::storage_exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); - } - - return StatusCode::AS_FILE_NOT_FOUND; -} - -std::vector AzureStorageAdapter::FindSubdirectories(std::string path) { - std::vector output; - - std::string::size_type prev_pos = 0, pos = 0; - - while ((pos = path.find('/', pos)) != std::string::npos) { - std::string substring(path.substr(prev_pos, pos - prev_pos)); - - output.push_back(substring); - - prev_pos = ++pos; - } - - output.push_back(path.substr(prev_pos, pos - prev_pos)); - - return output; -} - -StatusCode AzureStorageFile::parseFilePath(const std::string& path) { - // azure://share/directory/file - // azure://share/directory - // azure://share/ - if (path.back() == '/') { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Path can not end with '/'", path); - return StatusCode::AS_INVALID_PATH; - } - - fullUri_ = path; - int share_start = 0; - // File or directory path - if (path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) != std::string::npos) { - share_start = path.find(AzureFileSystem::AZURE_URL_FILE_PREFIX) + AzureFileSystem::AZURE_URL_FILE_PREFIX.size(); - } else if (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos) { - // Blob path - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Wrong object type. azfs:// prefix in path required, found az://:", path); - return StatusCode::AS_INVALID_PATH; - } else { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Missing azfs:// prefix in path:", path); - return StatusCode::AS_INVALID_PATH; - } - - int share_end = path.find_first_of("/", share_start); - int file_start = path.find_last_of("/"); - int file_end = path.length(); - - fullPath_ = path.substr(share_end + 1, file_end - share_end - 1); - - subdirs_ = FindSubdirectories(fullPath_); - - if (share_end > share_start) { - share_ = path.substr(share_start, share_end - share_start); - - directory_ = path.substr(share_end + 1, file_start - share_end - 1); - - // No file or no directory - if (share_end == file_start) { - file_ = ""; - } else { - // No file - if (file_start == file_end) { - file_ = ""; - } else { - file_ = path.substr(file_start + 1, file_end - file_start); - } - } - } else { - // No directory and no file - share_ = path.substr(share_start); - directory_ = ""; - file_ = ""; - } - - // No share - if (share_.empty()) { - return StatusCode::AS_INVALID_PATH; - } - - return StatusCode::OK; -} - -std::shared_ptr AzureStorageFactory::getNewAzureStorageObject(const std::string& path, as::cloud_storage_account account) { - if (isBlobStoragePath(path)) - return std::make_shared(path, account); - - return std::make_shared(path, account); -} - -bool AzureStorageFactory::isBlobStoragePath(std::string path) { - return (path.find(AzureFileSystem::AZURE_URL_BLOB_PREFIX) != std::string::npos); -} - -} // namespace ovms diff --git a/src/ovms_lib/azurestorage.hpp b/src/ovms_lib/azurestorage.hpp deleted file mode 100644 index 3a8b6bd79b..0000000000 --- a/src/ovms_lib/azurestorage.hpp +++ /dev/null @@ -1,195 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "status.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -#pragma GCC diagnostic ignored "-Wreorder" -#pragma GCC diagnostic ignored "-Wunused-value" -#include - -#include "was/blob.h" -#include "was/common.h" -#include "was/storage_account.h" -#pragma GCC diagnostic pop - -namespace ovms { - -namespace as = azure::storage; -using files_list_t = std::set; - -class AzureStorageAdapter { -public: - AzureStorageAdapter() {} - - virtual StatusCode fileExists(bool* exists) = 0; - virtual StatusCode isDirectory(bool* is_directory) = 0; - virtual StatusCode fileModificationTime(int64_t* mtime_ns) = 0; - virtual StatusCode getDirectoryContents(files_list_t* contents) = 0; - virtual StatusCode getDirectorySubdirs(files_list_t* subdirs) = 0; - virtual StatusCode getDirectoryFiles(files_list_t* files) = 0; - virtual StatusCode readTextFile(std::string* contents) = 0; - virtual StatusCode downloadFileFolder(const std::string& local_path) = 0; - virtual StatusCode deleteFileFolder() = 0; - virtual StatusCode downloadFile(const std::string& local_path) = 0; - virtual StatusCode downloadFileFolderTo(const std::string& local_path) = 0; - virtual StatusCode checkPath(const std::string& path) = 0; - - std::string joinPath(std::initializer_list segments); - StatusCode CreateLocalDir(const std::string& path); - bool isAbsolutePath(const std::string& path); - std::vector FindSubdirectories(std::string path); - -protected: - const std::string extractAzureStorageExceptionMessage(const as::storage_exception& e); - -private: - virtual StatusCode parseFilePath(const std::string& path) = 0; -}; - -class AzureStorageBlob : public AzureStorageAdapter { -public: - AzureStorageBlob(const std::string& path, as::cloud_storage_account account); - - StatusCode checkPath(const std::string& path) override; - - StatusCode fileExists(bool* exists) override; - - StatusCode isDirectory(bool* is_directory) override; - - StatusCode fileModificationTime(int64_t* mtime_ns) override; - - StatusCode getDirectoryContents(files_list_t* contents) override; - - StatusCode getDirectorySubdirs(files_list_t* subdirs) override; - - StatusCode getDirectoryFiles(files_list_t* files) override; - - StatusCode readTextFile(std::string* contents) override; - - StatusCode downloadFileFolder(const std::string& local_path) override; - - StatusCode deleteFileFolder() override; - - StatusCode downloadFile(const std::string& local_path) override; - - StatusCode downloadFileFolderTo(const std::string& local_path) override; - -private: - std::string getLastPathPart(const std::string& path); - - StatusCode parseFilePath(const std::string& path) override; - - std::string getNameFromPath(std::string& path); - - bool isPathValidationOk_; - - std::string fullPath_; - - std::string fullUri_; - - std::string blockpath_; - - std::vector subdirs_; - - std::string container_; - - as::cloud_blob_container as_container_; - - as::cloud_block_blob as_block_blob_; - - as::cloud_blob as_blob_; - - as::cloud_storage_account account_; - - as::cloud_blob_client as_blob_client_; -}; - -class AzureStorageFile : public AzureStorageAdapter { -public: - AzureStorageFile(const std::string& path, as::cloud_storage_account account); - - StatusCode checkPath(const std::string& path) override; - - StatusCode fileExists(bool* exists) override; - - StatusCode isDirectory(bool* is_directory) override; - - StatusCode fileModificationTime(int64_t* mtime_ns) override; - - StatusCode getDirectoryContents(files_list_t* contents) override; - - StatusCode getDirectorySubdirs(files_list_t* subdirs) override; - - StatusCode getDirectoryFiles(files_list_t* files) override; - - StatusCode readTextFile(std::string* contents) override; - - StatusCode downloadFileFolder(const std::string& local_path) override; - - StatusCode deleteFileFolder() override; - - StatusCode downloadFile(const std::string& local_path) override; - - StatusCode downloadFileFolderTo(const std::string& local_path) override; - -private: - StatusCode parseFilePath(const std::string& path) override; - - bool isPathValidationOk_; - - std::string fullPath_; - - std::string fullUri_; - - std::string file_; - - std::string directory_; - - std::vector subdirs_; - - std::string share_; - - as::cloud_storage_account account_; - - as::cloud_file_client as_file_client_; - - as::cloud_file_share as_share_; - - as::cloud_file_directory as_directory_; - - as::cloud_file as_file1_; -}; - -class AzureStorageFactory { -public: - std::shared_ptr getNewAzureStorageObject(const std::string& path, as::cloud_storage_account account); - -private: - bool isBlobStoragePath(std::string path); -}; - -} // namespace ovms diff --git a/src/ovms_lib/binaryutils.cpp b/src/ovms_lib/binaryutils.cpp deleted file mode 100644 index 97130add96..0000000000 --- a/src/ovms_lib/binaryutils.cpp +++ /dev/null @@ -1,447 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "binaryutils.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include -#include -#include -#include - -#include - -#include "logging.hpp" -#include "opencv2/opencv.hpp" -#include "status.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "kfs_grpc_inference_service.hpp" -#pragma GCC diagnostic pop - -namespace ovms { - -int getMatTypeFromTensorPrecision(ovms::Precision tensorPrecision) { - switch (tensorPrecision) { - case ovms::Precision::FP32: - return CV_32F; - case ovms::Precision::FP64: - return CV_64F; - case ovms::Precision::FP16: - return CV_16F; - case ovms::Precision::I16: - return CV_16S; - case ovms::Precision::U8: - return CV_8U; - case ovms::Precision::I8: - return CV_8S; - case ovms::Precision::U16: - return CV_16U; - case ovms::Precision::I32: - return CV_32S; - default: - return -1; - } -} - -bool isPrecisionEqual(int matPrecision, ovms::Precision tensorPrecision) { - int convertedTensorPrecision = getMatTypeFromTensorPrecision(tensorPrecision); - if (convertedTensorPrecision == matPrecision) { - return true; - } - return false; -} - -cv::Mat convertStringToMat(const std::string& image) { - std::vector data(image.begin(), image.end()); - cv::Mat dataMat(data, true); - - try { - return cv::imdecode(dataMat, cv::IMREAD_UNCHANGED); - } catch (const cv::Exception& e) { - SPDLOG_DEBUG("Error during string_val to mat conversion: {}", e.what()); - return cv::Mat{}; - } -} - -Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision requestedPrecision) { - int type = getMatTypeFromTensorPrecision(requestedPrecision); - if (type == -1) { - SPDLOG_DEBUG("Error during binary input conversion: not supported precision: {}", toString(requestedPrecision)); - return StatusCode::INVALID_PRECISION; - } - - src.convertTo(dst, type); - return StatusCode::OK; -} - -Status validateLayout(const std::shared_ptr& tensorInfo) { - static const std::string binarySupportedLayout = "N...HWC"; - if (!tensorInfo->getLayout().createIntersection(Layout(binarySupportedLayout), tensorInfo->getShape().size()).has_value()) { - SPDLOG_DEBUG("Endpoint needs to be compatible with {} to support binary image inputs, actual: {}", - binarySupportedLayout, - tensorInfo->getLayout()); - return StatusCode::UNSUPPORTED_LAYOUT; - } - return StatusCode::OK; -} - -bool resizeNeeded(const cv::Mat& image, const dimension_value_t height, const dimension_value_t width) { - if (height != image.rows || width != image.cols) { - return true; - } - return false; -} - -Status resizeMat(const cv::Mat& src, cv::Mat& dst, const dimension_value_t height, const dimension_value_t width) { - cv::resize(src, dst, cv::Size(width, height)); - return StatusCode::OK; -} - -Status validateNumberOfChannels(const std::shared_ptr& tensorInfo, - const cv::Mat input, - cv::Mat* firstBatchImage) { - - // At this point we can either have nhwc format or pretendant to be nhwc but with ANY layout in pipeline info - Dimension numberOfChannels; - if (tensorInfo->getShape().size() == 4) { - numberOfChannels = tensorInfo->getShape()[3]; - } else if (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5) { - numberOfChannels = tensorInfo->getShape()[4]; - } else { - return StatusCode::INVALID_NO_OF_CHANNELS; - } - if (numberOfChannels.isAny() && firstBatchImage) { - numberOfChannels = firstBatchImage->channels(); - } - if (numberOfChannels.isAny()) { - return StatusCode::OK; - } - if (!numberOfChannels.match(input.channels())) { - SPDLOG_DEBUG("Binary data sent to input: {} has invalid number of channels. Expected: {} Actual: {}", - tensorInfo->getMappedName(), - numberOfChannels.toString(), - input.channels()); - return StatusCode::INVALID_NO_OF_CHANNELS; - } - - return StatusCode::OK; -} - -Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* firstBatchImage) { - if (input.cols == firstBatchImage->cols && input.rows == firstBatchImage->rows) { - return StatusCode::OK; - } - SPDLOG_DEBUG("Each binary image in request needs to have resolution matched. First cols: {}, rows: {}, current cols: {}, rows: {}", - firstBatchImage->cols, firstBatchImage->rows, input.cols, input.rows); - return StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH; -} - -bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, - const int batchSize) { - if (!tensorInfo->getBatchSize().has_value()) { - return true; - } - return !tensorInfo->getBatchSize().value().match(batchSize); -} - -Status validateInput(const std::shared_ptr& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage, bool enforceResolutionAlignment) { - // Binary inputs are supported for any endpoint that is compatible with N...HWC layout. - // With unknown layout, there is no way to deduce expected endpoint input resolution. - // This forces binary utility to create tensors with resolution inherited from first batch of binary input image (request). - // In case of any dimension in endpoint shape is dynamic, we need to validate images against first image resolution. - // Otherwise we can omit that, and proceed to image resize. - if (firstBatchImage && enforceResolutionAlignment) { - auto status = validateResolutionAgainstFirstBatchImage(input, firstBatchImage); - if (!status.ok()) { - return status; - } - } - return validateNumberOfChannels(tensorInfo, input, firstBatchImage); -} - -Status validateTensor(const std::shared_ptr& tensorInfo, - const tensorflow::TensorProto& src) { - auto status = validateLayout(tensorInfo); - if (!status.ok()) { - return status; - } - // 4 for default pipelines, 5 for pipelines with demultiplication at entry - bool isShapeLengthValid = tensorInfo->getShape().size() == 4 || - (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5); - if (!isShapeLengthValid) { - return StatusCode::INVALID_SHAPE; - } - - if (checkBatchSizeMismatch(tensorInfo, src.string_val_size())) { - SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", - tensorInfo->getMappedName(), - tensorInfo->getBatchSize().has_value() ? tensorInfo->getBatchSize().value().toString() : std::string{"none"}, - src.string_val_size()); - return StatusCode::INVALID_BATCH_SIZE; - } - - for (size_t i = 0; i < src.string_val_size(); i++) { - if (src.string_val(i).size() <= 0) { - return StatusCode::STRING_VAL_EMPTY; - } - } - - return StatusCode::OK; -} - -Status validateTensor(const std::shared_ptr& tensorInfo, - const ::inference::ModelInferRequest::InferInputTensor& src) { - auto status = validateLayout(tensorInfo); - if (!status.ok()) { - return status; - } - // 4 for default pipelines, 5 for pipelines with demultiplication at entry - bool isShapeLengthValid = tensorInfo->getShape().size() == 4 || - (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5); - if (!isShapeLengthValid) { - return StatusCode::INVALID_SHAPE; - } - - if (checkBatchSizeMismatch(tensorInfo, src.contents().bytes_contents_size())) { - SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", - tensorInfo->getMappedName(), - tensorInfo->getBatchSize().has_value() ? tensorInfo->getBatchSize().value().toString() : std::string{"none"}, - src.contents().bytes_contents_size()); - return StatusCode::INVALID_BATCH_SIZE; - } - - for (size_t i = 0; i < src.contents().bytes_contents_size(); i++) { - if (src.contents().bytes_contents(i).size() <= 0) { - return StatusCode::BYTES_CONTENTS_EMPTY; - } - } - - if (src.contents().bytes_contents_size() <= 0) { - return StatusCode::BYTES_CONTENTS_EMPTY; - } - - return StatusCode::OK; -} - -Dimension getTensorInfoHeightDim(const std::shared_ptr& tensorInfo) { - size_t numberOfShapeDimensions = tensorInfo->getShape().size(); - if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { - throw std::logic_error("wrong number of shape dimensions"); - } - size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 1 : /*N?HWC*/ 2; - return tensorInfo->getShape()[position]; -} - -Dimension getTensorInfoWidthDim(const std::shared_ptr& tensorInfo) { - size_t numberOfShapeDimensions = tensorInfo->getShape().size(); - if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) { - throw std::logic_error("wrong number of shape dimensions"); - } - size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 2 : /*N?HWC*/ 3; - return tensorInfo->getShape()[position]; -} - -void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& image) { - if (height.isAny()) { - height = image.rows; - } else if (height.isDynamic()) { - if (height.match(image.rows)) { - height = image.rows; - } else { - if (image.rows > height.getMaxValue()) { - height = height.getMaxValue(); - } else { - height = height.getMinValue(); - } - } - } - if (width.isAny()) { - width = image.cols; - } else if (width.isDynamic()) { - if (width.match(image.cols)) { - width = image.cols; - } else { - if (image.cols > width.getMaxValue()) { - width = width.getMaxValue(); - } else { - width = width.getMinValue(); - } - } - } -} - -bool isResizeSupported(const std::shared_ptr& tensorInfo) { - for (const auto& dim : tensorInfo->getShape()) { - if (dim.isAny()) { - return false; - } - } - if (tensorInfo->getLayout() != "NHWC" && - tensorInfo->getLayout() != "N?HWC" && - tensorInfo->getLayout() != Layout::getUnspecifiedLayout()) { - return false; - } - return true; -} - -const std::string& getBinaryInput(const tensorflow::TensorProto& tensor, size_t i) { - return tensor.string_val(i); -} - -const std::string& getBinaryInput(const ::inference::ModelInferRequest::InferInputTensor& tensor, size_t i) { - return tensor.contents().bytes_contents(i); -} - -size_t getBinaryInputsSize(const tensorflow::TensorProto& tensor) { - return tensor.string_val_size(); -} - -size_t getBinaryInputsSize(const ::inference::ModelInferRequest::InferInputTensor& tensor) { - return tensor.contents().bytes_contents_size(); -} - -template -Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo) { - Dimension targetHeight = getTensorInfoHeightDim(tensorInfo); - Dimension targetWidth = getTensorInfoWidthDim(tensorInfo); - - // Enforce resolution alignment against first image in the batch if resize is not supported. - bool resizeSupported = isResizeSupported(tensorInfo); - bool enforceResolutionAlignment = !resizeSupported; - - for (int i = 0; i < getBinaryInputsSize(src); i++) { - cv::Mat image = convertStringToMat(getBinaryInput(src, i)); - if (image.data == nullptr) - return StatusCode::IMAGE_PARSING_FAILED; - - cv::Mat* firstImage = images.size() == 0 ? nullptr : &images.at(0); - auto status = validateInput(tensorInfo, image, firstImage, enforceResolutionAlignment); - if (status != StatusCode::OK) { - return status; - } - - if (i == 0) { - updateTargetResolution(targetHeight, targetWidth, image); - } - - if (!isPrecisionEqual(image.depth(), tensorInfo->getPrecision())) { - cv::Mat imageCorrectPrecision; - status = convertPrecision(image, imageCorrectPrecision, tensorInfo->getPrecision()); - - if (status != StatusCode::OK) { - return status; - } - image = std::move(imageCorrectPrecision); - } - if (!targetHeight.isStatic() || !targetWidth.isStatic()) { - return StatusCode::INTERNAL_ERROR; - } - if (resizeNeeded(image, targetHeight.getStaticValue(), targetWidth.getStaticValue())) { - if (!resizeSupported) { - return StatusCode::INVALID_SHAPE; - } - cv::Mat imageResized; - status = resizeMat(image, imageResized, targetHeight.getStaticValue(), targetWidth.getStaticValue()); - if (!status.ok()) { - return status; - } - image = std::move(imageResized); - } - - // if (i == 0 && src.contents().bytes_contents_size() > 1) { - // // Multiply src.string_val_size() * image resolution * precision size - // } - - images.push_back(image); - } - - return StatusCode::OK; -} -shape_t getShapeFromImages(const std::vector& images, const std::shared_ptr& tensorInfo) { - shape_t dims; - dims.push_back(images.size()); - if (tensorInfo->isInfluencedByDemultiplexer()) { - dims.push_back(1); - } - dims.push_back(images[0].rows); - dims.push_back(images[0].cols); - dims.push_back(images[0].channels()); - return dims; -} - -ov::Tensor createTensorFromMats(const std::vector& images, const std::shared_ptr& tensorInfo) { - ov::Shape shape = getShapeFromImages(images, tensorInfo); - ov::element::Type precision = tensorInfo->getOvPrecision(); - ov::Tensor tensor(precision, shape); - char* ptr = (char*)tensor.data(); - for (cv::Mat image : images) { - memcpy(ptr, (char*)image.data, image.total() * image.elemSize()); - ptr += (image.total() * image.elemSize()); - } - return tensor; -} - -ov::Tensor convertMatsToTensor(std::vector& images, const std::shared_ptr& tensorInfo) { - switch (tensorInfo->getPrecision()) { - case ovms::Precision::FP32: - case ovms::Precision::I32: - case ovms::Precision::FP64: - case ovms::Precision::I8: - case ovms::Precision::U8: - case ovms::Precision::FP16: - case ovms::Precision::U16: - case ovms::Precision::I16: - return createTensorFromMats(images, tensorInfo); - case ovms::Precision::MIXED: - case ovms::Precision::Q78: - case ovms::Precision::BIN: - case ovms::Precision::BOOL: - case ovms::Precision::CUSTOM: - default: - return ov::Tensor(); - } -} - -template -Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo) { - auto status = validateTensor(tensorInfo, src); - if (status != StatusCode::OK) { - return status; - } - - std::vector images; - - status = convertTensorToMatsMatchingTensorInfo(src, images, tensorInfo); - if (!status.ok()) { - return status; - } - - tensor = convertMatsToTensor(images, tensorInfo); - if (!tensor) { - return StatusCode::IMAGE_PARSING_FAILED; - } - return StatusCode::OK; -} - -template Status convertBinaryRequestTensorToOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); -template Status convertBinaryRequestTensorToOVTensor<::inference::ModelInferRequest::InferInputTensor>(const ::inference::ModelInferRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); -} // namespace ovms diff --git a/src/ovms_lib/binaryutils.hpp b/src/ovms_lib/binaryutils.hpp deleted file mode 100644 index b41806bb61..0000000000 --- a/src/ovms_lib/binaryutils.hpp +++ /dev/null @@ -1,26 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { -template -Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); -} // namespace ovms diff --git a/src/ovms_lib/cleaner_utils.hpp b/src/ovms_lib/cleaner_utils.hpp deleted file mode 100644 index 3ac0cea675..0000000000 --- a/src/ovms_lib/cleaner_utils.hpp +++ /dev/null @@ -1,43 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include "global_sequences_viewer.hpp" -#include "modelmanager.hpp" - -namespace ovms { -struct FunctorSequenceCleaner { - GlobalSequencesViewer& globalSequencesViewer; - - FunctorSequenceCleaner(GlobalSequencesViewer& globalSequencesViewer) : - globalSequencesViewer(globalSequencesViewer) {} - - virtual void cleanup() { - globalSequencesViewer.removeIdleSequences(); - } -}; - -struct FunctorResourcesCleaner { - ModelManager& modelManager; - - FunctorResourcesCleaner(ModelManager& modelManager) : - modelManager(modelManager) {} - - virtual void cleanup() { - modelManager.cleanupResources(); - } -}; -} // namespace ovms diff --git a/src/ovms_lib/config.cpp b/src/ovms_lib/config.cpp deleted file mode 100644 index 39c3b05a41..0000000000 --- a/src/ovms_lib/config.cpp +++ /dev/null @@ -1,303 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "config.hpp" - -#include -#include -#include -#include -#include - -#include -#include - -#include "logging.hpp" -#include "version.hpp" - -namespace ovms { - -const uint AVAILABLE_CORES = std::thread::hardware_concurrency(); -const uint MAX_PORT_NUMBER = std::numeric_limits::max(); - -const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; -const std::string DEFAULT_REST_WORKERS_STRING{std::to_string(DEFAULT_REST_WORKERS)}; -const uint64_t MAX_REST_WORKERS = 10'000; - -Config& Config::parse(int argc, char** argv) { - try { - options = std::make_unique(argv[0], "OpenVINO Model Server"); - - // clang-format off - options->add_options() - ("h, help", - "Show this help message and exit") - ("version", - "Show binary version") - ("port", - "gRPC server port", - cxxopts::value()->default_value("9178"), - "PORT") - ("grpc_bind_address", - "Network interface address to bind to for the gRPC API", - cxxopts::value()->default_value("0.0.0.0"), - "GRPC_BIND_ADDRESS") - ("rest_port", - "REST server port, the REST server will not be started if rest_port is blank or set to 0", - cxxopts::value()->default_value("0"), - "REST_PORT") - ("rest_bind_address", - "Network interface address to bind to for the REST API", - cxxopts::value()->default_value("0.0.0.0"), - "REST_BIND_ADDRESS") - ("grpc_workers", - "Number of gRPC servers. Default 1. Increase for multi client, high throughput scenarios", - cxxopts::value()->default_value("1"), - "GRPC_WORKERS") - ("rest_workers", - "Number of worker threads in REST server - has no effect if rest_port is not set. Default value depends on number of CPUs. ", - cxxopts::value()->default_value(DEFAULT_REST_WORKERS_STRING.c_str()), - "REST_WORKERS") - ("log_level", - "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", - cxxopts::value()->default_value("INFO"), "LOG_LEVEL") - ("log_path", - "Optional path to the log file", - cxxopts::value(), "LOG_PATH") -#ifdef MTR_ENABLED - ("trace_path", - "Path to the trace file", - cxxopts::value(), "TRACE_PATH") -#endif - ("grpc_channel_arguments", - "A comma separated list of arguments to be passed to the grpc server. (e.g. grpc.max_connection_age_ms=2000)", - cxxopts::value(), "GRPC_CHANNEL_ARGUMENTS") - ("file_system_poll_wait_seconds", - "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", - cxxopts::value()->default_value("1"), - "FILE_SYSTEM_POLL_WAIT_SECONDS") - ("sequence_cleaner_poll_wait_minutes", - "Time interval between two consecutive sequence cleanup scans. Default is 5. Zero value disables sequence cleaner.", - cxxopts::value()->default_value("5"), - "SEQUENCE_CLEANER_POLL_WAIT_MINUTES") - ("custom_node_resources_cleaner_interval", - "Time interval between two consecutive resources cleanup scans. Default is 1. Must be greater than 0.", - cxxopts::value()->default_value("1"), - "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL") - ("cache_dir", - "Overrides model cache directory. By default cache files are saved into /opt/cache if the directory is present. When enabled, first model load will produce cache files.", - cxxopts::value(), - "CACHE_DIR") - ("cpu_extension", - "A path to shared library containing custom CPU layer implementation. Default: empty.", - cxxopts::value()->default_value(""), - "CPU_EXTENSION"); - options->add_options("multi model") - ("config_path", - "Absolute path to json configuration file", - cxxopts::value(), "CONFIG_PATH"); - - options->add_options("single model") - ("model_name", - "Name of the model", - cxxopts::value(), - "MODEL_NAME") - ("model_path", - "Absolute path to model, as in tf serving", - cxxopts::value(), - "MODEL_PATH") - ("batch_size", - "Resets models batchsize, int value or auto. This parameter will be ignored if shape is set", - cxxopts::value(), - "BATCH_SIZE") - ("shape", - "Resets models shape (model must support reshaping). If set, batch_size parameter is ignored", - cxxopts::value(), - "SHAPE") - ("layout", - "Resets model layout.", - cxxopts::value(), - "LAYOUT") - ("model_version_policy", - "Model version policy", - cxxopts::value(), - "MODEL_VERSION_POLICY") - ("nireq", - "Size of inference request queue for model executions. Recommended to be >= parallel executions. Default value calculated by OpenVINO based on available resources. Request for 0 is treated as request for default value", - cxxopts::value(), - "NIREQ") - ("target_device", - "Target device to run the inference", - cxxopts::value()->default_value("CPU"), - "TARGET_DEVICE") - ("plugin_config", - "A dictionary of plugin configuration keys and their values, eg \"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"1\\\"}\". Default throughput streams for CPU and GPU are calculated by OpenVINO", - cxxopts::value(), - "PLUGIN_CONFIG") - ("stateful", - "Flag indicating model is stateful", - cxxopts::value()->default_value("false"), - "STATEFUL") - ("idle_sequence_cleanup", - "Flag indicating if model is subject to sequence cleaner scans", - cxxopts::value()->default_value("true"), - "IDLE_SEQUENCE_CLEANUP") - ("low_latency_transformation", - "Flag indicating that Model Server should perform low latency transformation on that model", - cxxopts::value()->default_value("false"), - "LOW_LATENCY_TRANSFORMATION") - ("max_sequence_number", - "Determines how many sequences can be processed concurrently by one model instance. When that value is reached, attempt to start a new sequence will result in error.", - cxxopts::value(), - "MAX_SEQUENCE_NUMBER"); - - // clang-format on - - result = std::make_unique(options->parse(argc, argv)); - - if (result->count("version")) { - std::string project_name(PROJECT_NAME); - std::string project_version(PROJECT_VERSION); - std::cout << project_name + " " + project_version << std::endl; - std::cout << "OpenVINO backend " << OPENVINO_NAME << std::endl; - exit(EX_OK); - } - - if (result->count("help") || result->arguments().size() == 0) { - std::cout << options->help({"", "multi model", "single model"}) << std::endl; - exit(EX_OK); - } - - validate(); - } catch (const cxxopts::OptionException& e) { - std::cerr << "error parsing options: " << e.what() << std::endl; - exit(EX_USAGE); - } - - return instance(); -} - -bool Config::check_hostname_or_ip(const std::string& input) { - if (input.size() > 255) { - return false; - } - bool all_numeric = true; - for (char c : input) { - if (c == '.') { - continue; - } - if (!::isdigit(c)) { - all_numeric = false; - } - } - if (all_numeric) { - std::regex valid_ip_regex("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"); - return std::regex_match(input, valid_ip_regex); - } else { - std::regex valid_hostname_regex("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$"); - return std::regex_match(input, valid_hostname_regex); - } -} - -void Config::validate() { - // cannot set both config path & model_name/model_path - if (result->count("config_path") && (result->count("model_name") || result->count("model_path"))) { - std::cerr << "Use either config_path or model_path with model_name" << std::endl; - exit(EX_USAGE); - } - - if (!result->count("config_path") && !(result->count("model_name") && result->count("model_path"))) { - std::cerr << "Use config_path or model_path with model_name" << std::endl; - exit(EX_USAGE); - } - - if (result->count("config_path") && (result->count("batch_size") || result->count("shape") || - result->count("nireq") || result->count("model_version_policy") || result->count("target_device") || - result->count("plugin_config"))) { - std::cerr << "Model parameters in CLI are exclusive with the config file" << std::endl; - exit(EX_USAGE); - } - - // check grpc_workers value - if (result->count("grpc_workers") && ((this->grpcWorkers() > AVAILABLE_CORES) || (this->grpcWorkers() < 1))) { - std::cerr << "grpc_workers count should be from 1 to CPU core count : " << AVAILABLE_CORES << std::endl; - exit(EX_USAGE); - } - - // check rest_workers value - if (result->count("rest_workers") && ((this->restWorkers() > MAX_REST_WORKERS) || (this->restWorkers() < 2))) { - std::cerr << "rest_workers count should be from 2 to " << MAX_REST_WORKERS << std::endl; - exit(EX_USAGE); - } - - if (result->count("rest_workers") && (this->restWorkers() != DEFAULT_REST_WORKERS) && this->restPort() == 0) { - std::cerr << "rest_workers is set but rest_port is not set. rest_port is required to start rest servers" << std::endl; - exit(EX_USAGE); - } - - // check docker ports - if (result->count("port") && ((this->port() > MAX_PORT_NUMBER) || (this->port() < 0))) { - std::cerr << "port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); - } - if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { - std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); - } - - // check bind addresses: - if (result->count("rest_bind_address") && check_hostname_or_ip(this->restBindAddress()) == false) { - std::cerr << "rest_bind_address has invalid format: proper hostname or IP address expected." << std::endl; - exit(EX_USAGE); - } - if (result->count("grpc_bind_address") && check_hostname_or_ip(this->grpcBindAddress()) == false) { - std::cerr << "grpc_bind_address has invalid format: proper hostname or IP address expected." << std::endl; - exit(EX_USAGE); - } - if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { - std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); - } - - // port and rest_port cannot be the same - if (this->port() == this->restPort()) { - std::cerr << "port and rest_port cannot have the same values" << std::endl; - exit(EX_USAGE); - } - - // check cpu_extension path: - if (result->count("cpu_extension") && !std::filesystem::exists(this->cpuExtensionLibraryPath())) { - std::cerr << "File path provided as an --cpu_extension parameter does not exists in the filesystem: " << this->cpuExtensionLibraryPath() << std::endl; - exit(EX_USAGE); - } - - // check log_level values - if (result->count("log_level")) { - std::vector v({"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"}); - if (std::find(v.begin(), v.end(), this->logLevel()) == v.end()) { - std::cerr << "log_level should be one of: TRACE, DEBUG, INFO, WARNING, ERROR" << std::endl; - exit(EX_USAGE); - } - } - - // check stateful flags: - if ((result->count("low_latency_transformation") || result->count("max_sequence_number") || result->count("idle_sequence_cleanup")) && !result->count("stateful")) { - std::cerr << "Setting low_latency_transformation, max_sequence_number and idle_sequence_cleanup require setting stateful flag for the model." << std::endl; - exit(EX_USAGE); - } - return; -} - -} // namespace ovms diff --git a/src/ovms_lib/config.hpp b/src/ovms_lib/config.hpp deleted file mode 100644 index ac52e39d9e..0000000000 --- a/src/ovms_lib/config.hpp +++ /dev/null @@ -1,394 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "modelconfig.hpp" - -namespace ovms { -/** - * @brief Provides all the configuration options from command line - */ -class Config { -private: - /** - * @brief A default constructor is private - */ - Config() = default; - - /** - * @brief Private copying constructor - */ - Config(const Config&) = delete; - - /** - * @brief cxxopts options dictionary definition - */ - std::unique_ptr options; - - /** - * @brief cxxopts contains parsed parameters - */ - std::unique_ptr result; - - /** - * @brief - */ - const std::string empty; - -public: - /** - * @brief Gets the instance of the config - */ - static Config& instance() { - static Config instance; - - return instance; - } - - /** - * @brief Parse the commandline parameters - * - * @param argc - * @param argv - * @return Config& - */ - Config& parse(int argc, char** argv); - - /** - * @brief Validate passed arguments - * - * @return void - */ - void validate(); - - /** - * @brief checks if input is a proper hostname or IP address value - * - * @return bool - */ - static bool check_hostname_or_ip(const std::string& input); - - /** - * @brief Get the config path - * - * @return std::string - */ - const std::string& configPath() const { - if (result->count("config_path")) - return result->operator[]("config_path").as(); - return empty; - } - - /** - * @brief Gets the grpc port - * - * @return uint64_t - */ - uint64_t port() const { - return result->operator[]("port").as(); - } - - /** - * @brief Get the gRPC network interface address to bind to - * - * @return const std::string - */ - const std::string cpuExtensionLibraryPath() const { - if (result != nullptr && result->count("cpu_extension")) { - return result->operator[]("cpu_extension").as(); - } - return ""; - } - - /** - * @brief Get the gRPC network interface address to bind to - * - * @return const std::string& - */ - const std::string grpcBindAddress() const { - if (result->count("grpc_bind_address")) - return result->operator[]("grpc_bind_address").as(); - return "0.0.0.0"; - } - /** - * @brief Gets the REST port - * - * @return uint64_t - */ - uint64_t restPort() const { - return result->operator[]("rest_port").as(); - } - - /** - * @brief Get the rest network interface address to bind to - * - * @return const std::string& - */ - const std::string restBindAddress() const { - if (result->count("rest_bind_address")) - return result->operator[]("rest_bind_address").as(); - return "0.0.0.0"; - } - - /** - * @brief Gets the gRPC workers count - * - * @return uint - */ - uint grpcWorkers() const { - return result->operator[]("grpc_workers").as(); - } - - /** - * @brief Gets the rest workers count - * - * @return uint - */ - uint restWorkers() const { - return result->operator[]("rest_workers").as(); - } - - /** - * @brief Get the model name - * - * @return const std::string& - */ - const std::string& modelName() const { - if (result->count("model_name")) - return result->operator[]("model_name").as(); - return empty; - } - - /** - * @brief Gets the model path - * - * @return const std::string& - */ - const std::string& modelPath() const { - if (result->count("model_path")) - return result->operator[]("model_path").as(); - return empty; - } - - /** - * @brief Gets the batch size - * - * @return const std::string& - */ - const std::string& batchSize() const { - if (!result->count("batch_size")) { - static const std::string d = "0"; - return d; - } - return result->operator[]("batch_size").as(); - } - - /** - * @brief Get the shape - * - * @return const std::string& - */ - const std::string& shape() const { - if (result->count("shape")) - return result->operator[]("shape").as(); - return empty; - } - - /** - * @brief Get the layout - * - * @return const std::string& - */ - const std::string& layout() const { - if (result->count("layout")) - return result->operator[]("layout").as(); - return empty; - } - - /** - * @brief Get the shape - * - * @return const std::string& - */ - const std::string& modelVersionPolicy() const { - if (result->count("model_version_policy")) - return result->operator[]("model_version_policy").as(); - return empty; - } - - /** - * @brief Get the nireq - * - * @return uint - */ - uint32_t nireq() const { - if (!result->count("nireq")) { - return 0; - } - return result->operator[]("nireq").as(); - } - - /** - * @brief Get the target device - * - * @return const std::string& - */ - const std::string& targetDevice() const { - return result->operator[]("target_device").as(); - } - - /** - * @brief Get the plugin config - * - * @return const std::string& - */ - const std::string& pluginConfig() const { - if (result->count("plugin_config")) - return result->operator[]("plugin_config").as(); - return empty; - } - - /** - * @brief Get stateful flag - * - * @return bool - */ - bool stateful() const { - return result->operator[]("stateful").as(); - } - - /** - * @brief Get idle sequence cleanup flag - * - * @return uint - */ - bool idleSequenceCleanup() const { - return result->operator[]("idle_sequence_cleanup").as(); - } - - /** - * @brief Get low latency transformation flag - * - * @return bool - */ - bool lowLatencyTransformation() const { - return result->operator[]("low_latency_transformation").as(); - } - - /** - * @brief Get max number of sequences that can be processed concurrently - * - * @return uint - */ - uint32_t maxSequenceNumber() const { - if (!result->count("max_sequence_number")) { - return DEFAULT_MAX_SEQUENCE_NUMBER; - } - return result->operator[]("max_sequence_number").as(); - } - - /** - * @brief Get the log level - * - * @return const std::string& - */ - const std::string& logLevel() const { - if (result->count("log_level")) - return result->operator[]("log_level").as(); - return empty; - } - - /** - * @brief Get the log path - * - * @return const std::string& - */ - const std::string& logPath() const { - if (result->count("log_path")) - return result->operator[]("log_path").as(); - return empty; - } - -#ifdef MTR_ENABLED - /** - * @brief Get the log path - * - * @return const std::string& - */ - const std::string& tracePath() const { - if (result->count("trace_path")) - return result->operator[]("trace_path").as(); - return empty; - } -#endif - - /** - * @brief Get the plugin config - * - * @return const std::string& - */ - const std::string& grpcChannelArguments() const { - if (result->count("grpc_channel_arguments")) - return result->operator[]("grpc_channel_arguments").as(); - return empty; - } - - /** - * @brief Get the filesystem poll wait time in seconds - * - * @return uint - */ - uint filesystemPollWaitSeconds() const { - return result->operator[]("file_system_poll_wait_seconds").as(); - } - - /** - * @brief Get the sequence cleanup poll wait time in minutes - * - * @return uint32_t - */ - uint32_t sequenceCleanerPollWaitMinutes() const { - return result->operator[]("sequence_cleaner_poll_wait_minutes").as(); - } - - /** - * @brief Get the resources cleanup poll wait time in seconds - * - * @return uint32_t - */ - uint32_t resourcesCleanerPollWaitSeconds() const { - return result->operator[]("custom_node_resources_cleaner_interval").as(); - } - - /** - * @brief Model cache directory - * - * @return const std::string& - */ - const std::string cacheDir() const { - if (result != nullptr && result->count("cache_dir")) { - return result->operator[]("cache_dir").as(); - } - return ""; - } -}; -} // namespace ovms diff --git a/src/ovms_lib/custom_node.cpp b/src/ovms_lib/custom_node.cpp deleted file mode 100644 index 3feb6c3c6e..0000000000 --- a/src/ovms_lib/custom_node.cpp +++ /dev/null @@ -1,103 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "custom_node.hpp" - -#include - -#include "custom_node_output_allocator.hpp" -#include "customnodesession.hpp" -#include "logging.hpp" -#include "node_library.hpp" -#include "node_library_utils.hpp" - -namespace ovms { - -CustomNode::CustomNode( - const std::string& nodeName, - const NodeLibrary& library, - const parameters_t& parameters, - const std::unordered_map& nodeOutputNameAlias, - std::optional demultiplyCount, - std::set gatherFromNode, - std::shared_ptr customNodeLibraryInternalManager) : - Node(nodeName, demultiplyCount, gatherFromNode), - library(library), - parameters(parameters), - nodeOutputNameAlias(nodeOutputNameAlias), - libraryParameters(createCustomNodeParamArray(this->parameters)), - customNodeLibraryInternalManager(customNodeLibraryInternalManager) { -} - -Status CustomNode::execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) { - auto& nodeSession = getNodeSession(sessionKey); - auto& customNodeSession = static_cast(nodeSession); - return customNodeSession.execute(notifyEndQueue, *this, this->library, this->libraryParameters, this->parameters.size(), getCNLIMWrapperPtr(customNodeLibraryInternalManager)); -} - -Status CustomNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { - auto& customNodeSession = static_cast(nodeSession); - const auto& sessionMetadata = nodeSession.getNodeSessionMetadata(); - SessionResult sessionResults{sessionMetadata, {}}; - auto it = nodeSessionOutputs.emplace(sessionMetadata.getSessionKey(), std::move(sessionResults)); - if (!it.second) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to put node: {} session: {} results in node session outputs", - getName(), nodeSession.getSessionKey()); - customNodeSession.release(); - return StatusCode::INTERNAL_ERROR; - } - auto& metadataTensorResultsPair = it.first->second; - auto& tensorResults = metadataTensorResultsPair.second; - return this->fetchResults( - tensorResults, - nodeSession.getSessionKey()); -} - -Status CustomNode::fetchResults(TensorWithSourceMap& outputs, session_key_t sessionKey) { - auto& session = static_cast(this->getNodeSession(sessionKey)); - session.clearInputs(); - - for (const auto& node : this->next) { - for (const auto& pair : node.get().getMappingByDependency(*this)) { - const auto& output_name = pair.first; - if (outputs.find(output_name) != outputs.end()) { - continue; - } - const auto& realOutputName = this->getRealOutputName(output_name); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Getting custom node output tensor with name: {}", - getName(), sessionKey, realOutputName); - - ov::Tensor resultTensor; - auto status = session.fetchResult(realOutputName, resultTensor); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} Custom node output with name {} is missing", - getName(), sessionKey, realOutputName); - return StatusCode::NODE_LIBRARY_MISSING_OUTPUT; - } - - outputs.emplace(std::make_pair(output_name, TensorWithSource(std::move(resultTensor)))); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Tensor with name {} has been prepared under alias {}", - getName(), sessionKey, realOutputName, output_name); - } - } - - return StatusCode::OK; -} - -std::unique_ptr CustomNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { - return std::make_unique(metadata, getName(), previous.size(), collapsingDetails); -} - -} // namespace ovms diff --git a/src/ovms_lib/custom_node.hpp b/src/ovms_lib/custom_node.hpp deleted file mode 100644 index 358dbc4af1..0000000000 --- a/src/ovms_lib/custom_node.hpp +++ /dev/null @@ -1,65 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include "custom_node_interface.h" // NOLINT -#include "custom_node_library_internal_manager_wrapper.hpp" -#include "node.hpp" -#include "nodeinfo.hpp" -#include "pipelineeventqueue.hpp" - -namespace ovms { - -class NodeLibrary; - -class CustomNode : public Node { - NodeLibrary library; - parameters_t parameters; - std::unordered_map nodeOutputNameAlias; - - std::unique_ptr libraryParameters = nullptr; - - std::shared_ptr customNodeLibraryInternalManager; - -public: - CustomNode( - const std::string& nodeName, - const NodeLibrary& library, - const parameters_t& parameters, - const std::unordered_map& nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, - std::set gatherFromNode = {}, - std::shared_ptr customNodeLibraryInternalManager = nullptr); - - Status execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) override; - - Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; - Status fetchResults(TensorWithSourceMap& outputs, session_key_t sessionKey); - - const std::string& getRealOutputName(const std::string& alias) const { - auto it = nodeOutputNameAlias.find(alias); - return it != nodeOutputNameAlias.end() ? it->second : alias; - } - - std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; -}; - -} // namespace ovms diff --git a/src/ovms_lib/custom_node_interface.h b/src/ovms_lib/custom_node_interface.h deleted file mode 100644 index 96ab2ecb58..0000000000 --- a/src/ovms_lib/custom_node_interface.h +++ /dev/null @@ -1,79 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -typedef enum { - UNSPECIFIED, - FP32, - FP16, - U8, - I8, - I16, - U16, - I32, - FP64, - I64 -} CustomNodeTensorPrecision; - -struct CustomNodeTensor { - const char* name; - uint8_t* data; - uint64_t dataBytes; - uint64_t* dims; - uint64_t dimsCount; - CustomNodeTensorPrecision precision; -}; - -struct CustomNodeTensorInfo { - const char* name; - uint64_t* dims; - uint64_t dimsCount; - CustomNodeTensorPrecision precision; -}; - -struct CustomNodeParam { - const char *key, *value; -}; - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief Custom node library initialize enables creation of resources to be reused between predictions. - * Potential use cases include optimized temporary buffers allocation. - * Using initialize is optional and not required for custom node to work. - * CustomNodeLibraryInternalManager should be created here if initialize is used. - * On initialize failure status not equal to zero is returned and error log is printed. - */ -int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount); -/** - * @brief Custom node library deinitialize enables destruction of resources that were used between predictions. - * Using deinitialize is optional and not required for custom node to work. - * CustomNodeLibraryInternalManager should be destroyed here if deinitialize is used. - * On deinitialize failure only error log is printed. - */ -int deinitialize(void* customNodeLibraryInternalManager); -int execute(const struct CustomNodeTensor* inputs, int inputsCount, struct CustomNodeTensor** outputs, int* outputsCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); -int getInputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); -int getOutputsInfo(struct CustomNodeTensorInfo** info, int* infoCount, const struct CustomNodeParam* params, int paramsCount, void* customNodeLibraryInternalManager); -int release(void* ptr, void* customNodeLibraryInternalManager); - -#ifdef __cplusplus -} -#endif diff --git a/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp b/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp deleted file mode 100644 index 849d1a271c..0000000000 --- a/src/ovms_lib/custom_node_library_internal_manager_wrapper.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "custom_node_library_internal_manager_wrapper.hpp" - -namespace ovms { - -void* getCNLIMWrapperPtr(const std::shared_ptr& wrapper) { - if (wrapper == nullptr) { - return nullptr; - } - return wrapper->ptr; -} - -} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp b/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp deleted file mode 100644 index 8b68bc33f7..0000000000 --- a/src/ovms_lib/custom_node_library_internal_manager_wrapper.hpp +++ /dev/null @@ -1,40 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include "node_library.hpp" - -namespace ovms { - -struct CNLIMWrapper { - void* ptr; - deinitialize_fn deinitialize = nullptr; - - CNLIMWrapper(void* CNLIM, deinitialize_fn deinitialize) : - ptr(CNLIM), - deinitialize(deinitialize) {} - - ~CNLIMWrapper() { - deinitialize(ptr); - } -}; - -void* getCNLIMWrapperPtr(const std::shared_ptr& wrapper); - -} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_manager.cpp b/src/ovms_lib/custom_node_library_manager.cpp deleted file mode 100644 index 2eeb5eb8d1..0000000000 --- a/src/ovms_lib/custom_node_library_manager.cpp +++ /dev/null @@ -1,135 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "custom_node_library_manager.hpp" - -#include - -#include - -#include "filesystem.hpp" -#include "logging.hpp" -#include "status.hpp" - -namespace ovms { - -Status CustomNodeLibraryManager::loadLibrary(const std::string& name, const std::string& basePath) { - if (FileSystem::isPathEscaped(basePath)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", basePath); - return StatusCode::PATH_INVALID; - } - - auto it = libraries.find(name); - if (it != libraries.end() && it->second.basePath == basePath) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom node library name: {} is already loaded", name); - return StatusCode::NODE_LIBRARY_ALREADY_LOADED; - } - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Loading custom node library name: {}; base_path: {}", name, basePath); - - void* handle = dlopen(basePath.c_str(), RTLD_LAZY | RTLD_LOCAL); - char* error = dlerror(); - if (handle == NULL) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Library name: {} failed to open base_path: {} with error: {}", name, basePath, error); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_OPEN; - } - - initialize_fn initialize = reinterpret_cast(dlsym(handle, "initialize")); - error = dlerror(); - if (error || initialize == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - deinitialize_fn deinitialize = reinterpret_cast(dlsym(handle, "deinitialize")); - error = dlerror(); - if (error || deinitialize == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - execute_fn execute = reinterpret_cast(dlsym(handle, "execute")); - error = dlerror(); - if (error || execute == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - metadata_fn getInputsInfo = reinterpret_cast(dlsym(handle, "getInputsInfo")); - error = dlerror(); - if (error || getInputsInfo == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - metadata_fn getOutputsInfo = reinterpret_cast(dlsym(handle, "getOutputsInfo")); - error = dlerror(); - if (error || getOutputsInfo == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - release_fn release = reinterpret_cast(dlsym(handle, "release")); - error = dlerror(); - if (error || release == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load library name: {} with error: {}", name, error); - dlclose(handle); - return StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM; - } - - libraries[name] = NodeLibrary{ - initialize, - deinitialize, - execute, - getInputsInfo, - getOutputsInfo, - release, - basePath}; - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Successfully loaded custom node library name: {}; base_path: {}", name, basePath); - return StatusCode::OK; -} - -Status CustomNodeLibraryManager::getLibrary(const std::string& name, NodeLibrary& library) const { - auto it = libraries.find(name); - if (it == libraries.end()) { - return StatusCode::NODE_LIBRARY_MISSING; - } else { - library = it->second; - return StatusCode::OK; - } -} - -void CustomNodeLibraryManager::unloadLibrariesRemovedFromConfig(const std::set& librariesInConfig) { - std::set librariesCurrentlyLoaded; - for (auto& library : libraries) { - librariesCurrentlyLoaded.emplace(library.first); - } - std::set librariesToUnload; - std::set_difference( - librariesCurrentlyLoaded.begin(), librariesCurrentlyLoaded.end(), - librariesInConfig.begin(), librariesInConfig.end(), - std::inserter(librariesToUnload, librariesToUnload.end())); - for (auto& library : librariesToUnload) { - libraries.erase(library); - } -} - -} // namespace ovms diff --git a/src/ovms_lib/custom_node_library_manager.hpp b/src/ovms_lib/custom_node_library_manager.hpp deleted file mode 100644 index 0a899f0e57..0000000000 --- a/src/ovms_lib/custom_node_library_manager.hpp +++ /dev/null @@ -1,36 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include "node_library.hpp" -#include "status.hpp" - -namespace ovms { - -class CustomNodeLibraryManager { - std::unordered_map libraries; - -public: - Status loadLibrary(const std::string& name, const std::string& basePath); - Status getLibrary(const std::string& name, NodeLibrary& library) const; - void unloadLibrariesRemovedFromConfig(const std::set& librariesInConfig); -}; - -} // namespace ovms diff --git a/src/ovms_lib/custom_node_output_allocator.cpp b/src/ovms_lib/custom_node_output_allocator.cpp deleted file mode 100644 index a4a4b6e536..0000000000 --- a/src/ovms_lib/custom_node_output_allocator.cpp +++ /dev/null @@ -1,54 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "custom_node_output_allocator.hpp" - -#include "logging.hpp" - -namespace ovms { -bool operator==(const CustomNodeTensor& t1, const CustomNodeTensor& t2) { - return (t1.name == t2.name) && - (t1.data == t2.data) && - (t1.dataBytes == t2.dataBytes) && - (t1.dims == t2.dims) && - (t1.dimsCount == t2.dimsCount) && - (t1.precision == t2.precision); -} -CustomNodeOutputAllocator::CustomNodeOutputAllocator(struct CustomNodeTensor tensor, NodeLibrary nodeLibrary, void* customNodeLibraryInternalManager) : - tensor(tensor), - nodeLibrary(nodeLibrary), - customNodeLibraryInternalManager(customNodeLibraryInternalManager) {} -void* CustomNodeOutputAllocator::allocate(const size_t bytes, const size_t alignment) { - return (void*)tensor.data; -} -void CustomNodeOutputAllocator::deallocate(void* handle, const size_t bytes, size_t alignment) { - bool succeeded = nodeLibrary.release(tensor.data, customNodeLibraryInternalManager) == 0; - if (false == succeeded) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to release custom node tensor:{} buffer using library:{}", tensor.name, nodeLibrary.basePath); - } -} -bool CustomNodeOutputAllocator::is_equal(const CustomNodeOutputAllocator& other) const { - return (customNodeLibraryInternalManager == other.customNodeLibraryInternalManager) && - (nodeLibrary == other.nodeLibrary) && - (tensor == other.tensor); -} -bool CustomNodeOutputAllocator::is_equal(const AllocatorImpl& other) const { - const CustomNodeOutputAllocator* otherPtr = dynamic_cast(&other); - if (otherPtr == nullptr) { - return false; - } - return this->is_equal(*otherPtr); -} -} // namespace ovms diff --git a/src/ovms_lib/custom_node_output_allocator.hpp b/src/ovms_lib/custom_node_output_allocator.hpp deleted file mode 100644 index b2b0a84e43..0000000000 --- a/src/ovms_lib/custom_node_output_allocator.hpp +++ /dev/null @@ -1,39 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "custom_node_interface.h" // NOLINT -#include "node_library.hpp" - -namespace ovms { - -bool operator==(const CustomNodeTensor& t1, const CustomNodeTensor& t2); - -class CustomNodeOutputAllocator : public ov::AllocatorImpl { - struct CustomNodeTensor tensor; - NodeLibrary nodeLibrary; - void* customNodeLibraryInternalManager; - -public: - CustomNodeOutputAllocator(struct CustomNodeTensor tensor, NodeLibrary nodeLibrary, void* customNodeLibraryInternalManager); - void* allocate(const size_t bytes, const size_t alignment = alignof(max_align_t)) override; - void deallocate(void* handle, const size_t bytes, size_t alignment = alignof(max_align_t)) override; - bool is_equal(const CustomNodeOutputAllocator& other) const; - bool is_equal(const AllocatorImpl& other) const override; -}; -} // namespace ovms diff --git a/src/ovms_lib/customloaderconfig.hpp b/src/ovms_lib/customloaderconfig.hpp deleted file mode 100644 index 8251acc2fb..0000000000 --- a/src/ovms_lib/customloaderconfig.hpp +++ /dev/null @@ -1,145 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include -#include -#include -#include -#include - -#include "status.hpp" -#include "stringutils.hpp" - -namespace ovms { - -/** - * @brief This class represents Custom Loader configuration - */ -class CustomLoaderConfig { -private: - /** - * @brief Custom Loader Name - */ - std::string loaderName; - - /** - * @brief Custom Loader Library Path - */ - std::string libraryPath; - - /** - * @brief Custom Loader Config Path - */ - std::string loaderConfigFile; - -public: - /** - * @brief Construct a new Custom Loader Config object - * - * @param name - * @param libraryPath - * @param configPath - */ - CustomLoaderConfig(const std::string& loaderName = "", - const std::string& libraryPath = "", - const std::string& loaderConfigFile = "") : - loaderName(loaderName), - libraryPath(libraryPath), - loaderConfigFile(loaderConfigFile) { - } - - void clear() { - loaderName.clear(); - libraryPath.clear(); - loaderConfigFile.clear(); - } - - /** - * @brief Get the name - * - * @return const std::string& - */ - const std::string& getLoaderName() const { - return this->loaderName; - } - - /** - * @brief Set the name - * - * @param name - */ - void setLoaderName(const std::string& loaderName) { - this->loaderName = loaderName; - } - - /** - * @brief Get the Library Path - * - * @return const std::string& - */ - const std::string& getLibraryPath() const { - return this->libraryPath; - } - - /** - * @brief Set the Library Path - * - * @param libraryPath - */ - void setLibraryPath(const std::string& libraryPath) { - this->libraryPath = libraryPath; - } - - /** - * @brief Get the Config Path - * - * @return const std::string& - */ - const std::string& getLoaderConfigFile() const { - return this->loaderConfigFile; - } - - /** - * @brief Set the Config Path - * - * @param configPath - */ - void setLoaderConfigFile(const std::string& loaderConfigFile) { - this->loaderConfigFile = loaderConfigFile; - } - - /** - * @brief Parses all settings from a JSON node - * - * @return Status - */ - Status parseNode(const rapidjson::Value& v) { - try { - this->setLoaderName(v["loader_name"].GetString()); - this->setLibraryPath(v["library_path"].GetString()); - if (v.HasMember("loader_config_file")) - this->setLoaderConfigFile(v["loader_config_file"].GetString()); - } catch (...) { - SPDLOG_ERROR("There was an error parsing the custom loader config"); - return StatusCode::JSON_INVALID; - } - return StatusCode::OK; - } -}; -} // namespace ovms diff --git a/src/ovms_lib/customloaderinterface.hpp b/src/ovms_lib/customloaderinterface.hpp deleted file mode 100644 index bd44c2df72..0000000000 --- a/src/ovms_lib/customloaderinterface.hpp +++ /dev/null @@ -1,117 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -namespace ovms { - -enum class CustomLoaderStatus { - OK, /*!< Success */ - MODEL_TYPE_IR, /*!< When model buffers are returned, they belong to IR model */ - MODEL_TYPE_ONNX, /*!< When model buffers are returned, they belong to ONXX model */ - MODEL_TYPE_BLOB, /*!< When model buffers are returned, they belong to Blob */ - MODEL_LOAD_ERROR, /*!< Error while loading the model */ - MODEL_BLACKLISTED, /*!< Model is blacklisted. Do not load */ - INTERNAL_ERROR /*!< generic error */ -}; - -/** - * @brief This class is the custom loader interface base class. - * Custom Loader implementation shall derive from this base calss - * and implement interface functions and define the virtual functions. - * Based on the config file, OVMS loads a model using specified custom loader - */ -class CustomLoaderInterface { -public: - /** - * @brief Constructor - */ - CustomLoaderInterface() { - } - /** - * @brief Destructor - */ - virtual ~CustomLoaderInterface() { - } - - /** - * @brief Initialize the custom loader - * - * @param loader config file defined under custom loader config in the config file - * - * @return status - */ - virtual CustomLoaderStatus loaderInit(const std::string& loaderConfigFile) = 0; - - /** - * @brief Load the model by the custom loader - * - * @param model name required to be loaded - defined under model config in the config file - * @param base path where the required model files are present - * @param version of the model - * @param loader config parameters json as string - * @param vector of uint8_t of model - * @param vector of uint8_t of weights - * @return status (On success, the return value will specify the type of model (IR,ONNX,BLOB) read into vectors) - */ - virtual CustomLoaderStatus loadModel(const std::string& modelName, - const std::string& basePath, - const int version, - const std::string& loaderOptions, - std::vector& modelBuffer, - std::vector& weights) = 0; - - /** - * @brief Get the model black list status - * - * @param model name for which black list status is required - * @param version for which the black list status is required - * @return blacklist status OK or MODEL_BLACKLISTED - */ - virtual CustomLoaderStatus getModelBlacklistStatus(const std::string& modelName, const int version) { - return CustomLoaderStatus::OK; - } - - /** - * @brief Unload model resources by custom loader once model is unloaded by OVMS - * - * @param model name which is been unloaded - * @param version which is been unloaded - * @return status - */ - virtual CustomLoaderStatus unloadModel(const std::string& modelName, const int version) = 0; - - /** - * @brief Retire the model from customloader when OVMS retires the model - * - * @param model name which is being retired - * @return status - */ - virtual CustomLoaderStatus retireModel(const std::string& modelName) = 0; - - /** - * @brief Deinitialize the custom loader - * - */ - virtual CustomLoaderStatus loaderDeInit() = 0; -}; - -// the types of the class factories -typedef CustomLoaderInterface* createCustomLoader_t(); - -} // namespace ovms diff --git a/src/ovms_lib/customloaders.cpp b/src/ovms_lib/customloaders.cpp deleted file mode 100644 index 77aa1193ad..0000000000 --- a/src/ovms_lib/customloaders.cpp +++ /dev/null @@ -1,90 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "customloaders.hpp" - -#include - -#include "customloaderinterface.hpp" - -namespace ovms { - -Status CustomLoaders::add(std::string name, std::shared_ptr loaderInterface, void* library) { - auto loaderIt = newCustomLoaderInterfacePtrs.emplace(name, std::make_pair(library, loaderInterface)); - // if the loader already exists, print an error message - if (!loaderIt.second) { - SPDLOG_ERROR("The loader {} already exists in the config file", name); - return StatusCode::CUSTOM_LOADER_EXISTS; - } - - return StatusCode::OK; -} - -Status CustomLoaders::remove(const std::string& name) { - SPDLOG_INFO("Removing loader {} from loaders list", name); - auto loaderIt = customLoaderInterfacePtrs.find(name); - - if (loaderIt == customLoaderInterfacePtrs.end()) { - return StatusCode::CUSTOM_LOADER_NOT_PRESENT; - } - - customLoaderInterfacePtrs.erase(loaderIt); - return StatusCode::OK; -} - -std::shared_ptr CustomLoaders::find(const std::string& name) { - SPDLOG_DEBUG("Looking for loader {} in loaders list", name); - auto loaderIt = customLoaderInterfacePtrs.find(name); - - if (loaderIt == customLoaderInterfacePtrs.end()) { - return nullptr; - } - - return (loaderIt->second).second; -} - -Status CustomLoaders::move(const std::string& name) { - SPDLOG_INFO("Moving loader {} from old to new loaders list", name); - auto loaderIt = customLoaderInterfacePtrs.find(name); - - if (loaderIt == customLoaderInterfacePtrs.end()) { - return StatusCode::INTERNAL_ERROR; - } - - newCustomLoaderInterfacePtrs.insert({name, customLoaderInterfacePtrs[name]}); - customLoaderInterfacePtrs.erase(loaderIt); - return StatusCode::OK; -} - -Status CustomLoaders::finalize() { - // By now the remaining loaders in current list are not there in new config. Delete them - for (auto it = customLoaderInterfacePtrs.begin(); it != customLoaderInterfacePtrs.end(); it++) { - SPDLOG_INFO("Loader {} is not there in new list.. deleting the same", it->first); - auto loaderPtr = (it->second).second; - loaderPtr->loaderDeInit(); - } - - SPDLOG_INFO("Clearing the list"); - customLoaderInterfacePtrs.clear(); - - SPDLOG_INFO("Adding new list to the old list"); - // now assign new map to servicing map. - customLoaderInterfacePtrs.insert(newCustomLoaderInterfacePtrs.begin(), newCustomLoaderInterfacePtrs.end()); - newCustomLoaderInterfacePtrs.clear(); - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/customloaders.hpp b/src/ovms_lib/customloaders.hpp deleted file mode 100644 index 7d8b4333e6..0000000000 --- a/src/ovms_lib/customloaders.hpp +++ /dev/null @@ -1,92 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include "customloaderinterface.hpp" -#include "status.hpp" - -namespace ovms { -/** - * @brief Provides all customloaders - */ -class CustomLoaders { -private: - /** - * @brief A default constructor is private - */ - CustomLoaders() = default; - - /** - * @brief Private copying constructor - */ - CustomLoaders(const CustomLoaders&) = delete; - - std::map>> customLoaderInterfacePtrs; - std::map>> newCustomLoaderInterfacePtrs; - - std::vector currentCustomLoaderNames; - -public: - /** - * @brief Gets the instance of the CustomLoaders - */ - static CustomLoaders& instance() { - static CustomLoaders instance; - return instance; - } - - /** - * @brief insert a new customloader - * - * @return status - */ - Status add(std::string name, std::shared_ptr loaderInsterface, void* library); - - /** - * @brief remove an existing customLoader referenced by it's name - * - * @return status - */ - Status remove(const std::string& name); - - /** - * @brief find an existing customLoader referenced by it's name - * - * @return pointer to customloader Interface if found, else NULL - */ - std::shared_ptr find(const std::string& name); - - /** - * @brief move the existing loader from serviced map to new map. - * - * @return status - */ - Status move(const std::string& name); - - /** - * @brief finalize the customloaders list after parsing config - * - * @return status - */ - Status finalize(); -}; -} // namespace ovms diff --git a/src/ovms_lib/customnodesession.cpp b/src/ovms_lib/customnodesession.cpp deleted file mode 100644 index 70b20ac90d..0000000000 --- a/src/ovms_lib/customnodesession.cpp +++ /dev/null @@ -1,250 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "customnodesession.hpp" - -#include -#include -#include -#include - -#include "custom_node_output_allocator.hpp" -#include "logging.hpp" -#include "node.hpp" -#include "node_library.hpp" -#include "node_library_utils.hpp" -#include "nodeinputhandler.hpp" -#include "pipelineeventqueue.hpp" -#include "profiler.hpp" -#include "timer.hpp" - -namespace ovms { - -CustomNodeSession::CustomNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : - NodeSession(metadata, nodeName, inputsCount, collapsingDetails) {} - -CustomNodeSession::CustomNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : - NodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails) {} - -CustomNodeSession::~CustomNodeSession() = default; - -static std::unordered_map createOwnedShapesCopy(const TensorMap& tensorMap) { - std::unordered_map tensorsDims; - for (auto& [name, tensor] : tensorMap) { - shape_t tensorDims = tensor.get_shape(); - tensorsDims.emplace(name, std::move(tensorDims)); - } - return tensorsDims; -} - -Status CustomNodeSession::execute(PipelineEventQueue& notifyEndQueue, Node& node, const NodeLibrary& library, std::unique_ptr& parameters, int parametersCount, void* customNodeLibraryInternalManager) { - OVMS_PROFILE_FUNCTION(); - const auto& tensorMap = this->inputHandler->getInputs(); - auto inputTensorsCount = tensorMap.size(); - // this is a hack to overcome OV 1.0 -> 2.0 API change where we do not get reference to - // tensor shape now but a copy. Hence we have to extend the lifetime of dims vector - auto tensorsDims = createOwnedShapesCopy(tensorMap); - auto inputTensors = createCustomNodeTensorArray(tensorMap, tensorsDims); - struct CustomNodeTensor* outputTensors = nullptr; - int outputTensorsCount = 0; - this->timer->start("execution"); - OVMS_PROFILE_SYNC_BEGIN("Custom Node Library execute()"); - int result = library.execute( - inputTensors.get(), - inputTensorsCount, - &outputTensors, - &outputTensorsCount, - parameters.get(), - parametersCount, - customNodeLibraryInternalManager); - OVMS_PROFILE_SYNC_END("Custom Node Library execute()"); - this->timer->stop("execution"); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Custom node execution processing time for node {}; session: {} - {} ms", - this->getName(), - this->getSessionKey(), - this->timer->elapsed("execution") / 1000); - - // If result is not 0, it means execution has failed. - // In this case shared library is responsible for cleaning up resources (memory). - if (result != 0) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has failed custom node execution with return code: {}", getName(), getSessionKey(), result); - notifyEndQueue.push({node, getSessionKey()}); - return StatusCode::NODE_LIBRARY_EXECUTION_FAILED; - } - // In other cases we are responsible of cleaning whatever is possible. - if (outputTensors == nullptr) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has corrupted outputs handle", getName(), getSessionKey()); - notifyEndQueue.push({node, getSessionKey()}); - return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED; - } - - if (outputTensorsCount <= 0) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; has corrupted number of outputs", getName(), getSessionKey()); - library.release(outputTensors, customNodeLibraryInternalManager); - notifyEndQueue.push({node, getSessionKey()}); - return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT; - } - - // At this point this is important we do not exit before finishing the loop. - // There will be memory leak if any tensor is not converted into ov::Tensor. - // ov::Tensor destructor is responsible for cleaning up resources. - Status status = StatusCode::OK; - for (int i = 0; i < outputTensorsCount; i++) { - ov::Tensor resultTensor; - auto result = this->createTensor(&outputTensors[i], resultTensor, library, customNodeLibraryInternalManager); - if (outputTensors[i].name == nullptr) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; failed tensor conversion - missing output name", getName(), getSessionKey()); - status = StatusCode::NODE_LIBRARY_OUTPUT_MISSING_NAME; - continue; - } - if (!result.ok()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; failed to convert {}: to tensor", getName(), getSessionKey(), outputTensors[i].name); - if (status.ok()) { - status = result; - } - continue; - } - this->resultTensors.emplace(std::string(outputTensors[i].name), std::move(resultTensor)); - } - - library.release(outputTensors, customNodeLibraryInternalManager); - notifyEndQueue.push({node, getSessionKey()}); - return status; -} - -Status CustomNodeSession::fetchResult(const std::string& name, ov::Tensor& resultTensor) { - auto it = resultTensors.find(name); - if (it == resultTensors.end()) { - return StatusCode::NODE_LIBRARY_MISSING_OUTPUT; - } - resultTensor = it->second; - return StatusCode::OK; -} - -void CustomNodeSession::releaseTensorResources(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) { - if (tensor->data) { - library.release(tensor->data, customNodeLibraryInternalManager); - } - if (tensor->dims) { - library.release(tensor->dims, customNodeLibraryInternalManager); - } -} - -class TensorResourcesGuard { - const struct CustomNodeTensor* tensor; - const NodeLibrary& library; - bool persistData = false; - void* customNodeLibraryInternalManager; - -public: - TensorResourcesGuard(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) : - tensor(tensor), - library(library), - customNodeLibraryInternalManager(customNodeLibraryInternalManager) {} - ~TensorResourcesGuard() { - if (tensor->data && !persistData) { - library.release(tensor->data, customNodeLibraryInternalManager); - } - if (tensor->dims) { - library.release(tensor->dims, customNodeLibraryInternalManager); - } - } - void setPersistData() { - this->persistData = true; - } -}; - -Status CustomNodeSession::createTensor(const struct CustomNodeTensor* tensor, ov::Tensor& resultTensor, const NodeLibrary& library, void* customNodeLibraryInternalManager) { - TensorResourcesGuard tensorResourcesGuard(tensor, library, customNodeLibraryInternalManager); - - auto precision = ovmsPrecisionToIE2Precision(toInferenceEnginePrecision(tensor->precision)); - if (precision == ov::element::Type_t::undefined) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; Unspecified output precision:{} from custom node tensor: {}", - this->getName(), - this->getSessionKey(), - precision, - tensor->name); - return StatusCode::NODE_LIBRARY_INVALID_PRECISION; - } - - if (tensor->dims == nullptr || tensor->dimsCount == 0) { - std::string error; - if (tensor->dims == nullptr) { - error = "shape handle is null"; - } else if (tensor->dimsCount == 0) { - error = "shape dimensions number is equal to 0"; - } - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; error: {}", - this->getName(), - this->getSessionKey(), - error); - return StatusCode::NODE_LIBRARY_INVALID_SHAPE; - } - shape_t shape(tensor->dims, tensor->dims + tensor->dimsCount); - - size_t expectedElementsCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies()); - size_t expectedDataLength = expectedElementsCount *= ov::element::Type(precision).size(); - if (tensor->data == nullptr || tensor->dataBytes != expectedDataLength) { - std::stringstream error; - if (tensor->data == nullptr) { - error << "data handle is null"; - } else if (tensor->dataBytes != expectedDataLength) { - error << "not expected data length: expected: " << expectedDataLength << " vs " << tensor->dataBytes; - } - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node {}; session: {}; error: {}", - this->getName(), - this->getSessionKey(), - error.str()); - return StatusCode::NODE_LIBRARY_INVALID_CONTENT_SIZE; - } - auto allocatorImpl = std::make_shared(*tensor, library, customNodeLibraryInternalManager); - auto allocator = ov::Allocator(allocatorImpl); - try { - switch (tensor->precision) { - case CustomNodeTensorPrecision::FP32: - case CustomNodeTensorPrecision::I32: - case CustomNodeTensorPrecision::I8: - case CustomNodeTensorPrecision::U8: - case CustomNodeTensorPrecision::FP16: - case CustomNodeTensorPrecision::I16: - case CustomNodeTensorPrecision::U16: - case CustomNodeTensorPrecision::FP64: - case CustomNodeTensorPrecision::I64: - resultTensor = ov::Tensor(ov::element::Type(ovmsPrecisionToIE2Precision(toInferenceEnginePrecision(tensor->precision))), ov::Shape(shape), allocator); - break; - case CustomNodeTensorPrecision::UNSPECIFIED: - return StatusCode::INTERNAL_ERROR; - } - } catch (const ov::Exception& e) { - Status status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_LOGGER_ERROR(dag_executor_logger, "{}: {}", status.string(), e.what()); - return status; - } catch (std::logic_error& e) { - Status status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_LOGGER_ERROR(dag_executor_logger, "{}: {}", status.string(), e.what()); - return status; - } - tensorResourcesGuard.setPersistData(); - return StatusCode::OK; -} - -void CustomNodeSession::clearInputs() { - this->inputHandler->clearInputs(); -} - -void CustomNodeSession::release() { -} - -} // namespace ovms diff --git a/src/ovms_lib/customnodesession.hpp b/src/ovms_lib/customnodesession.hpp deleted file mode 100644 index 2c0db19b85..0000000000 --- a/src/ovms_lib/customnodesession.hpp +++ /dev/null @@ -1,59 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include - -#include "custom_node_interface.h" // NOLINT -#include "nodesession.hpp" -#include "pipelineeventqueue.hpp" -#include "status.hpp" -#include "tensormap.hpp" - -namespace ovms { - -class Node; -class NodeLibrary; - -class CustomNodeSession : public NodeSession { - TensorMap resultTensors; - -public: - CustomNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); - CustomNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); - virtual ~CustomNodeSession(); - - Status execute( - PipelineEventQueue& notifyEndQueue, - Node& node, - const NodeLibrary& library, - std::unique_ptr& parameters, - int parametersCount, - void* customNodeLibraryInternalManager); - - Status fetchResult(const std::string& name, ov::Tensor& resultTensor); - - void clearInputs(); - void release() override; - -private: - static void releaseTensorResources(const struct CustomNodeTensor* tensor, const NodeLibrary& library, void* customNodeLibraryInternalManager); - Status createTensor(const struct CustomNodeTensor* tensor, ov::Tensor& resultTensor, const NodeLibrary& library, void* customNodeLibraryInternalManager); -}; -} // namespace ovms diff --git a/src/ovms_lib/deserialization.cpp b/src/ovms_lib/deserialization.cpp deleted file mode 100644 index be4e5a6111..0000000000 --- a/src/ovms_lib/deserialization.cpp +++ /dev/null @@ -1,73 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "deserialization.hpp" - -namespace ovms { - -template <> -Status InputSink::give(const std::string& name, ov::Tensor& tensor) { - Status status; - try { - requester.set_tensor(name, tensor); - // OV implementation the ov::Exception is not - // a base class for all other exceptions thrown from OV. - // OV can throw exceptions derived from std::logic_error. - } catch (const ov::Exception& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } catch (std::logic_error& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } - - return status; -} - -ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo) { - ov::Shape shape; - for (size_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::element::Type precision = tensorInfo->getOvPrecision(); - return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.tensor_content().data()))); -} - -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo, - const std::string& buffer) { - ov::Shape shape; - for (size_t i = 0; i < requestInput.shape_size(); i++) { - shape.push_back(requestInput.shape().at(i)); - } - ov::element::Type precision = tensorInfo->getOvPrecision(); - ov::Tensor tensor(precision, shape, const_cast(reinterpret_cast(buffer.data()))); - return tensor; -} -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo) { - ov::Shape shape; - for (size_t i = 0; i < requestInput.shape_size(); i++) { - shape.push_back(requestInput.shape().at(i)); - } - ov::element::Type precision = tensorInfo->getOvPrecision(); - ov::Tensor tensor(precision, shape); - return tensor; -} - -} // namespace ovms diff --git a/src/ovms_lib/deserialization.hpp b/src/ovms_lib/deserialization.hpp deleted file mode 100644 index 6cb7f666fd..0000000000 --- a/src/ovms_lib/deserialization.hpp +++ /dev/null @@ -1,405 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" - -#include "kfs_grpc_inference_service.hpp" -#pragma GCC diagnostic pop - -#include "binaryutils.hpp" -#include "profiler.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo); - -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo, - const std::string& buffer); -ov::Tensor makeTensor(const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo); - -class ConcreteTensorProtoDeserializator { -public: - static ov::Tensor deserializeTensorProto( - const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo, - const std::string* buffer) { - if (nullptr != buffer) { - switch (tensorInfo->getPrecision()) { - case ovms::Precision::FP64: - case ovms::Precision::FP32: - case ovms::Precision::FP16: - case ovms::Precision::I64: - case ovms::Precision::I32: - case ovms::Precision::I16: - case ovms::Precision::I8: - case ovms::Precision::U64: - case ovms::Precision::U32: - case ovms::Precision::U16: - case ovms::Precision::BOOL: - case ovms::Precision::U8: { - return makeTensor(requestInput, tensorInfo, *buffer); - } - case ovms::Precision::U1: - case ovms::Precision::CUSTOM: - case ovms::Precision::UNDEFINED: - case ovms::Precision::DYNAMIC: - case ovms::Precision::MIXED: - case ovms::Precision::Q78: - default: - return ov::Tensor(); - } - } else { - switch (tensorInfo->getPrecision()) { - // bool_contents - case ovms::Precision::BOOL: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - bool* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().bool_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - /// int_contents - case ovms::Precision::I8: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - int8_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().int_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - case ovms::Precision::I16: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - int16_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().int_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - case ovms::Precision::I32: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - int32_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().int_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - /// int64_contents - case ovms::Precision::I64: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - int64_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().int64_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - // uint_contents - case ovms::Precision::U8: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - uint8_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().uint_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - case ovms::Precision::U16: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - uint16_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().uint_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - case ovms::Precision::U32: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - uint32_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().uint_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - // uint64_contents - case ovms::Precision::U64: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - uint64_t* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().uint64_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - // fp32_contents - case ovms::Precision::FP32: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - float* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().fp32_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - // fp64_contentes - case ovms::Precision::FP64: { - ov::Tensor tensor = makeTensor(requestInput, tensorInfo); - double* ptr = reinterpret_cast(tensor.data()); - size_t i = 0; - for (auto& number : requestInput.contents().fp64_contents()) { - ptr[i++] = *(const_cast(reinterpret_cast(&number))); - } - return tensor; - break; - } - case ovms::Precision::FP16: - case ovms::Precision::U1: - case ovms::Precision::CUSTOM: - case ovms::Precision::UNDEFINED: - case ovms::Precision::DYNAMIC: - case ovms::Precision::MIXED: - case ovms::Precision::Q78: - case ovms::Precision::BIN: - default: - return ov::Tensor(); - } - } - } - - static ov::Tensor deserializeTensorProto( - const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo) { - switch (tensorInfo->getPrecision()) { - case ovms::Precision::FP32: - case ovms::Precision::I32: - case ovms::Precision::FP64: - case ovms::Precision::I64: - case ovms::Precision::U8: - case ovms::Precision::I16: - case ovms::Precision::I8: { - return makeTensor(requestInput, tensorInfo); - } - case ovms::Precision::FP16: { - ov::Shape shape; - for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::Tensor tensor(ov::element::f16, shape); - // Needs conversion due to zero padding for each value: - // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 - uint16_t* ptr = (uint16_t*)tensor.data(); - auto size = static_cast(requestInput.half_val_size()); - for (size_t i = 0; i < size; i++) { - ptr[i] = requestInput.half_val(i); - } - return tensor; - } - case ovms::Precision::U16: { - ov::Shape shape; - for (std::int64_t i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - shape.push_back(requestInput.tensor_shape().dim(i).size()); - } - ov::Tensor tensor(ov::element::u16, shape); - // Needs conversion due to zero padding for each value: - // https://github.com/tensorflow/tensorflow/blob/v2.2.0/tensorflow/core/framework/tensor.proto#L55 - uint16_t* ptr = (uint16_t*)tensor.data(); - auto size = static_cast(requestInput.int_val_size()); - for (size_t i = 0; i < size; i++) { - ptr[i] = requestInput.int_val(i); - } - return tensor; - } - case ovms::Precision::U32: - case ovms::Precision::U64: - default: - return ov::Tensor(); - } - } -}; - -template -ov::Tensor deserializeTensorProto( - const tensorflow::TensorProto& requestInput, - const std::shared_ptr& tensorInfo) { - return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo); -} - -template -ov::Tensor deserializeTensorProto( - const ::inference::ModelInferRequest::InferInputTensor& requestInput, - const std::shared_ptr& tensorInfo, - const std::string* buffer) { - return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo, buffer); -} - -template -class InputSink { - Requester requester; - -public: - InputSink(Requester requester) : - requester(requester) {} - Status give(const std::string& name, ov::Tensor& tensor); -}; - -template -Status deserializePredictRequest( - const tensorflow::serving::PredictRequest& request, - const tensor_map_t& inputMap, - Sink& inputSink, bool isPipeline) { - OVMS_PROFILE_FUNCTION(); - Status status; - for (const auto& pair : inputMap) { - try { - const auto& name = pair.first; - auto tensorInfo = pair.second; - auto requestInputItr = request.inputs().find(name); - if (requestInputItr == request.inputs().end()) { - SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); - return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); - } - auto& requestInput = requestInputItr->second; - ov::Tensor tensor; - - if (requestInput.dtype() == tensorflow::DataType::DT_STRING) { - SPDLOG_DEBUG("Request contains binary input: {}", name); - status = convertBinaryRequestTensorToOVTensor(requestInput, tensor, tensorInfo); - if (!status.ok()) { - SPDLOG_DEBUG("Binary inputs conversion failed."); - return status; - } - } else { - tensor = deserializeTensorProto( - requestInput, tensorInfo); - } - - if (!tensor) { - status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; - SPDLOG_DEBUG(status.string()); - return status; - } - const std::string ovTensorName = isPipeline ? name : tensorInfo->getName(); - status = inputSink.give(ovTensorName, tensor); - if (!status.ok()) { - SPDLOG_DEBUG("Feeding input:{} to inference performer failed:{}", ovTensorName, status.string()); - return status; - } - // OV implementation the ov::Exception is not - // a base class for all other exceptions thrown from OV. - // OV can throw exceptions derived from std::logic_error. - } catch (const ov::Exception& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } catch (std::logic_error& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } - } - return status; -} - -template -Status deserializePredictRequest( - const ::inference::ModelInferRequest& request, - const tensor_map_t& inputMap, - Sink& inputSink, bool isPipeline) { - OVMS_PROFILE_FUNCTION(); - Status status; - bool deserializeFromSharedInputContents = request.raw_input_contents().size() > 0; - for (const auto& pair : inputMap) { - try { - const auto& name = pair.first; - auto tensorInfo = pair.second; - auto requestInputItr = std::find_if(request.inputs().begin(), request.inputs().end(), [&name](const ::inference::ModelInferRequest::InferInputTensor& tensor) { return tensor.name() == name; }); - if (requestInputItr == request.inputs().end()) { - SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); - return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); - } - ov::Tensor tensor; - - if (requestInputItr->datatype() == "BYTES") { - SPDLOG_DEBUG("Request contains binary input: {}", name); - status = convertBinaryRequestTensorToOVTensor(*requestInputItr, tensor, tensorInfo); - if (!status.ok()) { - SPDLOG_DEBUG("Binary inputs conversion failed."); - return status; - } - } else { - auto inputIndex = requestInputItr - request.inputs().begin(); - auto bufferLocation = deserializeFromSharedInputContents ? &request.raw_input_contents()[inputIndex] : nullptr; - - tensor = deserializeTensorProto(*requestInputItr, tensorInfo, bufferLocation); - if (!tensor) { - status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; - SPDLOG_DEBUG(status.string()); - return status; - } - } - - const std::string ovTensorName = isPipeline ? name : tensorInfo->getName(); - status = inputSink.give(ovTensorName, tensor); - if (!status.ok()) { - SPDLOG_DEBUG("Feeding input:{} to inference performer failed:{}", ovTensorName, status.string()); - return status; - } - // OV implementation the ov::Exception is not - // a base class for all other exceptions thrown from OV. - // OV can throw exceptions derived from std::logic_error. - } catch (const ov::Exception& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } catch (std::logic_error& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } - } - return status; -} -} // namespace ovms diff --git a/src/ovms_lib/dl_node.cpp b/src/ovms_lib/dl_node.cpp deleted file mode 100644 index fd6f8ce2e4..0000000000 --- a/src/ovms_lib/dl_node.cpp +++ /dev/null @@ -1,134 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "dl_node.hpp" - -#include -#include - -#include "dlnodesession.hpp" -#include "logging.hpp" -#include "modelmanager.hpp" -#include "ov_utils.hpp" -#include "ovinferrequestsqueue.hpp" -#include "prediction_service_utils.hpp" -#include "timer.hpp" - -namespace ovms { - -const uint WAIT_FOR_STREAM_ID_TIMEOUT_MICROSECONDS = 1; - -Status DLNode::execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) { - auto& nodeSession = getNodeSession(sessionKey); - auto& dlNodeSession = static_cast(nodeSession); - return dlNodeSession.execute(notifyEndQueue, WAIT_FOR_STREAM_ID_TIMEOUT_MICROSECONDS, *this); -} - -Status DLNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { - auto& dlNodeSession = static_cast(nodeSession); - const auto& sessionMetadata = nodeSession.getNodeSessionMetadata(); - SessionResult sessionResults{sessionMetadata, {}}; - auto it = nodeSessionOutputs.emplace(sessionMetadata.getSessionKey(), std::move(sessionResults)); - if (!it.second) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to put node: {} session: {} results in node session outputs", - getName(), nodeSession.getSessionKey()); - return StatusCode::INTERNAL_ERROR; - } - auto& metadataTensorResultsPair = it.first->second; - auto& tensorResults = metadataTensorResultsPair.second; - Status status; - const uint waitTimeMicroseconds = 1; - auto& inferRequest = dlNodeSession.getInferRequest(waitTimeMicroseconds); - auto& model = dlNodeSession.getModelInstance(); - status = this->fetchResults(tensorResults, inferRequest, model, nodeSession.getSessionKey()); - return status; -} - -Status DLNode::fetchResults(TensorWithSourceMap& outputs, ov::InferRequest& inferRequest, ModelInstance& model, session_key_t sessionKey) { - ReleaseSessionGuard releaseSessionGuard(this->getNodeSession(sessionKey)); - // Wait for tensor results - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Waiting for infer request to finish", getName(), sessionKey); - try { - inferRequest.wait(); - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} IE exception occured during infer request wait: {}", getName(), sessionKey, e.what()); - return StatusCode::INTERNAL_ERROR; - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} session: {} exception occured during infer request wait: {}", getName(), sessionKey, e.what()); - return StatusCode::INTERNAL_ERROR; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} infer request finished", getName(), sessionKey); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Inference processing time for node {}; model name: {}; session: {} - {} ms", - this->getName(), - model.getName(), - sessionKey, - this->getNodeSession(sessionKey).getTimer().elapsed("inference") / 1000); - - static_cast(this->getNodeSession(sessionKey)).clearInputs(); - - // Fill outputs map with result tensors. Fetch only those that are required in following nodes. - for (const auto& node : this->next) { - for (const auto& pair : node.get().getMappingByDependency(*this)) { - const auto& output_name = pair.first; - if (outputs.find(output_name) != outputs.end()) { - continue; - } - - try { - std::string realModelOutputName; - if (!getRealOutputName(model, output_name, &realModelOutputName).ok()) { - SPDLOG_LOGGER_WARN(dag_executor_logger, "Node: {} session: {} Cannot find real model output name for alias: {}", getName(), sessionKey, output_name); - return StatusCode::INTERNAL_ERROR; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Getting tensor from model: {}, inferRequestStreamId: {}, tensorName: {}", - getName(), sessionKey, modelName, sessionKey, realModelOutputName); - const auto tensor = inferRequest.get_tensor(realModelOutputName); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Creating copy of tensor from model: {}, tensorName: {}", - getName(), sessionKey, modelName, realModelOutputName); - ov::Tensor copiedTensor; - auto status = tensorClone(copiedTensor, tensor); - if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Could not clone result tensor; node: {}; session: {}; model name: {}; output: {}", - getName(), - this->modelName, - realModelOutputName); - return status; - } - outputs.emplace(std::make_pair(output_name, TensorWithSource(std::move(copiedTensor)))); - } catch (const ov::Exception& e) { - Status status = StatusCode::OV_INTERNAL_SERIALIZATION_ERROR; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session:{} Error during getting tensor {}; exception message: {}", getName(), sessionKey, status.string(), e.what()); - return status; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} Tensor with name {} has been prepared", getName(), sessionKey, output_name); - } - } - return StatusCode::OK; -} - -void DLNode::release(session_key_t sessionId) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Release node: {} sessionKey: {}", getName(), sessionId); - getNodeSession(sessionId).release(); -} -bool DLNode::tryDisarm(const session_key_t& sessionKey, const uint microseconds) { - return getNodeSession(sessionKey).tryDisarm(microseconds); -} - -std::unique_ptr DLNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { - return std::make_unique(metadata, getName(), previous.size(), collapsingDetails, - this->modelManager, this->modelName, this->modelVersion.value_or(0)); -} - -} // namespace ovms diff --git a/src/ovms_lib/dl_node.hpp b/src/ovms_lib/dl_node.hpp deleted file mode 100644 index a0b4f219bf..0000000000 --- a/src/ovms_lib/dl_node.hpp +++ /dev/null @@ -1,98 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "executingstreamidguard.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelversion.hpp" -#include "node.hpp" -#include "nodestreamidguard.hpp" - -namespace ovms { - -class ModelManager; - -class DLNode : public Node { -protected: - std::string modelName; - std::optional modelVersion; - ModelManager& modelManager; - const std::unordered_map nodeOutputNameAlias; - - std::shared_ptr model; - std::unique_ptr nodeStreamIdGuard; - std::unique_ptr modelUnloadGuard; - -public: - DLNode(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, - std::unordered_map nodeOutputNameAlias = {}, - std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}) : - Node(nodeName, demultiplyCount, gatherFromNode), - modelName(modelName), - modelVersion(modelVersion), - modelManager(modelManager), - nodeOutputNameAlias(nodeOutputNameAlias) { - } - - Status execute(session_key_t sessionKey, PipelineEventQueue& notifyEndQueue) override; - - Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; - -private: - Status fetchResults(TensorWithSourceMap& outputs, ov::InferRequest& inferRequest, ModelInstance& model, session_key_t sessionKey); - -public: - void release(session_key_t sessionId) override; - -private: - Status getRealInputName(ModelInstance& model, const std::string& alias, std::string* result) const { - auto it = model.getInputsInfo().find(alias); - if (it == model.getInputsInfo().end()) { - return StatusCode::INVALID_MISSING_INPUT; - } - *result = it->second->getName(); - return StatusCode::OK; - } - - Status getRealOutputName(ModelInstance& model, const std::string& alias, std::string* result) const { - auto it = nodeOutputNameAlias.find(alias); - const auto& modelOutputName = it != nodeOutputNameAlias.end() ? it->second : alias; - auto jt = model.getOutputsInfo().find(modelOutputName); - if (jt == model.getOutputsInfo().end()) { - return StatusCode::INVALID_MISSING_OUTPUT; - } - *result = jt->second->getName(); - return StatusCode::OK; - } - - Status executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest& infer_request); - bool tryDisarm(const session_key_t& sessionKey, const uint microseconds = 1) override; - -protected: - std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; -}; - -} // namespace ovms diff --git a/src/ovms_lib/dlnodesession.cpp b/src/ovms_lib/dlnodesession.cpp deleted file mode 100644 index abc0d37721..0000000000 --- a/src/ovms_lib/dlnodesession.cpp +++ /dev/null @@ -1,345 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "dlnodesession.hpp" - -#include -#include - -#include "logging.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "nodeinputhandler.hpp" -#include "nodeoutputhandler.hpp" -#include "nodestreamidguard.hpp" -#include "ov_utils.hpp" -#include "profiler.hpp" -#include "shape.hpp" -#include "tensorinfo.hpp" -#include "timer.hpp" - -namespace ovms { -DLNodeSession::DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : - NodeSession(metadata, nodeName, inputsCount, collapsingDetails), - modelManager(manager), - modelName(modelName), - modelVersion(modelVersion) {} - -DLNodeSession::DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : - NodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails), - modelManager(manager), - modelName(modelName), - modelVersion(modelVersion) {} - -DLNodeSession::~DLNodeSession() = default; - -void DLNodeSession::clearInputs() { - this->inputHandler->clearInputs(); -} - -ModelInstance& DLNodeSession::getModelInstance() { - return *this->model; -} - -ov::InferRequest& DLNodeSession::getInferRequest(const uint microseconds) { - auto& inferRequestsQueue = this->model->getInferRequestsQueue(); - auto streamIdOpt = this->nodeStreamIdGuard->tryGetId(microseconds); - if (!streamIdOpt) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to get streamId on already executed node: {} session: {}", getName(), getSessionKey()); - throw std::logic_error("Stream id is empty on already executed node"); - } - return inferRequestsQueue.getInferRequest(streamIdOpt.value()); -} - -Status DLNodeSession::requestExecuteRequiredResources() { - OVMS_PROFILE_FUNCTION(); - Status status = modelManager.getModelInstance( - modelName, - modelVersion, - this->model, - this->modelUnloadGuard); - - if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Getting modelInstance failed for node: {} session: {} with: {}", getName(), getSessionKey(), status.string()); - return status; - } - - status = prepareInputsAndModelForInference(); - if (!status.ok()) { - return status; - } - this->nodeStreamIdGuard = std::make_unique(model->getInferRequestsQueue()); - return status; -} - -Status DLNodeSession::prepareInputsAndModelForInference() { - OVMS_PROFILE_FUNCTION(); - std::optional requestedBatchSize = std::nullopt; - std::map requestedReshapes; - - // Validate each tensor against its OV tensor info - const auto& inputsInfo = this->model->getInputsInfo(); - for (const auto& kv : this->inputHandler->getInputs()) { - const auto& name = kv.first; - auto& tensor = kv.second; - - auto it = inputsInfo.find(name); - if (it == inputsInfo.end()) { - std::stringstream ss; - ss << "Required input: " << name; - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Missing input with specific name - {}", getName(), details); - return Status(StatusCode::INVALID_MISSING_INPUT, details); - } - auto& inputInfo = *it->second; - auto status = validate(tensor, inputInfo); - if (status.ok()) { - continue; - } - - // If precision is incorrect, perform conversion - if (status == StatusCode::INVALID_PRECISION) { - return status; - } - - // If batch size is incorrect, perform model batch size change if allowed (shape mode=auto or batch size=auto) - if (status == StatusCode::INVALID_BATCH_SIZE) { - if (this->model->getModelConfig().getBatchingMode() == Mode::AUTO) { - requestedBatchSize = tensor.get_shape()[0]; - } else if (this->model->getModelConfig().isShapeAuto(name)) { - requestedReshapes[name] = tensor.get_shape(); - } else { - return status; - } - } - - // If shape is incorrect, perform reshape if allowed (mode=auto) - if (status == StatusCode::INVALID_SHAPE) { - if (!this->model->getModelConfig().isShapeAuto(name)) { - return status; - } - requestedReshapes[name] = tensor.get_shape(); - } - } - if (requestedReshapes.size() > 0) { - auto status = this->model->reloadModel(std::nullopt, requestedReshapes, this->modelUnloadGuard); - if (!status.ok()) { - return status; - } - } else if (requestedBatchSize.has_value()) { - auto status = this->model->reloadModel(requestedBatchSize, {}, this->modelUnloadGuard); - if (!status.ok()) { - return status; - } - } - return StatusCode::OK; -} - -Status DLNodeSession::validate(const ov::Tensor& tensor, const TensorInfo& tensorInfo) { - OVMS_PROFILE_FUNCTION(); - if (ovmsPrecisionToIE2Precision(tensorInfo.getPrecision()) != tensor.get_element_type()) { - std::stringstream ss; - ss << "Node: " << getName() << " input: " << tensorInfo.getName() - << " Invalid precision -" - << " Expected: " << tensorInfo.getPrecisionAsString() - << "; Actual: " << toString(ovElementTypeToOvmsPrecision(tensor.get_element_type())); - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); - return Status(StatusCode::INVALID_PRECISION, details); - } - - // If batch size differs, check if remaining dimensions are equal - const auto& dims = tensor.get_shape(); - const auto batchIndex = tensorInfo.getLayout().getBatchIndex(); - if (!batchIndex.has_value() || batchIndex.value() >= tensorInfo.getShape().size() || batchIndex.value() >= dims.size()) { - std::stringstream ss; - ss << "Node: " << getName() << " input: " << tensorInfo.getName() - << " Invalid batch size index"; - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); - return Status(StatusCode::INVALID_BATCH_DIMENSION, details); - } - if (!tensorInfo.getShape()[batchIndex.value()].match(dims[batchIndex.value()])) { - // If remaining dimensions are equal, it is invalid batch size - std::stringstream ss; - if (tensorInfo.getShape().match(dims, batchIndex.value())) { - ss << "Node: " << getName() << " input: " << tensorInfo.getName() - << " Invalid batch size -" - << " Expected: " << tensorInfo.getShape()[batchIndex.value()].toString() - << "; Actual: " << dims[batchIndex.value()]; - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } else { - // Otherwise whole shape is incorrect - ss << "Node: " << getName() << " input: " << tensorInfo.getName() - << " Invalid shape -" - << " Expected: " << tensorInfo.getShape().toString() - << "; Actual: " << TensorInfo::shapeToString(dims); - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); - return Status(StatusCode::INVALID_SHAPE, details); - } - } - - if (!tensorInfo.getShape().match(dims)) { - std::stringstream ss; - ss << "Node: " << getName() << " input: " << tensorInfo.getName() - << " Invalid shape -" - << " Expected: " << tensorInfo.getShape().toString() - << "; Actual: " << TensorInfo::shapeToString(dims); - const std::string details = ss.str(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, details); - return Status(StatusCode::INVALID_SHAPE, details); - } - - return StatusCode::OK; -} - -Status DLNodeSession::execute(PipelineEventQueue& notifyEndQueue, uint waitForStreamIdTimeoutMicroseconds, Node& node) { - OVMS_PROFILE_FUNCTION(); - Status status; - if (this->nodeStreamIdGuard == nullptr) { - status = requestExecuteRequiredResources(); - if (!status.ok()) { - notifyEndQueue.push({node, getSessionKey()}); - return status; - } - } - auto streamIdOpt = this->nodeStreamIdGuard->tryGetId(waitForStreamIdTimeoutMicroseconds); - if (!streamIdOpt) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Could not acquire stream Id right away", getName()); - return StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET; - } - auto& inferRequestsQueue = this->model->getInferRequestsQueue(); - auto& inferRequest = inferRequestsQueue.getInferRequest(streamIdOpt.value()); - status = setInputsForInference(inferRequest); - if (!status.ok()) { - notifyEndQueue.push({node, getSessionKey()}); - return status; - } - status = executeInference(notifyEndQueue, inferRequest, node); - if (!status.ok()) { - notifyEndQueue.push({node, getSessionKey()}); - return status; - } - return status; -} - -Status DLNodeSession::getRealInputName(const std::string& alias, std::string* result) const { - auto it = this->model->getInputsInfo().find(alias); - if (it == this->model->getInputsInfo().end()) { - return StatusCode::INVALID_MISSING_INPUT; - } - *result = it->second->getName(); - return StatusCode::OK; -} - -Status DLNodeSession::setInputsForInference(ov::InferRequest& inferRequest) { - OVMS_PROFILE_FUNCTION(); - Status status = StatusCode::OK; - try { - // Prepare inference request, fill with input tensors - for (const auto& [name, tensor] : this->inputHandler->getInputs()) { - std::string realModelInputName; - if (!getRealInputName(name, &realModelInputName).ok()) { - SPDLOG_LOGGER_WARN(dag_executor_logger, "DLNode::{} [Node name: {}]; cannot find real model input name for alias: {}", - __FUNCTION__, getName(), name); - return StatusCode::INTERNAL_ERROR; - } - // Workaround for GPU. - if (this->model->getModelConfig().isDeviceUsed("GPU")) { - ov::Tensor clonedTensor; - status = ovms::tensorClone(clonedTensor, tensor); - if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor clone error: {}", getName(), status.string()); - return status; - } - OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::set_tensor"); - inferRequest.set_tensor(realModelInputName, clonedTensor); - OVMS_PROFILE_SYNC_END("ov::InferRequest::set_tensor"); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] tensor name: {} cloned before GPU inference", getName(), name); - } else { - OVMS_PROFILE_SCOPE("ov::InferRequest::set_tensor"); - inferRequest.set_tensor(realModelInputName, tensor); - } - } - // OV implementation the ov::Exception is not - // a base class for all other exceptions thrown from OV. - // OV can throw exceptions derived from std::logic_error. - } catch (const ov::Exception& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; exception message: {}", getName(), status.string(), e.what()); - } catch (std::logic_error& e) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; exception message: {}", getName(), status.string(), e.what()); - } catch (...) { - status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] {}; with unknown exception", getName(), status.string()); - } - return status; -} - -Status DLNodeSession::executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest& inferRequest, Node& node) { - OVMS_PROFILE_FUNCTION(); - try { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Setting completion callback for node name: {}", this->getName()); - inferRequest.set_callback([this, ¬ifyEndQueue, &inferRequest, &node](std::exception_ptr exception_ptr) { - OVMS_PROFILE_ASYNC_END("async inference", this); - this->timer->stop("inference"); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Completion callback received for node name: {}", this->getName()); - // After inference is completed, input tensors are not needed anymore - this->inputHandler->clearInputs(); - notifyEndQueue.push({node, getSessionKey()}); - inferRequest.set_callback([](std::exception_ptr exception_ptr) {}); // reset callback on infer request - }); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Starting infer async for node name: {}", getName()); - this->timer->start("inference"); - OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::start_async"); - inferRequest.start_async(); - OVMS_PROFILE_SYNC_END("ov::InferRequest::start_async"); - OVMS_PROFILE_ASYNC_BEGIN("async inference", this); - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Exception occured when starting async inference or setting completion callback on model: {}, error: {}", - getName(), getModelName(), e.what()); - return StatusCode::OV_INTERNAL_INFERENCE_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Exception occured when starting async inference or setting completion callback on model: {}, error: {}", - getName(), getModelName(), e.what()); - return StatusCode::OV_INTERNAL_INFERENCE_ERROR; - } catch (...) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "[Node: {}] Unknown exception occured when starting async inference or setting completion callback on model: {}", - getName(), getModelName()); - return StatusCode::OV_INTERNAL_INFERENCE_ERROR; - } - return StatusCode::OK; -} - -void DLNodeSession::release() { - this->nodeStreamIdGuard.reset(); - this->model.reset(); - this->modelUnloadGuard.reset(); -} - -bool DLNodeSession::tryDisarm(uint microseconds) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to disarm stream id guard of node: {}", getName()); - if (this->nodeStreamIdGuard == nullptr) { - return true; - } - return this->nodeStreamIdGuard->tryDisarm(microseconds); -} -} // namespace ovms diff --git a/src/ovms_lib/dlnodesession.hpp b/src/ovms_lib/dlnodesession.hpp deleted file mode 100644 index ca9a798a3e..0000000000 --- a/src/ovms_lib/dlnodesession.hpp +++ /dev/null @@ -1,73 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include - -#include "modelversion.hpp" -#include "nodesession.hpp" -#include "pipelineeventqueue.hpp" -#include "status.hpp" - -namespace ovms { - -class ModelManager; -class ModelInstance; -class Node; -class NodeStreamIdGuard; -class ModelInstanceUnloadGuard; -class TensorInfo; - -class DLNodeSession : public NodeSession { - std::shared_ptr model; - std::unique_ptr nodeStreamIdGuard; - std::unique_ptr modelUnloadGuard; - - ModelManager& modelManager; - const std::string& modelName; - const model_version_t modelVersion; - -public: - DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); - DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); - virtual ~DLNodeSession(); - - ov::InferRequest& getInferRequest(const uint microseconds); - ModelInstance& getModelInstance(); - -private: - Status requestExecuteRequiredResources(); - -public: - Status prepareInputsAndModelForInference(); - Status validate(const ov::Tensor& tensor, const TensorInfo& info); - Status execute(PipelineEventQueue& notifyEndQueue, uint waitForStreamIdTimeoutMicroseconds, Node& node); - Status executeInference(PipelineEventQueue& notifyEndQueue, ov::InferRequest&, Node& node); - Status setInputsForInference(ov::InferRequest& inferRequest); - Status getRealInputName(const std::string& alias, std::string* result) const; - void release() override; - - void clearInputs(); - - const std::string& getModelName() { return modelName; } - bool tryDisarm(uint microseconds) override; -}; -} // namespace ovms diff --git a/src/ovms_lib/entry_node.cpp b/src/ovms_lib/entry_node.cpp deleted file mode 100644 index a01ceea217..0000000000 --- a/src/ovms_lib/entry_node.cpp +++ /dev/null @@ -1,177 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "entry_node.hpp" - -#include -#include -#include -#include -#include - -#include "binaryutils.hpp" -#include "deserialization.hpp" -#include "logging.hpp" -#include "ov_utils.hpp" -#include "predict_request_validation_utils.hpp" -#include "profiler.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -namespace ovms { - -template -Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) { - OVMS_PROFILE_FUNCTION(); - // this should be created in EntryNode::SetInputs, or special method for entry node called - // in event loop can be done in future release while implementing dynamic demultiplexing at - // entry node - NodeSessionMetadata metadata; - auto nodeSession = getNodeSession(metadata); // call to create session - if (!nodeSession) { - notifyEndQueue.push(NodeSessionKeyPair(*this, nodeSession->getSessionKey())); - return StatusCode::INTERNAL_ERROR; - } - notifyEndQueue.push(NodeSessionKeyPair(*this, nodeSession->getSessionKey())); - return StatusCode::OK; -} - -template -Status EntryNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { - OVMS_PROFILE_FUNCTION(); - TensorWithSourceMap outputs; - auto status = fetchResults(outputs); - if (!status.ok()) { - return status; - } - SessionResult metaOutputsPair{nodeSession.getNodeSessionMetadata(), std::move(outputs)}; - auto it = nodeSessionOutputs.emplace(nodeSession.getSessionKey(), std::move(metaOutputsPair)); - if (!it.second) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Failed to set entry node session results."); - return StatusCode::UNKNOWN_ERROR; - } - return StatusCode::OK; -} - -template -Status EntryNode::fetchResults(TensorWithSourceMap& outputs) { - auto status = validate(); - if (!status.ok()) { - return status; - } - InputSink inputSink(outputs); - bool isPipeline = true; - return deserializePredictRequest(*request, inputsInfo, inputSink, isPipeline); -} - -template <> -Status InputSink::give(const std::string& name, ov::Tensor& tensor) { - requester.emplace(std::make_pair(name, TensorWithSource(tensor))); - return StatusCode::OK; -} - -template <> -Status EntryNode::isInputBinary(const std::string& name, bool& isBinary) const { - auto it = request->inputs().find(name); - if (it == request->inputs().end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Error during checking binary input; input: {} does not exist", name); - return StatusCode::INTERNAL_ERROR; - } - isBinary = it->second.string_val_size() > 0; - return StatusCode::OK; -} -template <> -Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const { - auto it = request->inputs().begin(); - while (it != request->inputs().end()) { - if (it->name() == name) { - break; - } - ++it; - } - if (it == request->inputs().end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Error during checking binary input; input: {} does not exist", name); - return StatusCode::INTERNAL_ERROR; - } - isBinary = it->contents().bytes_contents_size() > 0; - return StatusCode::OK; -} - -template -Status EntryNode::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) { - bool isBinary = false; - auto status = this->isInputBinary(tensorName, isBinary); - if (!status.ok()) { - return status; - } - if (isBinary) { - return Node::createShardedTensor(dividedTensor, precision, shape, tensor, i, step, metadata, tensorName); - } - - // if condition is perf optimization - // when demultiplying from entry node from tensor content we can skip allocation for sharded tensors - // and reuse memory from original tensor since its memory is kept for whole duration of predict request - if ((precision == Precision::FP32) || - (precision == Precision::I32) || - (precision == Precision::FP64) || - (precision == Precision::I64) || - (precision == Precision::I8) || - (precision == Precision::U8) || - (precision == Precision::I16)) { - dividedTensor = createSharedTensor(ovmsPrecisionToIE2Precision(precision), shape, (void*)((char*)(tensor.data()) + i * step)); - } else { - return Node::createShardedTensor(dividedTensor, precision, shape, tensor, i, step, metadata, tensorName); - } - return StatusCode::OK; -} - -template <> -const Status EntryNode::validate() { - static const std::set optionalInputNames = {}; - return request_validation_utils::validate( - *request, - inputsInfo, - request->model_spec().name(), - 1, - optionalInputNames); // Pipelines are not versioned and always reports version 1 -} -template <> -const Status EntryNode<::inference::ModelInferRequest>::validate() { - static const std::set optionalInputNames = {}; - return request_validation_utils::validate( - *request, - inputsInfo, - request->model_name(), - 1, - optionalInputNames); // Pipelines are not versioned and always reports version 1 -} - -template Status EntryNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status EntryNode<::inference::ModelInferRequest>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status EntryNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status EntryNode<::inference::ModelInferRequest>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status EntryNode::fetchResults(TensorWithSourceMap& outputs); -template Status EntryNode<::inference::ModelInferRequest>::fetchResults(TensorWithSourceMap& outputs); -template Status EntryNode::isInputBinary(const std::string& name, bool& isBinary) const; -template Status EntryNode<::inference::ModelInferRequest>::isInputBinary(const std::string& name, bool& isBinary) const; -template Status EntryNode::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); -template Status EntryNode<::inference::ModelInferRequest>::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); -template const Status EntryNode::validate(); -template const Status EntryNode<::inference::ModelInferRequest>::validate(); -} // namespace ovms diff --git a/src/ovms_lib/entry_node.hpp b/src/ovms_lib/entry_node.hpp deleted file mode 100644 index 57c22f92a2..0000000000 --- a/src/ovms_lib/entry_node.hpp +++ /dev/null @@ -1,63 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include - -#include - -#include "logging.hpp" -#include "node.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -const std::string ENTRY_NODE_NAME = "request"; - -template -class EntryNode : public Node { - const RequestType* request; - const tensor_map_t inputsInfo; - -public: - EntryNode(const RequestType* request, - const tensor_map_t& inputsInfo, - std::optional demultiplyCount = std::nullopt) : - Node(ENTRY_NODE_NAME, demultiplyCount), - request(request), - inputsInfo(inputsInfo) {} - - Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override; - - Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; - -protected: - Status fetchResults(TensorWithSourceMap& outputs); - Status createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) override; - -public: - // Entry nodes have no dependency - void addDependency(Node&, const Aliases&) override { - throw std::logic_error("This node cannot have dependency"); - } - - Status isInputBinary(const std::string& name, bool& isBinary) const; - - const Status validate(); -}; - -} // namespace ovms diff --git a/src/ovms_lib/executingstreamidguard.hpp b/src/ovms_lib/executingstreamidguard.hpp deleted file mode 100644 index c5642afded..0000000000 --- a/src/ovms_lib/executingstreamidguard.hpp +++ /dev/null @@ -1,39 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include "ovinferrequestsqueue.hpp" - -namespace ovms { - -struct ExecutingStreamIdGuard { - ExecutingStreamIdGuard(ovms::OVInferRequestsQueue& inferRequestsQueue) : - inferRequestsQueue_(inferRequestsQueue), - id_(inferRequestsQueue_.getIdleStream().get()), - inferRequest(inferRequestsQueue.getInferRequest(id_)) {} - ~ExecutingStreamIdGuard() { - inferRequestsQueue_.returnStream(id_); - } - int getId() { return id_; } - ov::InferRequest& getInferRequest() { return inferRequest; } - -private: - ovms::OVInferRequestsQueue& inferRequestsQueue_; - const int id_; - ov::InferRequest& inferRequest; -}; - -} // namespace ovms diff --git a/src/ovms_lib/exit_node.cpp b/src/ovms_lib/exit_node.cpp deleted file mode 100644 index 5558b2fb1b..0000000000 --- a/src/ovms_lib/exit_node.cpp +++ /dev/null @@ -1,77 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "exit_node.hpp" - -#include -#include - -#include "logging.hpp" -#include "ov_utils.hpp" -#include "serialization.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop - -#include "exitnodesession.hpp" - -namespace ovms { -template -Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) { - OVMS_PROFILE_FUNCTION(); - auto& exitNodeSession = static_cast&>(nodeSession); - return this->fetchResults(exitNodeSession.getInputTensors()); -} - -template -Status ExitNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) { - OVMS_PROFILE_FUNCTION(); - notifyEndQueue.push(NodeSessionKeyPair(*this, sessionId)); - return StatusCode::OK; -} - -template <> -Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { - auto it = outputSource.find(name); - if (it == outputSource.end()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Failed to find expected pipeline output when serializing response: {}", name); - return StatusCode::INTERNAL_ERROR; - } - tensor = it->second; - return StatusCode::OK; -} - -template -Status ExitNode::fetchResults(const TensorMap& inputTensors) { - OutputGetter outputGetter(inputTensors); - return serializePredictResponse(outputGetter, this->outputsInfo, this->response, getOutputMapKeyName); -} - -template -std::unique_ptr ExitNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { - return std::make_unique>(metadata, getName(), previous.size(), collapsingDetails, response); -} - -template Status ExitNode<::inference::ModelInferResponse>::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status ExitNode::fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs); -template Status ExitNode<::inference::ModelInferResponse>::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status ExitNode::execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue); -template Status ExitNode<::inference::ModelInferResponse>::fetchResults(const TensorMap& inputTensors); -template Status ExitNode::fetchResults(const TensorMap& inputTensors); -template std::unique_ptr ExitNode<::inference::ModelInferResponse>::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); -template std::unique_ptr ExitNode::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); -} // namespace ovms diff --git a/src/ovms_lib/exit_node.hpp b/src/ovms_lib/exit_node.hpp deleted file mode 100644 index d3500736f3..0000000000 --- a/src/ovms_lib/exit_node.hpp +++ /dev/null @@ -1,63 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "node.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -const std::string EXIT_NODE_NAME = "response"; - -template -class ExitNode : public Node { - ResponseType* response; - const tensor_map_t outputsInfo; - -public: - ExitNode(ResponseType* response, const tensor_map_t& outputsInfo, std::set gatherFromNode = {}) : - Node(EXIT_NODE_NAME, std::nullopt, gatherFromNode), - response(response), - outputsInfo(outputsInfo) { - } - - // Exit node does not have execute logic. - // It serializes its received input tensors to proto in ::fetchResults - Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) override; - -protected: - Status fetchResults(const TensorMap& outputs); - -public: - Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) override; - - // Exit nodes have no dependants - void addDependant(Node& node) override { - throw std::logic_error("This node cannot have dependant"); - } - - std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) override; -}; - -} // namespace ovms diff --git a/src/ovms_lib/exitnodesession.cpp b/src/ovms_lib/exitnodesession.cpp deleted file mode 100644 index 715d466608..0000000000 --- a/src/ovms_lib/exitnodesession.cpp +++ /dev/null @@ -1,46 +0,0 @@ -//***************************************************************************** -// Copyright 2021-2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "exitnodesession.hpp" - -#include - -#include "gatherexitnodeinputhandler.hpp" - -namespace ovms { - -template -ExitNodeSession::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ResponseType* response) : - NodeSession(metadata, nodeName, inputsCount, collapsingDetails) { - if (collapsingDetails.collapsedSessionNames.size() != 0) { - this->inputHandler = std::make_unique>(inputsCount, collapsingDetails, response); - } -} - -template -const TensorMap& ExitNodeSession::getInputTensors() const { - return this->inputHandler->getInputs(); -} - -template -ExitNodeSession::~ExitNodeSession() = default; - -template ExitNodeSession::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, tensorflow::serving::PredictResponse* response); -template ExitNodeSession<::inference::ModelInferResponse>::ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ::inference::ModelInferResponse* response); - -template const TensorMap& ExitNodeSession::getInputTensors() const; -template const TensorMap& ExitNodeSession<::inference::ModelInferResponse>::getInputTensors() const; - -} // namespace ovms diff --git a/src/ovms_lib/exitnodesession.hpp b/src/ovms_lib/exitnodesession.hpp deleted file mode 100644 index fb7aac0da2..0000000000 --- a/src/ovms_lib/exitnodesession.hpp +++ /dev/null @@ -1,37 +0,0 @@ -//***************************************************************************** -// Copyright 2021-2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include - -#include "nodeinputhandler.hpp" -#include "nodesession.hpp" -#include "nodesessionmetadata.hpp" -#include "tensormap.hpp" - -namespace ovms { - -template -class ExitNodeSession : public NodeSession { -public: - ExitNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ResponseType* response); - virtual ~ExitNodeSession(); - const TensorMap& getInputTensors() const; -}; - -} // namespace ovms diff --git a/src/ovms_lib/filesystem.hpp b/src/ovms_lib/filesystem.hpp deleted file mode 100644 index 8f14be21c4..0000000000 --- a/src/ovms_lib/filesystem.hpp +++ /dev/null @@ -1,196 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include - -#include "model_version_policy.hpp" -#include "status.hpp" - -namespace ovms { - -namespace fs = std::filesystem; - -using files_list_t = std::set; -class FileSystem { -public: - /** - * @brief Destroy the File System object - * - */ - virtual ~FileSystem() {} - - /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ - virtual StatusCode fileExists(const std::string& path, bool* exists) = 0; - - /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ - virtual StatusCode isDirectory(const std::string& path, bool* is_dir) = 0; - - /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ - virtual StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) = 0; - - /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ - virtual StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) = 0; - - /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ - virtual StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) = 0; - - /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ - virtual StatusCode readTextFile(const std::string& path, std::string* contents) = 0; - - /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ - virtual StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) = 0; - - /** - * @brief Download model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ - virtual StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) = 0; - - /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ - - virtual StatusCode deleteFileFolder(const std::string& path) = 0; - /** - * @brief Create a Temp Path - * - * @param local_path - * @return StatusCode - */ - static StatusCode createTempPath(std::string* local_path) { - std::string file_template = "/tmp/fileXXXXXX"; - char* tmp_folder = mkdtemp(const_cast(file_template.c_str())); - if (tmp_folder == nullptr) { - SPDLOG_ERROR("Failed to create local temp folder: {} {}", file_template, strerror(errno)); - return StatusCode::FILESYSTEM_ERROR; - } - fs::permissions(tmp_folder, - fs::perms::others_all | fs::perms::group_all, - fs::perm_options::remove); - - *local_path = std::string(tmp_folder); - - return StatusCode::OK; - } - - static bool isPathEscaped(const std::string& path) { - return std::string::npos != path.find("../") || std::string::npos != path.find("/.."); - } - - std::string appendSlash(const std::string& name) { - if (name.empty() || (name.back() == '/')) { - return name; - } - - return (name + "/"); - } - - bool isAbsolutePath(const std::string& path) { - return !path.empty() && (path[0] == '/'); - } - - std::string joinPath(std::initializer_list segments) { - std::string joined; - - for (const auto& seg : segments) { - if (joined.empty()) { - joined = seg; - } else if (isAbsolutePath(seg)) { - if (joined[joined.size() - 1] == '/') { - joined.append(seg.substr(1)); - } else { - joined.append(seg); - } - } else { - if (joined[joined.size() - 1] != '/') { - joined.append("/"); - } - joined.append(seg); - } - } - - return joined; - } - - StatusCode CreateLocalDir(const std::string& path) { - int status = - mkdir(const_cast(path.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); - if (status == -1) { - SPDLOG_ERROR("Failed to create local folder: {} {} ", path, strerror(errno)); - return StatusCode::PATH_INVALID; - } - return StatusCode::OK; - } - - static const std::vector acceptedFiles; -}; - -} // namespace ovms diff --git a/src/ovms_lib/gatherexitnodeinputhandler.cpp b/src/ovms_lib/gatherexitnodeinputhandler.cpp deleted file mode 100644 index 08e73a81d1..0000000000 --- a/src/ovms_lib/gatherexitnodeinputhandler.cpp +++ /dev/null @@ -1,53 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "gatherexitnodeinputhandler.hpp" - -#include - -#include "logging.hpp" - -namespace ovms { - -Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& bufferOut, const std::string& name, size_t size) { - OVMS_PROFILE_FUNCTION(); - tensorflow::TensorProto tensorProto; - auto [it, isInserted] = response->mutable_outputs()->insert(google::protobuf::MapPair(name, std::move(tensorProto))); - if (!isInserted) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); - return StatusCode::INTERNAL_ERROR; - } - it->second.mutable_tensor_content()->resize(size); - bufferOut = it->second.mutable_tensor_content()->data(); - return StatusCode::OK; -} - -Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& bufferOut, const std::string& name, size_t size) { - OVMS_PROFILE_FUNCTION(); - for (int i = 0; i < response->outputs_size(); i++) { - if (response->mutable_outputs(i)->name() == name) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to prepare consolidated tensor, tensor with name {} already prepared", name); - return StatusCode::INTERNAL_ERROR; - } - } - auto* proto = response->add_outputs(); - proto->set_name(name); - auto* content = response->add_raw_output_contents(); - content->resize(size); - bufferOut = content->data(); - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/gatherexitnodeinputhandler.hpp b/src/ovms_lib/gatherexitnodeinputhandler.hpp deleted file mode 100644 index 43756052f2..0000000000 --- a/src/ovms_lib/gatherexitnodeinputhandler.hpp +++ /dev/null @@ -1,68 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include - -#include "gathernodeinputhandler.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" - -#include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" -#include "src/kfserving_api/grpc_predict_v2.pb.h" -#pragma GCC diagnostic pop - -#include "logging.hpp" -#include "profiler.hpp" - -namespace ovms { - -Status prepareConsolidatedTensorImpl(tensorflow::serving::PredictResponse* response, char*& tensorOut, const std::string& name, size_t size); -Status prepareConsolidatedTensorImpl(::inference::ModelInferResponse* response, char*& tensorOut, const std::string& name, size_t size); - -template -class GatherExitNodeInputHandler : public GatherNodeInputHandler { - ResponseType* response; - - Status prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const override { - OVMS_PROFILE_FUNCTION(); - auto numOfElements = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - size_t numOfBytes = numOfElements * ov::element::Type(precision).size(); - char* buffer = nullptr; - auto status = prepareConsolidatedTensorImpl(response, buffer, name, numOfBytes); - if (!status.ok()) { - return status; - } - if (!buffer) { - return StatusCode::INTERNAL_ERROR; - } - tensorOut = ov::Tensor(precision, shape, buffer); - return StatusCode::OK; - } - -public: - GatherExitNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails, ResponseType* response) : - GatherNodeInputHandler(inputsMissingCount, collapsingDetails), - response(response) {} -}; - -} // namespace ovms diff --git a/src/ovms_lib/gathernodeinputhandler.cpp b/src/ovms_lib/gathernodeinputhandler.cpp deleted file mode 100644 index 08a79fb059..0000000000 --- a/src/ovms_lib/gathernodeinputhandler.cpp +++ /dev/null @@ -1,116 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "gathernodeinputhandler.hpp" - -#include -#include - -#include "logging.hpp" -#include "nodesessionmetadata.hpp" -#include "ov_utils.hpp" -#include "profiler.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -GatherNodeInputHandler::GatherNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails) : - NodeInputHandler(inputsMissingCount), - collapsingDetails(std::make_unique(collapsingDetails)) { - remainingDependencies = std::accumulate( - collapsingDetails.collapsedSessionSizes.begin(), - collapsingDetails.collapsedSessionSizes.end(), - remainingDependencies, - std::multiplies()); -} - -Status GatherNodeInputHandler::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { - auto inputsShardsIt = shardsStorage.find(inputName); - if (inputsShardsIt == shardsStorage.end()) { - shard_map_t shardMap{{shardId, tensor.getActualTensor()}}; - auto itDidInsertPair = shardsStorage.emplace(inputName, std::move(shardMap)); - if (!itDidInsertPair.second) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to insert the same input: {} twice with the same shardId: {}", inputName, shardId); - return StatusCode::INTERNAL_ERROR; - } - } else { - auto itDidEmplacePair = inputsShardsIt->second.emplace(shardId, tensor.getActualTensor()); - if (!itDidEmplacePair.second) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to put the same input: {} shard: {} twice", inputName, shardId); - return StatusCode::INTERNAL_ERROR; - } - } - if (tensor.hasSource()) { - sourceTensorRefs.push_back(tensor.getSourceTensor()); - } - return StatusCode::OK; -} - -Status GatherNodeInputHandler::notifyFinishedDependency() { - OVMS_PROFILE_FUNCTION(); - NodeInputHandler::notifyFinishedDependency(); - if (remainingDependencies > 0) { - return StatusCode::OK; - } - for (auto& [inputName, shardMap] : shardsStorage) { - OVMS_PROFILE_SCOPE("Gather Tensor"); - const auto shardsCount = shardMap.size(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Consolidating: {} shards for input: {}", shardsCount, inputName); - session_id_t firstShardId = 0; - auto firstShard = shardMap.at(firstShardId); - auto firstShardDims = firstShard.get_shape(); - auto precision = firstShard.get_element_type(); - auto newDims = firstShardDims; - newDims.insert(newDims.begin(), - collapsingDetails->collapsedSessionSizes.begin(), - collapsingDetails->collapsedSessionSizes.end()); - ov::Tensor consolidatedTensor; - auto status = prepareConsolidatedTensor(consolidatedTensor, inputName, precision, newDims); - if (!status.ok()) { - return status; - } - for (auto& [shardId, tensor] : shardMap) { - OVMS_PROFILE_SCOPE("Copy Shard"); - if ((tensor.get_element_type() != precision) || - (tensor.get_shape() != firstShardDims)) { - std::stringstream firstShardShapeStream; - firstShardShapeStream << firstShardDims; - auto currentShardShape = tensor.get_shape(); - std::stringstream currentShardShapeStream; - currentShardShapeStream << currentShardShape; - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to consolidate tensor: {}; shards in gather node. First shard has different tensor precision: {}; or shape: {}; than current shard precision: {}; shape: {};", - inputName, - toString(ovElementTypeToOvmsPrecision(precision)), - firstShardShapeStream.str(), - toString(ovElementTypeToOvmsPrecision(tensor.get_element_type())), - currentShardShapeStream.str()); - return StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS; - } - const auto memstep = tensor.get_byte_size(); - size_t offset = shardId * memstep; - memcpy((char*)consolidatedTensor.data() + offset, - tensor.data(), - memstep); - } - inputTensors.insert({inputName, consolidatedTensor}); - } - return StatusCode::OK; -} - -Status GatherNodeInputHandler::prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const { - return createSharedTensor(tensorOut, precision, shape); -} - -} // namespace ovms diff --git a/src/ovms_lib/gathernodeinputhandler.hpp b/src/ovms_lib/gathernodeinputhandler.hpp deleted file mode 100644 index 4d4f73d8db..0000000000 --- a/src/ovms_lib/gathernodeinputhandler.hpp +++ /dev/null @@ -1,48 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include - -#include "nodeinputhandler.hpp" -#include "precision.hpp" -#include "session_id.hpp" -#include "shape.hpp" - -namespace ovms { - -using shard_map_t = std::unordered_map; - -class CollapseDetails; - -class GatherNodeInputHandler : public NodeInputHandler { - std::unordered_map shardsStorage; - std::unique_ptr collapsingDetails; - -public: - GatherNodeInputHandler(uint32_t inputsMissingCount, const CollapseDetails& collapsingDetails); - Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) override; - Status notifyFinishedDependency() override; - -protected: - virtual Status prepareConsolidatedTensor(ov::Tensor& tensorOut, const std::string& name, ov::element::Type_t precision, const ov::Shape& shape) const; -}; -} // namespace ovms diff --git a/src/ovms_lib/gcsfilesystem.cpp b/src/ovms_lib/gcsfilesystem.cpp deleted file mode 100644 index 1821228a52..0000000000 --- a/src/ovms_lib/gcsfilesystem.cpp +++ /dev/null @@ -1,407 +0,0 @@ -// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "gcsfilesystem.hpp" - -#include -#include -#include -#include -#include - -#include "logging.hpp" -#include "stringutils.hpp" - -namespace ovms { - -namespace fs = std::filesystem; -namespace gcs = google::cloud::storage; - -const std::string GCSFileSystem::GCS_URL_PREFIX = "gs://"; - -StatusCode GCSFileSystem::parsePath(const std::string& path, - std::string* bucket, std::string* object) { - int bucket_start = path.find(GCS_URL_PREFIX) + GCS_URL_PREFIX.size(); - int bucket_end = path.find("/", bucket_start); - if (bucket_end > bucket_start) { - *bucket = path.substr(bucket_start, bucket_end - bucket_start); - *object = path.substr(bucket_end + 1); - } else { - *bucket = path.substr(bucket_start); - *object = ""; - } - if (bucket->empty()) { - return StatusCode::GCS_BUCKET_NOT_FOUND; - } - return StatusCode::OK; -} - -namespace { - -google::cloud::storage::ClientOptions createDefaultOrAnonymousClientOptions() { - if (std::getenv("GOOGLE_APPLICATION_CREDENTIALS") == nullptr) { - auto credentials = - google::cloud::storage::v1::oauth2::CreateAnonymousCredentials(); - if (!credentials) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to create default GCS credentials"); - throw std::runtime_error("Unable to create default GCS credentials"); - } - auto options = google::cloud::storage::ClientOptions(credentials); - return options; - } else { - auto credentials = - google::cloud::storage::v1::oauth2::GoogleDefaultCredentials(); - if (!credentials) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to create default GCS credentials"); - throw std::runtime_error("Unable to create default GCS credentials"); - } - auto options = google::cloud::storage::ClientOptions(*credentials); - return options; - } -} - -} // namespace - -GCSFileSystem::GCSFileSystem() : - client_{createDefaultOrAnonymousClientOptions()} { - SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem default ctor"); -} - -GCSFileSystem::GCSFileSystem(const gcs::v1::ClientOptions& options) : - client_{options, gcs::StrictIdempotencyPolicy()} { - SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem ctor with custom options"); -} - -GCSFileSystem::~GCSFileSystem() { SPDLOG_LOGGER_TRACE(gcs_logger, "GCSFileSystem dtor"); } - -StatusCode GCSFileSystem::fileExists(const std::string& path, bool* exists) { - *exists = false; - std::string bucket, object; - - auto status = this->parsePath(path, &bucket, &object); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to parse path: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - - google::cloud::StatusOr object_metadata = - client_.GetObjectMetadata(bucket, object); - if (object_metadata) { - *exists = true; - return StatusCode::OK; - } - bool is_directory; - auto dir_status = this->isDirectory(path, &is_directory); - if (dir_status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "isDirectory failed: {} -> {}", path, - ovms::Status(status).string()); - return dir_status; - } - *exists = is_directory; - SPDLOG_LOGGER_TRACE(gcs_logger, "fileExits {} -> {}", path, is_directory); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::isDirectory(const std::string& path, - bool* is_directory) { - *is_directory = false; - std::string bucket, object; - auto status = this->parsePath(path, &bucket, &object); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to parse path: {} -> {}", path, - ovms::Status(status).string()); - return status; - } - if (path.empty()) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Path {} is empty", path); - *is_directory = true; - return StatusCode::OK; - } - try { - for (auto&& meta : - client_.ListObjects(bucket, gcs::Prefix(appendSlash(object)))) { - if (meta) { - *is_directory = true; - return StatusCode::OK; - } - } - } catch (std::exception& ex) { - SPDLOG_LOGGER_DEBUG(gcs_logger, "GCS list objects exception {}", ex.what()); - SPDLOG_LOGGER_ERROR(gcs_logger, "Invalid or missing GCS credentials, or directory does not exist - {}", path); - } - - return StatusCode::OK; -} - -StatusCode -GCSFileSystem::getDirectoryContents(const std::string& path, - std::set* contents) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Getting directory contents {}", path); - std::string bucket, directory_path, full_directory; - auto status = this->parsePath(path, &bucket, &directory_path); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to get directory content {} -> {}", path, - ovms::Status(status).string()); - return status; - } - full_directory = appendSlash(directory_path); - try { - for (auto&& meta : client_.ListObjects(bucket, gcs::Prefix(full_directory))) { - if (!meta) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to get directory content -> object metadata " - "is empty. Error: {}", - meta.status().message()); - return StatusCode::GCS_INVALID_ACCESS; - } - // ignore self: - if (meta->name() == full_directory) { - continue; - } - - // keep only basename: - std::string name = meta->name(); - int name_start = name.find(full_directory) + full_directory.size(); - int name_end = name.find("/", name_start); - contents->insert(name.substr(name_start, name_end - name_start)); - } - } catch (std::exception& ex) { - SPDLOG_LOGGER_DEBUG(gcs_logger, "GCS list objects exception {}", ex.what()); - SPDLOG_LOGGER_ERROR(gcs_logger, "Invalid or missing GCS credentials, or directory does not exist - {}", full_directory); - return StatusCode::GCS_INVALID_ACCESS; - } - - SPDLOG_LOGGER_TRACE(gcs_logger, "Directory contents fetched, items: {}", contents->size()); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::getDirectorySubdirs(const std::string& path, - std::set* subdirs) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory subdirs: {}", path); - auto status = this->getDirectoryContents(path, subdirs); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory subdir content {} -> {}", path, - ovms::Status(status).string()); - return status; - } - for (auto item = subdirs->begin(); item != subdirs->end();) { - bool is_directory; - auto status = this->isDirectory(joinPath({path, *item}), &is_directory); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory subdir content {} -> {}", path, - ovms::Status(status).string()); - return status; - } - if (!is_directory) { - item = subdirs->erase(item); - } else { - ++item; - } - } - SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory subdirs ok: {}", path); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::getDirectoryFiles(const std::string& path, - std::set* files) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Listing directory: {}", path); - auto status = this->getDirectoryContents(path, files); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory content {} -> {}", path, - ovms::Status(status).string()); - return status; - } - for (auto item = files->begin(); item != files->end();) { - bool is_directory; - auto status = this->isDirectory(joinPath({path, *item}), &is_directory); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to list directory content {} -> {}", path, - ovms::Status(status).string()); - return status; - } - if (is_directory) { - item = files->erase(item); - } else { - ++item; - } - } - SPDLOG_LOGGER_TRACE(gcs_logger, "listing directory ok for {}", path); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::readTextFile(const std::string& path, - std::string* contents) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Downloading file {}", path); - bool exists; - auto status = fileExists(path, &exists); - if (status != StatusCode::OK) { - return status; - } - if (!exists) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Downloading file -> file does not exist at {}", path); - return StatusCode::GCS_FILE_NOT_FOUND; - } - std::string bucket, object; - status = parsePath(path, &bucket, &object); - if (status != StatusCode::OK) { - return status; - } - gcs::ObjectReadStream stream = client_.ReadObject(bucket, object); - if (!stream) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Downloading file has failed: ", path); - return StatusCode::GCS_FILE_INVALID; - } - std::string data = ""; - char c = 0; - while (stream.get(c)) { - data += c; - } - *contents = data; - SPDLOG_LOGGER_TRACE(gcs_logger, "File {} has been downloaded (bytes={})", path, - data.size()); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::downloadFile(const std::string& remote_path, - const std::string& local_path) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Saving file {} to {}", remote_path, local_path); - std::string contents; - auto read_status = this->readTextFile(remote_path, &contents); - if (read_status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to get object at {}", remote_path); - return read_status; - } - std::ofstream output_file(local_path.c_str(), std::ios::binary); - output_file << contents; - output_file.close(); - return StatusCode::OK; -} - -StatusCode GCSFileSystem::downloadModelVersions(const std::string& path, - std::string* local_path, - const std::vector& versions) { - auto sc = createTempPath(local_path); - if (sc != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to create a temporary path {}", sc); - return sc; - } - - StatusCode result = StatusCode::OK; - for (auto& ver : versions) { - std::string versionpath = path; - if (!endsWith(versionpath, "/")) { - versionpath.append("/"); - } - versionpath.append(std::to_string(ver)); - std::string lpath = *local_path; - if (!endsWith(lpath, "/")) { - lpath.append("/"); - } - lpath.append(std::to_string(ver)); - fs::create_directory(lpath); - auto status = downloadFileFolder(versionpath, lpath); - if (status != StatusCode::OK) { - result = status; - SPDLOG_LOGGER_ERROR(gcs_logger, "Failed to download model version {}", versionpath); - } - } - - return result; -} - -StatusCode GCSFileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { - SPDLOG_LOGGER_TRACE(gcs_logger, "Downloading dir {} and saving to {}", path, local_path); - bool is_dir; - auto status = this->isDirectory(path, &is_dir); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "File/folder does not exist at {}", path); - return StatusCode::GCS_FILE_NOT_FOUND; - } - if (!is_dir) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Path is not a directory: {}", path); - return StatusCode::GCS_FILE_NOT_FOUND; - } - - std::set dirs; - status = getDirectorySubdirs(path, &dirs); - if (status != StatusCode::OK) { - return status; - } - - std::set files; - status = getDirectoryFiles(path, &files); - if (status != StatusCode::OK) { - return status; - } - - for (auto&& d : dirs) { - std::string remote_dir_path = joinPath({path, d}); - std::string local_dir_path = joinPath({local_path, d}); - SPDLOG_LOGGER_TRACE(gcs_logger, "Processing directory {} from {} -> {}", d, remote_dir_path, - local_dir_path); - auto mkdir_status = CreateLocalDir(local_dir_path); - if (mkdir_status != StatusCode::OK) { - return status; - } - auto download_dir_status = - this->downloadFileFolder(remote_dir_path, local_dir_path); - if (download_dir_status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to download directory from {} to {}", - remote_dir_path, local_dir_path); - return download_dir_status; - } - } - - for (auto&& f : files) { - if (std::any_of(acceptedFiles.begin(), acceptedFiles.end(), [&f](const std::string& x) { - return f.size() > 0 && endsWith(f, x); - })) { - std::string remote_file_path = joinPath({path, f}); - std::string local_file_path = joinPath({local_path, f}); - SPDLOG_LOGGER_TRACE(gcs_logger, "Processing file {} from {} -> {}", f, remote_file_path, - local_file_path); - auto download_status = - this->downloadFile(remote_file_path, local_file_path); - if (download_status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to save file from {} to {}", remote_file_path, - local_file_path); - return download_status; - } - } - } - return StatusCode::OK; -} - -StatusCode GCSFileSystem::deleteFileFolder(const std::string& path) { - SPDLOG_LOGGER_DEBUG(gcs_logger, "Deleting local file folder {}", path); - if (::remove(path.c_str()) == 0) { - return StatusCode::OK; - } else { - SPDLOG_LOGGER_ERROR(gcs_logger, "Unable to remove local path: {}", path); - return StatusCode::FILE_INVALID; - } -} - -} // namespace ovms diff --git a/src/ovms_lib/gcsfilesystem.hpp b/src/ovms_lib/gcsfilesystem.hpp deleted file mode 100644 index 3af4c4a7fe..0000000000 --- a/src/ovms_lib/gcsfilesystem.hpp +++ /dev/null @@ -1,165 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include "google/cloud/storage/client.h" - -#include "filesystem.hpp" -#include "status.hpp" - -namespace ovms { - -class GCSFileSystem : public FileSystem { -public: - /** - * @brief Construct a new GCSFileSystem object - * - */ - GCSFileSystem(); - /** - * @brief Construct a new GCSFileSystem object - * - * @param options - */ - GCSFileSystem(const google::cloud::storage::v1::ClientOptions& options); - - /** - * @brief Destroy the GCSFileSystem object - * - */ - virtual ~GCSFileSystem(); - - /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ - StatusCode fileExists(const std::string& path, bool* exists) override; - - /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ - StatusCode isDirectory(const std::string& path, bool* is_dir) override; - - /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode getDirectoryContents(const std::string& path, - files_list_t* contents) override; - - /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ - StatusCode getDirectorySubdirs(const std::string& path, - files_list_t* subdirs) override; - - /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ - StatusCode getDirectoryFiles(const std::string& path, - files_list_t* files) override; - - /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode readTextFile(const std::string& path, - std::string* contents) override; - - /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ - StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; - - /** - * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ - StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; - - /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ - StatusCode deleteFileFolder(const std::string& path) override; - - static const std::string GCS_URL_PREFIX; - -private: - /** - * @brief - * - * @param path - * @param bucket - * @param object - * @return StatusCode - */ - StatusCode parsePath(const std::string& path, std::string* bucket, - std::string* object); - - /** - * - * @brief - * - * @param remote_path - * @param local_path - */ - StatusCode downloadFile(const std::string& remote_path, - const std::string& local_path); - - /** - * @brief - * - */ - google::cloud::storage::Client client_; -}; - -} // namespace ovms diff --git a/src/ovms_lib/get_model_metadata_impl.cpp b/src/ovms_lib/get_model_metadata_impl.cpp deleted file mode 100644 index 33f7fd1e9b..0000000000 --- a/src/ovms_lib/get_model_metadata_impl.cpp +++ /dev/null @@ -1,202 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "get_model_metadata_impl.hpp" - -#include - -#include "modelmanager.hpp" -#include "pipelinedefinition.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "tfs_frontend/tfs_utils.hpp" - -using google::protobuf::util::JsonPrintOptions; -using google::protobuf::util::MessageToJsonString; - -namespace ovms { -GetModelMetadataImpl::GetModelMetadataImpl(ovms::Server& ovmsServer) : - ovmsServer(ovmsServer) {} - -Status GetModelMetadataImpl::getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) const { - auto status = validate(request); - if (!status.ok()) { - return status; - } - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return getModelStatus(request, response, manager); -} - -Status GetModelMetadataImpl::getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, - ModelManager& manager) { - const auto& name = request->model_spec().name(); - model_version_t version = request->model_spec().has_version() ? request->model_spec().version().value() : 0; - - auto model = manager.findModelByName(name); - if (model == nullptr) { - SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - return buildResponse(*pipelineDefinition, response, manager); - } - - std::shared_ptr instance = nullptr; - if (version != 0) { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; version {}", name, version); - instance = model->getModelInstanceByVersion(version); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, version); - return StatusCode::MODEL_VERSION_MISSING; - } - } else { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; default version", name); - instance = model->getDefaultModelInstance(); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; default version is missing", name); - return StatusCode::MODEL_VERSION_MISSING; - } - } - - return buildResponse(instance, response); -} - -Status GetModelMetadataImpl::validate( - const tensorflow::serving::GetModelMetadataRequest* request) { - - if (!request->has_model_spec()) { - return StatusCode::MODEL_SPEC_MISSING; - } - - if (request->metadata_field_size() != 1) { - return StatusCode::INVALID_SIGNATURE_DEF; - } - - const auto& signature = request->metadata_field().at(0); - - if (signature != "signature_def") { - return StatusCode::INVALID_SIGNATURE_DEF; - } - - return StatusCode::OK; -} - -void GetModelMetadataImpl::convert( - const tensor_map_t& from, - proto_signature_map_t* to) { - for (const auto& [name, tensor] : from) { - auto& input = (*to)[name]; - - input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); - - // Since this method is used for models and pipelines we cannot rely on tensor getMappedName(). - // In both cases we can rely on tensor_map key values as final names. - *input.mutable_name() = name; - *input.mutable_tensor_shape() = tensorflow::TensorShapeProto(); - - for (auto dim : tensor->getShape()) { - if (dim.isStatic()) { - input.mutable_tensor_shape()->add_dim()->set_size(dim.getStaticValue()); - } else { - input.mutable_tensor_shape()->add_dim()->set_size(DYNAMIC_DIMENSION); - } - } - } -} - -Status GetModelMetadataImpl::buildResponse( - std::shared_ptr instance, - tensorflow::serving::GetModelMetadataResponse* response) { - - std::unique_ptr unloadGuard; - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = instance->waitForLoaded(0, unloadGuard); - if (!status.ok()) { - return status; - } - - response->Clear(); - response->mutable_model_spec()->set_name(instance->getName()); - response->mutable_model_spec()->mutable_version()->set_value(instance->getVersion()); - - tensorflow::serving::SignatureDefMap def; - convert(instance->getInputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); - convert(instance->getOutputsInfo(), ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); - - (*response->mutable_metadata())["signature_def"].PackFrom(def); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::buildResponse( - PipelineDefinition& pipelineDefinition, - tensorflow::serving::GetModelMetadataResponse* response, - const ModelManager& manager) { - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - std::unique_ptr unloadGuard; - auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); - if (!status.ok()) { - return status; - } - - const tensor_map_t& inputs = pipelineDefinition.getInputsInfo(); - const tensor_map_t& outputs = pipelineDefinition.getOutputsInfo(); - - response->Clear(); - response->mutable_model_spec()->set_name(pipelineDefinition.getName()); - response->mutable_model_spec()->mutable_version()->set_value(1); - - tensorflow::serving::SignatureDefMap def; - convert(inputs, ((*def.mutable_signature_def())["serving_default"]).mutable_inputs()); - convert(outputs, ((*def.mutable_signature_def())["serving_default"]).mutable_outputs()); - - (*response->mutable_metadata())["signature_def"].PackFrom(def); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::createGrpcRequest(std::string model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request) { - request->mutable_model_spec()->set_name(model_name); - if (model_version.has_value()) { - request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); - } - request->mutable_metadata_field()->Add("signature_def"); - return StatusCode::OK; -} - -Status GetModelMetadataImpl::serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output) { - JsonPrintOptions opts; - opts.add_whitespace = true; - opts.always_print_primitive_fields = true; - const auto& status = MessageToJsonString(*response, output, opts); - if (!status.ok()) { - SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); - return StatusCode::JSON_SERIALIZATION_ERROR; - } - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/get_model_metadata_impl.hpp b/src/ovms_lib/get_model_metadata_impl.hpp deleted file mode 100644 index 5a31d054ba..0000000000 --- a/src/ovms_lib/get_model_metadata_impl.hpp +++ /dev/null @@ -1,71 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -using proto_signature_map_t = google::protobuf::Map; - -class ModelInstance; -class ModelManager; -class PipelineDefinition; -class Server; - -class GetModelMetadataImpl { - ovms::Server& ovmsServer; - -public: - GetModelMetadataImpl(ovms::Server& ovmsServer); - static Status validate( - const tensorflow::serving::GetModelMetadataRequest* request); - - static void convert( - const tensor_map_t& from, - proto_signature_map_t* to); - - static Status buildResponse( - std::shared_ptr instance, - tensorflow::serving::GetModelMetadataResponse* response); - static Status buildResponse( - PipelineDefinition& pipelineDefinition, - tensorflow::serving::GetModelMetadataResponse* response, - const ModelManager& manager); - - Status getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) const; - static Status getModelStatus( - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response, - ModelManager& manager); - static Status createGrpcRequest(std::string model_name, std::optional model_version, tensorflow::serving::GetModelMetadataRequest* request); - static Status serializeResponse2Json(const tensorflow::serving::GetModelMetadataResponse* response, std::string* output); -}; - -} // namespace ovms diff --git a/src/ovms_lib/global_sequences_viewer.cpp b/src/ovms_lib/global_sequences_viewer.cpp deleted file mode 100644 index d0d4f43d78..0000000000 --- a/src/ovms_lib/global_sequences_viewer.cpp +++ /dev/null @@ -1,73 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "global_sequences_viewer.hpp" - -#include -#include -#include - -#include "logging.hpp" -#include "model.hpp" -#include "modelversion.hpp" -#include "statefulmodelinstance.hpp" -#include "status.hpp" - -namespace ovms { - -static std::string separator = "_"; - -Status GlobalSequencesViewer::registerForCleanup(std::string modelName, model_version_t modelVersion, std::shared_ptr sequenceManager) { - std::string registration_id = modelName + separator + std::to_string(modelVersion); - std::unique_lock viewerLock(viewerMutex); - if (registeredSequenceManagers.find(registration_id) != registeredSequenceManagers.end()) { - SPDLOG_LOGGER_ERROR(sequence_manager_logger, "Model: {}, version: {}, cannot register model instance in sequence cleaner. Already registered.", modelName, modelVersion); - return StatusCode::INTERNAL_ERROR; - } else { - registeredSequenceManagers.emplace(registration_id, sequenceManager); - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, has been successfully registered in sequence cleaner", modelName, modelVersion); - } - - return StatusCode::OK; -} - -Status GlobalSequencesViewer::unregisterFromCleanup(std::string modelName, model_version_t modelVersion) { - std::string registration_id = modelName + separator + std::to_string(modelVersion); - std::unique_lock viewerLock(viewerMutex); - auto it = registeredSequenceManagers.find(registration_id); - if (it != registeredSequenceManagers.end()) { - registeredSequenceManagers.erase(it); - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, has been successfully unregistered from sequence cleaner", modelName, modelVersion); - } else { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model: {}, version: {}, cannot unregister model instance from sequence cleaner. It has not been registered.", modelName, modelVersion); - return StatusCode::INTERNAL_ERROR; - } - return StatusCode::OK; -} - -Status GlobalSequencesViewer::removeIdleSequences() { - std::unique_lock viewerLock(viewerMutex); - for (auto it = registeredSequenceManagers.begin(); it != registeredSequenceManagers.end();) { - auto sequenceManager = it->second; - auto status = sequenceManager->removeIdleSequences(); - it++; - if (status.getCode() != ovms::StatusCode::OK) - return status; - } - - return ovms::StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/global_sequences_viewer.hpp b/src/ovms_lib/global_sequences_viewer.hpp deleted file mode 100644 index b15ef71bee..0000000000 --- a/src/ovms_lib/global_sequences_viewer.hpp +++ /dev/null @@ -1,46 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "sequence_manager.hpp" -#include "status.hpp" - -namespace ovms { -const uint32_t DEFAULT_SEQUENCE_CLEANER_INTERVAL = 5; // in minutes -class GlobalSequencesViewer { -private: - // used to block parallel access to registered sequence managers map - std::mutex viewerMutex; - - std::map> registeredSequenceManagers; - -public: - Status removeIdleSequences(); - - Status registerForCleanup(std::string modelName, model_version_t modelVersion, std::shared_ptr sequenceManager); - - Status unregisterFromCleanup(std::string modelName, model_version_t modelVersion); -}; -} // namespace ovms diff --git a/src/ovms_lib/grpcservermodule.cpp b/src/ovms_lib/grpcservermodule.cpp deleted file mode 100644 index 6a18f5bd02..0000000000 --- a/src/ovms_lib/grpcservermodule.cpp +++ /dev/null @@ -1,183 +0,0 @@ -//**************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "grpcservermodule.hpp" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "config.hpp" -#include "kfs_grpc_inference_service.hpp" -#include "logging.hpp" -#include "model_service.hpp" -#include "modelmanager.hpp" -#include "prediction_service.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "stringutils.hpp" -#include "version.hpp" - -using grpc::ServerBuilder; - -namespace ovms { -} // namespace ovms -using namespace ovms; -static const int GIGABYTE = 1024 * 1024 * 1024; - -bool isPortAvailable(uint64_t port) { - struct sockaddr_in addr; - int s = socket(AF_INET, SOCK_STREAM, 0); - if (s == -1) { - return false; - } - - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(port); - - if (bind(s, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - close(s); - return false; - } - close(s); - return true; -} - -struct GrpcChannelArgument { - std::string key; - std::string value; -}; - -// Parses a comma separated list of gRPC channel arguments into list of -// ChannelArgument. -Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std::vector& result) { - const std::vector channel_arguments = tokenize(channel_arguments_str, ','); - - for (const std::string& channel_argument : channel_arguments) { - std::vector key_val = tokenize(channel_argument, '='); - if (key_val.size() != 2) { - return StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT; - } - erase_spaces(key_val[0]); - erase_spaces(key_val[1]); - result.push_back({key_val[0], key_val[1]}); - } - - return StatusCode::OK; -} - -uint getGRPCServersCount(const ovms::Config& config) { - const char* environmentVariableBuffer = std::getenv("GRPC_SERVERS"); - if (environmentVariableBuffer) { - auto result = stou32(environmentVariableBuffer); - if (result && result.value() > 0) { - return result.value(); - } - } - - return std::max(1, config.grpcWorkers()); -} - -GRPCServerModule::GRPCServerModule(Server& server) : - server(server), - tfsPredictService(this->server), - tfsModelService(this->server), - kfsGrpcInferenceService(this->server) {} -int GRPCServerModule::start(const ovms::Config& config) { - state = ModuleState::STARTED_INITIALIZE; - SPDLOG_INFO("{} starting", GRPC_SERVER_MODULE_NAME); - std::vector channel_arguments; - auto status = parseGrpcChannelArgs(config.grpcChannelArguments(), channel_arguments); - if (!status.ok()) { - SPDLOG_ERROR("grpc channel arguments passed in wrong format: {}", config.grpcChannelArguments()); - return EXIT_FAILURE; - } - - ServerBuilder builder; - builder.SetMaxReceiveMessageSize(GIGABYTE); - builder.SetMaxSendMessageSize(GIGABYTE); - builder.AddListeningPort(config.grpcBindAddress() + ":" + std::to_string(config.port()), grpc::InsecureServerCredentials()); - builder.RegisterService(&tfsPredictService); - builder.RegisterService(&tfsModelService); - builder.RegisterService(&kfsGrpcInferenceService); - for (const GrpcChannelArgument& channel_argument : channel_arguments) { - // gRPC accept arguments of two types, int and string. We will attempt to - // parse each arg as int and pass it on as such if successful. Otherwise we - // will pass it as a string. gRPC will log arguments that were not accepted. - SPDLOG_DEBUG("setting grpc channel argument {}: {}", channel_argument.key, channel_argument.value); - try { - int i = std::stoi(channel_argument.value); - builder.AddChannelArgument(channel_argument.key, i); - } catch (std::invalid_argument const& e) { - builder.AddChannelArgument(channel_argument.key, channel_argument.value); - } catch (std::out_of_range const& e) { - SPDLOG_WARN("Out of range parameter {} : {}", channel_argument.key, channel_argument.value); - } - } - uint grpcServersCount = getGRPCServersCount(config); - servers.reserve(grpcServersCount); - SPDLOG_DEBUG("Starting gRPC servers: {}", grpcServersCount); - - if (!isPortAvailable(config.port())) { - SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); - return EXIT_FAILURE; - } - for (uint i = 0; i < grpcServersCount; ++i) { - std::unique_ptr server = builder.BuildAndStart(); - if (server == nullptr) { - SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); - return EXIT_FAILURE; - } - servers.push_back(std::move(server)); - } - state = ModuleState::INITIALIZED; - SPDLOG_INFO("{} started", GRPC_SERVER_MODULE_NAME); - // FIXME this should be reenabled when functional tests are switched to wait - // for servablemanager module start log - // #KFS_CLEANUP - // SPDLOG_INFO("Server started on port {}", config.port()); - return EXIT_SUCCESS; -} - -void GRPCServerModule::shutdown() { - state = ModuleState::STARTED_SHUTDOWN; - SPDLOG_INFO("{} shutting down", GRPC_SERVER_MODULE_NAME); - for (const auto& server : servers) { - server->Shutdown(); - SPDLOG_INFO("Shutdown gRPC server"); - } - state = ModuleState::SHUTDOWN; - SPDLOG_INFO("{} shutdown", GRPC_SERVER_MODULE_NAME); -} - -const GetModelMetadataImpl& GRPCServerModule::getTFSModelMetadataImpl() const { - return this->tfsPredictService.getTFSModelMetadataImpl(); -} -KFSInferenceServiceImpl& GRPCServerModule::getKFSGrpcImpl() const { - return this->kfsGrpcInferenceService; -} diff --git a/src/ovms_lib/grpcservermodule.hpp b/src/ovms_lib/grpcservermodule.hpp deleted file mode 100644 index 8ab7be17d0..0000000000 --- a/src/ovms_lib/grpcservermodule.hpp +++ /dev/null @@ -1,46 +0,0 @@ -//**************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include - -#include - -#include "model_service.hpp" -#include "prediction_service.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" - -namespace ovms { -class Config; - -class GRPCServerModule : public Module { - Server& server; - PredictionServiceImpl tfsPredictService; - ModelServiceImpl tfsModelService; - mutable KFSInferenceServiceImpl kfsGrpcInferenceService; - std::vector> servers; - -public: - GRPCServerModule(Server& server); - int start(const ovms::Config& config) override; - void shutdown() override; - - const GetModelMetadataImpl& getTFSModelMetadataImpl() const; - KFSInferenceServiceImpl& getKFSGrpcImpl() const; -}; -} // namespace ovms diff --git a/src/ovms_lib/http_rest_api_handler.cpp b/src/ovms_lib/http_rest_api_handler.cpp deleted file mode 100644 index 1628d22fb6..0000000000 --- a/src/ovms_lib/http_rest_api_handler.cpp +++ /dev/null @@ -1,606 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "http_rest_api_handler.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "config.hpp" -#include "filesystem.hpp" -#include "get_model_metadata_impl.hpp" -#include "grpcservermodule.hpp" -#include "kfs_grpc_inference_service.hpp" -#include "model_service.hpp" -#include "modelinstanceunloadguard.hpp" -#include "pipelinedefinition.hpp" -#include "prediction_service_utils.hpp" -#include "rest_parser.hpp" -#include "rest_utils.hpp" -#include "server.hpp" -#include "timer.hpp" - -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - -namespace ovms { - -const std::string HttpRestApiHandler::predictionRegexExp = - R"((.?)\/v1\/models\/([^\/:]+)(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?:(classify|regress|predict))"; -const std::string HttpRestApiHandler::modelstatusRegexExp = - R"((.?)\/v1\/models(?:\/([^\/:]+))?(?:(?:\/versions\/(\d+))|(?:\/labels\/(\w+)))?(?:\/(metadata))?)"; -const std::string HttpRestApiHandler::configReloadRegexExp = R"((.?)\/v1\/config\/reload)"; -const std::string HttpRestApiHandler::configStatusRegexExp = R"((.?)\/v1\/config)"; - -const std::string HttpRestApiHandler::kfs_modelreadyRegexExp = - R"(/v2/models/([^/]+)(?:/versions/([0-9]+))?(?:/(ready)))"; -const std::string HttpRestApiHandler::kfs_modelmetadataRegexExp = - R"(/v2/models/([^/]+)(?:/versions/([0-9]+))?(?:/)?)"; -HttpRestApiHandler::HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ms) : - predictionRegex(predictionRegexExp), - modelstatusRegex(modelstatusRegexExp), - configReloadRegex(configReloadRegexExp), - configStatusRegex(configStatusRegexExp), - kfs_modelreadyRegex(kfs_modelreadyRegexExp), - kfs_modelmetadataRegex(kfs_modelmetadataRegexExp), - timeout_in_ms(timeout_in_ms), - ovmsServer(ovmsServer), - - kfsGrpcImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getKFSGrpcImpl()), - grpcGetModelMetadataImpl(dynamic_cast(this->ovmsServer.getModule(GRPC_SERVER_MODULE_NAME))->getTFSModelMetadataImpl()) { - registerAll(); -} - -Status HttpRestApiHandler::parseModelVersion(std::string& model_version_str, std::optional& model_version) { - if (!model_version_str.empty()) { - try { - model_version = std::stoll(model_version_str.c_str()); - } catch (std::exception& e) { - SPDLOG_ERROR("Couldn't parse model version {}", model_version_str); - return StatusCode::REST_COULD_NOT_PARSE_VERSION; - } - } - return StatusCode::OK; -} - -void HttpRestApiHandler::registerHandler(RequestType type, std::function f) { - handlers[type] = f; -} - -void HttpRestApiHandler::registerAll() { - registerHandler(Predict, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - if (request_components.processing_method == "predict") { - return processPredictRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, request_body, &response); - } else { - SPDLOG_WARN("Requested REST resource not found"); - return (Status)StatusCode::REST_NOT_FOUND; - } - }); - - registerHandler(GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - return processModelMetadataRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, &response); - }); - registerHandler(GetModelStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - return processModelStatusRequest(request_components.model_name, request_components.model_version, - request_components.model_version_label, &response); - }); - registerHandler(ConfigReload, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& manager = servableManagerModule->getServableManager(); - return processConfigReloadRequest(response, manager); - }); - registerHandler(ConfigStatus, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& manager = servableManagerModule->getServableManager(); - return processConfigStatusRequest(response, manager); - }); - registerHandler(KFS_GetModelReady, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { - return processModelReadyKFSRequest(request_components, response, request_body); - }); - registerHandler(KFS_GetModelMetadata, [this](const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) -> Status { - return processModelReadyKFSRequest(request_components, response, request_body); - }); -} - -Status HttpRestApiHandler::dispatchToProcessor( - const std::string& request_body, - std::string* response, - const HttpRequestComponents& request_components) { - - auto handler = handlers.find(request_components.type); - if (handler != handlers.end()) { - return handler->second(request_components, *response, request_body); - } else { - return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; - } - return StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE; -} - -Status HttpRestApiHandler::processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - ::inference::ModelReadyRequest grpc_request; - ::inference::ModelReadyResponse grpc_response; - Status status; - std::string modelName(request_components.model_name); - std::string modelVersion(std::to_string(request_components.model_version.value_or(0))); - grpc_request.set_name(modelName); - grpc_request.set_version(modelVersion); - SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", modelName, modelVersion); - - kfsGrpcImpl.ModelReady(nullptr, &grpc_request, &grpc_response); - std::string output; - google::protobuf::util::JsonPrintOptions opts; - google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); - response = output; - return StatusCode::OK; -} - -Status HttpRestApiHandler::processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body) { - ::inference::ModelMetadataRequest grpc_request; - ::inference::ModelMetadataResponse grpc_response; - Status status; - std::string modelName(request_components.model_name); - std::string modelVersion(std::to_string(request_components.model_version.value_or(0))); - grpc_request.set_name(modelName); - grpc_request.set_version(modelVersion); - SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", modelName, modelVersion); - kfsGrpcImpl.ModelMetadata(nullptr, &grpc_request, &grpc_response); - std::string output; - google::protobuf::util::JsonPrintOptions opts; - google::protobuf::util::MessageToJsonString(grpc_response, &output, opts); - response = output; - return StatusCode::OK; -} - -Status HttpRestApiHandler::parseRequestComponents(HttpRequestComponents& requestComponents, - const std::string_view http_method, - const std::string& request_path) { - std::smatch sm; - requestComponents.http_method = http_method; - - if (http_method != "POST" && http_method != "GET") { - return StatusCode::REST_UNSUPPORTED_METHOD; - } - - if (FileSystem::isPathEscaped(request_path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", request_path); - return StatusCode::PATH_INVALID; - } - - if (http_method == "POST") { - if (std::regex_match(request_path, sm, predictionRegex)) { - requestComponents.type = Predict; - requestComponents.model_name = sm[2]; - - std::string model_version_str = sm[3]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - - std::string model_version_label_str = sm[4]; - if (!model_version_label_str.empty()) { - requestComponents.model_version_label = model_version_label_str; - } - - requestComponents.processing_method = sm[5]; - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, configReloadRegex)) { - requestComponents.type = ConfigReload; - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, modelstatusRegex)) - return StatusCode::REST_UNSUPPORTED_METHOD; - } else if (http_method == "GET") { - if (std::regex_match(request_path, sm, modelstatusRegex)) { - requestComponents.model_name = sm[2]; - - std::string model_version_str = sm[3]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - - std::string model_version_label_str = sm[4]; - if (!model_version_label_str.empty()) { - requestComponents.model_version_label = model_version_label_str; - } - - requestComponents.model_subresource = sm[5]; - if (!requestComponents.model_subresource.empty() && requestComponents.model_subresource == "metadata") { - requestComponents.type = GetModelMetadata; - } else { - requestComponents.type = GetModelStatus; - } - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, configStatusRegex)) { - requestComponents.type = ConfigStatus; - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, kfs_modelmetadataRegex)) { - requestComponents.model_name = sm[1]; - std::string model_version_str = sm[2]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - requestComponents.type = KFS_GetModelMetadata; - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, kfs_modelreadyRegex)) { - requestComponents.model_name = sm[1]; - std::string model_version_str = sm[2]; - auto status = parseModelVersion(model_version_str, requestComponents.model_version); - if (!status.ok()) - return status; - requestComponents.type = KFS_GetModelReady; - return StatusCode::OK; - } - if (std::regex_match(request_path, sm, predictionRegex)) - return StatusCode::REST_UNSUPPORTED_METHOD; - } - return StatusCode::REST_INVALID_URL; -} - -Status HttpRestApiHandler::processRequest( - const std::string_view http_method, - const std::string_view request_path, - const std::string& request_body, - std::vector>* headers, - std::string* response) { - - std::smatch sm; - std::string request_path_str(request_path); - if (FileSystem::isPathEscaped(request_path_str)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", request_path); - return StatusCode::PATH_INVALID; - } - - headers->clear(); - response->clear(); - headers->push_back({"Content-Type", "application/json"}); - - HttpRequestComponents requestComponents; - auto status = parseRequestComponents(requestComponents, http_method, request_path_str); - if (!status.ok()) - return status; - return dispatchToProcessor(request_body, response, requestComponents); -} - -Status HttpRestApiHandler::processPredictRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::optional& modelVersionLabel, - const std::string& request, - std::string* response) { - // model_version_label currently is not in use - - Timer timer; - timer.start("total"); - using std::chrono::microseconds; - - SPDLOG_DEBUG("Processing REST request for model: {}; version: {}", - modelName, modelVersion.value_or(0)); - - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& modelManager = servableManagerModule->getServableManager(); - Order requestOrder; - tensorflow::serving::PredictResponse responseProto; - Status status; - - if (modelManager.modelExists(modelName)) { - SPDLOG_DEBUG("Found model with name: {}. Searching for requested version...", modelName); - status = processSingleModelRequest(modelName, modelVersion, request, requestOrder, responseProto); - } else if (modelManager.pipelineDefinitionExists(modelName)) { - SPDLOG_DEBUG("Found pipeline with name: {}", modelName); - status = processPipelineRequest(modelName, request, requestOrder, responseProto); - } else { - SPDLOG_WARN("Model or pipeline matching request parameters not found - name: {}, version: {}", modelName, modelVersion.value_or(0)); - status = StatusCode::MODEL_NAME_MISSING; - } - if (!status.ok()) - return status; - - status = makeJsonFromPredictResponse(responseProto, response, requestOrder); - if (!status.ok()) - return status; - - timer.stop("total"); - SPDLOG_DEBUG("Total REST request processing time: {} ms", timer.elapsed("total") / 1000); - return StatusCode::OK; -} - -Status HttpRestApiHandler::processSingleModelRequest(const std::string& modelName, - const std::optional& modelVersion, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto) { - - std::shared_ptr modelInstance; - std::unique_ptr modelInstanceUnloadGuard; - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& modelManager = servableManagerModule->getServableManager(); - auto status = modelManager.getModelInstance( - modelName, - modelVersion.value_or(0), - modelInstance, - modelInstanceUnloadGuard); - - if (!status.ok()) { - SPDLOG_WARN("Requested model instance - name: {}, version: {} - does not exist.", modelName, modelVersion.value_or(0)); - return status; - } - Timer timer; - timer.start("parse"); - RestParser requestParser(modelInstance->getInputsInfo()); - status = requestParser.parse(request.c_str()); - if (!status.ok()) { - return status; - } - requestOrder = requestParser.getOrder(); - timer.stop("parse"); - SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed("parse") / 1000); - - tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); - requestProto.mutable_model_spec()->set_name(modelName); - if (modelVersion.has_value()) { - requestProto.mutable_model_spec()->mutable_version()->set_value(modelVersion.value()); - } - status = modelInstance->infer(&requestProto, &responseProto, modelInstanceUnloadGuard); - return status; -} - -Status HttpRestApiHandler::getPipelineInputs(const std::string& modelName, ovms::tensor_map_t& inputs) { - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& modelManager = servableManagerModule->getServableManager(); - auto pipelineDefinition = modelManager.getPipelineFactory().findDefinitionByName(modelName); - if (!pipelineDefinition) { - return StatusCode::MODEL_MISSING; - } - std::unique_ptr unloadGuard; - Status status = pipelineDefinition->waitForLoaded(unloadGuard); - if (!status.ok()) { - return status; - } - - inputs = pipelineDefinition->getInputsInfo(); - return StatusCode::OK; -} - -Status HttpRestApiHandler::processPipelineRequest(const std::string& modelName, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto) { - - std::unique_ptr pipelinePtr; - - Timer timer; - timer.start("parse"); - ovms::tensor_map_t inputs; - auto status = getPipelineInputs(modelName, inputs); - if (!status.ok()) { - return status; - } - - RestParser requestParser(inputs); - status = requestParser.parse(request.c_str()); - if (!status.ok()) { - return status; - } - requestOrder = requestParser.getOrder(); - timer.stop("parse"); - SPDLOG_DEBUG("JSON request parsing time: {} ms", timer.elapsed("parse") / 1000); - - tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); - requestProto.mutable_model_spec()->set_name(modelName); - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& manager = servableManagerModule->getServableManager(); - status = manager.createPipeline(pipelinePtr, modelName, &requestProto, &responseProto); - if (!status.ok()) { - return status; - } - status = pipelinePtr->execute(); - return status; -} - -Status HttpRestApiHandler::processModelMetadataRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response) { - // model_version_label currently is not in use - tensorflow::serving::GetModelMetadataRequest grpc_request; - tensorflow::serving::GetModelMetadataResponse grpc_response; - Status status; - std::string modelName(model_name); - status = grpcGetModelMetadataImpl.createGrpcRequest(modelName, model_version, &grpc_request); - if (!status.ok()) { - return status; - } - status = grpcGetModelMetadataImpl.getModelStatus(&grpc_request, &grpc_response); - if (!status.ok()) { - return status; - } - status = grpcGetModelMetadataImpl.serializeResponse2Json(&grpc_response, response); - if (!status.ok()) { - return status; - } - return StatusCode::OK; -} - -Status HttpRestApiHandler::processModelStatusRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response) { - // model_version_label currently is not in use - SPDLOG_DEBUG("Processing model status request"); - tensorflow::serving::GetModelStatusRequest grpc_request; - tensorflow::serving::GetModelStatusResponse grpc_response; - Status status; - std::string modelName(model_name); - status = GetModelStatusImpl::createGrpcRequest(modelName, model_version, &grpc_request); - if (!status.ok()) { - return status; - } - // TODO #KFS_CLEANUP - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO #KFS_CLEANUP - auto& manager = servableManagerModule->getServableManager(); - status = GetModelStatusImpl::getModelStatus(&grpc_request, &grpc_response, manager); - if (!status.ok()) { - return status; - } - status = GetModelStatusImpl::serializeResponse2Json(&grpc_response, response); - if (!status.ok()) { - return status; - } - return StatusCode::OK; -} - -std::string createErrorJsonWithMessage(std::string message) { - return "{\n\t\"error\": \"" + message + "\"\n}"; -} - -Status HttpRestApiHandler::processConfigReloadRequest(std::string& response, ModelManager& manager) { - SPDLOG_DEBUG("Processing config reload request started."); - Status status; - auto& config = ovms::Config::instance(); - - bool reloadNeeded = false; - if (manager.getConfigFilename() != "") { - status = manager.configFileReloadNeeded(reloadNeeded); - if (!reloadNeeded) { - if (status == StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED) { - response = createErrorJsonWithMessage("Config file not found or cannot open."); - return status; - } - } - } - - if (reloadNeeded) { - status = manager.loadConfig(config.configPath()); - if (!status.ok()) { - response = createErrorJsonWithMessage("Reloading config file failed. Check server logs for more info."); - return status; - } - } else { - if (!status.ok()) { - status = manager.loadConfig(config.configPath()); - if (!status.ok()) { - response = createErrorJsonWithMessage("Reloading config file failed. Check server logs for more info."); - return status; - } - reloadNeeded = true; - } - } - - status = manager.updateConfigurationWithoutConfigFile(); - if (!status.ok()) { - response = createErrorJsonWithMessage("Reloading models versions failed. Check server logs for more info."); - return status; - } - if (status == StatusCode::OK_RELOADED) { - reloadNeeded = true; - } - - std::map modelsStatuses; - status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager); - if (!status.ok()) { - response = createErrorJsonWithMessage("Retrieving all model statuses failed. Check server logs for more info."); - return status; - } - - status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); - if (!status.ok()) { - response = createErrorJsonWithMessage("Serializing model statuses to json failed. Check server logs for more info."); - return status; - } - - if (!reloadNeeded) { - SPDLOG_DEBUG("Config file reload was not needed."); - return StatusCode::OK_NOT_RELOADED; - } - return StatusCode::OK_RELOADED; -} - -Status HttpRestApiHandler::processConfigStatusRequest(std::string& response, ModelManager& manager) { - SPDLOG_DEBUG("Processing config status request started."); - Status status; - - std::map modelsStatuses; - status = GetModelStatusImpl::getAllModelsStatuses(modelsStatuses, manager); - if (!status.ok()) { - response = createErrorJsonWithMessage("Retrieving all model statuses failed."); - return status; - } - - status = GetModelStatusImpl::serializeModelsStatuses2Json(modelsStatuses, response); - if (!status.ok()) { - response = createErrorJsonWithMessage("Serializing model statuses to json failed."); - return status; - } - - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/http_rest_api_handler.hpp b/src/ovms_lib/http_rest_api_handler.hpp deleted file mode 100644 index 4d76bbfa87..0000000000 --- a/src/ovms_lib/http_rest_api_handler.hpp +++ /dev/null @@ -1,190 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "modelmanager.hpp" -#include "rest_parser.hpp" -#include "status.hpp" - -namespace ovms { -class KFSInferenceServiceImpl; -class GetModelMetadataImpl; -class Server; -enum RequestType { Predict, - GetModelStatus, - GetModelMetadata, - ConfigReload, - ConfigStatus, - KFS_GetModelReady, - KFS_GetModelMetadata }; -struct HttpRequestComponents { - RequestType type; - std::string_view http_method; - std::string model_name; - std::optional model_version; - std::optional model_version_label; - std::string processing_method; - std::string model_subresource; -}; - -class HttpRestApiHandler { -public: - static const std::string predictionRegexExp; - static const std::string modelstatusRegexExp; - static const std::string configReloadRegexExp; - static const std::string configStatusRegexExp; - - static const std::string kfs_modelreadyRegexExp; - static const std::string kfs_modelmetadataRegexExp; - /** - * @brief Construct a new HttpRest Api Handler - * - * @param timeout_in_ms - */ - HttpRestApiHandler(ovms::Server& ovmsServer, int timeout_in_ms); - - Status parseRequestComponents(HttpRequestComponents& components, - const std::string_view http_method, - const std::string& request_path); - - Status parseModelVersion(std::string& model_version_str, std::optional& model_version); - - void registerHandler(RequestType type, std::function); - void registerAll(); - - Status dispatchToProcessor( - const std::string& request_body, - std::string* response, - const HttpRequestComponents& request_components); - - /** - * @brief Process Request - * - * @param http_method - * @param request_path - * @param request_body - * @param headers - * @param resposnse - * - * @return StatusCode - */ - Status processRequest( - const std::string_view http_method, - const std::string_view request_path, - const std::string& request_body, - std::vector>* headers, - std::string* response); - - /** - * @brief Process predict request - * - * @param modelName - * @param modelVersion - * @param modelVersionLabel - * @param request - * @param response - * - * @return StatusCode - */ - Status processPredictRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::optional& modelVersionLabel, - const std::string& request, - std::string* response); - - Status processSingleModelRequest( - const std::string& modelName, - const std::optional& modelVersion, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto); - - Status processPipelineRequest( - const std::string& modelName, - const std::string& request, - Order& requestOrder, - tensorflow::serving::PredictResponse& responseProto); - - /** - * @brief Process Model Metadata request - * - * @param model_name - * @param model_version - * @param model_version_label - * @param response - * - * @return StatusCode - */ - Status processModelMetadataRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response); - - /** - * @brief Process Model Status request - * - * @param model_name - * @param model_version - * @param model_version_label - * @param response - * @return StatusCode - */ - Status processModelStatusRequest( - const std::string_view model_name, - const std::optional& model_version, - const std::optional& model_version_label, - std::string* response); - - Status processConfigReloadRequest(std::string& response, ModelManager& manager); - - Status processConfigStatusRequest(std::string& response, ModelManager& manager); - Status processModelMetadataKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); - Status processModelReadyKFSRequest(const HttpRequestComponents& request_components, std::string& response, const std::string& request_body); - -private: - const std::regex predictionRegex; - const std::regex modelstatusRegex; - const std::regex configReloadRegex; - const std::regex configStatusRegex; - - const std::regex kfs_modelreadyRegex; - const std::regex kfs_modelmetadataRegex; - - std::map> handlers; - int timeout_in_ms; - - ovms::Server& ovmsServer; - ovms::KFSInferenceServiceImpl& kfsGrpcImpl; - const GetModelMetadataImpl& grpcGetModelMetadataImpl; - - Status getPipelineInputs(const std::string& modelName, ovms::tensor_map_t& inputs); -}; - -} // namespace ovms diff --git a/src/ovms_lib/http_server.cpp b/src/ovms_lib/http_server.cpp deleted file mode 100644 index 1304d4e425..0000000000 --- a/src/ovms_lib/http_server.cpp +++ /dev/null @@ -1,131 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "http_server.hpp" - -#include -#include -#include -#include -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/util/net_http/server/public/httpserver.h" -#include "tensorflow_serving/util/net_http/server/public/response_code_enum.h" -#include "tensorflow_serving/util/net_http/server/public/server_request_interface.h" -#include "tensorflow_serving/util/threadpool_executor.h" -#pragma GCC diagnostic pop - -#include "http_rest_api_handler.hpp" -#include "status.hpp" - -namespace ovms { - -namespace net_http = tensorflow::serving::net_http; - -class RequestExecutor final : public net_http::EventExecutor { -public: - explicit RequestExecutor(int num_threads) : - executor_(tensorflow::Env::Default(), "httprestserver", num_threads) {} - - void Schedule(std::function fn) override { executor_.Schedule(fn); } - -private: - tensorflow::serving::ThreadPoolExecutor executor_; -}; - -class RestApiRequestDispatcher { -public: - RestApiRequestDispatcher(ovms::Server& ovmsServer, int timeout_in_ms) { - handler_ = std::make_unique(ovmsServer, timeout_in_ms); - } - - net_http::RequestHandler dispatch(net_http::ServerRequestInterface* req) { - return [this](net_http::ServerRequestInterface* req) { - this->processRequest(req); - }; - } - -private: - void processRequest(net_http::ServerRequestInterface* req) { - SPDLOG_DEBUG("REST request {}", req->uri_path()); - std::string body; - int64_t num_bytes = 0; - auto request_chunk = req->ReadRequestBytes(&num_bytes); - while (request_chunk != nullptr) { - body.append(std::string_view(request_chunk.get(), num_bytes)); - request_chunk = req->ReadRequestBytes(&num_bytes); - } - - std::vector> headers; - std::string output; - SPDLOG_DEBUG("Processing HTTP request: {} {} body: {} bytes", - req->http_method(), - req->uri_path(), - body.size()); - const auto status = handler_->processRequest(req->http_method(), req->uri_path(), body, &headers, &output); - if (!status.ok() && output.empty()) { - output.append("{\"error\": \"" + status.string() + "\"}"); - } - const auto http_status = status.http(); - for (const auto& kv : headers) { - req->OverwriteResponseHeader(kv.first, kv.second); - } - req->WriteResponseString(output); - if (http_status != net_http::HTTPStatusCode::OK && http_status != net_http::HTTPStatusCode::CREATED) { - SPDLOG_DEBUG("Processing HTTP/REST request failed: {} {}. Reason: {}", - req->http_method(), - req->uri_path(), - status.string()); - } - req->ReplyWithStatus(http_status); - } - - std::unique_ptr handler_; -}; - -std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms) { - auto options = std::make_unique(); - options->AddPort(static_cast(port)); - options->SetAddress(address); - options->SetExecutor(std::make_unique(num_threads)); - - auto server = net_http::CreateEvHTTPServer(std::move(options)); - if (server == nullptr) { - SPDLOG_ERROR("Failed to create http server"); - return nullptr; - } - - std::shared_ptr dispatcher = - std::make_shared(ovmsServer, timeout_in_ms); - - net_http::RequestHandlerOptions handler_options; - server->RegisterRequestDispatcher( - [dispatcher](net_http::ServerRequestInterface* req) { - return dispatcher->dispatch(req); - }, - handler_options); - - if (server->StartAcceptingRequests()) { - SPDLOG_INFO("REST server listening on port {} with {} threads", port, num_threads); - return server; - } - - return nullptr; -} -} // namespace ovms diff --git a/src/ovms_lib/http_server.hpp b/src/ovms_lib/http_server.hpp deleted file mode 100644 index 88117e3048..0000000000 --- a/src/ovms_lib/http_server.hpp +++ /dev/null @@ -1,41 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/util/net_http/server/public/httpserver_interface.h" -#pragma GCC diagnostic pop - -namespace ovms { -class Server; - -using http_server = tensorflow::serving::net_http::HTTPServerInterface; - -/** - * @brief Creates a and starts Http Server - * - * @param port - * @param num_threads - * @param timeout_in_m not implemented - * - * @return std::unique_ptr - */ -std::unique_ptr createAndStartHttpServer(const std::string& address, int port, int num_threads, ovms::Server& ovmsServer, int timeout_in_ms = -1); -} // namespace ovms diff --git a/src/ovms_lib/kfs_grpc_inference_service.cpp b/src/ovms_lib/kfs_grpc_inference_service.cpp deleted file mode 100644 index 4d803f2756..0000000000 --- a/src/ovms_lib/kfs_grpc_inference_service.cpp +++ /dev/null @@ -1,343 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "kfs_grpc_inference_service.hpp" - -#include -#include -#include - -#include "deserialization.hpp" -#include "modelinstance.hpp" -#include "modelmanager.hpp" -#include "ovinferrequestsqueue.hpp" -#include "pipelinedefinition.hpp" -#include "prediction_service_utils.hpp" -#include "serialization.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "tensorinfo.hpp" -#include "timer.hpp" -#include "version.hpp" - -namespace ovms { - -Status KFSInferenceServiceImpl::getModelInstance(const ::inference::ModelInferRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; // TODO consider other + add details - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - model_version_t requestedVersion = 0; - if (!request->model_version().empty()) { - auto versionRead = stoi64(request->model_version()); - if (versionRead) { - requestedVersion = versionRead.value(); - } else { - SPDLOG_DEBUG("requested model: name {}; with version in invalid format: {}", request->model_name(), request->model_version()); - return StatusCode::MODEL_VERSION_INVALID_FORMAT; - } - } - return manager.getModelInstance(request->model_name(), requestedVersion, modelInstance, modelInstanceUnloadGuardPtr); -} - -Status KFSInferenceServiceImpl::getPipeline(const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, - std::unique_ptr& pipelinePtr) { - OVMS_PROFILE_FUNCTION(); - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; // TODO consider other + add details - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return manager.createPipeline(pipelinePtr, request->model_name(), request, response); -} - -const std::string PLATFORM = "OpenVINO"; - -::grpc::Status KFSInferenceServiceImpl::ServerLive(::grpc::ServerContext* context, const ::inference::ServerLiveRequest* request, ::inference::ServerLiveResponse* response) { - (void)context; - (void)request; - (void)response; - bool isLive = this->ovmsServer.isLive(); - SPDLOG_DEBUG("Requested Server liveness state: {}", isLive); - response->set_live(isLive); - return grpc::Status::OK; -} - -::grpc::Status KFSInferenceServiceImpl::ServerReady(::grpc::ServerContext* context, const ::inference::ServerReadyRequest* request, ::inference::ServerReadyResponse* response) { - (void)context; - (void)request; - (void)response; - bool isReady = this->ovmsServer.isReady(); - SPDLOG_DEBUG("Requested Server readiness state: {}", isReady); - response->set_ready(isReady); - return grpc::Status::OK; -} - -Status KFSInferenceServiceImpl::getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager) { - // Return in response true/false - // if no version requested give response for default version - const auto& name = request->name(); - const auto& versionString = request->version(); - auto model = manager.findModelByName(name); - SPDLOG_DEBUG("ModelReady requested name: {}, version: {}", name, versionString); - if (model == nullptr) { - SPDLOG_DEBUG("ModelReady requested model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { - return Status(StatusCode::MODEL_NAME_MISSING); - } - return buildResponse(*pipelineDefinition, response); - } - std::shared_ptr instance = nullptr; - if (!versionString.empty()) { - SPDLOG_DEBUG("ModelReady requested model: name {}; version {}", name, versionString); - model_version_t requestedVersion = 0; - auto versionRead = stoi64(versionString); - if (versionRead) { - requestedVersion = versionRead.value(); - } else { - SPDLOG_DEBUG("ModelReady requested model: name {}; with version in invalid format: {}", name, versionString); - return Status(StatusCode::MODEL_VERSION_INVALID_FORMAT); - } - instance = model->getModelInstanceByVersion(requestedVersion); - if (instance == nullptr) { - SPDLOG_DEBUG("ModelReady requested model {}; version {} is missing", name, versionString); - return Status(StatusCode::MODEL_VERSION_MISSING); - } - } else { - SPDLOG_DEBUG("ModelReady requested model: name {}; default version", name); - instance = model->getDefaultModelInstance(); - if (instance == nullptr) { - SPDLOG_DEBUG("ModelReady requested model {}; version {} is missing", name, versionString); - return Status(StatusCode::MODEL_VERSION_MISSING); - } - } - return buildResponse(instance, response); -} - -::grpc::Status KFSInferenceServiceImpl::ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) { - (void)context; - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return this->getModelReady(request, response, manager).grpc(); -} - -::grpc::Status KFSInferenceServiceImpl::ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) { - (void)context; - (void)request; - (void)response; - response->set_name(PROJECT_NAME); - response->set_version(PROJECT_VERSION); - return grpc::Status::OK; -} - -::grpc::Status KFSInferenceServiceImpl::ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) { - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - const auto& name = request->name(); - const auto& versionString = request->version(); - - auto model = manager.findModelByName(name); - if (model == nullptr) { - SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { - return Status(StatusCode::MODEL_NAME_MISSING).grpc(); - } - return buildResponse(*pipelineDefinition, response); - } - std::shared_ptr instance = nullptr; - if (!versionString.empty()) { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; version {}", name, versionString); - model_version_t requestedVersion = 0; - auto versionRead = stoi64(versionString); - if (versionRead) { - requestedVersion = versionRead.value(); - } else { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; with version in invalid format: {}", name, versionString); - return Status(StatusCode::MODEL_VERSION_INVALID_FORMAT).grpc(); - } - instance = model->getModelInstanceByVersion(requestedVersion); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, versionString); - return Status(StatusCode::MODEL_VERSION_MISSING).grpc(); - } - } else { - SPDLOG_DEBUG("GetModelMetadata requested model: name {}; default version", name); - instance = model->getDefaultModelInstance(); - if (instance == nullptr) { - SPDLOG_DEBUG("GetModelMetadata requested model {}; version {} is missing", name, versionString); - return Status(StatusCode::MODEL_VERSION_MISSING).grpc(); - } - } - return buildResponse(*model, *instance, response).grpc(); -} - -::grpc::Status KFSInferenceServiceImpl::ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) { - (void)context; - OVMS_PROFILE_FUNCTION(); - Timer timer; - timer.start("total"); - using std::chrono::microseconds; - SPDLOG_DEBUG("Processing gRPC request for model: {}; version: {}", - request->model_name(), - request->model_version()); - - std::shared_ptr modelInstance; - std::unique_ptr pipelinePtr; - - std::unique_ptr modelInstanceUnloadGuard; - auto status = getModelInstance(request, modelInstance, modelInstanceUnloadGuard); - if (status == StatusCode::MODEL_NAME_MISSING) { - SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", request->model_name()); - status = getPipeline(request, response, pipelinePtr); - } - if (!status.ok()) { - SPDLOG_DEBUG("Getting modelInstance or pipeline failed. {}", status.string()); - return status.grpc(); - } - - if (pipelinePtr) { - status = pipelinePtr->execute(); - } else { - status = modelInstance->infer(request, response, modelInstanceUnloadGuard); - } - - if (!status.ok()) { - return status.grpc(); - } - - timer.stop("total"); - SPDLOG_DEBUG("Total gRPC request processing time: {} ms", timer.elapsed("total") / 1000); - return grpc::Status::OK; -} - -Status KFSInferenceServiceImpl::buildResponse( - std::shared_ptr instance, - ::inference::ModelReadyResponse* response) { - response->set_ready(instance->getStatus().getState() == ModelVersionState::AVAILABLE); - return StatusCode::OK; -} - -Status KFSInferenceServiceImpl::buildResponse( - PipelineDefinition& pipelineDefinition, - ::inference::ModelReadyResponse* response) { - response->set_ready(pipelineDefinition.getStatus().isAvailable()); - return StatusCode::OK; -} - -void addReadyVersions(Model& model, - ::inference::ModelMetadataResponse* response) { - auto modelVersions = model.getModelVersionsMapCopy(); - for (auto& [modelVersion, modelInstance] : modelVersions) { - if (modelInstance.getStatus().getState() == ModelVersionState::AVAILABLE) - response->add_versions(std::to_string(modelVersion)); - } -} - -Status KFSInferenceServiceImpl::buildResponse( - Model& model, - ModelInstance& instance, - ::inference::ModelMetadataResponse* response) { - - std::unique_ptr unloadGuard; - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = instance.waitForLoaded(0, unloadGuard); - if (!status.ok()) { - return status; - } - - response->Clear(); - response->set_name(instance.getName()); - addReadyVersions(model, response); - response->set_platform(PLATFORM); - - for (const auto& input : instance.getInputsInfo()) { - convert(input, response->add_inputs()); - } - - for (const auto& output : instance.getOutputsInfo()) { - convert(output, response->add_outputs()); - } - - return StatusCode::OK; -} - -KFSInferenceServiceImpl::KFSInferenceServiceImpl(const Server& server) : - ovmsServer(server) {} - -Status KFSInferenceServiceImpl::buildResponse( - PipelineDefinition& pipelineDefinition, - ::inference::ModelMetadataResponse* response) { - - std::unique_ptr unloadGuard; - - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); - if (!status.ok()) { - return status; - } - - response->Clear(); - response->set_name(pipelineDefinition.getName()); - response->add_versions("1"); - response->set_platform(PLATFORM); - - for (const auto& input : pipelineDefinition.getInputsInfo()) { - convert(input, response->add_inputs()); - } - - for (const auto& output : pipelineDefinition.getOutputsInfo()) { - convert(output, response->add_outputs()); - } - - return StatusCode::OK; -} - -void KFSInferenceServiceImpl::convert( - const std::pair>& from, - ::inference::ModelMetadataResponse::TensorMetadata* to) { - to->set_name(from.first); - to->set_datatype(from.second->getPrecisionAsKFSPrecision()); - for (auto dim : from.second->getShape()) { - if (dim.isStatic()) { - to->add_shape(dim.getStaticValue()); - } else { - to->add_shape(DYNAMIC_DIMENSION); - } - } -} - -} // namespace ovms diff --git a/src/ovms_lib/kfs_grpc_inference_service.hpp b/src/ovms_lib/kfs_grpc_inference_service.hpp deleted file mode 100644 index 24c4b486fc..0000000000 --- a/src/ovms_lib/kfs_grpc_inference_service.hpp +++ /dev/null @@ -1,68 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" -#include "src/kfserving_api/grpc_predict_v2.pb.h" -#include "status.hpp" - -namespace ovms { - -using inference::GRPCInferenceService; -class Model; -class ModelManager; -class ModelInstance; -class ModelInstanceUnloadGuard; -class Pipeline; -class Server; -class Status; -class TensorInfo; -class PipelineDefinition; - -class KFSInferenceServiceImpl final : public GRPCInferenceService::Service { - const Server& ovmsServer; - -public: - KFSInferenceServiceImpl(const Server& server); - ::grpc::Status ServerLive(::grpc::ServerContext* context, const ::inference::ServerLiveRequest* request, ::inference::ServerLiveResponse* response) override; - ::grpc::Status ServerReady(::grpc::ServerContext* context, const ::inference::ServerReadyRequest* request, ::inference::ServerReadyResponse* response) override; - ::grpc::Status ModelReady(::grpc::ServerContext* context, const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response) override; - ::grpc::Status ServerMetadata(::grpc::ServerContext* context, const ::inference::ServerMetadataRequest* request, ::inference::ServerMetadataResponse* response) override; - ::grpc::Status ModelMetadata(::grpc::ServerContext* context, const ::inference::ModelMetadataRequest* request, ::inference::ModelMetadataResponse* response) override; - ::grpc::Status ModelInfer(::grpc::ServerContext* context, const ::inference::ModelInferRequest* request, ::inference::ModelInferResponse* response) override; - static Status buildResponse(Model& model, ModelInstance& instance, ::inference::ModelMetadataResponse* response); - static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelMetadataResponse* response); - static Status buildResponse(std::shared_ptr instance, ::inference::ModelReadyResponse* response); - static Status buildResponse(PipelineDefinition& pipelineDefinition, ::inference::ModelReadyResponse* response); - static void convert(const std::pair>& from, ::inference::ModelMetadataResponse::TensorMetadata* to); - static Status getModelReady(const ::inference::ModelReadyRequest* request, ::inference::ModelReadyResponse* response, const ModelManager& manager); - -protected: - Status getModelInstance(const ::inference::ModelInferRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr); - Status getPipeline(const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, - std::unique_ptr& pipelinePtr); -}; - -} // namespace ovms diff --git a/src/ovms_lib/layout.cpp b/src/ovms_lib/layout.cpp deleted file mode 100644 index 92b7ea2903..0000000000 --- a/src/ovms_lib/layout.cpp +++ /dev/null @@ -1,169 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "layout.hpp" - -#include -#include -#include - -#include "logging.hpp" - -const char* DEFAULT_LAYOUT = "N..."; -const char* UNSPECIFIED_LAYOUT = "..."; - -namespace ovms { -const Layout& Layout::getDefaultLayout() { - static const Layout defaultLayout{DEFAULT_LAYOUT}; - return defaultLayout; -} - -const Layout& Layout::getUnspecifiedLayout() { - static const Layout unspecifiedLayout{UNSPECIFIED_LAYOUT}; - return unspecifiedLayout; -} - -Layout::Layout(const std::string& str) : - std::string(str) { - this->batchIndex = retrieveBatchIndex(); -} - -const std::optional& Layout::getBatchIndex() const { - return this->batchIndex; -} - -std::optional Layout::retrieveBatchIndex() const { - auto status = validate(); - if (!status.ok()) { - return std::nullopt; - } - auto batchPos = this->find(BATCH_DIMENSION_LETTER); - auto etcPos = this->find(ETC_LAYOUT_DELIMETER); - if (static_cast(*this) == UNSPECIFIED_LAYOUT) { - // we want to treat ANY layout as having BS on 0 position - // otherwise in any case we extract this we have to check - // against layout ... - return 0; - } - if (batchPos == std::string::npos) { - return std::nullopt; - } - if (etcPos != std::string::npos && batchPos > etcPos) { - return std::nullopt; - } - return batchPos; -} - -Status Layout::validate() const { - if (this->find_first_not_of(ALLOWED_DIMENSION_LETTERS_AND_CHARS) != std::string::npos) - return StatusCode::LAYOUT_WRONG_FORMAT; // Cannot contain other letters - - for (char c : ALLOWED_DIMENSION_LETTERS) { - if (std::count(this->begin(), this->end(), c) > 1) { - return StatusCode::LAYOUT_WRONG_FORMAT; // Can contain NCHWD only single time - } - } - - size_t dotCount = 0; - bool firstEtcAppeared = false; - bool fullEtcAppeared = false; - for (char c : (*this)) { - if (c == ETC_CHAR) { - if (fullEtcAppeared) { - return StatusCode::LAYOUT_WRONG_FORMAT; // Cannot appear multiple times - } - firstEtcAppeared = true; - dotCount++; - if (dotCount >= 3) { - fullEtcAppeared = true; - firstEtcAppeared = false; - } - } else if (firstEtcAppeared) { - return StatusCode::LAYOUT_WRONG_FORMAT; // Dots separated - } - } - if (firstEtcAppeared && !fullEtcAppeared) { - return StatusCode::LAYOUT_WRONG_FORMAT; // Dots not completed - } - - return StatusCode::OK; -} - -std::optional Layout::createIntersection(const Layout& other, size_t numberOfDimensions) const { - Layout lhs = (*this); - Layout rhs = other; - - if (lhs.containsEtc()) { - size_t knownDimensions = std::count_if(lhs.begin(), lhs.end(), [](char c) { return c != ETC_CHAR; }); - if (knownDimensions > numberOfDimensions) - return std::nullopt; - size_t unknownDimensions = numberOfDimensions - knownDimensions; - lhs.replace(lhs.find(ETC_LAYOUT_DELIMETER), ETC_LAYOUT_DELIMETER.size(), std::string(unknownDimensions, UNDEFINED_DIMENSION_CHAR)); - } - - if (rhs.containsEtc()) { - size_t knownDimensions = std::count_if(rhs.begin(), rhs.end(), [](char c) { return c != ETC_CHAR; }); - if (knownDimensions > numberOfDimensions) - return std::nullopt; - size_t unknownDimensions = numberOfDimensions - knownDimensions; - rhs.replace(rhs.find(ETC_LAYOUT_DELIMETER), ETC_LAYOUT_DELIMETER.size(), std::string(unknownDimensions, UNDEFINED_DIMENSION_CHAR)); - } - - if (lhs.size() != rhs.size() || lhs.size() != numberOfDimensions) - return std::nullopt; - - for (size_t i = 0; i < lhs.size(); i++) { - if (lhs[i] == rhs[i]) - continue; - if (rhs[i] != UNDEFINED_DIMENSION_CHAR && lhs.find(rhs[i]) != std::string::npos) - return std::nullopt; - if (lhs[i] == UNDEFINED_DIMENSION_CHAR) { - lhs[i] = rhs[i]; - continue; - } - if (rhs[i] == UNDEFINED_DIMENSION_CHAR) - continue; - return std::nullopt; - } - - return lhs; -} - -Layout Layout::fromOvLayout(const ov::Layout& layout) { - std::string strCopy = layout.to_string(); - strCopy.erase(std::remove_if(strCopy.begin(), strCopy.end(), - [](char c) -> bool { - return c == '[' || c == ']' || c == ','; - }), - strCopy.end()); - return Layout(strCopy); -} - -bool Layout::containsEtc() const { - return this->find(ETC_LAYOUT_DELIMETER) != std::string::npos; -} - -std::string::size_type Layout::getNumberOfKnownDimensions() const { - return std::count_if(this->begin(), this->end(), [](char c) { return ALLOWED_DIMENSION_LETTERS.find(c) != std::string::npos || c == UNDEFINED_DIMENSION_CHAR; }); -} - -bool Layout::isCompatible(const Shape& shape) const { - if (this->containsEtc()) { - return this->getNumberOfKnownDimensions() <= shape.size(); - } - return this->getNumberOfKnownDimensions() == shape.size(); -} - -} // namespace ovms diff --git a/src/ovms_lib/layout.hpp b/src/ovms_lib/layout.hpp deleted file mode 100644 index 42fb215ba3..0000000000 --- a/src/ovms_lib/layout.hpp +++ /dev/null @@ -1,59 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "shape.hpp" -#include "status.hpp" - -namespace ovms { - -static const std::string ALLOWED_DIMENSION_LETTERS = "NCHWD"; -static const char ETC_CHAR = '.'; -static const char UNDEFINED_DIMENSION_CHAR = '?'; -static const std::string ALLOWED_DIMENSION_LETTERS_AND_CHARS = ALLOWED_DIMENSION_LETTERS + ETC_CHAR + UNDEFINED_DIMENSION_CHAR; -static const std::string ETC_LAYOUT_DELIMETER = "..."; -static const std::string BATCH_DIMENSION_LETTER = "N"; - -class Layout : public std::string { - std::optional batchIndex = std::nullopt; - - std::optional retrieveBatchIndex() const; - bool containsEtc() const; - - std::string::size_type getNumberOfKnownDimensions() const; - -public: - Layout() = default; - Layout(const std::string& str); - - static Layout fromOvLayout(const ov::Layout& layout); - - const std::optional& getBatchIndex() const; - Status validate() const; - std::optional createIntersection(const Layout& other, size_t numberOfDimensions) const; - static const Layout& getDefaultLayout(); - static const Layout& getUnspecifiedLayout(); - - bool isCompatible(const Shape& shape) const; -}; - -} // namespace ovms diff --git a/src/ovms_lib/layout_configuration.cpp b/src/ovms_lib/layout_configuration.cpp deleted file mode 100644 index 7462e3f76d..0000000000 --- a/src/ovms_lib/layout_configuration.cpp +++ /dev/null @@ -1,98 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "layout_configuration.hpp" - -#include -#include -#include - -#include "stringutils.hpp" - -namespace ovms { - -LayoutConfiguration::LayoutConfiguration(const char* layout) : - LayoutConfiguration(std::string(layout)) { -} - -LayoutConfiguration::LayoutConfiguration(const std::string& layout) : - LayoutConfiguration(layout, layout) { -} - -LayoutConfiguration::LayoutConfiguration(const std::string& tensorLayout, const std::string& modelLayout) : - tensor(Layout(tensorLayout)), - model(Layout(modelLayout)) { -} - -bool LayoutConfiguration::isSet() const { - return !tensor.empty() || !model.empty(); -} - -Status LayoutConfiguration::fromString(const std::string& configurationStr, LayoutConfiguration& configOut) { - std::string configurationCopy = configurationStr; - erase_spaces(configurationCopy); - - std::transform(configurationCopy.begin(), configurationCopy.end(), configurationCopy.begin(), ::toupper); - - if (configurationCopy.find_first_not_of(ALLOWED_DIMENSION_LETTERS_AND_CHARS + LAYOUT_CONFIGURATION_DELIMETER) != std::string::npos) - return StatusCode::LAYOUT_WRONG_FORMAT; - - size_t delimCount = std::count(configurationCopy.begin(), configurationCopy.end(), LAYOUT_CONFIGURATION_DELIMETER); - if (delimCount > 1) - return StatusCode::LAYOUT_WRONG_FORMAT; - - if (delimCount == 0) { - configOut = LayoutConfiguration(configurationCopy); - } else { - std::vector tokens = tokenize(configurationCopy, LAYOUT_CONFIGURATION_DELIMETER); - if (tokens.size() > 2) - return StatusCode::LAYOUT_WRONG_FORMAT; - else if (tokens.size() == 2) - configOut = LayoutConfiguration(tokens[0], tokens[1]); - else if (tokens.size() == 1) - configOut = LayoutConfiguration(tokens[0]); - else - return StatusCode::LAYOUT_WRONG_FORMAT; - } - return StatusCode::OK; -} - -std::string LayoutConfiguration::toString() const { - std::stringstream ss; - if (tensor.empty()) { - ss << model; - } else { - ss << tensor << LAYOUT_CONFIGURATION_DELIMETER << model; - } - return ss.str(); -} - -const Layout& LayoutConfiguration::getTensorLayout() const { - return this->tensor; -} - -const Layout& LayoutConfiguration::getModelLayout() const { - return this->model; -} - -bool LayoutConfiguration::operator==(const LayoutConfiguration& rhs) const { - return this->tensor == rhs.tensor && this->model == rhs.model; -} - -bool LayoutConfiguration::operator!=(const LayoutConfiguration& rhs) const { - return !(this->operator==(rhs)); -} - -} // namespace ovms diff --git a/src/ovms_lib/layout_configuration.hpp b/src/ovms_lib/layout_configuration.hpp deleted file mode 100644 index 11d56cffba..0000000000 --- a/src/ovms_lib/layout_configuration.hpp +++ /dev/null @@ -1,52 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include "layout.hpp" -#include "status.hpp" - -namespace ovms { - -static const char LAYOUT_CONFIGURATION_DELIMETER = ':'; - -class LayoutConfiguration { - Layout tensor; - Layout model; - -public: - LayoutConfiguration() = default; - LayoutConfiguration(const char* layout); - LayoutConfiguration(const std::string& layout); - LayoutConfiguration(const std::string& tensorLayout, const std::string& modelLayout); - - const Layout& getTensorLayout() const; - const Layout& getModelLayout() const; - - bool isSet() const; - - bool operator==(const LayoutConfiguration& rhs) const; - bool operator!=(const LayoutConfiguration& rhs) const; - - static Status fromString(const std::string& configurationStr, LayoutConfiguration& configOut); - std::string toString() const; -}; - -using layout_configurations_map_t = std::unordered_map; - -} // namespace ovms diff --git a/src/ovms_lib/localfilesystem.cpp b/src/ovms_lib/localfilesystem.cpp deleted file mode 100644 index 3620abb5e1..0000000000 --- a/src/ovms_lib/localfilesystem.cpp +++ /dev/null @@ -1,173 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "localfilesystem.hpp" - -#include -#include -#include -#include - -#include - -#if defined(__APPLE__) || defined(__NetBSD__) -#define st_mtim st_mtimespec -#endif - -namespace ovms { - -namespace fs = std::filesystem; -constexpr uint64_t NANOS_PER_SECOND = 1000000000; - -const std::vector FileSystem::acceptedFiles = {".bin", ".onnx", ".xml", "mapping_config.json"}; - -StatusCode LocalFileSystem::fileExists(const std::string& path, bool* exists) { - try { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - *exists = fs::exists(path); - } catch (fs::filesystem_error& e) { - SPDLOG_DEBUG("Couldn't access path {}", e.what()); - return StatusCode::PATH_INVALID; - } - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::isDirectory(const std::string& path, bool* is_dir) { - try { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - *is_dir = fs::is_directory(path); - } catch (fs::filesystem_error& e) { - SPDLOG_DEBUG("Couldn't access path {}", e.what()); - return StatusCode::PATH_INVALID; - } - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::getDirectoryContents(const std::string& path, files_list_t* contents) { - try { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - for (const auto& entry : fs::directory_iterator(path)) { - contents->insert(entry.path().string()); - } - } catch (fs::filesystem_error& e) { - SPDLOG_DEBUG("Couldn't access path {}", e.what()); - return StatusCode::PATH_INVALID; - } - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::getDirectorySubdirs(const std::string& path, files_list_t* subdirs) { - try { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - for (const auto& entry : fs::directory_iterator(path)) { - if (entry.is_directory()) { - subdirs->insert(entry.path().filename().string()); - } - } - } catch (fs::filesystem_error& e) { - SPDLOG_DEBUG("Couldn't access path {}", e.what()); - return StatusCode::PATH_INVALID; - } - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::getDirectoryFiles(const std::string& path, files_list_t* files) { - try { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - for (const auto& entry : fs::directory_iterator(path)) { - if (!entry.is_directory()) { - files->insert(entry.path().string()); - } - } - } catch (fs::filesystem_error& e) { - SPDLOG_DEBUG("Couldn't access path {}", e.what()); - return StatusCode::PATH_INVALID; - } - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::readTextFile(const std::string& path, std::string* contents) { - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - std::ifstream input(path, std::ios::in | std::ios::binary); - if (!input) { - SPDLOG_DEBUG("Couldn't access path {}", path); - return StatusCode::PATH_INVALID; - } - - input.seekg(0, std::ios::end); - contents->resize(input.tellg()); - input.seekg(0, std::ios::beg); - input.read(&(*contents)[0], contents->size()); - input.close(); - - return StatusCode::OK; -} - -StatusCode LocalFileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { - // For LocalFileSystem there is no need to download - return StatusCode::OK; -} - -StatusCode LocalFileSystem::downloadModelVersions(const std::string& path, - std::string* local_path, - const std::vector& versions) { - *local_path = path; - return StatusCode::OK; -} - -StatusCode LocalFileSystem::deleteFileFolder(const std::string& path) { - std::error_code errorCode; - std::filesystem::path p = path; - std::filesystem::path parentPath = p.parent_path(); - if (isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - if (!std::filesystem::remove_all(path, errorCode)) { - return StatusCode::PATH_INVALID; - } - // delete empty folder with model version - if (std::filesystem::is_empty(parentPath)) { - SPDLOG_DEBUG("Deleting empty folder: ()", parentPath.string()); - std::filesystem::remove(parentPath); - } - - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/localfilesystem.hpp b/src/ovms_lib/localfilesystem.hpp deleted file mode 100644 index 08f4affaea..0000000000 --- a/src/ovms_lib/localfilesystem.hpp +++ /dev/null @@ -1,116 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include "filesystem.hpp" -#include "status.hpp" - -namespace ovms { - -class LocalFileSystem : public FileSystem { -public: - /** - * @brief Destroy the Local File System object - * - */ - ~LocalFileSystem() {} - - /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ - StatusCode fileExists(const std::string& path, bool* exists) override; - - /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ - StatusCode isDirectory(const std::string& path, bool* is_dir) override; - - /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; - - /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ - StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; - - /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ - StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; - - /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode readTextFile(const std::string& path, std::string* contents) override; - - /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ - StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; - - /** - * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ - StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; - - /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ - StatusCode deleteFileFolder(const std::string& path) override; -}; - -} // namespace ovms diff --git a/src/ovms_lib/log b/src/ovms_lib/log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/ovms_lib/logging.cpp b/src/ovms_lib/logging.cpp deleted file mode 100644 index 009bfd73e6..0000000000 --- a/src/ovms_lib/logging.cpp +++ /dev/null @@ -1,87 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "logging.hpp" - -#include - -namespace ovms { - -std::shared_ptr gcs_logger = std::make_shared("gcs"); -std::shared_ptr azurestorage_logger = std::make_shared("azurestorage"); -std::shared_ptr s3_logger = std::make_shared("s3"); -std::shared_ptr modelmanager_logger = std::make_shared("modelmanager"); -std::shared_ptr dag_executor_logger = std::make_shared("dag_executor"); -std::shared_ptr sequence_manager_logger = std::make_shared("sequence_manager"); - -const std::string default_pattern = "[%Y-%m-%d %T.%e][%t][%n][%l][%s:%#] %v"; - -void set_log_level(const std::string log_level, std::shared_ptr logger) { - logger->set_level(spdlog::level::info); - if (!log_level.empty()) { - if (log_level == "DEBUG") { - logger->set_level(spdlog::level::debug); - logger->flush_on(spdlog::level::trace); - } else if (log_level == "ERROR") { - logger->set_level(spdlog::level::err); - logger->flush_on(spdlog::level::err); - } else if (log_level == "TRACE") { - logger->set_level(spdlog::level::trace); - logger->flush_on(spdlog::level::trace); - } - } -} - -void register_loggers(const std::string log_level, std::vector sinks) { - auto serving_logger = std::make_shared("serving", begin(sinks), end(sinks)); - serving_logger->set_pattern(default_pattern); - gcs_logger->set_pattern(default_pattern); - azurestorage_logger->set_pattern(default_pattern); - s3_logger->set_pattern(default_pattern); - modelmanager_logger->set_pattern(default_pattern); - dag_executor_logger->set_pattern(default_pattern); - sequence_manager_logger->set_pattern(default_pattern); - for (auto sink : sinks) { - gcs_logger->sinks().push_back(sink); - azurestorage_logger->sinks().push_back(sink); - s3_logger->sinks().push_back(sink); - modelmanager_logger->sinks().push_back(sink); - dag_executor_logger->sinks().push_back(sink); - sequence_manager_logger->sinks().push_back(sink); - } - set_log_level(log_level, serving_logger); - set_log_level(log_level, gcs_logger); - set_log_level(log_level, azurestorage_logger); - set_log_level(log_level, s3_logger); - set_log_level(log_level, modelmanager_logger); - set_log_level(log_level, dag_executor_logger); - set_log_level(log_level, sequence_manager_logger); - spdlog::set_default_logger(serving_logger); -} - -void configure_logger(const std::string log_level, const std::string log_path) { - static bool wasRun = false; - if (wasRun) - return; - wasRun = true; - std::vector sinks; - sinks.push_back(std::make_shared()); - if (!log_path.empty()) { - sinks.push_back(std::make_shared(log_path)); - } - register_loggers(log_level, sinks); -} - -} // namespace ovms diff --git a/src/ovms_lib/logging.hpp b/src/ovms_lib/logging.hpp deleted file mode 100644 index 31a481f440..0000000000 --- a/src/ovms_lib/logging.hpp +++ /dev/null @@ -1,36 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include -#include - -namespace ovms { - -extern std::shared_ptr gcs_logger; -extern std::shared_ptr azurestorage_logger; -extern std::shared_ptr s3_logger; -extern std::shared_ptr modelmanager_logger; -extern std::shared_ptr dag_executor_logger; -extern std::shared_ptr sequence_manager_logger; - -void configure_logger(const std::string log_level, const std::string log_path); - -} // namespace ovms diff --git a/src/ovms_lib/main.cpp b/src/ovms_lib/main.cpp deleted file mode 100644 index 3f3d9b8c58..0000000000 --- a/src/ovms_lib/main.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//***************************************************************************** -// Copyright 2018-2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "server.hpp" - -using ovms::Server; - -int main(int argc, char** argv) { - Server& server = Server::instance(); - return server.start(argc, argv); -} diff --git a/src/ovms_lib/main2.cpp b/src/ovms_lib/main2.cpp deleted file mode 100644 index 0c60c1a2a6..0000000000 --- a/src/ovms_lib/main2.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include -#include - -#include "server.hpp" - -using ovms::Server; - -int main(int argc, char** argv) { - Server& server = Server::instance(); - std::thread t([&server, &argv, &argc](){ - std::cout << server.start(argc, argv) << std::endl; - }); - t.join(); - return 0; -} diff --git a/src/ovms_lib/model.cpp b/src/ovms_lib/model.cpp deleted file mode 100644 index 9a751a37df..0000000000 --- a/src/ovms_lib/model.cpp +++ /dev/null @@ -1,305 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "model.hpp" - -#include -#include -#include -#include -#include - -#include "customloaders.hpp" -#include "localfilesystem.hpp" -#include "logging.hpp" -#include "pipelinedefinition.hpp" - -namespace ovms { - -StatusCode downloadModels(std::shared_ptr& fs, ModelConfig& config, std::shared_ptr versions) { - if (versions->size() == 0) { - return StatusCode::OK; - } - - std::string localPath; - SPDLOG_INFO("Getting model from {}", config.getBasePath()); - auto sc = fs->downloadModelVersions(config.getBasePath(), &localPath, *versions); - if (sc != StatusCode::OK) { - SPDLOG_ERROR("Couldn't download model from {}", config.getBasePath()); - return sc; - } - config.setLocalPath(localPath); - SPDLOG_INFO("Model downloaded to {}", config.getLocalPath()); - - return StatusCode::OK; -} - -void Model::subscribe(PipelineDefinition& pd) { - subscriptionManager.subscribe(pd); -} - -void Model::unsubscribe(PipelineDefinition& pd) { - subscriptionManager.unsubscribe(pd); -} - -bool Model::isAnyVersionSubscribed() const { - if (subscriptionManager.isSubscribed()) { - return true; - } - for (const auto& [name, instance] : modelVersions) { - if (instance->getSubscribtionManager().isSubscribed()) { - return true; - } - } - return false; -} - -const std::map Model::getModelVersionsMapCopy() const { - std::shared_lock lock(modelVersionsMtx); - std::map modelInstancesMapCopy; - for (auto& [modelVersion, modelInstancePtr] : modelVersions) { - modelInstancesMapCopy.insert({modelVersion, *modelInstancePtr}); - } - return modelInstancesMapCopy; -} - -const std::map>& Model::getModelVersions() const { - return modelVersions; -} - -void Model::updateDefaultVersion(int ignoredVersion) { - model_version_t newDefaultVersion = 0; - SPDLOG_INFO("Updating default version for model: {}, from: {}", getName(), defaultVersion); - for (const auto& [version, versionInstance] : modelVersions) { - if (version != ignoredVersion && - version > newDefaultVersion && - ModelVersionState::AVAILABLE == versionInstance->getStatus().getState()) { - newDefaultVersion = version; - } - } - defaultVersion = newDefaultVersion; - if (newDefaultVersion) { - SPDLOG_INFO("Updated default version for model: {}, to: {}", getName(), newDefaultVersion); - } else { - SPDLOG_INFO("Model: {} will not have default version since no version is available.", getName()); - } -} - -const std::shared_ptr Model::getDefaultModelInstance() const { - std::shared_lock lock(modelVersionsMtx); - auto defaultVersion = getDefaultVersion(); - const auto modelInstanceIt = modelVersions.find(defaultVersion); - - if (modelVersions.end() == modelInstanceIt) { - SPDLOG_WARN("Default version: {} for model: {} not found", defaultVersion, getName()); - return nullptr; - } - return modelInstanceIt->second; -} - -std::shared_ptr Model::modelInstanceFactory(const std::string& modelName, const model_version_t modelVersion, ov::Core& ieCore) { - if (isStateful()) { - SPDLOG_DEBUG("Creating new stateful model instance - model name: {}; model version: {};", modelName, modelVersion); - return std::move(std::static_pointer_cast( - std::make_shared(modelName, modelVersion, ieCore, this->globalSequencesViewer))); - } else { - SPDLOG_DEBUG("Creating new model instance - model name: {}; model version: {};", modelName, modelVersion); - return std::move(std::make_shared(modelName, modelVersion, ieCore)); - } -} - -Status Model::addVersion(const ModelConfig& config, ov::Core& ieCore) { - const auto& version = config.getVersion(); - std::shared_ptr modelInstance = modelInstanceFactory(config.getName(), version, ieCore); - - std::unique_lock lock(modelVersionsMtx); - modelVersions.emplace(version, modelInstance); - lock.unlock(); - auto status = modelInstance->loadModel(config); - if (!status.ok()) { - return status; - } - updateDefaultVersion(); - subscriptionManager.notifySubscribers(); - return StatusCode::OK; -} - -Status Model::addVersions(std::shared_ptr versionsToStart, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed) { - Status result = StatusCode::OK; - downloadModels(fs, config, versionsToStart); - versionsFailed->clear(); - for (const auto version : *versionsToStart) { - SPDLOG_INFO("Will add model: {}; version: {} ...", getName(), version); - config.setVersion(version); - config.parseModelMapping(); - auto status = addVersion(config, ieCore); - if (!status.ok()) { - SPDLOG_ERROR("Error occurred while loading model: {}; version: {}; error: {}", - getName(), - version, - status.string()); - versionsFailed->push_back(version); - result = status; - cleanupModelTmpFiles(config); - } - } - return result; -} - -Status Model::retireVersions(std::shared_ptr versionsToRetire) { - Status result = StatusCode::OK; - for (const auto version : *versionsToRetire) { - SPDLOG_INFO("Will unload model: {}; version: {} ...", getName(), version); - auto modelVersion = getModelInstanceByVersion(version); - if (!modelVersion) { - Status status = StatusCode::UNKNOWN_ERROR; - SPDLOG_ERROR("Error occurred while unloading model: {}; version: {}; error: {}", - getName(), - version, - status.string()); - result = status; - continue; - } - cleanupModelTmpFiles(modelVersion->getModelConfig()); - updateDefaultVersion(version); - modelVersion->retireModel(); - } - subscriptionManager.notifySubscribers(); - return result; -} - -Status Model::cleanupFailedLoad(std::shared_ptr versionsToCleanUp) { - Status result = StatusCode::OK; - for (const auto version : *versionsToCleanUp) { - SPDLOG_INFO("Will clean up model: {}; version: {} ...", getName(), version); - auto modelVersion = getModelInstanceByVersion(version); - if (!modelVersion) { - Status status = StatusCode::UNKNOWN_ERROR; - SPDLOG_ERROR("Error occurred while cleaning up model: {}; version: {}; error: {}", - getName(), - version, - status.string()); - result = status; - continue; - } - cleanupModelTmpFiles(modelVersion->getModelConfig()); - updateDefaultVersion(version); - modelVersion->cleanupFailedLoad(); - } - subscriptionManager.notifySubscribers(); - return result; -} - -void Model::retireAllVersions() { - if (!(customLoaderName.empty())) { - auto& customloaders = ovms::CustomLoaders::instance(); - auto loaderPtr = customloaders.find(customLoaderName); - if (loaderPtr != nullptr) { - loaderPtr->retireModel(name); - } else { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Could not find custom loader for model: {} but it is using custom loader: {}", getName(), customLoaderName); - } - } - - for (const auto versionModelInstancePair : modelVersions) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Will unload model: {}; version: {} ...", getName(), versionModelInstancePair.first); - cleanupModelTmpFiles(versionModelInstancePair.second->getModelConfig()); - versionModelInstancePair.second->retireModel(); - updateDefaultVersion(); - } - subscriptionManager.notifySubscribers(); -} - -void Model::cleanupAllVersions() { - if (!(customLoaderName.empty())) { - auto& customloaders = ovms::CustomLoaders::instance(); - auto loaderPtr = customloaders.find(customLoaderName); - if (loaderPtr != nullptr) { - loaderPtr->retireModel(name); - } else { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Could not find custom loader for model: {} but it is using custom loader: {}", getName(), customLoaderName); - } - } - - for (const auto versionModelInstancePair : modelVersions) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Will unload model: {}; version: {} ...", getName(), versionModelInstancePair.first); - cleanupModelTmpFiles(versionModelInstancePair.second->getModelConfig()); - versionModelInstancePair.second->cleanupFailedLoad(); - updateDefaultVersion(); - } - subscriptionManager.notifySubscribers(); -} - -Status Model::reloadVersions(std::shared_ptr versionsToReload, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed) { - Status result = StatusCode::OK; - for (const auto version : *versionsToReload) { - SPDLOG_INFO("Will reload model: {}; version: {} ...", getName(), version); - config.setVersion(version); - auto status = config.parseModelMapping(); - if ((!status.ok()) && (status != StatusCode::FILE_INVALID)) { - SPDLOG_ERROR("Error while parsing model mapping for model {}; error: {}", getName(), status.string()); - } - - auto modelVersion = getModelInstanceByVersion(version); - - if (!modelVersion) { - SPDLOG_ERROR("Error occurred while reloading model: {}; version: {}; Model version was not found", - getName(), - version); - result = StatusCode::UNKNOWN_ERROR; - continue; - } - if (modelVersion->getStatus().getState() == ModelVersionState::END || - modelVersion->getStatus().getState() == ModelVersionState::LOADING || - modelVersion->getModelConfig().getBasePath() != config.getBasePath()) { - downloadModels(fs, config, versionsToReload); - } else { - config.setLocalPath(modelVersion->getModelConfig().getLocalPath()); - } - status = modelVersion->reloadModel(config); - if (!status.ok()) { - SPDLOG_ERROR("Error occurred while loading model: {}; version: {}; error: {}", - getName(), - version, - status.string()); - result = status; - versionsFailed->push_back(version); - continue; - } - updateDefaultVersion(); - } - subscriptionManager.notifySubscribers(); - return result; -} - -Status Model::cleanupModelTmpFiles(const ModelConfig& config) { - auto lfstatus = StatusCode::OK; - - if (config.isCloudStored()) { - LocalFileSystem lfs; - lfstatus = lfs.deleteFileFolder(config.getPath()); - if (lfstatus != StatusCode::OK) { - SPDLOG_ERROR("Error occurred while deleting local copy of cloud model: {} reason: {}", - config.getLocalPath(), - lfstatus); - } else { - SPDLOG_DEBUG("Model removed from: {}", config.getPath()); - } - } - - return lfstatus; -} - -} // namespace ovms diff --git a/src/ovms_lib/model.hpp b/src/ovms_lib/model.hpp deleted file mode 100644 index 5d513072e6..0000000000 --- a/src/ovms_lib/model.hpp +++ /dev/null @@ -1,246 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "filesystem.hpp" -#include "modelchangesubscription.hpp" -#include "modelinstance.hpp" -#include "statefulmodelinstance.hpp" - -namespace ovms { -class PipelineDefinition; -/* * @brief This class represent inference models - */ -class Model { -private: - /** - * @brief Mutex for protecting concurrent modfying and accessing modelVersions - */ - mutable std::shared_mutex modelVersionsMtx; - - /** - * @brief Flag indicating whether model is stateful or not - */ - bool stateful; - - GlobalSequencesViewer* globalSequencesViewer; - - /** - * @brief Update default version - * - * @param ignoredVersion Version to exclude from being selected as the default version - */ - void updateDefaultVersion(int ignoredVersion = 0); - -protected: - /** - * @brief Model name - */ - std::string name; - - /** - * @brief Holds different versions of model - */ - std::map> modelVersions; - - /** - * @brief Model default version - * - */ - model_version_t defaultVersion = 0; - - /** - * @brief Get default version - * - * @return default version - */ - const model_version_t getDefaultVersion() const { - SPDLOG_DEBUG("Getting default version for model: {}, {}", getName(), defaultVersion); - return defaultVersion; - } - - /** - * @brief Adds a new version of ModelInstance to the list of versions - * - * @param config model configuration - * - * @return status - */ - virtual Status addVersion(const ModelConfig& config, ov::Core& ieCore); - - /** - * @brief ModelInstances factory - * - * @return modelInstance - */ - virtual std::shared_ptr modelInstanceFactory(const std::string& modelName, const model_version_t modelVersion, ov::Core& ieCore); - - ModelChangeSubscription subscriptionManager; - - /** - * @brief Holds the custom loader interface Name - * - */ - std::string customLoaderName; - -public: - /** - * @brief Constructor - */ - Model(const std::string& name, bool stateful, GlobalSequencesViewer* globalSequencesViewer) : - stateful(stateful), - globalSequencesViewer(globalSequencesViewer), - name(name), - defaultVersion(0), - subscriptionManager(std::string("model: ") + name) {} - - /** - * @brief Destroy the Model object - * - */ - virtual ~Model() {} - - /** - * @brief Gets the model name - * - * @return model name - */ - const std::string& getName() const { - return name; - } - - const bool isStateful() const { - return stateful; - } - - /** - * @brief Gets the default ModelInstance - * - * @return ModelInstance - */ - const std::shared_ptr getDefaultModelInstance() const; - - /** - * @brief Gets model versions instances - * - * @return model versions instances - */ - const std::map>& getModelVersions() const; - - /** - * @brief Gets model versions instances - * - * @return model versions instances - */ - const std::map getModelVersionsMapCopy() const; - - /** - * @brief Finds ModelInstance with specific version - * - * @param version of the model to search for - * - * @return specific model version - */ - const std::shared_ptr getModelInstanceByVersion(const model_version_t& version) const { - std::shared_lock lock(modelVersionsMtx); - auto it = modelVersions.find(version); - return it != modelVersions.end() ? it->second : nullptr; - } - - /** - * @brief Adds new versions of ModelInstance - * - * @param config model configuration - * - * @return status - */ - Status addVersions(std::shared_ptr versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed); - - /** - * @brief Retires versions of Model - * - * @param versions versions to retire - * - * @return status - */ - Status retireVersions(std::shared_ptr versions); - - /** - * @brief Cleans up versions of Model - * - * @param versions versions to clean up - * - * @return status - */ - Status cleanupFailedLoad(std::shared_ptr versions); - - /** - * @brief Retires all versions of Model - */ - void retireAllVersions(); - - /** - * @brief Cleans up all versions of Model - */ - void cleanupAllVersions(); - - /** - * @brief Reloads versions of Model - * - * @param config model configuration - * - * @return status - */ - Status reloadVersions(std::shared_ptr versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr versionsFailed); - - void subscribe(PipelineDefinition& pd); - void unsubscribe(PipelineDefinition& pd); - /** - * @brief Set the custom loader name - * - * @param custom loader name - * - */ - - bool isAnyVersionSubscribed() const; - - void setCustomLoaderName(const std::string name) { - customLoaderName = name; - } - - /** - * @brief Reset the custom loader name - * - */ - void resetCustomLoaderName() { - customLoaderName.clear(); - } - - /** - * @brief Delete temporary model files - * - */ - static Status cleanupModelTmpFiles(const ModelConfig& config); -}; -} // namespace ovms diff --git a/src/ovms_lib/model_service.cpp b/src/ovms_lib/model_service.cpp deleted file mode 100644 index 5cfbb7105f..0000000000 --- a/src/ovms_lib/model_service.cpp +++ /dev/null @@ -1,220 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "model_service.hpp" - -#include -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - -#include "modelmanager.hpp" -#include "pipelinedefinition.hpp" -#include "servablemanagermodule.hpp" -#include "status.hpp" - -using google::protobuf::util::JsonPrintOptions; -using google::protobuf::util::MessageToJsonString; - -namespace ovms { - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status) { - SPDLOG_DEBUG("add_status_to_response version={} status={}", version, model_version_status.getStateString()); - auto status_to_fill = response->add_model_version_status(); - status_to_fill->set_state(static_cast(static_cast(model_version_status.getState()))); - status_to_fill->set_version(version); - status_to_fill->clear_status(); - status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(model_version_status.getErrorCode()))); - status_to_fill->mutable_status()->set_error_message(model_version_status.getErrorMsg()); -} - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, const model_version_t version, const PipelineDefinitionStatus& pipeline_status) { - auto [state, error_code] = pipeline_status.convertToModelStatus(); - SPDLOG_DEBUG("add_status_to_response state={} error_code", state, error_code); - auto status_to_fill = response->add_model_version_status(); - status_to_fill->set_state(static_cast(static_cast(state))); - status_to_fill->set_version(version); - status_to_fill->clear_status(); - status_to_fill->mutable_status()->set_error_code(static_cast(static_cast(error_code))); - status_to_fill->mutable_status()->set_error_message(ModelVersionStatusErrorCodeToString(error_code)); -} - -::grpc::Status ModelServiceImpl::GetModelStatus( - ::grpc::ServerContext* context, const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response) { - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return grpc::Status(grpc::StatusCode::NOT_FOUND, SERVABLE_MANAGER_MODULE_NAME + " module not started yet"); - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return GetModelStatusImpl::getModelStatus(request, response, manager).grpc(); -} - -Status GetModelStatusImpl::createGrpcRequest(std::string model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request) { - request->mutable_model_spec()->set_name(model_name); - if (model_version.has_value()) { - request->mutable_model_spec()->mutable_version()->set_value(model_version.value()); - } - return StatusCode::OK; -} - -Status GetModelStatusImpl::serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output) { - JsonPrintOptions opts; - opts.add_whitespace = true; - opts.always_print_primitive_fields = true; - const auto& status = MessageToJsonString(*response, output, opts); - if (!status.ok()) { - SPDLOG_ERROR("Failed to convert proto to json. Error: ", status.ToString()); - return StatusCode::JSON_SERIALIZATION_ERROR; - } - return StatusCode::OK; -} - -Status GetModelStatusImpl::getModelStatus( - const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response, - ModelManager& manager) { - SPDLOG_DEBUG("model_service: request: {}", request->DebugString()); - - bool has_requested_version = request->model_spec().has_version(); - auto requested_version = request->model_spec().version().value(); - std::string requested_model_name = request->model_spec().name(); - auto model_ptr = manager.findModelByName(requested_model_name); - if (!model_ptr) { - SPDLOG_DEBUG("GetModelStatus: Model {} is missing, trying to find pipeline with such name", requested_model_name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(requested_model_name); - if (!pipelineDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - - addStatusToResponse(response, pipelineDefinition->getVersion(), pipelineDefinition->getStatus()); - SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); - SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); - return StatusCode::OK; - } - - SPDLOG_DEBUG("requested model: {}, has_version: {} (version: {})", requested_model_name, has_requested_version, requested_version); - if (has_requested_version && requested_version != 0) { - // return details only for a specific version of requested model; NOT_FOUND otherwise. If requested_version == 0, default is returned. - std::shared_ptr model_instance = model_ptr->getModelInstanceByVersion(requested_version); - if (!model_instance) { - SPDLOG_DEBUG("requested model {} in version {} was not found.", requested_model_name, requested_version); - return StatusCode::MODEL_VERSION_MISSING; - } - const auto& status = model_instance->getStatus(); - SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, requested_version, status.getStateString()); - addStatusToResponse(response, requested_version, status); - } else { - // return status details of all versions of a requested model. - auto modelVersionsInstances = model_ptr->getModelVersionsMapCopy(); - for (const auto& [modelVersion, modelInstance] : modelVersionsInstances) { - const auto& status = modelInstance.getStatus(); - SPDLOG_DEBUG("adding model {} - {} :: {} to response", requested_model_name, modelVersion, status.getStateString()); - addStatusToResponse(response, modelVersion, status); - } - } - SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); - SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); - return StatusCode::OK; -} - -Status GetModelStatusImpl::getAllModelsStatuses(std::map& modelsStatuses, ModelManager& manager) { - std::shared_lock lock(manager.modelsMtx); - std::map modelsStatusesTmp; - - const std::map>& models = manager.getModels(); - for (auto const& model : models) { - std::optional noValueModelVersion; - tensorflow::serving::GetModelStatusRequest request; - GetModelStatusImpl::createGrpcRequest(model.first, noValueModelVersion, &request); - tensorflow::serving::GetModelStatusResponse response; - auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager); - if (status != StatusCode::OK) { - // For now situation when getModelStatus return status other than OK cannot occur because we never remove models and pipelines from model manager. - // However, if something in this matter will change we should handle this somehow. - continue; - } - modelsStatusesTmp.insert({model.first, response}); - } - lock.unlock(); - - const std::vector& pipelinesNames = manager.getPipelineFactory().getPipelinesNames(); - for (auto const& pipelineName : pipelinesNames) { - std::optional noValueModelVersion; - tensorflow::serving::GetModelStatusRequest request; - GetModelStatusImpl::createGrpcRequest(pipelineName, noValueModelVersion, &request); - tensorflow::serving::GetModelStatusResponse response; - auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager); - if (status != StatusCode::OK) { - // Same situation like with models. - continue; - } - modelsStatusesTmp.insert({pipelineName, response}); - } - - modelsStatuses.merge(modelsStatusesTmp); - return StatusCode::OK; -} - -Status GetModelStatusImpl::serializeModelsStatuses2Json(const std::map& modelsStatuses, std::string& output) { - std::string outputTmp; - if (modelsStatuses.begin() == modelsStatuses.end()) { - output = "{}"; - return StatusCode::OK; - } - - outputTmp += "{\n"; - for (auto modelStatus = modelsStatuses.begin(); modelStatus != modelsStatuses.end(); modelStatus++) { - outputTmp += ("\"" + modelStatus->first + "\" : \n"); - std::string responseStr; - auto status = GetModelStatusImpl::serializeResponse2Json(&modelStatus->second, &responseStr); - if (status != StatusCode::OK) { - return status; - } - responseStr.pop_back(); - outputTmp += responseStr; - if (std::next(modelStatus) != modelsStatuses.end()) { - outputTmp += (",\n"); - } - } - outputTmp += "\n}"; - output = outputTmp; - - return StatusCode::OK; -} - -ModelServiceImpl::ModelServiceImpl(ovms::Server& ovmsServer) : - ovmsServer(ovmsServer) {} - -::grpc::Status ModelServiceImpl::HandleReloadConfigRequest( - ::grpc::ServerContext* context, const tensorflow::serving::ReloadConfigRequest* request, - tensorflow::serving::ReloadConfigResponse* response) { - SPDLOG_INFO("Requested HandleReloadConfigRequest - but this service is reloading config automatically by itself, therefore this operation has no *EXTRA* affect."); - return grpc::Status::OK; // we're reloading config all the time; for a total client compatibility, this means returning success here. -} - -} // namespace ovms diff --git a/src/ovms_lib/model_service.hpp b/src/ovms_lib/model_service.hpp deleted file mode 100644 index d2df3d702c..0000000000 --- a/src/ovms_lib/model_service.hpp +++ /dev/null @@ -1,65 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/get_model_status.pb.h" -#include "tensorflow_serving/apis/model_service.grpc.pb.h" -#include "tensorflow_serving/apis/model_service.pb.h" -#pragma GCC diagnostic pop - -#include "modelmanager.hpp" -#include "status.hpp" - -namespace ovms { -class Server; - -void addStatusToResponse(tensorflow::serving::GetModelStatusResponse* response, model_version_t version, const ModelVersionStatus& model_version_status); - -class ModelServiceImpl final : public tensorflow::serving::ModelService::Service { - ovms::Server& ovmsServer; - -public: - ModelServiceImpl(ovms::Server& ovmsServer); - ::grpc::Status GetModelStatus(::grpc::ServerContext* context, - const tensorflow::serving::GetModelStatusRequest* request, - tensorflow::serving::GetModelStatusResponse* response) override; - ::grpc::Status HandleReloadConfigRequest(::grpc::ServerContext* context, - const tensorflow::serving::ReloadConfigRequest* request, - tensorflow::serving::ReloadConfigResponse* response) override; -}; - -class GetModelStatusImpl { - ovms::Server& ovmsServer; - -public: - GetModelStatusImpl(ovms::Server& ovmsServer); - static Status getModelStatus(const tensorflow::serving::GetModelStatusRequest* request, tensorflow::serving::GetModelStatusResponse* response, ModelManager& manager); - static Status createGrpcRequest(std::string model_name, const std::optional model_version, tensorflow::serving::GetModelStatusRequest* request); - static Status serializeResponse2Json(const tensorflow::serving::GetModelStatusResponse* response, std::string* output); - - static Status getAllModelsStatuses(std::map& models_versions, ModelManager& manager); - static Status serializeModelsStatuses2Json(const std::map& models_versions, std::string& output); -}; - -} // namespace ovms diff --git a/src/ovms_lib/model_version_policy.cpp b/src/ovms_lib/model_version_policy.cpp deleted file mode 100644 index 600d5aa365..0000000000 --- a/src/ovms_lib/model_version_policy.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "model_version_policy.hpp" - -#include -#include -#include -#include -#include - -namespace ovms { - -std::shared_ptr ModelVersionPolicy::getDefaultVersionPolicy() { - return std::make_shared(1); -} - -AllModelVersionPolicy::operator std::string() const { - return std::string("all"); -} - -std::vector SpecificModelVersionPolicy::filter(std::vector versions) const { - std::vector result; - std::sort(versions.begin(), versions.end()); - std::set_intersection( - versions.begin(), - versions.end(), - specificVersions.begin(), - specificVersions.end(), - std::back_inserter(result)); - return result; -} - -SpecificModelVersionPolicy::operator std::string() const { - std::stringstream versionStream; - versionStream << "specific: "; - std::copy(specificVersions.begin(), specificVersions.end(), std::ostream_iterator(versionStream, " ")); - return versionStream.str(); -} - -std::vector LatestModelVersionPolicy::filter(std::vector versions) const { - std::vector result; - std::sort(versions.begin(), versions.end(), std::greater()); - for (size_t i = 0; i < numVersions && i < versions.size(); i++) { - result.push_back(versions[i]); - } - return result; -} - -LatestModelVersionPolicy::operator std::string() const { - return std::string("latest: ") + std::to_string(numVersions); -} - -} // namespace ovms diff --git a/src/ovms_lib/model_version_policy.hpp b/src/ovms_lib/model_version_policy.hpp deleted file mode 100644 index f213126bf5..0000000000 --- a/src/ovms_lib/model_version_policy.hpp +++ /dev/null @@ -1,135 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include "modelversion.hpp" - -namespace ovms { - -/** - * @brief Base class for model version policy types - */ -class ModelVersionPolicy { -protected: - ModelVersionPolicy() {} - virtual ~ModelVersionPolicy() {} - -public: - /** - * @brief Strips out model versions list passed by parameter depending on internal state - * - * @param versions model versions to filter - * @return Filtered version list - */ - virtual model_versions_t filter(model_versions_t versions) const = 0; - - /** - * @brief Creates default model version policy, by default only one version (highest) should be served - * - * @param highestVersion highest version - * @return default version policy - */ - static std::shared_ptr getDefaultVersionPolicy(); - - /** - * @brief Converts ModelVersionPolicy to readable string - */ - virtual operator std::string() const = 0; -}; - -/** - * @brief Model version policy that enables all available versions - */ -class AllModelVersionPolicy : public ModelVersionPolicy { -public: - /** - * @brief Default constructor, nothing needs to be specified since all versions will be served - */ - AllModelVersionPolicy() {} - - /** - * @brief Filters passed versions depending on internal state - * - * @param versions model versions to filter - * @return Filtered version list - */ - model_versions_t filter(model_versions_t versions) const override { - return versions; - } - - operator std::string() const override; -}; - -/** - * @brief Model version policy for explicitely specifying which versions should be enabled - */ -class SpecificModelVersionPolicy : public ModelVersionPolicy { - model_versions_t specificVersions; - -public: - /** - * @brief Default constructor - * - * @param versions list of all model versions that should be served - */ - SpecificModelVersionPolicy(const model_versions_t& versions) : - specificVersions(versions) { - std::sort(specificVersions.begin(), specificVersions.end()); - } - - /** - * @brief Filters passed versions depending on internal state - * - * @param versions model versions to filter - * @return Filtered version list - */ - model_versions_t filter(model_versions_t versions) const override; - - operator std::string() const override; -}; - -/** - * @brief Model version policy for serving only X latest versions - */ -class LatestModelVersionPolicy : public ModelVersionPolicy { - size_t numVersions; - -public: - /** - * @brief Default constructor - * - * @param numVersions number of latest versions to be served - */ - LatestModelVersionPolicy(size_t numVersions = 1) : - numVersions(numVersions) {} - - /** - * @brief Filters passed versions depending on internal state - * - * @param versions model versions to filter - * @return Filtered version list - */ - model_versions_t filter(model_versions_t versions) const override; - - operator std::string() const override; -}; - -} // namespace ovms diff --git a/src/ovms_lib/modelchangesubscription.cpp b/src/ovms_lib/modelchangesubscription.cpp deleted file mode 100644 index 804850a3b4..0000000000 --- a/src/ovms_lib/modelchangesubscription.cpp +++ /dev/null @@ -1,57 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelchangesubscription.hpp" - -#include -#include - -#include "pipelinedefinition.hpp" - -namespace ovms { -void ModelChangeSubscription::subscribe(PipelineDefinition& pd) { - SPDLOG_INFO("Subscription to {} from {}", ownerName, pd.getName()); - if (subscriptions.find(pd.getName()) != subscriptions.end()) { - std::stringstream ss; - ss << "Tried to subscribe pipeline:" << pd.getName() << " to:" << ownerName; - ss << ", but this pipeline was already subscribed"; - SPDLOG_ERROR(ss.str().c_str()); - throw std::logic_error(ss.str()); - } - subscriptions.insert({pd.getName(), pd}); -} - -void ModelChangeSubscription::unsubscribe(PipelineDefinition& pd) { - SPDLOG_INFO("Subscription to {} from {} removed", ownerName, pd.getName()); - auto numberOfErased = subscriptions.erase(pd.getName()); - if (0 == numberOfErased) { - std::stringstream ss; - ss << "Tried to unsubscribe pipeline:" << pd.getName() << " to:" << ownerName; - ss << ", but this pipeline was never subscribed"; - SPDLOG_ERROR(ss.str().c_str()); - throw std::logic_error(ss.str()); - } -} - -void ModelChangeSubscription::notifySubscribers() { - if (subscriptions.size() == 0) { - return; - } - SPDLOG_INFO("Notified subscribers of: {}", ownerName); - for (auto& [pipelineName, pipelineDefinition] : subscriptions) { - pipelineDefinition.notifyUsedModelChanged(ownerName); - } -} -} // namespace ovms diff --git a/src/ovms_lib/modelchangesubscription.hpp b/src/ovms_lib/modelchangesubscription.hpp deleted file mode 100644 index 427b3cea03..0000000000 --- a/src/ovms_lib/modelchangesubscription.hpp +++ /dev/null @@ -1,42 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include -#include - -namespace ovms { - -class PipelineDefinition; - -class ModelChangeSubscription { - const std::string ownerName; - std::unordered_map subscriptions; - -public: - ModelChangeSubscription(const std::string& ownerName) : - ownerName(ownerName) {} - - void subscribe(PipelineDefinition& pd); - - void unsubscribe(PipelineDefinition& pd); - - void notifySubscribers(); - - bool isSubscribed() const { return subscriptions.size() > 0; } -}; -} // namespace ovms diff --git a/src/ovms_lib/modelconfig.cpp b/src/ovms_lib/modelconfig.cpp deleted file mode 100644 index b3c95ff4d3..0000000000 --- a/src/ovms_lib/modelconfig.cpp +++ /dev/null @@ -1,669 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelconfig.hpp" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "logging.hpp" -#include "schema.hpp" -#include "stringutils.hpp" - -namespace ovms { - -bool ModelConfig::isDeviceUsed(const std::string& device) const { - if (this->isSingleDeviceUsed(device)) - return true; - if (targetDevice.find(device) != std::string::npos) - return true; - if (targetDevice == "AUTO") - return true; - return false; -} - -bool ModelConfig::isReloadRequired(const ModelConfig& rhs) const { - if (this->name != rhs.name) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to name mismatch", this->name); - return true; - } - if (this->stateful != rhs.stateful) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to stateful mismatch", this->name); - return true; - } - if (this->idleSequenceCleanup != rhs.idleSequenceCleanup) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to idleSequenceCleanup mismatch", this->name); - return true; - } - if (this->maxSequenceNumber != rhs.maxSequenceNumber) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to maxSequenceNumber mismatch", this->name); - return true; - } - if (this->lowLatencyTransformation != rhs.lowLatencyTransformation) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to lowLatencyTransformation mismatch", this->name); - return true; - } - if (this->basePath != rhs.basePath) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to original base path mismatch", this->name); - return true; - } - if (this->targetDevice != rhs.targetDevice) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to target device mismatch", this->name); - return true; - } - if (this->batchingMode != rhs.batchingMode) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to batching mode mismatch", this->name); - return true; - } - if (!isBatchSizeConfigurationEqual(rhs)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to batch size mismatch", this->name); - return true; - } - if (this->nireq != rhs.nireq) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to nireq mismatch", this->name); - return true; - } - if (this->pluginConfig != rhs.pluginConfig) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to plugin config mismatch", this->name); - return true; - } - if (!isLayoutConfigurationEqual(rhs)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to named layout mismatch", this->name); - return true; - } - if (!isShapeConfigurationEqual(rhs)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to shape configuration mismatch", this->name); - return true; - } - if (isCustomLoaderConfigChanged(rhs)) { - return true; - } - if (this->isAllowCacheSetToTrue() != rhs.isAllowCacheSetToTrue()) { - return true; - } - return false; -} - -bool ModelConfig::isCustomLoaderConfigChanged(const ModelConfig& rhs) const { - if (this->customLoaderOptionsConfigMap.size() != rhs.customLoaderOptionsConfigMap.size()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to custom loader config mismatch", this->name); - return true; - } - if (this->customLoaderOptionsConfigMap.size() > 0 && rhs.customLoaderOptionsConfigMap.size() > 0) { - if (!(this->customLoaderOptionsConfigMap == rhs.customLoaderOptionsConfigMap)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to custom loader config mismatch", this->name); - return true; - } - } - return false; -} - -bool ModelConfig::isBatchSizeConfigurationEqual(const ModelConfig& rhs) const { - if (this->batchSize.has_value() != rhs.batchSize.has_value()) { - return false; - } - if (this->batchSize.has_value() && this->batchSize.value() != rhs.batchSize.value()) { - return false; - } - return true; -} - -bool ModelConfig::isLayoutConfigurationEqual(const ModelConfig& rhs) const { - if (this->layout != rhs.layout) { - return false; - } - - if (this->layouts.size() != rhs.layouts.size()) { - return false; - } - for (const auto& [name, layoutConfig] : this->layouts) { - auto it = rhs.layouts.find(name); - if (it == rhs.layouts.end()) { - return false; - } - if (layoutConfig != it->second) { - return false; - } - } - return true; -} - -bool ModelConfig::isShapeConfigurationEqual(const ModelConfig& rhs) const { - if (this->shapes.size() != rhs.shapes.size()) { - return false; - } - for (const auto& [name, shape] : this->shapes) { - auto it = rhs.shapes.find(name); - if (it == rhs.shapes.end()) { - return false; - } - if (shape != it->second) { - return false; - } - } - return true; -} - -std::tuple> ModelConfig::extractBatchingParams(std::string configBatchSize) { - Mode batchingMode = FIXED; - std::optional effectiveBatchSize = std::nullopt; - if (configBatchSize == "auto") { - batchingMode = AUTO; - } else if (configBatchSize == "0") { - // do nothing - } else { - Dimension dim; - auto status = Dimension::fromString(configBatchSize, dim); - if (!status.ok()) { - SPDLOG_WARN("Wrong batch size parameter provided. Model batch size will be set to default."); - } else { - effectiveBatchSize = dim; - } - } - return std::tuple>{batchingMode, effectiveBatchSize}; -} - -Status ModelConfig::parseModelVersionPolicy(std::string command) { - rapidjson::Document node; - if (command == "") { - modelVersionPolicy = ModelVersionPolicy::getDefaultVersionPolicy(); - return StatusCode::OK; - } - - if (node.Parse(command.c_str()).HasParseError()) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - - if (!node.IsObject()) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - if (node.MemberCount() != 1) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - - auto m = node.FindMember("all"); - if (m != node.MemberEnd()) { - modelVersionPolicy = std::make_shared(); - return StatusCode::OK; - } - - m = node.FindMember("specific"); - if (m != node.MemberEnd()) { - auto& specific = m->value; - if (specific.MemberCount() != 1) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - m = specific.FindMember("versions"); - if (m == specific.MemberEnd()) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - std::vector versions; - for (auto& version : m->value.GetArray()) { - if (version.IsUint64() && version.GetUint64() > 0) { - versions.push_back(version.GetUint64()); - } else { - SPDLOG_WARN("Model policy specified in config contains invalid version. Version should be a number greater than 0."); - } - } - modelVersionPolicy = std::make_shared(versions); - return StatusCode::OK; - } - - m = node.FindMember("latest"); - if (m != node.MemberEnd()) { - auto& latest = m->value; - if (latest.MemberCount() != 1) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - m = latest.FindMember("num_versions"); - if (m == latest.MemberEnd()) { - return StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT; - } - modelVersionPolicy = std::make_shared(m->value.GetInt64()); - return StatusCode::OK; - } - - return StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY; -} - -Status ModelConfig::parsePluginConfig(const rapidjson::Value& node) { - if (!node.IsObject()) { - return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; - } - - for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { - if (it->value.IsString()) { - pluginConfig[it->name.GetString()] = it->value.GetString(); - } else if (it->value.IsInt64()) { - pluginConfig[it->name.GetString()] = std::to_string(it->value.GetInt64()); - } else if (it->value.IsDouble()) { - pluginConfig[it->name.GetString()] = std::to_string(it->value.GetDouble()); - } else { - return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; - } - } - - return StatusCode::OK; -} - -Status ModelConfig::parseShapeParameter(const rapidjson::Value& node) { - if (!node.IsObject()) { - return StatusCode::SHAPE_WRONG_FORMAT; - } - - shapes_info_map_t shapes; - for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { - if (!it->value.IsString()) { - return StatusCode::SHAPE_WRONG_FORMAT; - } - ShapeInfo shapeInfo; - auto status = parseShape(shapeInfo, it->value.GetString()); - if (!status.ok()) { - return status; - } - shapes[it->name.GetString()] = shapeInfo; - } - this->shapes = shapes; - - return StatusCode::OK; -} - -Status ModelConfig::parseShapeParameter(const std::string& command) { - this->shapes.clear(); - - if (command.empty()) { - return StatusCode::OK; - } - - // parse as string - if (command.front() == shapeLeft || command == "auto") { - ShapeInfo shapeInfo; - auto status = parseShape(shapeInfo, command); - if (!status.ok()) { - return status; - } - this->addShape(ANONYMOUS_INPUT_NAME, shapeInfo); - return StatusCode::OK; - } - - // parse as json - rapidjson::Document node; - if (command.empty()) { - return StatusCode::OK; - } - if (node.Parse(command.c_str()).HasParseError()) { - return StatusCode::SHAPE_WRONG_FORMAT; - } - return parseShapeParameter(node); -} - -Status ModelConfig::parseLayoutParameter(const rapidjson::Value& node) { - if (!node.IsObject()) { - return StatusCode::LAYOUT_WRONG_FORMAT; - } - layout_configurations_map_t layouts; - for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { - if (!it->value.IsString()) { - return StatusCode::LAYOUT_WRONG_FORMAT; - } - std::string layoutStr = it->value.GetString(); - std::transform(layoutStr.begin(), layoutStr.end(), layoutStr.begin(), ::toupper); - - LayoutConfiguration layout; - auto status = LayoutConfiguration::fromString(layoutStr, layout); - if (!status.ok()) { - return status; - } - layouts[it->name.GetString()] = layout; - } - setLayouts(layouts); - - return StatusCode::OK; -} - -Status ModelConfig::parseLayoutParameter(const std::string& command) { - this->layouts.clear(); - this->layout = LayoutConfiguration(); - if (command.empty()) { - return StatusCode::OK; - } - - std::string upperCaseCommand; - std::transform(command.begin(), command.end(), std::back_inserter(upperCaseCommand), ::toupper); - - erase_spaces(upperCaseCommand); - - if (*upperCaseCommand.begin() != '{') { - LayoutConfiguration layout; - auto status = LayoutConfiguration::fromString(upperCaseCommand, layout); - if (!status.ok()) { - return status; - } - setLayout(layout); - return StatusCode::OK; - } - - // parse as json - rapidjson::Document node; - if (node.Parse(command.c_str()).HasParseError()) { - return StatusCode::LAYOUT_WRONG_FORMAT; - } - return parseLayoutParameter(node); -} - -Status ModelConfig::parseShape(ShapeInfo& shapeInfo, const std::string& str) { - if (str == "auto") { - shapeInfo.shapeMode = AUTO; - return StatusCode::OK; - } - - shapeInfo.shapeMode = FIXED; - shapeInfo.shape = Shape(); - return Shape::fromString(str, shapeInfo.shape); -} - -Status ModelConfig::parseModelMapping() { - SPDLOG_DEBUG("Parsing model: {} mapping from path: {}", getName(), getPath()); - mappingInputs.clear(); - mappingOutputs.clear(); - std::filesystem::path path = this->getPath(); - path.append(MAPPING_CONFIG_JSON); - - std::ifstream ifs(path.c_str()); - if (!ifs.good()) { - return StatusCode::FILE_INVALID; - } - - rapidjson::Document doc; - rapidjson::IStreamWrapper isw(ifs); - if (doc.ParseStream(isw).HasParseError()) { - SPDLOG_ERROR("Configuration file is not a valid JSON file."); - return StatusCode::JSON_INVALID; - } - - if (validateJsonAgainstSchema(doc, MODELS_MAPPING_INPUTS_SCHEMA) != StatusCode::OK) { - SPDLOG_WARN("Couldn't load inputs object from file {}", path.c_str()); - } else { - // Process inputs - const auto itr = doc.FindMember("inputs"); - for (const auto& key : itr->value.GetObject()) { - SPDLOG_DEBUG("Loaded input mapping {} => {}", key.name.GetString(), key.value.GetString()); - mappingInputs[key.name.GetString()] = key.value.GetString(); - reversedMappingInputs[key.value.GetString()] = key.name.GetString(); - } - } - - if (validateJsonAgainstSchema(doc, MODELS_MAPPING_OUTPUTS_SCHEMA) != StatusCode::OK) { - SPDLOG_WARN("Couldn't load outputs object from file {}", path.c_str()); - } else { - // Process outputs - const auto it = doc.FindMember("outputs"); - for (const auto& key : it->value.GetObject()) { - SPDLOG_DEBUG("Loaded output mapping {} => {}", key.name.GetString(), key.value.GetString()); - mappingOutputs[key.name.GetString()] = key.value.GetString(); - reversedMappingOutputs[key.value.GetString()] = key.name.GetString(); - } - } - - return StatusCode::OK; -} - -Status ModelConfig::parseNode(const rapidjson::Value& v) { - this->setName(v["name"].GetString()); - this->setBasePath(v["base_path"].GetString()); - Status firstErrorStatus = StatusCode::OK; - - // Check for optional parameters - if (v.HasMember("batch_size")) { - if (v["batch_size"].IsString()) { - this->setBatchingParams(v["batch_size"].GetString()); - } else { - this->setBatchingParams(v["batch_size"].GetUint64()); - } - } - if (v.HasMember("target_device")) - this->setTargetDevice(v["target_device"].GetString()); - if (v.HasMember("version")) { - this->setVersion(v["version"].GetUint64()); - } - if (v.HasMember("nireq")) - this->setNireq(v["nireq"].GetUint64()); - - if (v.HasMember("shape")) { - // Legacy format as string - if (v["shape"].IsString()) { - ShapeInfo shapeInfo; - auto status = parseShape(shapeInfo, v["shape"].GetString()); - if (!status.ok()) { - if (!firstErrorStatus.ok()) { - firstErrorStatus = status; - } - SPDLOG_WARN("There was an error parsing shape {}", v["shape"].GetString()); - } else { - this->addShape(ANONYMOUS_INPUT_NAME, shapeInfo); - } - } else { - // Map of shapes - for (auto& s : v["shape"].GetObject()) { - ShapeInfo shapeInfo; - bool valid = true; - // check if legacy format is used - if (s.value.IsString()) { - auto status = ModelConfig::parseShape(shapeInfo, s.value.GetString()); - if (!status.ok()) { - if (!firstErrorStatus.ok()) { - firstErrorStatus = status; - } - SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); - valid = false; - } - } else { - for (auto& sh : s.value.GetArray()) { - if (sh.IsUint64()) { - size_t dim = sh.GetUint64(); - if (dim > std::numeric_limits::max()) { - SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); - if (!firstErrorStatus.ok()) { - firstErrorStatus = StatusCode::SHAPE_WRONG_FORMAT; - } - valid = false; - break; - } - shapeInfo.shape.add(Dimension{static_cast(dim)}); - } else { - SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); - if (!firstErrorStatus.ok()) { - firstErrorStatus = StatusCode::SHAPE_WRONG_FORMAT; - } - valid = false; - break; - } - } - } - if (s.name.GetString() != ANONYMOUS_INPUT_NAME) { - if (valid) { - this->addShape(s.name.GetString(), shapeInfo); - } - } else { - SPDLOG_WARN("Provided shape name: {} is forbidden and will be omitted", ANONYMOUS_INPUT_NAME); - } - } - } - } - - if (v.HasMember("layout")) { - if (v["layout"].IsString()) { - Status status = this->parseLayoutParameter(v["layout"].GetString()); - if (!status.ok()) { - return status; - } - } else { - Status status = this->parseLayoutParameter(v["layout"]); - if (!status.ok()) { - return status; - } - } - } - - if (v.HasMember("plugin_config")) { - auto status = parsePluginConfig(v["plugin_config"]); - if (!status.ok()) { - if (!firstErrorStatus.ok()) { - firstErrorStatus = status; - } - SPDLOG_ERROR("Couldn't parse plugin config"); - return status; - } - } - - if (v.HasMember("stateful")) - this->setStateful(v["stateful"].GetBool()); - - if (v.HasMember("low_latency_transformation")) { - if (!this->isStateful()) { - SPDLOG_ERROR("Low latency transformation parameter was set for non stateful model {}.", v["name"].GetString()); - return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; - } - this->setLowLatencyTransformation(v["low_latency_transformation"].GetBool()); - } - - if (v.HasMember("idle_sequence_cleanup")) { - if (!this->isStateful()) { - SPDLOG_ERROR("Idle sequence cleanup parameter was set for non stateful model {}.", v["name"].GetString()); - return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; - } - this->setIdleSequenceCleanup(v["idle_sequence_cleanup"].GetBool()); - } - - if (v.HasMember("max_sequence_number")) { - if (!this->isStateful()) { - SPDLOG_ERROR("Max sequence number parameter was set for non stateful model {}.", v["name"].GetString()); - return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; - } - if (!v["max_sequence_number"].IsUint()) { - SPDLOG_ERROR("Sequence maximum number parameter was set above unsigned int value for model {}.", v["name"].GetString()); - return StatusCode::INVALID_MAX_SEQUENCE_NUMBER; - } - this->setMaxSequenceNumber(v["max_sequence_number"].GetUint()); - } - - if (v.HasMember("model_version_policy")) { - rapidjson::StringBuffer buffer; - buffer.Clear(); - rapidjson::Writer writer(buffer); - v["model_version_policy"].Accept(writer); - const auto& status = parseModelVersionPolicy(buffer.GetString()); - if (!status.ok()) { - if (!firstErrorStatus.ok()) { - firstErrorStatus = status; - } - SPDLOG_WARN("Couldn't parse model version policy. {}", status.string()); - } - } else { - modelVersionPolicy = ModelVersionPolicy::getDefaultVersionPolicy(); - } - - SPDLOG_DEBUG("Specified model parameters:"); - SPDLOG_DEBUG("model_basepath: {}", getBasePath()); - SPDLOG_DEBUG("model_name: {}", getName()); - SPDLOG_DEBUG("batch_size: {}", getBatchSize().has_value() ? getBatchSize().value().toString() : "not configured"); - if (isShapeAnonymous()) { - SPDLOG_DEBUG("shape: {}", std::string(getShapes().begin()->second)); - } else { - SPDLOG_DEBUG("shape:"); - for (auto& [shapeInput, shapeValue] : getShapes()) { - SPDLOG_DEBUG(" {}: {}", shapeInput, std::string(shapeValue)); - } - } - if (getModelVersionPolicy()) { - SPDLOG_DEBUG("model_version_policy: {}", std::string(*getModelVersionPolicy())); - } - SPDLOG_DEBUG("nireq: {}", getNireq()); - SPDLOG_DEBUG("target_device: {}", getTargetDevice()); - SPDLOG_DEBUG("plugin_config:"); - for (auto& [pluginParameter, pluginValue] : getPluginConfig()) { - SPDLOG_DEBUG(" {}: {}", pluginParameter, pluginValue.as()); - } - - bool batchSizeSet = (getBatchingMode() != FIXED || getBatchSize().has_value()); - bool shapeSet = (getShapes().size() > 0); - - SPDLOG_DEBUG("Batch size set: {}, shape set: {}", batchSizeSet, shapeSet); - if (batchSizeSet && shapeSet) { - SPDLOG_WARN("Both shape and batch size have been defined. Batch size parameter will be ignored."); - setBatchingMode(FIXED); - setBatchSize(std::nullopt); - } - - SPDLOG_DEBUG("stateful: {}", isStateful()); - if (isStateful()) { - SPDLOG_DEBUG("idle_sequence_cleanup: {}", getIdleSequenceCleanup()); - SPDLOG_DEBUG("max_sequence_number: {}", getMaxSequenceNumber()); - SPDLOG_DEBUG("low_latency_transformation: {}", isLowLatencyTransformationUsed()); - } - - // Model Cache options - if (v.HasMember("allow_cache")) { - setAllowCache(v["allow_cache"].GetBool()); - SPDLOG_DEBUG("allow_cache: {}", v["allow_cache"].GetBool()); - } - - // if the config has models which require custom loader to be used, then load the same here - if (v.HasMember("custom_loader_options")) { - if (!parseCustomLoaderOptionsConfig(v["custom_loader_options"]).ok()) { - SPDLOG_ERROR("Couldn't parse custom loader options config"); - } - } - return StatusCode::OK; -} - -Status ModelConfig::parseCustomLoaderOptionsConfig(const rapidjson::Value& node) { - if (!node.IsObject()) { - return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; - } - for (auto it = node.MemberBegin(); it != node.MemberEnd(); ++it) { - if (!it->value.IsString()) { - return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; - } - addCustomLoaderOption(it->name.GetString(), it->value.GetString()); - } - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - node.Accept(writer); - customLoaderOptionsStr = buffer.GetString(); - - return StatusCode::OK; -} - -std::string ModelConfig::layoutConfigurationToString() const { - if (getLayout().isSet()) { - return getLayout().toString(); - } - std::stringstream ss; - for (const auto& [name, layoutCfg] : getLayouts()) { - ss << name << " " << layoutCfg.toString() << "; "; - } - return ss.str(); -} - -} // namespace ovms diff --git a/src/ovms_lib/modelconfig.hpp b/src/ovms_lib/modelconfig.hpp deleted file mode 100644 index 21a4be8c98..0000000000 --- a/src/ovms_lib/modelconfig.hpp +++ /dev/null @@ -1,992 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "layout_configuration.hpp" -#include "model_version_policy.hpp" -#include "shape.hpp" -#include "status.hpp" - -namespace ovms { - -using mapping_config_t = std::unordered_map; -using plugin_config_t = std::map; -using custom_loader_options_config_t = std::map; - -const std::string ANONYMOUS_INPUT_NAME = "ANONYMOUS_INPUT_NAME"; -const std::string MAPPING_CONFIG_JSON = "mapping_config.json"; -const uint32_t DEFAULT_MAX_SEQUENCE_NUMBER = 500; - -/** - * @brief This class represents model configuration - */ -class ModelConfig { -private: - /** - * @brief Model name - */ - std::string name; - - /** - * @brief Model uri path - */ - std::string basePath; - - /** - * @brief Model local destination path on disk after downloading from online storage - */ - std::string localPath; - - /** - * @brief Target device - */ - std::string targetDevice; - - /** - * @brief Batching mode - */ - Mode batchingMode; - - /** - * @brief Batch size - */ - std::optional batchSize; - - /** - * @brief Model version policy - */ - std::shared_ptr modelVersionPolicy; - - /** - * @brief Nireq - */ - uint64_t nireq; - - /** - * @brief Flag determining if model is stateful - */ - bool stateful; - - /** - * @brief Flag determining if model will be a subject to sequence cleaner scans - */ - bool idleSequenceCleanup; - - /** - * @brief Flag determining if model will use low latency transformation - */ - bool lowLatencyTransformation; - - /** - * @brief Number of maximum frames in one sequence - */ - uint32_t maxSequenceNumber; - - /** - * @brief Model cache directory - */ - std::string cacheDir; - - /** - * @brief Flag determining if allow cache option is set to true - */ - bool isAllowCacheTrue = false; - - /** - * @brief Model version - */ - model_version_t version = -1; - - /** - * @brief Plugin config - */ - plugin_config_t pluginConfig; - - /** - * @brief Layout for single input - */ - LayoutConfiguration layout; - - /** - * @brief Map of shapes - */ - shapes_info_map_t shapes; - - /** - * @brief Map of layouts - */ - layout_configurations_map_t layouts; - - /** - * @brief Input mapping configuration - */ - mapping_config_t mappingInputs; - - /** - * @brief Output mapping configuration - */ - mapping_config_t mappingOutputs; - - /** - * @brief Reversed input mapping configuration - */ - mapping_config_t reversedMappingInputs; - /** - * @brief Reversed output mapping configuration - */ - mapping_config_t reversedMappingOutputs; - - /** - * @brief Shape left opening bracket in string format - */ - static const char shapeLeft = '('; - - /** - * @brief Shape right opening bracket in string format - */ - static const char shapeRight = ')'; - - /** - * @brief Shape delimeter in string format - */ - static const char shapeDelimeter = ','; - - /** - * @brief Allowed configurable layouts - */ - static const std::set configAllowedLayouts; - - /** - * @brief custom_loader_options config as map - */ - custom_loader_options_config_t customLoaderOptionsConfigMap; - - /** - * @brief custom_loader_options config as string - */ - std::string customLoaderOptionsStr; - -public: - /** - * @brief Construct a new Model Config object - * - * @param name - * @param basePath - * @param targetDevice - * @param configBatchSize - * @param nireq - */ - ModelConfig(const std::string& name = "", - const std::string& basePath = "", - const std::string& targetDevice = "CPU", - const std::string& configBatchSize = "0", - uint64_t nireq = 0, - bool stateful = false, - bool idleSequenceCleanup = true, - bool lowLatencyTransformation = false, - uint32_t maxSequenceNumber = DEFAULT_MAX_SEQUENCE_NUMBER, - const std::string& cacheDir = "", - model_version_t version = 0, - const std::string& localPath = "") : - name(name), - basePath(basePath), - localPath(localPath), - targetDevice(targetDevice), - modelVersionPolicy(ModelVersionPolicy::getDefaultVersionPolicy()), - nireq(nireq), - stateful(stateful), - idleSequenceCleanup(idleSequenceCleanup), - lowLatencyTransformation(lowLatencyTransformation), - maxSequenceNumber(maxSequenceNumber), - cacheDir(cacheDir), - version(version), - pluginConfig({}), - layout(""), - shapes({}), - layouts({}), - mappingInputs({}), - mappingOutputs({}) { - setBatchingParams(configBatchSize); - } - - /** - * @brief Compares two ModelConfig instances and decides if models should be reloaded - * - * @param rhs - * - * @return true if configs are equal false otherwise - */ - bool isReloadRequired(const ModelConfig& rhs) const; - - /** - * @brief Compares two ModelConfig instances and decides if customloader configuration changed - * - * @param rhs - * - * @return true if customloader configuration has changed - */ - bool isCustomLoaderConfigChanged(const ModelConfig& rhs) const; - - /** - * @brief Compares two ModelConfig instances for batch size configuration - * - * @param rhs - * - * @return true if configurations are equal false otherwise - */ - bool isBatchSizeConfigurationEqual(const ModelConfig& rhs) const; - - /** - * @brief Compares two ModelConfig instances for layout configuration - * - * @param rhs - * - * @return true if configurations are equal false otherwise - */ - bool isLayoutConfigurationEqual(const ModelConfig& rhs) const; - - /** - * @brief Compares two ModelConfig instances for shape configuration - * - * @param rhs - * - * @return true if configurations are equal false otherwise - */ - bool isShapeConfigurationEqual(const ModelConfig& rhs) const; - - /** - * @brief Get the name - * - * @return const std::string& - */ - const std::string& getName() const { - return this->name; - } - - /** - * @brief Set the name - * - * @param name - */ - void setName(const std::string& name) { - this->name = name; - } - - /** - * @brief Get local path to specific model version where .xml and .bin is located for loading - * - * @return std::string - * */ - const std::string getPath() const { - return getLocalPath() + "/" + std::to_string(version); - } - - /** - * @brief Get the base path - * - * @return const std::string& - */ - const std::string& getBasePath() const { - return this->basePath; - } - - /** - * @brief Set the base path - * - * @param basePath - */ - void setBasePath(const std::string& basePath) { - this->basePath = basePath; - } - - /** - * @brief Get the local path - * - * @return const std::string& - */ - const std::string& getLocalPath() const { - return this->localPath; - } - - /** - * @brief Set the local path - * - * @param localPath - */ - void setLocalPath(const std::string& localPath) { - this->localPath = localPath; - } - - /** - * @brief Get the target device - * - * @return const std::string& - */ - const std::string& getTargetDevice() const { - return this->targetDevice; - } - - /** - * @brief Set the target device - * - * @param target device - */ - void setTargetDevice(const std::string& targetDevice) { - this->targetDevice = targetDevice; - } - - /** - * @brief Get the cache directory - * - * @return const std::string& - */ - const std::string& getCacheDir() const { - return this->cacheDir; - } - - /** - * @brief Set the cache directory - * - * @param cache directory - */ - void setCacheDir(const std::string& cacheDir) { - this->cacheDir = cacheDir; - } - - /** - * @brief Get the allow cache flag - * - * @return bool - */ - bool isAllowCacheSetToTrue() const { - return this->isAllowCacheTrue; - } - - /** - * @brief Set the allow cache flag - * - * @param allow cache flag - */ - void setAllowCache(const bool& allowCache) { - this->isAllowCacheTrue = allowCache; - } - - /** - * @brief Checks if given device is used as single target device. - * - * @param bool - */ - bool isSingleDeviceUsed(const std::string& device) const { - return this->targetDevice == device; - } - - bool isDeviceUsed(const std::string& device) const; - - /** - * @brief Get the batching mode - * - * @return ovms::Mode - */ - Mode getBatchingMode() const { - return this->batchingMode; - } - - bool isDynamicParameterEnabled() const { - return this->getBatchingMode() == Mode::AUTO || this->anyShapeSetToAuto(); - } - - /** - * @brief Get the batch size - * - * @return size_t - */ - std::optional getBatchSize() const { - return this->batchSize; - } - - /** - * @brief Set the batching mode - * - * @param batchingMode - */ - void setBatchingMode(Mode batchingMode) { - this->batchingMode = batchingMode; - } - - /** - * @brief Set the batch size - * - * @param batchSize - */ - void setBatchSize(std::optional batchSize) { - this->batchSize = batchSize; - } - - /** - * @brief Set batching mode to FIXED and batch size to value provided in a parameter. - * - * @param configBatchSize - */ - void setBatchingParams(size_t configBatchSize) { - setBatchingMode(FIXED); - setBatchSize(configBatchSize); - } - - /** - * @brief Extracts batching mode and batch size value from string provided in a parameter. - * - * @param configBatchSize - */ - void setBatchingParams(const std::string& configBatchSize) { - auto [batchingMode, effectiveBatchSize] = extractBatchingParams(configBatchSize); - setBatchingMode(batchingMode); - setBatchSize(effectiveBatchSize); - } - - /** - * @brief Extract batching mode and effective batch size from string. - * - * @param configBatchSize - */ - static std::tuple> extractBatchingParams(std::string configBatchSize); - - /** - * @brief Get the model version policy - * - * @return std::shared_ptr - */ - std::shared_ptr getModelVersionPolicy() { - return this->modelVersionPolicy; - } - - /** - * @brief Set the model version policy - * - * @param modelVersionPolicy - */ - void setModelVersionPolicy(std::shared_ptr modelVersionPolicy) { - this->modelVersionPolicy = modelVersionPolicy; - } - - /** - * @brief Parses string for model version policy - * - * @param string representing model version policy configuration - - * @return status - */ - Status parseModelVersionPolicy(std::string command); - - /** - * @brief Get the nireq - * - * @return uint64_t - */ - uint64_t getNireq() const { - return this->nireq; - } - - /** - * @brief Set the nireq - * - * @param nireq - */ - void setNireq(const uint64_t nireq) { - this->nireq = nireq; - } - - /** - * @brief Get the plugin config - * - * @return const std::string& - */ - const plugin_config_t& getPluginConfig() const { - return this->pluginConfig; - } - - /** - * @brief Set the plugin config - * - * @param pluginConfig - */ - void setPluginConfig(const plugin_config_t& pluginConfig) { - this->pluginConfig = pluginConfig; - } - - /** - * @brief Get stateful model flag - * - * @return bool - */ - const bool isStateful() const { - return this->stateful; - } - - /** - * @brief Set stateful model flag - * - * @return bool - */ - void setStateful(bool stateful) { - this->stateful = stateful; - } - - /** - * @brief Set stateful low latency transformation flag - * - * @return bool - */ - void setLowLatencyTransformation(bool lowLatencyTransformation) { - this->lowLatencyTransformation = lowLatencyTransformation; - } - - /** - * @brief Get stateful low latency transformation flag - * - * @return bool - */ - const bool isLowLatencyTransformationUsed() const { - return this->lowLatencyTransformation; - } - - /** - * @brief Get max number of sequences handled concurrently by the model - * - * @return uint - */ - uint64_t getMaxSequenceNumber() const { - return this->maxSequenceNumber; - } - - /** - * @brief Set max number of sequences handled concurrently by the model - * - * @return uint - */ - void setMaxSequenceNumber(const uint32_t maxSequenceNumber) { - this->maxSequenceNumber = maxSequenceNumber; - } - - /** - * @brief Get stateful sequence timeout - * - * @return uint - */ - bool getIdleSequenceCleanup() const { - return this->idleSequenceCleanup; - } - - /** - * @brief Set stateful sequence timeout - * - * @return uint - */ - void setIdleSequenceCleanup(const bool idleSequenceCleanup) { - this->idleSequenceCleanup = idleSequenceCleanup; - } - - /** - * @brief Parses json node for plugin config keys and values - * - * @param json node representing plugin_config - * - * @return status - */ - Status parsePluginConfig(const rapidjson::Value& node); - - /** - * @brief Parses string for plugin config keys and values - * - * @param string representing plugin_config - * - * @return status - */ - Status parsePluginConfig(std::string command) { - rapidjson::Document node; - if (command.empty()) { - return StatusCode::OK; - } - if (node.Parse(command.c_str()).HasParseError()) { - return StatusCode::PLUGIN_CONFIG_WRONG_FORMAT; - } - - return parsePluginConfig(node); - } - - /** - * @brief Parses value from json and extracts shapes info - * - * @param rapidjson::Value& node - * - * @return status - */ - Status parseShapeParameter(const rapidjson::Value& node); - - /** - * @brief Parses value from string and extracts shapes info - * - * @param string - * - * @return status - */ - Status parseShapeParameter(const std::string& command); - - /** - * @brief Parses value from json and extracts layouts info - * - * @param rapidjson::Value& node - * - * @return status - */ - Status parseLayoutParameter(const rapidjson::Value& node); - - /** - * @brief Parses value from string and extracts layouts info - * - * @param string - * - * @return status - */ - Status parseLayoutParameter(const std::string& command); - - /** - * @brief Returns true if any input shape specified in shapes map is in AUTO mode - * - * @return bool - */ - bool anyShapeSetToAuto() const { - for (const auto& [name, shapeInfo] : getShapes()) { - if (shapeInfo.shapeMode == AUTO) - return true; - } - return false; - } - - /** - * @brief Get the shapes - * - * @return const shapes_map_t& - */ - const shapes_info_map_t& getShapes() const { - return this->shapes; - } - - /** - * @brief Set the shapes - * - * @param shapes - */ - void setShapes(const shapes_info_map_t& shapes) { - this->shapes = shapes; - } - - /** - * @brief Returns true if shape with certain name is in AUTO mode - * - * @return bool - */ - bool isShapeAuto(const std::string& name) const { - auto it = getShapes().find(name); - if (it == getShapes().end()) { - it = getShapes().find(ANONYMOUS_INPUT_NAME); - } - if (it == getShapes().end()) { - return false; - } - return it->second.shapeMode == Mode::AUTO; - } - - bool isShapeAnonymous() const { - return getShapes().size() == 1 && getShapes().begin()->first == ANONYMOUS_INPUT_NAME; - } - - bool isShapeAnonymousFixed() const { - return isShapeAnonymous() && !isShapeAuto(ANONYMOUS_INPUT_NAME); - } - - bool isCloudStored() const { - return getLocalPath() != getBasePath(); - } - - /** - * @brief Sets the shape from the string representation - * - * @param shape - * @param str - * @return Status - */ - static Status parseShape(ShapeInfo& shapeInfo, const std::string& str); - - /** - * @brief Add a single named shape - * - * @param name - * @param shape - */ - void addShape(const std::string& name, const ShapeInfo& shapeInfo) { - this->shapes[name] = shapeInfo; - } - - void removeShape(const std::string& name) { - this->shapes.erase(name); - } - - /** - * @brief Get the layouts - * - * @return const std::string& - */ - const LayoutConfiguration& getLayout() const { - return this->layout; - } - - /** - * @brief Set the layout - * - * @param layout - */ - void setLayout(const LayoutConfiguration& layout) { - this->layout = layout; - this->layouts.clear(); - } - - /** - * @brief Get the layouts - * - * @return const layout_configurations_map_t& - */ - const layout_configurations_map_t& getLayouts() const { - return this->layouts; - } - - /** - * @brief Set the layouts - * - * @param layouts - */ - void setLayouts(const layout_configurations_map_t& layouts) { - this->layouts = layouts; - this->layout = LayoutConfiguration(); - } - - /** - * @brief Get the version - * - * @return const model_version_t& - */ - const model_version_t& getVersion() const { - return this->version; - } - - /** - * @brief Set the version - * - * @param version - */ - void setVersion(const model_version_t& version) { - this->version = version; - } - - /** - * @brief Get the mapping for inputs - * - * @return const mapping_config_t& - */ - const mapping_config_t& getMappingInputs() const { - return this->mappingInputs; - } - - /** - * @brief Get the mapping for outputs - * - * @return const mapping_config_t& - */ - const mapping_config_t& getMappingOutputs() const { - return this->mappingOutputs; - } - - /** - * @brief Get the reversed mapping for inputs - * - * @return const mapping_config_t& - */ - const mapping_config_t& getRealMappingInputs() const { - return this->reversedMappingInputs; - } - - /** - * @brief Get the reversed mapping for outputs - * - * @return const mapping_config_t& - */ - const mapping_config_t& getRealMappingOutputs() const { - return this->reversedMappingOutputs; - } - - /** - * @brief Get the mapping inputs by key - * - * @param key - * @return const std::string - */ - const std::string getMappingInputByKey(const std::string& key) const { - auto it = mappingInputs.find(key); - return it != mappingInputs.end() ? it->second : ""; - } - - /** - * @brief Get the mapping outputs by key - * - * @param key - * @return const std::string - */ - const std::string getMappingOutputByKey(const std::string& key) const { - auto it = mappingOutputs.find(key); - return it != mappingOutputs.end() ? it->second : ""; - } - - /** - * @brief Get the real inputs by value - * - * @param value - * @return const std::string - */ - const std::string getRealInputNameByValue(const std::string& value) const { - auto it = reversedMappingInputs.find(value); - return it != reversedMappingInputs.end() ? it->second : ""; - } - - /** - * @brief Get the real outputs by value - * - * @param value - * @return const std::string - */ - const std::string getRealOutputNameByValue(const std::string& value) const { - auto it = reversedMappingOutputs.find(value); - return it != reversedMappingOutputs.end() ? it->second : ""; - } - - /** - * @brief Set the mapping inputs - * - * @param mapping - */ - void setMappingInputs(const mapping_config_t& mapping) { - this->mappingInputs = mapping; - } - - /** - * @brief Set the mapping outputs - * - * @param mapping - */ - void setMappingOutputs(const mapping_config_t& mapping) { - this->mappingOutputs = mapping; - } - - /** - * @brief Set the reversed mapping inputs - * - * @param mapping - */ - void setRealMappingInputs(const mapping_config_t& mapping) { - this->reversedMappingInputs = mapping; - } - - /** - * @brief Set the reversed mapping outputs - * - * @param mapping - */ - void setRealMappingOutputs(const mapping_config_t& mapping) { - this->reversedMappingOutputs = mapping; - } - - /** - * @brief Parses mapping_config.json for mapping input/outputs in the model - * - * @return Status - */ - Status parseModelMapping(); - - /** - * @brief Parses all settings from a JSON node - * - * @return Status - */ - Status parseNode(const rapidjson::Value& v); - - /** - * @brief Returns true if model requires a custom loader to load - * - * @return bool - */ - const bool isCustomLoaderRequiredToLoadModel() const { - return (this->customLoaderOptionsConfigMap.size() > 0); - } - - /** - * @brief Get the custom loader option config - * - * @return const custom_loader_options_config_t& - */ - const custom_loader_options_config_t& getCustomLoaderOptionsConfigMap() const { - return this->customLoaderOptionsConfigMap; - } - - /** - * @brief Add custom loader option - * - * @param name - * @param value - */ - void addCustomLoaderOption(const std::string& name, const std::string& value) { - customLoaderOptionsConfigMap[name] = value; - } - - /** - * @brief Get the custom loader option config - * - * @return const std::string - */ - const std::string& getCustomLoaderOptionsConfigStr() const { - return this->customLoaderOptionsStr; - } - - /** - * @brief Parses json node for custom_loader_options config keys and values - * - * @param json node representing custom_loader_options_config - * - * @return status - */ - Status parseCustomLoaderOptionsConfig(const rapidjson::Value& node); - - std::string layoutConfigurationToString() const; -}; -} // namespace ovms diff --git a/src/ovms_lib/modelinstance.cpp b/src/ovms_lib/modelinstance.cpp deleted file mode 100644 index 39104d70b7..0000000000 --- a/src/ovms_lib/modelinstance.cpp +++ /dev/null @@ -1,1225 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelinstance.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "config.hpp" -#include "customloaders.hpp" -#include "deserialization.hpp" -#include "executingstreamidguard.hpp" -#include "filesystem.hpp" -#include "layout.hpp" -#include "layout_configuration.hpp" -#include "logging.hpp" -#include "ov_utils.hpp" -#include "predict_request_validation_utils.hpp" -#include "prediction_service_utils.hpp" -#include "profiler.hpp" -#include "serialization.hpp" -#include "shape.hpp" -#include "stringutils.hpp" -#include "tensorinfo.hpp" -#include "timer.hpp" - -namespace ovms { - -const char* CPU_THROUGHPUT_STREAMS = "CPU_THROUGHPUT_STREAMS"; -const char* NIREQ = "NIREQ"; - -const uint MAX_NIREQ_COUNT = 100000; - -const int DEFAULT_OV_STREAMS = std::thread::hardware_concurrency() / 4; - -const uint UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS = 10; - -void ModelInstance::subscribe(PipelineDefinition& pd) { - subscriptionManager.subscribe(pd); -} - -void ModelInstance::unsubscribe(PipelineDefinition& pd) { - subscriptionManager.unsubscribe(pd); -} - -Status getRequestedShape(const ModelConfig& config, const DynamicModelParameter& parameter, const std::string& name, Shape& shapeOut) { - Shape shape; - auto mappedName = config.getMappingInputByKey(name); - if (config.getBatchSize().has_value() || parameter.isBatchSizeRequested()) { - // leave shape untouched - } else if (config.isShapeAuto(name) && parameter.isShapeRequested(name)) { - auto status = Shape::fromFlatShape(parameter.getShape(name), shape); - if (!status.ok()) { - return status; - } - } else if (mappedName == "" && config.getShapes().count(name) && config.getShapes().at(name).shape.size()) { - shape = config.getShapes().at(name).shape; - } else if (config.getShapes().count(mappedName) && config.getShapes().at(mappedName).shape.size()) { - shape = config.getShapes().at(mappedName).shape; - } else if (config.getShapes().count(ANONYMOUS_INPUT_NAME) && config.getShapes().at(ANONYMOUS_INPUT_NAME).shape.size()) { - shape = config.getShapes().at(ANONYMOUS_INPUT_NAME).shape; - } - shapeOut = shape; - return StatusCode::OK; -} - -bool hasInputWithName(std::shared_ptr& model, const std::string& name) { - try { - model->input(name); - return true; - } catch (ov::Exception& e) { - return false; - } -} - -bool hasOutputWithName(std::shared_ptr& model, const std::string& name) { - try { - model->output(name); - return true; - } catch (ov::Exception& e) { - return false; - } -} - -Status validateConfigurationAgainstNetwork(const ModelConfig& config, std::shared_ptr& model) { - if (config.isShapeAnonymousFixed() && model->inputs().size() > 1) { - Status status = StatusCode::ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED; - SPDLOG_LOGGER_WARN(modelmanager_logger, status.string()); - return status; - } - if (config.getLayout().isSet() && model->inputs().size() > 1) { - Status status = StatusCode::ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED; - SPDLOG_LOGGER_WARN(modelmanager_logger, status.string()); - return status; - } - for (const auto& [name, _] : config.getShapes()) { - if (name == ANONYMOUS_INPUT_NAME) { - continue; - } - if (hasInputWithName(model, name) && config.getMappingInputByKey(name) != "") { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Config shape - {} is mapped by {}. Changes will not apply", name, config.getMappingInputByKey(name)); - return StatusCode::CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME; - } else if (!hasInputWithName(model, name) && !hasInputWithName(model, config.getRealInputNameByValue(name))) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Config shape - {} not found in model", name); - return StatusCode::CONFIG_SHAPE_IS_NOT_IN_MODEL; - } - } - for (const auto& [name, _] : config.getLayouts()) { - if (hasInputWithName(model, name) && config.getMappingInputByKey(name) != "") { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} is mapped by {}. Changes will not apply", name, config.getMappingInputByKey(name)); - return StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME; - } else if (hasOutputWithName(model, name) && config.getMappingOutputByKey(name) != "") { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} is mapped by {}. Changes will not apply", name, config.getMappingOutputByKey(name)); - return StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME; - } else if (!hasInputWithName(model, name) && !hasOutputWithName(model, name) && !hasInputWithName(model, config.getRealInputNameByValue(name)) && !hasOutputWithName(model, config.getRealOutputNameByValue(name))) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Config layout - {} not found in model", name); - return StatusCode::CONFIG_LAYOUT_IS_NOT_IN_MODEL; - } - } - return StatusCode::OK; -} - -const Layout ModelInstance::getReportedTensorLayout(const ModelConfig& config, const std::string& name, bool isInput) { - if (isInput) { - const auto& input = this->model->input(name); - auto networkSpecifiedLayout = getLayoutFromRTMap(input.get_rt_info()); - if (networkSpecifiedLayout.has_value()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting input layout from RTMap: {}; for tensor name: {}", networkSpecifiedLayout.value().to_string(), name); - return Layout::fromOvLayout(networkSpecifiedLayout.value()); - } - } else { - const auto& output = this->model->output(name); - auto networkSpecifiedLayout = getLayoutFromRTMap(output.get_rt_info()); - if (networkSpecifiedLayout.has_value()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting output layout from RTMap: {}; for tensor name: {}", networkSpecifiedLayout.value().to_string(), name); - return Layout::fromOvLayout(networkSpecifiedLayout.value()); - } - } - auto layout = Layout::getDefaultLayout(); - if (isInput && config.getLayout().isSet()) { - layout = config.getLayout().getTensorLayout(); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting layout from ModelConfig: {}; for tensor name: {}", layout, name); - return layout; - } else if (config.getLayouts().size() > 0) { - auto mappedName = config.getMappingInputByKey(name); - auto it = config.getLayouts().find(mappedName == "" ? name : mappedName); - if (it != config.getLayouts().end()) { - layout = it->second.getTensorLayout(); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting layout from ModelConfig: {}; for tensor name: {}", layout, name); - return layout; - } - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reporting default layout: {}; for tensor name: {}", layout, name); - return layout; -} - -Status applyLayoutConfiguration(const ModelConfig& config, std::shared_ptr& model, const std::string& modelName, model_version_t modelVersion) { - ov::preprocess::PrePostProcessor preproc(model); - - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Applying layout configuration: {}", config.layoutConfigurationToString()); - - for (const ov::Output& input : model->inputs()) { - try { - std::string name = input.get_any_name(); - std::string mappedName = config.getMappingInputByKey(name).empty() ? name : config.getMappingInputByKey(name); - if (config.getLayout().isSet()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding preprocessing step: Tensor Layout:{}; Network Layout:{}; single input", - modelName, - modelVersion, - config.getLayout().getTensorLayout(), - config.getLayout().getModelLayout()); - - preproc.input().tensor().set_layout(ov::Layout(config.getLayout().getTensorLayout())); - preproc.input().model().set_layout(ov::Layout(config.getLayout().getModelLayout())); - } else if (config.getLayouts().count(mappedName) > 0) { - auto& layout = config.getLayouts().at(mappedName); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding preprocessing step: Tensor Layout:{}; Network Layout:{}; input name: {}", - modelName, - modelVersion, - layout.getTensorLayout(), - layout.getModelLayout(), - mappedName); - - preproc.input(name).tensor().set_layout(ov::Layout(layout.getTensorLayout())); - preproc.input(name).model().set_layout(ov::Layout(layout.getModelLayout())); - } else { - auto inheritedModelLayout = getLayoutFromRTMap(input.get_rt_info()); - auto guessedModelLayout = Layout::getDefaultLayout(); - - ov::Layout targetModelLayout = inheritedModelLayout.has_value() ? inheritedModelLayout.value() : ov::Layout(guessedModelLayout); - - if (inheritedModelLayout.has_value()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (inherited from network); input name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (default); input name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); - } - preproc.input(name).model().set_layout(targetModelLayout); - } - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO with error:{}", - modelName, - modelVersion, - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO with error:{}", - modelName, - modelVersion, - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure input layout for model:{}; version:{}; from OpenVINO", - modelName, - modelVersion); - return StatusCode::UNKNOWN_ERROR; - } - } - - for (const ov::Output& output : model->outputs()) { - try { - std::string name = output.get_any_name(); - std::string mappedName = config.getMappingOutputByKey(name).empty() ? name : config.getMappingOutputByKey(name); - if (config.getLayouts().count(mappedName) > 0) { - auto& layout = config.getLayouts().at(mappedName); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Adding postprocessing step: Tensor Layout:{}; Network Layout:{}; output name: {}", - modelName, - modelVersion, - layout.getTensorLayout(), - layout.getModelLayout(), - mappedName); - preproc.output(name).tensor().set_layout(ov::Layout(layout.getTensorLayout())); - preproc.output(name).model().set_layout(ov::Layout(layout.getModelLayout())); - } else { - auto inheritedModelLayout = getLayoutFromRTMap(output.get_rt_info()); - auto guessedModelLayout = Layout::getDefaultLayout(); - - ov::Layout targetModelLayout = inheritedModelLayout.has_value() ? inheritedModelLayout.value() : ov::Layout(guessedModelLayout); - - if (inheritedModelLayout.has_value()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (inherited from network); output name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "model: {}, version: {}; Configuring layout: Tensor Layout:; Network Layout:{} (default); output name: {}", modelName, modelVersion, targetModelLayout.to_string(), name); - } - preproc.output(name).model().set_layout(targetModelLayout); - } - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO with error:{}", - modelName, - modelVersion, - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO with error:{}", - modelName, - modelVersion, - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to configure output layout for model:{}; version:{}; from OpenVINO", - modelName, - modelVersion); - return StatusCode::UNKNOWN_ERROR; - } - } - - try { - model = preproc.build(); - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot change layout"); - return StatusCode::MODEL_NOT_LOADED; - } - return StatusCode::OK; -} - -Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLayoutConfiguration, const DynamicModelParameter& parameter) { - Status status = validateConfigurationAgainstNetwork(config, this->model); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during configuration validation against model"); - return status; - } - if (needsToApplyLayoutConfiguration) { - status = applyLayoutConfiguration(config, this->model, getName(), getVersion()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during layout configuration"); - return status; - } - } - status = loadInputTensors(config, parameter); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during loading input tensors"); - return status; - } - status = loadOutputTensors(config); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during loading output tensors"); - return status; - } - return StatusCode::OK; -} - -Status ModelInstance::loadInputTensors(const ModelConfig& config, const DynamicModelParameter& parameter) { - this->inputsInfo.clear(); - - std::map modelShapes; - bool reshapeRequired = false; - - // First pass, gather reshape info. - for (const ov::Output& input : this->model->inputs()) { - std::string name; - try { - std::string name = input.get_any_name(); - ov::PartialShape shape = input.get_partial_shape(); - - Shape requestedShape; - auto status = getRequestedShape(config, parameter, name, requestedShape); - if (!status.ok()) { - return status; - } - if (requestedShape.size() > 0) { - shape = requestedShape.createPartialShape(); - } - - modelShapes[name] = shape; - if (input.get_partial_shape() != shape) { - reshapeRequired = true; - } - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO", - getName(), - getVersion()); - return StatusCode::UNKNOWN_ERROR; - } - } - - if (reshapeRequired) { - SPDLOG_DEBUG("model: {}, version: {}; reshaping inputs", getName(), getVersion()); - try { - model->reshape(modelShapes); - } catch (const ov::Exception& e) { - SPDLOG_WARN("OV does not support reshaping model: {} with provided shape", getName()); - SPDLOG_DEBUG("Description: {}", e.what()); - return StatusCode::RESHAPE_ERROR; - } catch (const std::exception& e) { - SPDLOG_WARN("OV does not support reshaping model: {} with provided shape", getName()); - SPDLOG_DEBUG("Description: {}", e.what()); - return StatusCode::RESHAPE_ERROR; - } - } else { - SPDLOG_DEBUG("model: {}, version: {}; reshaping inputs is not required", getName(), getVersion()); - } - - configureBatchSize(this->config, parameter); - - for (const ov::Output& input : this->model->inputs()) { - try { - std::string name = input.get_any_name(); - - ovms::Precision precision = ovElementTypeToOvmsPrecision(input.get_element_type()); - Shape shape(input.get_partial_shape()); - std::string mappingName = config.getMappingInputByKey(name); - const Layout layout = getReportedTensorLayout(config, name, true); - - if (!layout.isCompatible(shape)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Layout: {}; incompatible with shape: {}; for input name: {}", layout, shape.toString(), name); - return StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE; - } - - std::shared_ptr info = std::make_shared( - name, - mappingName, - precision, - shape, - layout); - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Input {}", info->asString()); - - this->inputsInfo[info->getMappedName()] = std::move(info); - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get input name for model:{}; version:{}; from OpenVINO", - getName(), - getVersion()); - return StatusCode::UNKNOWN_ERROR; - } - } - return StatusCode::OK; -} - -Status ModelInstance::loadOutputTensors(const ModelConfig& config) { - this->outputsInfo.clear(); - - for (const ov::Output& output : this->model->outputs()) { - try { - std::string name = output.get_any_name(); - - ovms::Precision precision = ovElementTypeToOvmsPrecision(output.get_element_type()); - Shape shape(output.get_partial_shape()); - std::string mappingName = config.getMappingOutputByKey(name); - const Layout layout = getReportedTensorLayout(config, name, false); - - if (!layout.isCompatible(shape)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Layout: {}; incompatible with shape: {}; for output name: {}", layout, shape.toString(), name); - return StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE; - } - - std::shared_ptr info = std::make_shared( - name, - mappingName, - precision, - shape, - layout); - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Output {}", info->asString()); - - this->outputsInfo[info->getMappedName()] = std::move(info); - } catch (const ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO with error:{}", - getName(), - getVersion(), - e.what()); - return StatusCode::UNKNOWN_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to get output name for model:{}; version:{}; from OpenVINO", - getName(), - getVersion()); - return StatusCode::UNKNOWN_ERROR; - } - } - - return StatusCode::OK; -} - -// Temporary methods. To be replaces with proper storage class. -bool dirExists(const std::string& path) { - if (FileSystem::isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return false; - } - DIR* dir = opendir(path.c_str()); - if (dir) { - closedir(dir); - return true; - } - - return false; -} - -std::string findFilePathWithExtension(const std::string& path, const std::string& extension) { - struct dirent* entry; - if (FileSystem::isPathEscaped(path)) { - SPDLOG_ERROR("Path {} escape with .. is forbidden.", path); - return std::string(); - } - DIR* dir = opendir(path.c_str()); - if (!dir) { - SPDLOG_WARN("Failed to opendir: {}", path); - return std::string(); - } - - while ((entry = readdir(dir)) != nullptr) { - auto name = std::string(entry->d_name); - if (endsWith(name, extension)) { - closedir(dir); - if (endsWith(name, "/")) { - return path + name; - } else { - return path + '/' + name; - } - } - } - closedir(dir); - - return std::string(); -} - -std::string ModelInstance::findModelFilePathWithExtension(const std::string& extension) const { - return findFilePathWithExtension(path, extension); -} - -uint ModelInstance::getNumOfParallelInferRequestsUnbounded(const ModelConfig& modelConfig) { - uint numberOfParallelInferRequests = 0; - if (modelConfig.getNireq() > 0) { - return modelConfig.getNireq(); - } - auto& ovmsConfig = ovms::Config::instance(); - if (ovmsConfig.nireq() > 0) { - // nireq is set globally for all models in ovms startup parameters - return ovmsConfig.nireq(); - } - std::string key = METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS); - try { - numberOfParallelInferRequests = compiledModel->get_property(key).as(); - } catch (const ov::Exception& ex) { - SPDLOG_WARN("Failed to query OPTIMAL_NUMBER_OF_INFER_REQUESTS with error {}. Using 1 nireq.", ex.what()); - numberOfParallelInferRequests = 1u; - } - return numberOfParallelInferRequests; -} - -uint ModelInstance::getNumOfParallelInferRequests(const ModelConfig& modelConfig) { - uint nireq = getNumOfParallelInferRequestsUnbounded(modelConfig); - if (nireq > MAX_NIREQ_COUNT) { - SPDLOG_WARN("Invalid nireq because its value was too high: {}. Maximum value: {}", nireq, MAX_NIREQ_COUNT); - return 0; - } else if (nireq < 1u) { - SPDLOG_WARN("Ignored configured nireq because it has to be above 0 and was: {}. Set to 1", nireq); - return 1u; - } - return nireq; -} - -std::shared_ptr ModelInstance::loadOVModelPtr(const std::string& modelFile) { - return ieCore.read_model(modelFile); -} - -Status ModelInstance::loadOVModel() { - auto& modelFile = modelFiles[0]; - SPDLOG_DEBUG("Try reading model file: {}", modelFile); - try { - model = loadOVModelPtr(modelFile); - } catch (std::exception& e) { - SPDLOG_ERROR("Error: {}; occurred during loading ov::Model model: {} version: {}", e.what(), getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } - return StatusCode::OK; -} - -Status ModelInstance::loadOVModelUsingCustomLoader() { - SPDLOG_DEBUG("Try reading model using a custom loader"); - try { - std::vector modelBinary; - std::vector weights; - - SPDLOG_INFO("loading ov::Model for model: {} basepath: {} <> {} version: {}", getName(), getPath(), this->config.getBasePath().c_str(), getVersion()); - - custom_loader_options_config_t customLoaderOptionsConfig = this->config.getCustomLoaderOptionsConfigMap(); - const std::string loaderName = customLoaderOptionsConfig["loader_name"]; - - auto& customloaders = ovms::CustomLoaders::instance(); - auto customLoaderInterfacePtr = customloaders.find(loaderName); - if (customLoaderInterfacePtr == nullptr) { - SPDLOG_INFO("Loader {} is not in loaded customloaders list", loaderName); - throw std::invalid_argument("customloader not exisiting"); - } - - CustomLoaderStatus res = customLoaderInterfacePtr->loadModel(this->config.getName(), - this->config.getBasePath(), - getVersion(), - this->config.getCustomLoaderOptionsConfigStr(), modelBinary, weights); - - if (res == CustomLoaderStatus::MODEL_LOAD_ERROR) { - return StatusCode::FILE_INVALID; - } - - if ((res == CustomLoaderStatus::INTERNAL_ERROR) || (res == CustomLoaderStatus::MODEL_BLACKLISTED)) { - return StatusCode::INTERNAL_ERROR; - } - - std::string strModel(modelBinary.begin(), modelBinary.end()); - - if (res == CustomLoaderStatus::MODEL_TYPE_IR) { - ov::Tensor tensorWts(ov::element::u8, ov::Shape{weights.size()}); - std::memcpy(tensorWts.data(), weights.data(), weights.size()); - model = ieCore.read_model(strModel, tensorWts); - } else if (res == CustomLoaderStatus::MODEL_TYPE_ONNX) { - model = ieCore.read_model(strModel, ov::Tensor()); - } else if (res == CustomLoaderStatus::MODEL_TYPE_BLOB) { - return StatusCode::INTERNAL_ERROR; - } - } catch (ov::Exception& e) { - SPDLOG_ERROR("Error: {}; occurred during loading ov::Model for model: {} version: {}", e.what(), getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } catch (std::exception& e) { - SPDLOG_ERROR("Error: {}; occurred during loading ov::Model for model: {} version: {}", e.what(), getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } - return StatusCode::OK; -} - -void ModelInstance::loadCompiledModelPtr(const plugin_config_t& pluginConfig) { - compiledModel = std::make_shared(ieCore.compile_model(model, targetDevice, pluginConfig)); -} - -plugin_config_t ModelInstance::prepareDefaultPluginConfig(const ModelConfig& config) { - plugin_config_t pluginConfig = config.getPluginConfig(); - // Do not add CPU_THROUGHPUT_AUTO when performance hint is specified. - bool isPerformanceHintSpecified = pluginConfig.count("PERFORMANCE_HINT") > 0; - if (isPerformanceHintSpecified) { - return pluginConfig; - } - // For CPU and GPU, if user did not specify, calculate CPU_THROUGHPUT_STREAMS automatically - if (config.isSingleDeviceUsed("CPU")) { - if (pluginConfig.count("CPU_THROUGHPUT_STREAMS") == 0) { - pluginConfig["CPU_THROUGHPUT_STREAMS"] = "CPU_THROUGHPUT_AUTO"; - } - } - if (config.isSingleDeviceUsed("GPU")) { - if (pluginConfig.count("GPU_THROUGHPUT_STREAMS") == 0) { - pluginConfig["GPU_THROUGHPUT_STREAMS"] = "GPU_THROUGHPUT_AUTO"; - } - } - return pluginConfig; -} - -Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { - plugin_config_t pluginConfig = prepareDefaultPluginConfig(config); - try { - loadCompiledModelPtr(pluginConfig); - } catch (ov::Exception& e) { - Status status = StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE; - SPDLOG_LOGGER_ERROR(modelmanager_logger, "{}; error: {}; model: {}; version: {}; device: {}", - status.string(), - e.what(), - getName(), - getVersion(), - config.getTargetDevice()); - return status; - } catch (std::exception& e) { - Status status = StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE; - SPDLOG_LOGGER_ERROR(modelmanager_logger, "{}; error: {}; model: {}; version: {}; device: {}", - status.string(), - e.what(), - getName(), - getVersion(), - config.getTargetDevice()); - return status; - } - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Plugin config for device: {}", targetDevice); - for (const auto pair : pluginConfig) { - const auto key = pair.first; - const auto value = pair.second; - SPDLOG_LOGGER_INFO(modelmanager_logger, "OVMS set plugin settings key: {}; value: {};", key, value.as()); - } - - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); - std::vector supportedConfigKeys; - try { - std::vector supportedConfigKeys2 = compiledModel->get_property(supportedConfigKey).as>(); - supportedConfigKeys = std::move(supportedConfigKeys2); - } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}; Error: {}", targetDevice, supportedConfigKey, e.what()); - return StatusCode::OK; - } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel metric key: {}", targetDevice, supportedConfigKey); - return StatusCode::OK; - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging model:{}; version: {};target device: {}; CompiledModel configuration", getName(), getVersion(), targetDevice); - for (auto& key : supportedConfigKeys) { - std::string value; - try { - auto paramValue = compiledModel->get_property(key); - value = paramValue.as(); - } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel config key: {}; Error: {}", targetDevice, key, e.what()); - continue; - } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting target device: {}, CompiledModel config key: {}", targetDevice, key); - continue; - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {}; version: {}; target device: {}, CompiledModel config key: {}, value: {}", getName(), getVersion(), targetDevice, key, value); - } - return StatusCode::OK; -} - -Status ModelInstance::fetchModelFilepaths() { - if (this->config.isCustomLoaderRequiredToLoadModel()) { - // not required if the model is loaded using a custom loader and can be returned from here - return StatusCode::OK; - } - - SPDLOG_DEBUG("Getting model files from path: {}", path); - if (!dirExists(path)) { - SPDLOG_ERROR("Missing model directory {}", path); - return StatusCode::PATH_INVALID; - } - - bool found = true; - for (auto extension : OV_MODEL_FILES_EXTENSIONS) { - auto file = findModelFilePathWithExtension(extension); - if (file.empty()) { - found = false; - } - modelFiles.push_back(file); - } - if (!found) { - found = true; - modelFiles.clear(); - for (auto extension : ONNX_MODEL_FILES_EXTENSIONS) { - auto file = findModelFilePathWithExtension(extension); - if (file.empty()) { - found = false; - } - modelFiles.push_back(file); - } - } - - if (!found) { - SPDLOG_ERROR("Could not find file for model: {} version: {} in path: {}", getName(), getVersion(), path); - return StatusCode::FILE_INVALID; - } - - return StatusCode::OK; -} - -Status ModelInstance::prepareInferenceRequestsQueue(const ModelConfig& config) { - uint numberOfParallelInferRequests = getNumOfParallelInferRequests(config); - if (numberOfParallelInferRequests == 0) { - return Status(StatusCode::INVALID_NIREQ, "Exceeded allowed nireq value"); - } - inferRequestsQueue = std::make_unique(*compiledModel, numberOfParallelInferRequests); - SPDLOG_INFO("Loaded model {}; version: {}; batch size: {}; No of InferRequests: {}", - getName(), - getVersion(), - getBatchSize().toString(), - numberOfParallelInferRequests); - return StatusCode::OK; -} - -void ModelInstance::configureBatchSize(const ModelConfig& config, const DynamicModelParameter& parameter) { - if (parameter.isBatchSizeRequested()) { - ov::set_batch(model, parameter.getBatchSize()); - } else if (config.getBatchSize().has_value()) { - ov::set_batch(model, config.getBatchSize().value().createPartialDimension()); - } -} - -Status ModelInstance::loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter) { - bool isLayoutConfigurationChanged = !config.isLayoutConfigurationEqual(this->config); - bool needsToApplyLayoutConfiguration = isLayoutConfigurationChanged || !this->model; - - subscriptionManager.notifySubscribers(); - this->path = config.getPath(); - this->targetDevice = config.getTargetDevice(); - this->config = config; - auto status = fetchModelFilepaths(); - - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - try { - status = setCacheOptions(this->config); - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - - if (!this->model || isLayoutConfigurationChanged) { - if (this->config.isCustomLoaderRequiredToLoadModel()) { - // loading the model using the custom loader - status = loadOVModelUsingCustomLoader(); - } else { - status = loadOVModel(); - } - } - - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - - status = loadTensors(this->config, needsToApplyLayoutConfiguration, parameter); - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - status = loadOVCompiledModel(this->config); - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - status = prepareInferenceRequestsQueue(this->config); - if (!status.ok()) { - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return status; - } - } catch (const ov::Exception& e) { - SPDLOG_ERROR("exception occurred while loading model: {}", e.what()); - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return StatusCode::MODEL_NOT_LOADED; - } catch (const std::exception& e) { - SPDLOG_ERROR("exception occurred while loading model: {}", e.what()); - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - return StatusCode::MODEL_NOT_LOADED; - } - this->status.setAvailable(); - modelLoadedNotify.notify_all(); - return status; -} - -Status ModelInstance::setCacheOptions(const ModelConfig& config) { - if (!config.getCacheDir().empty()) { - if (!config.isAllowCacheSetToTrue() && (config.isCustomLoaderRequiredToLoadModel() || config.anyShapeSetToAuto() || (config.getBatchingMode() == Mode::AUTO))) { - this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), ""}}); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has disabled caching", this->getName()); - this->cacheDisabled = true; - } else if (config.isAllowCacheSetToTrue() && config.isCustomLoaderRequiredToLoadModel()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model: {} has allow cache set to true while using custom loader", this->getName()); - return StatusCode::ALLOW_CACHE_WITH_CUSTOM_LOADER; - } else { - this->ieCore.set_property({{CONFIG_KEY(CACHE_DIR), config.getCacheDir()}}); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Model: {} has enabled caching", this->getName()); - } - } - return StatusCode::OK; -} - -Status ModelInstance::loadModel(const ModelConfig& config) { - std::lock_guard loadingLock(loadingMutex); - SPDLOG_INFO("Loading model: {}, version: {}, from path: {}, with target device: {} ...", - config.getName(), config.getVersion(), config.getPath(), config.getTargetDevice()); - if (config.getBatchingMode() == AUTO) { - SPDLOG_INFO("Batch size mode for model {} is set to auto", config.getName()); - } else if (config.anyShapeSetToAuto()) { - SPDLOG_INFO("Some inputs shapes for model {} are set to auto", config.getName()); - } - this->status = ModelVersionStatus(config.getName(), config.getVersion()); - this->status.setLoading(); - return loadModelImpl(config); -} - -Status ModelInstance::reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter) { - std::lock_guard loadingLock(loadingMutex); - this->status.setLoading(); - while (!canUnloadInstance()) { - SPDLOG_INFO("Waiting to reload model: {} version: {}. Blocked by: {} inferences in progress.", - getName(), getVersion(), predictRequestsHandlesCount); - std::this_thread::sleep_for(std::chrono::milliseconds(UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS)); - } - if ((this->config.isCustomLoaderRequiredToLoadModel()) && (isCustomLoaderConfigChanged)) { - // unloading and the loading back the model - isCustomLoaderConfigChanged = false; - retireModel(isCustomLoaderConfigChanged); - } - return loadModelImpl(config, parameter); -} - -Status ModelInstance::recoverFromReloadingError(const Status& status) { - SPDLOG_WARN("Failed to perform complete reload with requested dynamic parameter. Model: {} version: {} with error: {}. Reloading to previous configuration", - getName(), getVersion(), status.string()); - bool changeStatus{false}; - retireModel(changeStatus); - - auto recoveryStatus = reloadModel(config); - if (!recoveryStatus.ok()) { - SPDLOG_WARN("Failed to recover model: {} version: {} to previous configuration with error: {}", - getName(), getVersion(), recoveryStatus.string()); - } - return status; -} - -Status ModelInstance::reshapeWithFullReload(const Status& status, const DynamicModelParameter& parameter) { - SPDLOG_WARN("Failed to reload model: {} version: {} with error: {}. Trying to perform complete reload with requested dynamic parameter", - getName(), getVersion(), status.string()); - bool changeStatus{false}; - retireModel(changeStatus); - - auto recoveryStatus = reloadModel(config, parameter); - if (!recoveryStatus.ok()) { - SPDLOG_WARN("Failed to reload model: {} version: {} to previous configuration with error: {}", - getName(), getVersion(), recoveryStatus.string()); - } - return recoveryStatus; -} - -Status ModelInstance::reloadModel(std::optional batchSize, std::map requestShapes, std::unique_ptr& unloadGuard) { - // temporarily release current predictRequest lock on model loading - unloadGuard.reset(); - // block concurrent requests for reloading/unloading - assure that after reload predict request - // will block further requests for reloading/unloading until inference is performed - std::lock_guard loadingLock(loadingMutex); - SPDLOG_INFO("Will reload model: {} version: {}", getName(), getVersion()); - - DynamicModelParameter parameter; - if (batchSize.has_value() && batchSize.value().isStatic()) { - parameter = DynamicModelParameter(batchSize.value().getStaticValue()); - } else if (requestShapes.size() > 0) { - parameter = DynamicModelParameter(requestShapes); - } else { - SPDLOG_DEBUG("Error: requested model: {} version: {} reload with no batchsize and shapes set.", getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } - - auto status = reloadModel(config, parameter); - if (!status.ok()) { - status = this->reshapeWithFullReload(status, parameter); - if (!status.ok()) { - return this->recoverFromReloadingError(status); - } - } - unloadGuard = std::make_unique(*this); - return status; -} - -Status ModelInstance::reloadModelIfRequired( - Status validationStatus, - const std::optional& requestedBatchSize, - const std::map& requestedShapes, - std::unique_ptr& modelUnloadGuardPtr) { - Status status = validationStatus; - if (status.batchSizeChangeRequired()) { - try { - status = reloadModel(requestedBatchSize, {}, modelUnloadGuardPtr); - } catch (const std::exception& e) { - status = Status(StatusCode::INVALID_BATCH_DIMENSION, e.what()); - } - if (!status.ok()) { - SPDLOG_ERROR("Model: {}, version: {} reload (batch size change) failed. Status Code: {}, Error {}", - getName(), getVersion(), status.getCode(), status.string()); - } - } else if (status.reshapeRequired()) { - status = reloadModel(std::nullopt, requestedShapes, modelUnloadGuardPtr); - if (!status.ok() && status != StatusCode::RESHAPE_ERROR) { - SPDLOG_ERROR("Model: {}, version: {} reload (reshape) failed. Status Code: {}, Error: {}", - getName(), getVersion(), status.getCode(), status.string()); - } - } else if (!status.ok()) { - SPDLOG_DEBUG("Model: {}, version: {} validation of inferRequest failed. Status Code: {}, Error: {}", - getName(), getVersion(), status.getCode(), status.string()); - } - return status; -} - -Status ModelInstance::waitForLoaded(const uint waitForModelLoadedTimeoutMilliseconds, - std::unique_ptr& modelInstanceUnloadGuard) { - // order is important here for performance reasons - // assumption: model is already loaded for most of the calls - modelInstanceUnloadGuard = std::make_unique(*this); - if (getStatus().getState() == ModelVersionState::AVAILABLE) { - SPDLOG_DEBUG("Model: {}, version: {} already loaded", getName(), getVersion()); - return StatusCode::OK; - } - modelInstanceUnloadGuard.reset(); - - // wait several time since no guarantee that cv wakeup will be triggered before calling wait_for - const uint waitLoadedTimestepMilliseconds = 100; - const uint waitCheckpoints = waitForModelLoadedTimeoutMilliseconds / waitLoadedTimestepMilliseconds; - uint waitCheckpointsCounter = waitCheckpoints; - SPDLOG_DEBUG("Waiting for loaded state for model: {} version: {} with timestep: {} timeout: {} check count: {}", getName(), getVersion(), - waitLoadedTimestepMilliseconds, waitForModelLoadedTimeoutMilliseconds, waitCheckpointsCounter); - std::mutex cv_mtx; - std::unique_lock cv_lock(cv_mtx); - while (waitCheckpointsCounter-- > 0) { - if (modelLoadedNotify.wait_for(cv_lock, - std::chrono::milliseconds(waitLoadedTimestepMilliseconds), - [this]() { - return this->getStatus().getState() > ModelVersionState::LOADING; - })) { - SPDLOG_INFO("Waiting for model: {} version: {} loaded state for: {} time", - getName(), getVersion(), waitCheckpoints - waitCheckpointsCounter); - } - modelInstanceUnloadGuard = std::make_unique(*this); - if (getStatus().getState() == ModelVersionState::AVAILABLE) { - SPDLOG_INFO("Succesfully waited for model: {}, version: {}", getName(), getVersion()); - return StatusCode::OK; - } - modelInstanceUnloadGuard.reset(); - if (ModelVersionState::AVAILABLE < getStatus().getState()) { - SPDLOG_INFO("Stopped waiting for model: {} version: {} since it is unloading.", getName(), getVersion()); - return StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_INFO("Waiting for loaded state reached timeout for model: {} version: {}", - getName(), getVersion()); - if (getStatus().getState() > ModelVersionState::AVAILABLE) { - SPDLOG_DEBUG("Waiting for model: {}, version: {} ended since it started unloading.", getName(), getVersion()); - return StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE; - } else { - SPDLOG_DEBUG("Waiting for model: {}, version: {} ended due to timeout.", getName(), getVersion()); - return StatusCode::MODEL_VERSION_NOT_LOADED_YET; - } -} - -void ModelInstance::retireModel(bool isPermanent) { - std::lock_guard loadingLock(loadingMutex); - if (isPermanent) { - this->status.setUnloading(); - } else { - this->status.setLoading(); - } - unloadModelComponents(); - if (isPermanent) { - status.setEnd(); - } -} - -void ModelInstance::cleanupFailedLoad() { - std::lock_guard loadingLock(loadingMutex); - this->status.setLoading(ModelVersionStatusErrorCode::UNKNOWN); - unloadModelComponents(); -} - -void ModelInstance::unloadModelComponents() { - subscriptionManager.notifySubscribers(); - while (!canUnloadInstance()) { - SPDLOG_DEBUG("Waiting to unload model: {} version: {}. Blocked by: {} inferences in progres.", - getName(), getVersion(), predictRequestsHandlesCount); - std::this_thread::sleep_for(std::chrono::milliseconds(UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS)); - } - inferRequestsQueue.reset(); - compiledModel.reset(); - model.reset(); - outputsInfo.clear(); - inputsInfo.clear(); - modelFiles.clear(); - - if (this->config.isCustomLoaderRequiredToLoadModel()) { - custom_loader_options_config_t customLoaderOptionsConfig = this->config.getCustomLoaderOptionsConfigMap(); - const std::string loaderName = customLoaderOptionsConfig["loader_name"]; - auto& customloaders = ovms::CustomLoaders::instance(); - auto customLoaderInterfacePtr = customloaders.find(loaderName); - if (customLoaderInterfacePtr == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "The loader {} is no longer available for model: {} version : {}", - loaderName, getName(), getVersion()); - } else { - // once model is unloaded, notify custom loader object about the unload - customLoaderInterfacePtr->unloadModel(getName(), getVersion()); - } - } -} - -template -const Status ModelInstance::validate(const RequestType* request) { - OVMS_PROFILE_FUNCTION(); - static const std::set optionalInputNames = {}; - return request_validation_utils::validate( - *request, - getInputsInfo(), - getName(), - getVersion(), - optionalInputNames, - getModelConfig().getBatchingMode(), - getModelConfig().getShapes()); -} - -template const Status ModelInstance::validate(const ::inference::ModelInferRequest* request); -template const Status ModelInstance::validate(const tensorflow::serving::PredictRequest* request); - -Status ModelInstance::performInference(ov::InferRequest& inferRequest) { - OVMS_PROFILE_FUNCTION(); - try { - OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::start_async"); - inferRequest.start_async(); - OVMS_PROFILE_SYNC_END("ov::InferRequest::start_async"); - OVMS_PROFILE_SYNC_BEGIN("ov::InferRequest::wait"); - inferRequest.wait(); - OVMS_PROFILE_SYNC_END("ov::InferRequest::wait"); - } catch (const ov::Exception& e) { - Status status = StatusCode::OV_INTERNAL_INFERENCE_ERROR; - SPDLOG_ERROR("Async caught an exception {}: {}", status.string(), e.what()); - return status; - } - return StatusCode::OK; -} - -Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - using std::chrono::microseconds; - - auto status = validate(requestProto); - auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); - auto requestShapes = getRequestShapes(requestProto); - status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); - if (!status.ok()) - return status; - timer.start("get infer request"); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); - int executingInferId = executingStreamIdGuard.getId(); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop("get infer request"); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); - - timer.start("deserialize"); - InputSink inputSink(inferRequest); - bool isPipeline = false; - status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); - timer.stop("deserialize"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); - - timer.start("prediction"); - status = performInference(inferRequest); - timer.stop("prediction"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); - - timer.start("serialize"); - OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); - timer.stop("serialize"); - if (!status.ok()) - return status; - - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); - - return StatusCode::OK; -} - -Status ModelInstance::infer(const ::inference::ModelInferRequest* requestProto, - ::inference::ModelInferResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - using std::chrono::microseconds; - - auto status = validate(requestProto); - auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); - auto requestShapes = getRequestShapes(requestProto); - status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); - if (!status.ok()) - return status; - timer.start("get infer request"); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); - int executingInferId = executingStreamIdGuard.getId(); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop("get infer request"); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); - - timer.start("deserialize"); - InputSink inputSink(inferRequest); - bool isPipeline = false; - status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); - timer.stop("deserialize"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); - - timer.start("prediction"); - status = performInference(inferRequest); - timer.stop("prediction"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); - - timer.start("serialize"); - OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); - timer.stop("serialize"); - if (!status.ok()) - return status; - - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); - - return StatusCode::OK; -} - -const size_t ModelInstance::getBatchSizeIndex() const { - const auto& inputItr = this->inputsInfo.cbegin(); - if (inputItr == this->inputsInfo.cend()) { - throw std::logic_error("model has no inputs"); - } - const auto& input = inputItr->second; - const auto batchIndex = input->getLayout().getBatchIndex(); - if (!batchIndex.has_value()) { - throw std::logic_error("cannot get batch index"); - } - return batchIndex.value(); -} - -} // namespace ovms diff --git a/src/ovms_lib/modelinstance.hpp b/src/ovms_lib/modelinstance.hpp deleted file mode 100644 index 5f9316f910..0000000000 --- a/src/ovms_lib/modelinstance.hpp +++ /dev/null @@ -1,556 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "customloaderconfig.hpp" -#include "customloaderinterface.hpp" -#include "modelchangesubscription.hpp" -#include "modelconfig.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelversionstatus.hpp" -#include "ovinferrequestsqueue.hpp" -#include "sequence_processing_spec.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace inference { -class ModelInferRequest; -class ModelInferResponse; -} // namespace inference - -namespace ovms { - -class DynamicModelParameter { -public: - DynamicModelParameter() : - batchSize(0), - shapes({}) {} - DynamicModelParameter(int batchSize) : - batchSize(batchSize), - shapes({}) {} - DynamicModelParameter(const std::map& shapes) : - batchSize(0), - shapes(shapes) {} - - bool isBatchSizeRequested() const { return batchSize > 0; } - bool isShapeRequested(const std::string& name) const { return shapes.count(name) && shapes.at(name).size() > 0; } - - int getBatchSize() const { return batchSize; } - const shape_t& getShape(const std::string& name) const { return shapes.at(name); } - -private: - int batchSize; - std::map shapes; -}; - -class PipelineDefinition; - -/** - * @brief This class contains all the information about model - */ -class ModelInstance { -protected: - /** - * @brief Performs model loading - * - * @return status - */ - virtual Status loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); - - /** - * @brief OpenVINO Runtime Core object reference - */ - ov::Core& ieCore; - - /** - * @brief OpenVINO Runtime Model object - */ - std::shared_ptr model; - - /** - * @brief OpenVINO Runtime CompiledModel object - */ - std::shared_ptr compiledModel; - - /** - * @brief Model name - */ - const std::string name; - - /** - * @brief A path for the model - */ - std::string path; - - /** - * @brief A model version - */ - const model_version_t version = -1; - - /** - * @brief A model subscription manager - */ - ModelChangeSubscription subscriptionManager; - - /** - * @brief A model status - */ - ModelVersionStatus status; - - /** - * @brief Target device to run model - */ - std::string targetDevice; - - /** - * @brief Model batch size - */ - size_t batchSize = 0; - - /** - * @brief Stores required OpenVINO model files extensions to be able to load model - * Order is important, first file on the list is passed to LoadNetwork - */ - static constexpr std::array OV_MODEL_FILES_EXTENSIONS{".xml", ".bin"}; - - /** - * @brief Stores required onnx model files extensions to be able to load model - */ - static constexpr std::array ONNX_MODEL_FILES_EXTENSIONS{".onnx"}; - - /** - * @brief Notifies model instance users who wait for loading - */ - std::condition_variable modelLoadedNotify; - - /** - * @brief Holds currently loaded model configuration - */ - ModelConfig config; - - /** - * @brief Load OV CNNNetwork ptr - * - * @return CNNNetwork ptr - */ - virtual std::shared_ptr loadOVModelPtr(const std::string& modelFile); - - /** - * @brief Lock to disable concurrent modelinstance load/unload/reload - */ - std::recursive_mutex loadingMutex; - - /** - * @brief Load OV Engine - */ - void loadOVEngine(); - - /** - * @brief Loads OV CNNNetwork - * - * @return Status - */ - Status loadOVModel(); - - /** - * @brief Sets OV CompiledModelPtr - */ - virtual void loadCompiledModelPtr(const plugin_config_t& pluginConfig); - - /** - * @brief Loads OV CompiledModel - * - * @return Status - */ - virtual Status loadOVCompiledModel(const ModelConfig& config); - - /** - * @brief Prepares inferenceRequestsQueue - */ - Status prepareInferenceRequestsQueue(const ModelConfig& config); - - /** - * @brief Fetch model file paths - * - * @return Status - */ - Status fetchModelFilepaths(); - - const Layout getReportedTensorLayout(const ModelConfig& config, const std::string& name, bool isInput); - - /** - * @brief Find file path with extension in model path - * - * @return Returns filename with desired extension if exists otherwise empty string - */ - std::string findModelFilePathWithExtension(const std::string& extension) const; - - /** - * @brief Loads OV CNNNetwork Using the Custom Loader - * - * @return Status - */ - Status loadOVModelUsingCustomLoader(); - - template - const Status validate(const RequestType* request); - -private: - /** - * @brief Holds the information about inputs and it's parameters - */ - tensor_map_t inputsInfo; - - /** - * @brief Holds the information about outputs and it's parameters - */ - tensor_map_t outputsInfo; - - /** - * @brief Holds model required file names. First is loaded - */ - std::vector modelFiles; - - /** - * @brief OpenVINO inference execution stream pool - */ - std::unique_ptr inferRequestsQueue; - - /** - * @brief Holds current usage count in predict requests - * - * Needed for gating model unloading. - */ - std::atomic predictRequestsHandlesCount = 0; - - /** - * @brief Internal method for loading tensors - * - * @param config - */ - Status loadTensors(const ModelConfig& config, bool needsToApplyLayoutConfiguration, const DynamicModelParameter& parameter = DynamicModelParameter()); - - /** - * @brief Internal method for loading inputs - * - * @param config - */ - Status loadInputTensors(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); - - /** - * @brief Internal method for loading outputs - * - * @param config - */ - Status loadOutputTensors(const ModelConfig& config); - - /** - * @brief Flag determining if cache is disabled - */ - bool cacheDisabled = false; - - /** - * @brief Configures batchsize - */ - void configureBatchSize(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); - - uint32_t getNumOfParallelInferRequests(const ModelConfig& config); - uint32_t getNumOfParallelInferRequestsUnbounded(const ModelConfig& config); - - /** - * @brief Recover from any state model is put into when reload is requested - * - * @param status returned from reload operation - * - * @return Status - */ - Status recoverFromReloadingError(const Status& status); - - /** - * @brief Perform full model reload with dynamic parameter - * - * @param status returned from reload operation - * @param parameter requested dynamic parameter - * - * @return Status - */ - Status reshapeWithFullReload(const Status& status, const DynamicModelParameter& parameter); - - /** - * Variable to tell reload is due to customloader config change - */ - bool isCustomLoaderConfigChanged; - -public: - /** - * @brief A default constructor - */ - ModelInstance(const std::string& name, model_version_t version, ov::Core& ieCore) : - ieCore(ieCore), - name(name), - version(version), - subscriptionManager(std::string("model: ") + name + std::string(" version: ") + std::to_string(version)), - status(name, version) { isCustomLoaderConfigChanged = false; } - - /** - * @brief Destroy the Model Instance object - */ - virtual ~ModelInstance() = default; - - /** - * @brief Increases predict requests usage count - */ - void increasePredictRequestsHandlesCount() { - ++predictRequestsHandlesCount; - } - - /** - * @brief sets the flag in model instance indicating change in custom loader configuration. - */ - void setCustomLoaderConfigChangeFlag() { - isCustomLoaderConfigChanged = true; - } - - /** - * @brief Decreases predict requests usage count - */ - void decreasePredictRequestsHandlesCount() { - --predictRequestsHandlesCount; - } - - /** - * @brief Gets the model name - * - * @return model name - */ - virtual const std::string& getName() const { - return name; - } - - /** - * @brief Gets path for the model - * - * @return path - */ - const std::string& getPath() const { - return path; - } - - /** - * @brief Gets version - * - * @return version - */ - virtual model_version_t getVersion() const { - return version; - } - - /** - * @brief Gets model status - * - * @return status - */ - const ModelVersionStatus& getStatus() const { - return status; - } - - /** - * @brief Gets executing target device name - * - * @return target device name - */ - const std::string& getTargetDevice() { - return targetDevice; - } - - /** - * @brief Internal method for setting cache options - */ - Status setCacheOptions(const ModelConfig& config); - - /** - * @brief Check if cache is disabled - * - * @return cache disabled - */ - const bool isCacheDisabled() { - return cacheDisabled; - } - - /** - * @brief Gets batch size - * - * @return batch size - */ - virtual Dimension getBatchSize() const { - return Dimension(ov::get_batch(model)); - } - - const size_t getBatchSizeIndex() const; - - /** - * @brief Gets model config - * - * @return model config - */ - virtual const ModelConfig& getModelConfig() const { - return config; - } - - /** - * @brief Get the Inputs Info object - * - * @return const tensor_map_t& - */ - virtual const tensor_map_t& getInputsInfo() const { - return inputsInfo; - } - - /** - * @brief Get the Outputs Info object - * - * @return const tensor_map_t& - */ - virtual const tensor_map_t& getOutputsInfo() const { - return outputsInfo; - } - - /** - * @brief Check if can unload infer requests - * - * @return bool - */ - virtual bool canUnloadInstance() const { - return 0 == predictRequestsHandlesCount; - } - - /** - * @brief Get OV streams pool - * - * @return OVStreamsQueue - */ - OVInferRequestsQueue& getInferRequestsQueue() { - return *inferRequestsQueue; - } - - /** - * @brief Combines plugin config from user with default config calculated at runtime - * - * @return plugin config - */ - static plugin_config_t prepareDefaultPluginConfig(const ModelConfig& config); - - /** - * @brief Loads model version - * - * @param config model configuration - * - * @return Status - */ - virtual Status loadModel(const ModelConfig& config); - - /** - * @brief Reloads model version - * - * @param config model configuration - * - * @return Status - */ - virtual Status reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()); - - /** - * @brief Reloads model version with different batch size or shape - * - * @param batchSize new batch size - * @param shape new shape - * @param unloadGuard unloadGuardPtr - * - * @return Status - */ - virtual Status reloadModel(std::optional batchSize, std::map shape, std::unique_ptr& unloadGuardPtr); - - /** - * @brief Reloads model version if status of request validation indicates there's a need for reshape or batch size change - * - * @param validationStatus status of request validation - * @param requestProto pointer to the request proto - * @param modelUnloadGuardPtr unloadGuardPtr - * - * @return Status - */ - Status reloadModelIfRequired( - Status validationStatus, - const std::optional& requestedBatchSize, - const std::map& requestedShapes, - std::unique_ptr& modelUnloadGuardPtr); - - /** - * @brief Unloads model version - * @param isPermanent defines if the unload operation should be permanent and should change instance state to End after it is completed - * otherwise model might be unloaded temporarily so the instance state should be preserved as Loading - */ - virtual void retireModel(bool isPermanent = true); - - /** - * @brief Cleans model version that failed to load - */ - virtual void cleanupFailedLoad(); - - void unloadModelComponents(); - - /** - * @brief Wait for model to change to AVAILABLE state - * - * @param waitForModelLoadedTimeoutMilliseconds - * @param modelInstanceUnloadGuard - * - * @return Status - */ - Status waitForLoaded(const uint waitForModelLoadedTimeoutMilliseconds, - std::unique_ptr& modelInstanceUnloadGuard); - - void subscribe(PipelineDefinition& pd); - - void unsubscribe(PipelineDefinition& pd); - - const ModelChangeSubscription& getSubscribtionManager() const { return subscriptionManager; } - - Status performInference(ov::InferRequest& inferRequest); - - virtual Status infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr); - virtual Status infer(const ::inference::ModelInferRequest* requestProto, - ::inference::ModelInferResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr); -}; -} // namespace ovms diff --git a/src/ovms_lib/modelinstanceunloadguard.cpp b/src/ovms_lib/modelinstanceunloadguard.cpp deleted file mode 100644 index 0636e64d51..0000000000 --- a/src/ovms_lib/modelinstanceunloadguard.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelinstanceunloadguard.hpp" - -#include "modelinstance.hpp" - -namespace ovms { -ModelInstanceUnloadGuard::ModelInstanceUnloadGuard(ModelInstance& modelInstance) : - modelInstance(modelInstance) { - modelInstance.increasePredictRequestsHandlesCount(); -} - -ModelInstanceUnloadGuard::~ModelInstanceUnloadGuard() { - modelInstance.decreasePredictRequestsHandlesCount(); -} -} // namespace ovms diff --git a/src/ovms_lib/modelinstanceunloadguard.hpp b/src/ovms_lib/modelinstanceunloadguard.hpp deleted file mode 100644 index 55e5a0ba56..0000000000 --- a/src/ovms_lib/modelinstanceunloadguard.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -namespace ovms { -class ModelInstance; - -class ModelInstanceUnloadGuard { -public: - ModelInstanceUnloadGuard() = delete; - ModelInstanceUnloadGuard(ModelInstance& modelInstance); - ~ModelInstanceUnloadGuard(); - -private: - ModelInstance& modelInstance; -}; -} // namespace ovms diff --git a/src/ovms_lib/modelmanager.cpp b/src/ovms_lib/modelmanager.cpp deleted file mode 100644 index a0c77acccd..0000000000 --- a/src/ovms_lib/modelmanager.cpp +++ /dev/null @@ -1,1316 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelmanager.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "azurefilesystem.hpp" -#include "cleaner_utils.hpp" -#include "config.hpp" -#include "custom_node_library_manager.hpp" -#include "customloaders.hpp" -#include "entry_node.hpp" // need for ENTRY_NODE_NAME -#include "exit_node.hpp" // need for EXIT_NODE_NAME -#include "filesystem.hpp" -#include "gcsfilesystem.hpp" -#include "localfilesystem.hpp" -#include "logging.hpp" -#include "node_library.hpp" -#include "openssl/md5.h" -#include "ov_utils.hpp" -#include "pipeline.hpp" -#include "pipeline_factory.hpp" -#include "pipelinedefinition.hpp" -#include "s3filesystem.hpp" -#include "schema.hpp" -#include "stringutils.hpp" - -namespace ovms { - -static uint16_t MAX_CONFIG_JSON_READ_RETRY_COUNT = 2; - -ModelManager::ModelManager(const std::string& modelCacheDirectory) : - ieCore(std::make_unique()), - waitForModelLoadedTimeoutMs(DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS), - modelCacheDirectory(modelCacheDirectory) { - // Take --cache_dir from CLI - if (this->modelCacheDirectory.empty()) { - this->modelCacheDirectory = ovms::Config::instance().cacheDir(); - } - // If not enabled via CLI, check for /opt/cache existence. - if (this->modelCacheDirectory.empty()) { - if (std::filesystem::exists(DEFAULT_MODEL_CACHE_DIRECTORY)) { - this->modelCacheDirectory = DEFAULT_MODEL_CACHE_DIRECTORY; - } - } - // If cache dir enabled, check for write access. - if (!this->modelCacheDirectory.empty()) { - // Create directory if does not exist - if (!std::filesystem::exists(this->modelCacheDirectory)) { - std::filesystem::create_directories(this->modelCacheDirectory); - SPDLOG_LOGGER_WARN(modelmanager_logger, "Cache directory {} did not exist, created", this->modelCacheDirectory); - } - int result = access(this->modelCacheDirectory.c_str(), W_OK); - if (result != 0) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Cache directory {} is not writable; access() result: {}", this->modelCacheDirectory, result); - } else { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Model cache is enabled: {}", this->modelCacheDirectory); - } - } - this->customNodeLibraryManager = std::make_unique(); - if (ovms::Config::instance().cpuExtensionLibraryPath() != "") { - SPDLOG_INFO("Loading custom CPU extension from {}", ovms::Config::instance().cpuExtensionLibraryPath()); - try { - ieCore->add_extension(ovms::Config::instance().cpuExtensionLibraryPath()); - SPDLOG_INFO("Extension added."); - } catch (std::exception& ex) { - SPDLOG_CRITICAL("Custom CPU extension loading has failed! Reason: {}", ex.what()); - throw; - } catch (...) { - SPDLOG_CRITICAL("Custom CPU extension loading has failed with an unknown error!"); - throw; - } - } - this->logPluginConfiguration(); -} - -void ModelManager::logPluginConfiguration() { - auto availableDevices = ieCore->get_available_devices(); - SPDLOG_LOGGER_INFO(modelmanager_logger, "Available devices for Open VINO: {}", joins(availableDevices, std::string(", "))); - auto availablePlugins = availableDevices; - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); - for (const auto& plugin : availablePlugins) { - std::vector supportedConfigKeys; - try { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Logging plugin: {}; configuration", plugin); - std::vector supportedConfigKeys2 = ieCore->get_property(plugin, supportedConfigKey).as>(); - supportedConfigKeys = std::move(supportedConfigKeys2); - } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", plugin, supportedConfigKey, e.what()); - } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", plugin, supportedConfigKey); - } - for (auto& key : supportedConfigKeys) { - std::string value; - try { - auto paramValue = ieCore->get_property(plugin, key); - value = paramValue.as(); - } catch (std::exception& e) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; config key: {}; Error: {}", plugin, key, e.what()); - continue; - } catch (...) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; config key: {}", plugin, key); - continue; - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Global plugin: {}; key: {}; value: {}", plugin, key, value); - } - } -} - -ModelManager::~ModelManager() = default; - -Status ModelManager::start(const Config& config) { - watcherIntervalSec = config.filesystemPollWaitSeconds(); - sequenceCleaupIntervalMinutes = config.sequenceCleanerPollWaitMinutes(); - resourcesCleanupIntervalSec = config.resourcesCleanerPollWaitSeconds(); - if (resourcesCleanupIntervalSec < 1) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Parameter: custom_node_resources_cleaner_interval has to be greater than 0. Applying default value(1 second)"); - resourcesCleanupIntervalSec = 1; - } - Status status; - if (config.configPath() != "") { - status = startFromFile(config.configPath()); - } else { - status = startFromConfig(); - } - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't start model manager"); - return status; - } - - startWatcher(); - startCleaner(); - return status; -} - -void ModelManager::startWatcher() { - if ((!watcherStarted) && (watcherIntervalSec > 0)) { - std::future exitSignal = exitTrigger.get_future(); - std::thread t(std::thread(&ModelManager::watcher, this, std::move(exitSignal))); - watcherStarted = true; - monitor = std::move(t); - } -} - -void ModelManager::startCleaner() { - if ((!cleanerStarted)) { - std::future exitSignal = cleanerExitTrigger.get_future(); - std::thread t(std::thread(&ModelManager::cleanerRoutine, this, resourcesCleanupIntervalSec, sequenceCleaupIntervalMinutes, std::move(exitSignal))); - cleanerStarted = true; - cleanerThread = std::move(t); - } -} - -Status ModelManager::startFromConfig() { - auto& config = ovms::Config::instance(); - - auto [it, success] = servedModelConfigs.emplace( - config.modelName(), - ModelConfig{ - config.modelName(), - config.modelPath(), - config.targetDevice(), - config.batchSize(), - config.nireq(), - config.stateful(), - config.idleSequenceCleanup(), - config.lowLatencyTransformation(), - config.maxSequenceNumber(), - this->modelCacheDirectory}); - - if (!success) { - return StatusCode::UNKNOWN_ERROR; - } - - ModelConfig& modelConfig = it->second; - - auto status = modelConfig.parsePluginConfig(config.pluginConfig()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse plugin config"); - return status; - } - - status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); - return status; - } - - status = modelConfig.parseModelVersionPolicy(config.modelVersionPolicy()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse model version policy. {}", status.string()); - return status; - } - - status = modelConfig.parseShapeParameter(config.shape()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse shape parameter"); - return status; - } - - status = modelConfig.parseLayoutParameter(config.layout()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't parse layout parameter"); - return status; - } - - bool batchSizeSet = (modelConfig.getBatchingMode() != FIXED || modelConfig.getBatchSize() != 0); - bool shapeSet = (modelConfig.getShapes().size() > 0); - - SPDLOG_DEBUG("Batch size set: {}, shape set: {}", batchSizeSet, shapeSet); - if (batchSizeSet && shapeSet) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Both shape and batch size have been defined. Batch size parameter will be ignored."); - modelConfig.setBatchingMode(FIXED); - modelConfig.setBatchSize(std::nullopt); - } - - return reloadModelWithVersions(modelConfig); -} - -Status ModelManager::startFromFile(const std::string& jsonFilename) { - Status status = loadConfig(jsonFilename); - if (status == StatusCode::CONFIG_FILE_INVALID || status == StatusCode::JSON_INVALID) { - return status; - } - - return StatusCode::OK; -} - -void processNodeInputs(const std::string nodeName, const rapidjson::Value::ConstMemberIterator& itro, pipeline_connections_t& connections) { - for (const auto& nodeInput : itro->value.GetArray()) { - for (const auto& objectNameValue : nodeInput.GetObject()) { - const std::string inputName = objectNameValue.name.GetString(); - const std::string sourceNodeName = objectNameValue.value.GetObject()["node_name"].GetString(); - const std::string sourceOutputName = objectNameValue.value.GetObject()["data_item"].GetString(); - SPDLOG_DEBUG("Creating node dependencies mapping request. Node: {} input: {} <- SourceNode: {} output: {}", - nodeName, inputName, sourceNodeName, sourceOutputName); - if (connections.find(nodeName) == connections.end()) { - connections[nodeName] = { - {sourceNodeName, - {{sourceOutputName, inputName}}}}; - } else { - if (connections[nodeName].find(sourceNodeName) == connections[nodeName].end()) { - connections[nodeName].insert({sourceNodeName, - {{sourceOutputName, inputName}}}); - } else { - connections[nodeName][sourceNodeName].push_back({sourceOutputName, inputName}); - } - } - } - } -} - -void processPipelineInputs(const rapidjson::Value::ConstMemberIterator& pipelineInputsPtr, const std::string& nodeName, std::unordered_map& nodeOutputNameAlias, const std::string& pipelineName) { - for (const auto& pipelineInput : pipelineInputsPtr->value.GetArray()) { - const std::string pipelineInputName = pipelineInput.GetString(); - SPDLOG_DEBUG("Mapping node:{} output:{}, under alias:{}", - nodeName, pipelineInputName, pipelineInputName); - auto result = nodeOutputNameAlias.insert({pipelineInputName, pipelineInputName}); - if (!result.second) { - SPDLOG_ERROR("Pipeline {} has duplicated input declaration", pipelineName); - } - } -} - -void processNodeOutputs(const rapidjson::Value::ConstMemberIterator& nodeOutputsItr, const std::string& nodeName, const std::string& modelName, std::unordered_map& nodeOutputNameAlias) { - for (const auto& nodeOutput : nodeOutputsItr->value.GetArray()) { - const std::string modelOutputName = nodeOutput.GetObject()["data_item"].GetString(); - const std::string nodeOutputName = nodeOutput.GetObject()["alias"].GetString(); - SPDLOG_DEBUG("Mapping node: {} model_name: {} output: {}, under alias: {}", - nodeName, modelName, modelOutputName, nodeOutputName); - nodeOutputNameAlias[nodeOutputName] = modelOutputName; - } -} - -void processDLNodeConfig(const rapidjson::Value& nodeConfig, DLNodeInfo& info) { - info.modelName = nodeConfig["model_name"].GetString(); - if (nodeConfig.HasMember("version")) { - info.modelVersion = nodeConfig["version"].GetUint64(); - } -} - -#define IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status) \ - if (firstErrorStatus.ok()) { \ - firstErrorStatus = status; \ - } - -Status processCustomNodeConfig(const rapidjson::Value& nodeConfig, CustomNodeInfo& info, const std::string& pipelineName, ModelManager& manager) { - std::string libraryName = nodeConfig["library_name"].GetString(); - auto status = manager.getCustomNodeLibraryManager().getLibrary(libraryName, info.library); - if (!status.ok()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {} refers to non existing custom node library: {}", pipelineName, libraryName); - } - if (nodeConfig.HasMember("params")) { - for (const auto& param : nodeConfig["params"].GetObject()) { - info.parameters.emplace(param.name.GetString(), param.value.GetString()); - } - } - return StatusCode::OK; -} - -Status processPipelineConfig(rapidjson::Document& configJson, const rapidjson::Value& pipelineConfig, std::set& pipelinesInConfigFile, PipelineFactory& factory, ModelManager& manager) { - const std::string pipelineName = pipelineConfig["name"].GetString(); - if (pipelinesInConfigFile.find(pipelineName) != pipelinesInConfigFile.end()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated pipeline names: {} defined in config file. Only first definition will be loaded.", pipelineName); - return StatusCode::OK; - } - SPDLOG_LOGGER_INFO(modelmanager_logger, "Reading pipeline: {} configuration", pipelineName); - std::set demultiplexerNodes; - std::set gatheredDemultiplexerNodes; - std::optional demultiplyCountEntry = std::nullopt; - auto demultiplyCountEntryIt = pipelineConfig.FindMember("demultiply_count"); - if (demultiplyCountEntryIt != pipelineConfig.MemberEnd()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} does have demultiply at entry node", pipelineName); - int32_t parsedDemultiplyCount = pipelineConfig["demultiply_count"].GetInt(); - if (parsedDemultiplyCount == 0) { - parsedDemultiplyCount = -1; - SPDLOG_LOGGER_WARN(modelmanager_logger, "demultiply_count 0 will be deprecated. For dynamic count use -1."); - } - demultiplyCountEntry = parsedDemultiplyCount; - demultiplexerNodes.insert(ENTRY_NODE_NAME); - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} does not have demultiply at entry node", pipelineName); - } - - std::vector info; - NodeInfo entryInfo{NodeKind::ENTRY, ENTRY_NODE_NAME, "", std::nullopt, {}, demultiplyCountEntry}; - info.emplace_back(std::move(entryInfo)); - processPipelineInputs(pipelineConfig.FindMember("inputs"), ENTRY_NODE_NAME, info[0].outputNameAliases, pipelineName); - pipeline_connections_t connections; - - auto nodesItr = pipelineConfig.FindMember("nodes"); - for (const auto& nodeConfig : nodesItr->value.GetArray()) { - std::string nodeName; - nodeName = nodeConfig["name"].GetString(); - - const std::string nodeKindStr = nodeConfig["type"].GetString(); - NodeKind nodeKind; - auto status = toNodeKind(nodeKindStr, nodeKind); - if (!status.ok()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Parsing node kind failed: {} for pipeline: {}", nodeKindStr, pipelineName); - return status; - } - - DLNodeInfo dlNodeInfo; - CustomNodeInfo customNodeInfo; - if (nodeKind == NodeKind::DL) { - processDLNodeConfig(nodeConfig, dlNodeInfo); - } else if (nodeKind == NodeKind::CUSTOM) { - status = processCustomNodeConfig(nodeConfig, customNodeInfo, pipelineName, manager); - if (!status.ok()) { - return status; - } - } else { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline {} contains unknown node kind", pipelineName); - throw std::invalid_argument("unknown node kind"); - } - - auto nodeOutputsItr = nodeConfig.FindMember("outputs"); - if (nodeOutputsItr == nodeConfig.MemberEnd() || !nodeOutputsItr->value.IsArray()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {} does not have valid outputs configuration", pipelineName); - return status; - } - std::unordered_map nodeOutputNameAlias; // key:alias, value realName - processNodeOutputs(nodeOutputsItr, nodeName, dlNodeInfo.modelName, nodeOutputNameAlias); - std::optional demultiplyCount; - if (nodeConfig.HasMember("demultiply_count")) { - int32_t parsedDemultiplyCount = nodeConfig["demultiply_count"].GetInt(); - if (parsedDemultiplyCount == 0) { - parsedDemultiplyCount = -1; - SPDLOG_LOGGER_WARN(modelmanager_logger, "demultiply_count 0 will be deprecated. For dynamic count use -1."); - } - demultiplyCount = parsedDemultiplyCount; - demultiplexerNodes.insert(nodeName); - } - std::set gatherFromNode; - if (nodeConfig.HasMember("gather_from_node")) { - std::string nodeToGatherFrom = nodeConfig["gather_from_node"].GetString(); - gatherFromNode.insert(nodeToGatherFrom); - gatheredDemultiplexerNodes.insert(nodeToGatherFrom); - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Creating node: {} type: {} model_name: {} modelVersion: {}", - nodeName, nodeKindStr, dlNodeInfo.modelName, dlNodeInfo.modelVersion.value_or(0)); - info.emplace_back( - nodeKind, - nodeName, - dlNodeInfo.modelName, - dlNodeInfo.modelVersion, - nodeOutputNameAlias, - demultiplyCount, - gatherFromNode, - customNodeInfo.library, - customNodeInfo.parameters); - auto nodeInputItr = nodeConfig.FindMember("inputs"); - processNodeInputs(nodeName, nodeInputItr, connections); - } - const auto iteratorOutputs = pipelineConfig.FindMember("outputs"); - // pipeline outputs are node exit inputs - processNodeInputs(EXIT_NODE_NAME, iteratorOutputs, connections); - std::set nonGatheredDemultiplexerNodes; - std::set_difference(demultiplexerNodes.begin(), demultiplexerNodes.end(), - gatheredDemultiplexerNodes.begin(), gatheredDemultiplexerNodes.end(), - std::inserter(nonGatheredDemultiplexerNodes, nonGatheredDemultiplexerNodes.begin())); - info.emplace_back(std::move(NodeInfo(NodeKind::EXIT, EXIT_NODE_NAME, "", std::nullopt, {}, std::nullopt, nonGatheredDemultiplexerNodes))); - if (!factory.definitionExists(pipelineName)) { - SPDLOG_DEBUG("Pipeline:{} was not loaded so far. Triggering load", pipelineName); - auto status = factory.createDefinition(pipelineName, info, connections, manager); - pipelinesInConfigFile.insert(pipelineName); - return status; - } - SPDLOG_DEBUG("Pipeline:{} is already loaded. Triggering reload", pipelineName); - auto status = factory.reloadDefinition(pipelineName, - std::move(info), - std::move(connections), - manager); - pipelinesInConfigFile.insert(pipelineName); - return status; -} - -Status ModelManager::loadCustomNodeLibrariesConfig(rapidjson::Document& configJson) { - const auto doc = configJson.FindMember("custom_node_library_config_list"); - if (doc == configJson.MemberEnd()) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Configuration file doesn't have custom node libraries property."); - return StatusCode::OK; - } - std::set librariesInConfig; - for (const auto& libraryConfig : doc->value.GetArray()) { - librariesInConfig.emplace(libraryConfig.FindMember("name")->value.GetString()); - this->customNodeLibraryManager->loadLibrary( - libraryConfig.FindMember("name")->value.GetString(), - libraryConfig.FindMember("base_path")->value.GetString()); - } - this->customNodeLibraryManager->unloadLibrariesRemovedFromConfig(librariesInConfig); - return StatusCode::OK; -} - -Status ModelManager::loadPipelinesConfig(rapidjson::Document& configJson) { - const auto itrp = configJson.FindMember("pipeline_config_list"); - if (itrp == configJson.MemberEnd() || !itrp->value.IsArray()) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Configuration file doesn't have pipelines property."); - pipelineFactory.retireOtherThan({}, *this); - return StatusCode::OK; - } - std::set pipelinesInConfigFile; - Status firstErrorStatus = StatusCode::OK; - for (const auto& pipelineConfig : itrp->value.GetArray()) { - auto status = processPipelineConfig(configJson, pipelineConfig, pipelinesInConfigFile, pipelineFactory, *this); - if (status != StatusCode::OK) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - } - pipelineFactory.retireOtherThan(std::move(pipelinesInConfigFile), *this); - return firstErrorStatus; -} - -Status ModelManager::createCustomLoader(CustomLoaderConfig& loaderConfig) { - auto& customloaders = ovms::CustomLoaders::instance(); - std::string loaderName = loaderConfig.getLoaderName(); - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Check if loader is already loaded"); - if (customloaders.find(loaderName) == nullptr) { - // this is where library or custom loader is loaded - if (FileSystem::isPathEscaped(loaderConfig.getLibraryPath())) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", loaderConfig.getLibraryPath()); - return StatusCode::PATH_INVALID; - } - void* handleCL = dlopen(const_cast(loaderConfig.getLibraryPath().c_str()), RTLD_LAZY | RTLD_LOCAL); - if (!handleCL) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot open library: {} {}", loaderConfig.getLibraryPath(), dlerror()); - return StatusCode::CUSTOM_LOADER_LIBRARY_INVALID; - } - - // load the symbols - createCustomLoader_t* customObj = (createCustomLoader_t*)dlsym(handleCL, "createCustomLoader"); - const char* dlsym_error = dlerror(); - if (dlsym_error || (customObj == nullptr)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot load symbol create: {} ", dlsym_error); - return StatusCode::CUSTOM_LOADER_LIBRARY_LOAD_FAILED; - } - - std::shared_ptr customLoaderIfPtr{customObj()}; - try { - customLoaderIfPtr->loaderInit(loaderConfig.getLoaderConfigFile()); - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot create or initialize the custom loader. Failed with error {}", e.what()); - return StatusCode::CUSTOM_LOADER_INIT_FAILED; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Cannot create or initialize the custom loader"); - return StatusCode::CUSTOM_LOADER_INIT_FAILED; - } - customloaders.add(loaderName, customLoaderIfPtr, handleCL); - } else { - // Loader is already in the existing loaders. Move it to new loaders. - // Reload of customloader is not supported yet - customloaders.move(loaderName); - } - return StatusCode::OK; -} - -Status ModelManager::loadCustomLoadersConfig(rapidjson::Document& configJson) { - const auto itrp = configJson.FindMember("custom_loader_config_list"); - if (itrp == configJson.MemberEnd() || !itrp->value.IsArray()) { - return StatusCode::OK; - } - - Status firstErrorStatus = StatusCode::OK; - // Load Customer Loaders as per the configuration - SPDLOG_DEBUG("Using Customloader"); - for (const auto& configs : itrp->value.GetArray()) { - const std::string loaderName = configs["config"]["loader_name"].GetString(); - SPDLOG_INFO("Reading Custom Loader: {} configuration", loaderName); - - CustomLoaderConfig loaderConfig; - auto status = loaderConfig.parseNode(configs["config"]); - if (status != StatusCode::OK) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - SPDLOG_ERROR("Parsing loader: {} config failed", loaderName); - } - - auto retVal = createCustomLoader(loaderConfig); - if (retVal != StatusCode::OK) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(retVal); - SPDLOG_ERROR("Creation of loader: {} failed", loaderName); - } - } - // All loaders are the done. Finalize the list by deleting removed loaders in config - auto& customloaders = ovms::CustomLoaders::instance(); - customloaders.finalize(); - return firstErrorStatus; -} - -Status ModelManager::loadModelsConfig(rapidjson::Document& configJson, std::vector& gatedModelConfigs) { - const auto itr = configJson.FindMember("model_config_list"); - if (itr == configJson.MemberEnd() || !itr->value.IsArray()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file doesn't have models property."); - return StatusCode::JSON_INVALID; - } - Status firstErrorStatus = StatusCode::OK; - std::set modelsInConfigFile; - std::set modelsWithInvalidConfig; - std::unordered_map newModelConfigs; - for (const auto& configs : itr->value.GetArray()) { - ModelConfig modelConfig; - auto status = modelConfig.parseNode(configs["config"]); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_CONFIG_INVALID); - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Parsing model: {} config failed due to error: {}", modelConfig.getName(), status.string()); - modelsWithInvalidConfig.emplace(modelConfig.getName()); - continue; - } - - status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); - return status; - } - modelConfig.setCacheDir(this->modelCacheDirectory); - - const auto modelName = modelConfig.getName(); - if (pipelineDefinitionExists(modelName)) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model name: {} is already occupied by pipeline definition.", modelName); - continue; - } - if (modelsInConfigFile.find(modelName) != modelsInConfigFile.end()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); - SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated model names: {} defined in config file. Only first definition will be loaded.", modelName); - continue; - } - status = reloadModelWithVersions(modelConfig); - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - - modelsInConfigFile.emplace(modelName); - if (!status.ok()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cannot reload model: {} with versions due to error: {}", modelName, status.string()); - } - if (status == StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL || status == StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Will retry to reload model({}) after pipelines are revalidated", modelName); - auto it = this->servedModelConfigs.find(modelName); - if (it == this->servedModelConfigs.end()) { - continue; - } - gatedModelConfigs.emplace_back(std::move(modelConfig)); - newModelConfigs.emplace(modelName, std::move(it->second)); - this->servedModelConfigs.erase(modelName); - } else { - newModelConfigs.emplace(modelName, std::move(modelConfig)); - } - } - this->servedModelConfigs = std::move(newModelConfigs); - retireModelsRemovedFromConfigFile(modelsInConfigFile, modelsWithInvalidConfig); - return firstErrorStatus; -} - -Status ModelManager::tryReloadGatedModelConfigs(std::vector& gatedModelConfigs) { - Status firstErrorStatus = StatusCode::OK; - for (auto& modelConfig : gatedModelConfigs) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Trying to reload model({}) configuration", modelConfig.getName()); - auto status = reloadModelWithVersions(modelConfig); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - continue; - } - auto it = this->servedModelConfigs.find(modelConfig.getName()); - if (it == this->servedModelConfigs.end()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - continue; - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Successfully retried to load new model({}) configuration after unsubscribed from pipeline", modelConfig.getName()); - this->servedModelConfigs.at(modelConfig.getName()) = std::move(modelConfig); - } - return firstErrorStatus; -} - -class LoudFileInfoReporter { - std::stringstream ss; - -public: - LoudFileInfoReporter(const std::string& filename, std::ifstream& file) { - struct stat statTime; - - if (stat(filename.c_str(), &statTime) != 0) { - SPDLOG_ERROR("Failed to debug-read fileconfig"); - return; - } - ss << "FileInfoReporter: " << filename - << " time modification [s]: " << statTime.st_ctim.tv_sec - << " [ns]: " << statTime.st_ctim.tv_nsec << std::endl; - std::string some; - file.clear(); - file.seekg(0); - while (file) { - file >> some; - ss << some << std::endl; - } - file.clear(); - file.seekg(0); - } - void log() { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, ss.str()); - } -}; - -Status ModelManager::loadConfig(const std::string& jsonFilename) { - std::lock_guard loadingLock(configMtx); - configFilename = jsonFilename; - lastConfigFileMD5 = getConfigFileMD5(); - rapidjson::Document configJson; - - uint16_t counter = 0; - Status intermediateStatus; - do { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Loading configuration from {} for: {} time", jsonFilename, counter + 1); - std::ifstream ifs(jsonFilename.c_str()); - LoudFileInfoReporter loud(jsonFilename, ifs); - if (!ifs.good()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is invalid {}", jsonFilename); - intermediateStatus = StatusCode::CONFIG_FILE_INVALID; - loud.log(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - continue; - } - rapidjson::IStreamWrapper isw(ifs); - rapidjson::ParseResult parseResult = configJson.ParseStream(isw); - if (!parseResult) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is not a valid JSON file. Error: {}", - rapidjson::GetParseError_En(parseResult.Code())); - intermediateStatus = StatusCode::JSON_INVALID; - loud.log(); - std::this_thread::sleep_for(std::chrono::seconds(1)); - continue; - } - intermediateStatus = StatusCode::OK; - break; - } while (++counter < MAX_CONFIG_JSON_READ_RETRY_COUNT && !intermediateStatus.ok()); - if (!intermediateStatus.ok()) { - lastLoadConfigStatus = intermediateStatus; - return lastLoadConfigStatus; - } - - if (validateJsonAgainstSchema(configJson, MODELS_CONFIG_SCHEMA) != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Configuration file is not in valid configuration format"); - lastLoadConfigStatus = StatusCode::JSON_INVALID; - return lastLoadConfigStatus; - } - Status status; - Status firstErrorStatus = StatusCode::OK; - // load the custom loader config, if available - status = loadCustomLoadersConfig(configJson); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - std::vector gatedModelConfigs; - status = loadModelsConfig(configJson, gatedModelConfigs); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - status = loadCustomNodeLibrariesConfig(configJson); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - status = loadPipelinesConfig(configJson); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - status = tryReloadGatedModelConfigs(gatedModelConfigs); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - - lastLoadConfigStatus = firstErrorStatus; - return firstErrorStatus; -} - -void ModelManager::retireModelsRemovedFromConfigFile(const std::set& modelsExistingInConfigFile, const std::set& modelsWithInvalidConfig) { - std::set modelsCurrentlyLoaded; - for (auto& nameModelPair : getModels()) { - modelsCurrentlyLoaded.insert(nameModelPair.first); - } - std::vector modelsToUnloadAllVersions(getModels().size()); - auto it = std::set_difference( - modelsCurrentlyLoaded.begin(), modelsCurrentlyLoaded.end(), - modelsExistingInConfigFile.begin(), modelsExistingInConfigFile.end(), - modelsToUnloadAllVersions.begin()); - modelsToUnloadAllVersions.resize(it - modelsToUnloadAllVersions.begin()); - for (auto& modelName : modelsToUnloadAllVersions) { - if (modelsWithInvalidConfig.find(modelName) == modelsWithInvalidConfig.end()) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Retiring all versions of model: {}", modelName); - try { - models.at(modelName)->retireAllVersions(); - } catch (const std::out_of_range& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unknown error occurred when tried to retire all versions of model: {}", modelName); - } - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cleaning up all versions of model: {}", modelName); - try { - models.at(modelName)->cleanupAllVersions(); - } catch (const std::out_of_range& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unknown error occurred when tried to clean up all versions of model: {}", modelName); - } - } - } -} - -Status ModelManager::updateConfigurationWithoutConfigFile() { - std::lock_guard loadingLock(configMtx); - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Checking if something changed with model versions"); - bool reloadNeeded = false; - Status firstErrorStatus = StatusCode::OK; - Status status; - for (auto& [name, config] : servedModelConfigs) { - status = reloadModelWithVersions(config); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } else if (status == StatusCode::OK_RELOADED) { - reloadNeeded = true; - } - } - status = pipelineFactory.revalidatePipelines(*this); - if (!status.ok()) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); - } - - if (!firstErrorStatus.ok()) { - return firstErrorStatus; - } - - if (reloadNeeded) { - return StatusCode::OK_RELOADED; - } else { - return StatusCode::OK_NOT_RELOADED; - } -} - -std::string ModelManager::getConfigFileMD5() { - std::ifstream ifs; - ifs.open(configFilename); - std::stringstream strStream; - strStream << ifs.rdbuf(); - std::string str = strStream.str(); - ifs.close(); - - unsigned char result[MD5_DIGEST_LENGTH]; - MD5((unsigned char*)str.c_str(), str.size(), result); - std::string md5sum(reinterpret_cast(result), MD5_DIGEST_LENGTH); - return (md5sum); -} - -Status ModelManager::configFileReloadNeeded(bool& isNeeded) { - std::lock_guard loadingLock(configMtx); - - if (!std::ifstream(configFilename)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Config file not found or cannot open."); - isNeeded = false; - return StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED; - } - - std::string newmd5 = getConfigFileMD5(); - bool configFileModified = false; - if (lastConfigFileMD5 != newmd5) { - configFileModified = true; - } - - if (configFilename == "" || !configFileModified) { - isNeeded = false; - return lastLoadConfigStatus; - } else { - isNeeded = true; - } - - return StatusCode::OK; -} - -void ModelManager::watcher(std::future exitSignal) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Started model manager thread"); - - while (exitSignal.wait_for(std::chrono::seconds(watcherIntervalSec)) == std::future_status::timeout) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle begin"); - std::lock_guard loadingLock(configMtx); - bool isNeeded; - configFileReloadNeeded(isNeeded); - if (isNeeded) { - loadConfig(configFilename); - } - updateConfigurationWithoutConfigFile(); - - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle end"); - } - SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped model manager thread"); -} - -void ModelManager::cleanerRoutine(uint32_t resourcesCleanupIntervalSec, uint32_t sequenceCleanerIntervalMinutes, std::future cleanerExitSignal) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Started cleaner thread"); - - uint32_t resourcesCleanupIntervalMiliseconds = resourcesCleanupIntervalSec * 1000; - uint32_t sequenceCleanerIntervalMiliseconds = sequenceCleanerIntervalMinutes * 60 * 1000; - - FunctorResourcesCleaner functorResourcesCleaner{*this}; - FunctorSequenceCleaner functorSequenceCleaner{globalSequencesViewer}; - - ovms::cleanerRoutine(resourcesCleanupIntervalMiliseconds, functorResourcesCleaner, sequenceCleanerIntervalMiliseconds, functorSequenceCleaner, cleanerExitSignal); - - SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped cleaner thread"); -} - -void cleanerRoutine(uint32_t resourcesCleanupInterval, FunctorResourcesCleaner& functorResourcesCleaner, uint32_t sequenceCleanerInterval, FunctorSequenceCleaner& functorSequenceCleaner, std::future& cleanerExitSignal) { - uint32_t currentResourcesWaitTime = resourcesCleanupInterval; - uint32_t currentSequenceWaitTime = sequenceCleanerInterval; - bool shouldCheckForSequenceCleanup = sequenceCleanerInterval != 0; - uint32_t currentWaitTime = (!shouldCheckForSequenceCleanup || currentResourcesWaitTime < currentSequenceWaitTime) ? currentResourcesWaitTime : currentSequenceWaitTime; - - while (cleanerExitSignal.wait_for(std::chrono::milliseconds(currentWaitTime)) == std::future_status::timeout) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Cleanup check cycle begin"); - - currentResourcesWaitTime = (currentResourcesWaitTime - currentWaitTime) == 0 ? resourcesCleanupInterval : currentResourcesWaitTime - currentWaitTime; - if (shouldCheckForSequenceCleanup) - currentSequenceWaitTime = (currentSequenceWaitTime - currentWaitTime) == 0 ? sequenceCleanerInterval : currentSequenceWaitTime - currentWaitTime; - currentWaitTime = (!shouldCheckForSequenceCleanup || currentResourcesWaitTime < currentSequenceWaitTime) ? currentResourcesWaitTime : currentSequenceWaitTime; - - if (currentResourcesWaitTime == resourcesCleanupInterval) - functorResourcesCleaner.cleanup(); - if (currentSequenceWaitTime == sequenceCleanerInterval && shouldCheckForSequenceCleanup) - functorSequenceCleaner.cleanup(); - - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Cleanup check cycle end"); - } -} - -void ModelManager::cleanupResources() { - std::vector> toBeRemoved; - std::unique_lock resourcesLock(resourcesMtx); - // Move all resources that should be destroyed to temporary container - std::copy_if(resources.begin(), - resources.end(), - std::back_inserter(toBeRemoved), - [](auto& resource) { return resource.use_count() == 1; }); - resources.erase( - std::remove_if( - resources.begin(), - resources.end(), - [toBeRemoved](auto& resource) { return std::find(toBeRemoved.begin(), toBeRemoved.end(), resource) != toBeRemoved.end(); }), - resources.end()); - // Unlock mutex so new resources can be put into container owned by ModelManager - resourcesLock.unlock(); - // Temporary container will fall out of scope and therefore deinitialize should be called on every resource inside of it -} - -void ModelManager::join() { - if (watcherStarted) - exitTrigger.set_value(); - if (cleanerStarted) - cleanerExitTrigger.set_value(); - - if (watcherStarted) { - if (monitor.joinable()) { - monitor.join(); - watcherStarted = false; - SPDLOG_INFO("Shutdown model manager"); - } - } - - if (cleanerStarted) { - if (cleanerThread.joinable()) { - cleanerThread.join(); - cleanerStarted = false; - SPDLOG_INFO("Shutdown cleaner thread"); - } - } -} - -void ModelManager::getVersionsToChange( - const ModelConfig& newModelConfig, - const std::map>& modelVersionsInstances, - model_versions_t requestedVersions, - std::shared_ptr& versionsToStartIn, - std::shared_ptr& versionsToReloadIn, - std::shared_ptr& versionsToRetireIn) { - std::sort(requestedVersions.begin(), requestedVersions.end()); - model_versions_t registeredModelVersions; - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Currently registered model: {} versions count: {}", newModelConfig.getName(), modelVersionsInstances.size()); - for (const auto& [version, versionInstance] : modelVersionsInstances) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "model: {} version: {} state: {}", newModelConfig.getName(), version, ovms::ModelVersionStateToString(versionInstance->getStatus().getState())); - registeredModelVersions.push_back(version); - } - - if (newModelConfig.isCustomLoaderRequiredToLoadModel()) { - custom_loader_options_config_t customLoaderOptionsConfig = newModelConfig.getCustomLoaderOptionsConfigMap(); - const std::string loaderName = customLoaderOptionsConfig["loader_name"]; - - auto& customloaders = ovms::CustomLoaders::instance(); - auto loaderPtr = customloaders.find(loaderName); - if (loaderPtr != nullptr) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom Loader to be used : {}", loaderName); - - // check existing version for blacklist - for (const auto& [version, versionInstance] : modelVersionsInstances) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "The model {} checking for blacklist", versionInstance->getName()); - CustomLoaderStatus bres = loaderPtr->getModelBlacklistStatus(versionInstance->getName(), version); - if (bres != CustomLoaderStatus::OK) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "The model {} is blacklisted", versionInstance->getName()); - requestedVersions.erase(std::remove(requestedVersions.begin(), requestedVersions.end(), version), requestedVersions.end()); - } - } - } - } - - model_versions_t alreadyRegisteredVersionsWhichAreRequested(requestedVersions.size()); - model_versions_t::iterator it = std::set_intersection( - requestedVersions.begin(), requestedVersions.end(), - registeredModelVersions.begin(), registeredModelVersions.end(), - alreadyRegisteredVersionsWhichAreRequested.begin()); - alreadyRegisteredVersionsWhichAreRequested.resize(it - alreadyRegisteredVersionsWhichAreRequested.begin()); - - std::shared_ptr versionsToReload = std::make_shared(); - for (const auto& version : alreadyRegisteredVersionsWhichAreRequested) { - try { - if (modelVersionsInstances.at(version)->getStatus().willEndUnloaded() || - modelVersionsInstances.at(version)->getStatus().isFailedLoading() || - modelVersionsInstances.at(version)->getModelConfig().isReloadRequired(newModelConfig)) { - if (modelVersionsInstances.at(version)->getModelConfig().isCustomLoaderConfigChanged(newModelConfig)) { - modelVersionsInstances.at(version)->setCustomLoaderConfigChangeFlag(); - } - versionsToReload->push_back(version); - } - } catch (std::out_of_range& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Data race occured during versions update. Could not found version. Details: {}", e.what()); - } - } - - std::shared_ptr versionsToRetire = std::make_shared(registeredModelVersions.size()); - it = std::set_difference( - registeredModelVersions.begin(), registeredModelVersions.end(), - requestedVersions.begin(), requestedVersions.end(), - versionsToRetire->begin()); - versionsToRetire->resize(it - versionsToRetire->begin()); - try { - it = std::remove_if(versionsToRetire->begin(), - versionsToRetire->end(), - [&modelVersionsInstances](model_version_t version) { - return modelVersionsInstances.at(version)->getStatus().willEndUnloaded(); - }); - } catch (std::out_of_range& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Data race occured during versions update. Could not found version. Details: {}", e.what()); - } - versionsToRetire->resize(it - versionsToRetire->begin()); - - std::shared_ptr versionsToStart = std::make_shared(requestedVersions.size()); - it = std::set_difference( - requestedVersions.begin(), requestedVersions.end(), - registeredModelVersions.begin(), registeredModelVersions.end(), - versionsToStart->begin()); - versionsToStart->resize(it - versionsToStart->begin()); - - versionsToStartIn = std::move(versionsToStart); - versionsToReloadIn = std::move(versionsToReload); - versionsToRetireIn = std::move(versionsToRetire); -} - -Status ModelManager::checkStatefulFlagChange(const std::string& modelName, bool configStatefulFlag) { - std::unique_lock modelsLock(modelsMtx); - auto modelIt = models.find(modelName); - if (models.end() == modelIt) - return StatusCode::OK; // Model has not been loaded yet, so there are no restrictions regarding stateful flag setup - - auto model = models[modelName]; - if (model->isStateful() != configStatefulFlag) - return StatusCode::REQUESTED_MODEL_TYPE_CHANGE; - return StatusCode::OK; -} - -std::shared_ptr ModelManager::getModelIfExistCreateElse(const std::string& modelName, const bool isStateful) { - std::unique_lock modelsLock(modelsMtx); - auto modelIt = models.find(modelName); - if (models.end() == modelIt) { - models.insert({modelName, modelFactory(modelName, isStateful)}); - } - return models[modelName]; -} - -std::shared_ptr ModelManager::getFilesystem(const std::string& basePath) { - if (basePath.rfind(S3FileSystem::S3_URL_PREFIX, 0) == 0) { - Aws::SDKOptions options; - Aws::InitAPI(options); - return std::make_shared(options, basePath); - } - if (basePath.rfind(GCSFileSystem::GCS_URL_PREFIX, 0) == 0) { - return std::make_shared(); - } - if (basePath.rfind(AzureFileSystem::AZURE_URL_FILE_PREFIX, 0) == 0) { - return std::make_shared(); - } - if (basePath.rfind(AzureFileSystem::AZURE_URL_BLOB_PREFIX, 0) == 0) { - return std::make_shared(); - } - return std::make_shared(); -} - -Status ModelManager::readAvailableVersions(std::shared_ptr& fs, const std::string& base, model_versions_t& versions) { - files_list_t dirs; - - bool is_directory = false; - if (FileSystem::isPathEscaped(base)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Path {} escape with .. is forbidden.", base); - return StatusCode::PATH_INVALID; - } - - auto status = fs->isDirectory(base, &is_directory); - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't check directory: {}", base); - return status; - } - if (!is_directory) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Directory does not exist: {}", base); - return StatusCode::PATH_INVALID; - } - - status = fs->getDirectorySubdirs(base, &dirs); - - if (status != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Couldn't list directories in path: {}", base); - return status; - } - - for (const auto& entry : dirs) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Detected version folder: {}", entry); - try { - ovms::model_version_t version = std::stoll(entry); - if (version <= 0) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Expected version directory name to be a number greater than 0. Got: {}", version); - continue; - } - versions.push_back(version); - } catch (const std::invalid_argument& e) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Expected version directory name to be in number format. Got: {}", entry); - } catch (const std::out_of_range& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Directory name is out of range for supported version format. Got: {}", entry); - } - } - - if (0 == versions.size()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "No version found for model in path: {}", base); - } - - return StatusCode::OK; -} - -Status ModelManager::addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr versionsFailed) { - Status status = StatusCode::OK; - try { - status = model->addVersions(versionsToStart, config, fs, *ieCore, versionsFailed); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while loading model: {} versions; error: {}", - config.getName(), - status.string()); - return status; - } - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Exception occurred while loading model: {};", e.what()); - } - return status; -} - -Status ModelManager::reloadModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToReload, std::shared_ptr versionsFailed) { - Status status = StatusCode::OK; - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Reloading model versions"); - try { - auto status = model->reloadVersions(versionsToReload, config, fs, *ieCore, versionsFailed); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while reloading model: {}; versions; error: {}", - config.getName(), - status.string()); - - return status; - } - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Exception occurred while reloading model: {};", e.what()); - } - - return status; -} - -Status ModelManager::reloadModelWithVersions(ModelConfig& config) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Started applying config changes to model: {}", config.getName()); - - if (config.isStateful() && config.isDynamicParameterEnabled()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested setting dynamic parameters for stateful model {}. Dynamic shape and dynamic batch size not supported for stateful models.", config.getName()); - return StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL; - } - if (!config.isStateful()) { - if (config.isLowLatencyTransformationUsed()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested low latency transformation parameter for non stateful model {}.", config.getName()); - return StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER; - } - } - - auto status = checkStatefulFlagChange(config.getName(), config.isStateful()); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested model type change on model: {}. Stateful flag cannot be changed after model is loaded", config.getName()); - return StatusCode::REQUESTED_MODEL_TYPE_CHANGE; - } - - auto model = getModelIfExistCreateElse(config.getName(), config.isStateful()); - if (model->isAnyVersionSubscribed()) { - if (config.isDynamicParameterEnabled()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested setting dynamic parameters for model {} but it is used in pipeline. Cannot reload model configuration.", config.getName()); - return StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL; - } - if (config.isStateful()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested using stateful model {} but it is used in pipeline. Stateful model cannot be subscribed to pipeline.", config.getName()); - return StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL; - } - } - - auto fs = ModelManager::getFilesystem(config.getBasePath()); - std::vector availableVersions; - Status blocking_status = readAvailableVersions(fs, config.getBasePath(), availableVersions); - if (!blocking_status.ok()) { - return blocking_status; - } - auto requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); - std::shared_ptr versionsToStart; - std::shared_ptr versionsToReload; - std::shared_ptr versionsToRetire; - std::shared_ptr versionsFailed = std::make_shared(); - // first reset custom loader name to empty string so that any changes to name can be captured - model->resetCustomLoaderName(); - - if (config.isCustomLoaderRequiredToLoadModel()) { - custom_loader_options_config_t customLoaderOptionsConfig = config.getCustomLoaderOptionsConfigMap(); - const std::string loaderName = customLoaderOptionsConfig["loader_name"]; - - auto& customloaders = ovms::CustomLoaders::instance(); - auto loaderPtr = customloaders.find(loaderName); - if (loaderPtr != nullptr) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Custom Loader to be used : {}", loaderName); - model->setCustomLoaderName(loaderName); - } else { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Specified custom loader {} not found. In case any models are loaded, will be unloading them", loaderName); - model->retireAllVersions(); - return StatusCode::OK; - } - } - getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); - bool reloadNeeded = false; - if (versionsToStart->size() > 0 || versionsToReload->size() > 0 || versionsToRetire->size() > 0) { - reloadNeeded = true; - } - std::set allFailedVersions; - while (versionsToStart->size() > 0) { - blocking_status = addModelVersions(model, fs, config, versionsToStart, versionsFailed); - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Adding new versions. Status: {};", blocking_status.string()); - if (!blocking_status.ok()) { - for (const auto version : *versionsFailed) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Removing available version {} due to load failure; ", version); - if (std::binary_search(availableVersions.begin(), availableVersions.end(), version)) { - availableVersions.erase(std::remove(availableVersions.begin(), availableVersions.end(), version), availableVersions.end()); - } - allFailedVersions.insert(version); - } - requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); - getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); - } else { - break; - } - } - - if (versionsToReload->size() > 0) { - auto status = reloadModelVersions(model, fs, config, versionsToReload, versionsFailed); - if (!status.ok()) { - blocking_status = status; - } - } - - for (const auto version : *versionsFailed) { - SPDLOG_LOGGER_TRACE(modelmanager_logger, "Removing available version {} due to load failure.", version); - if (std::binary_search(availableVersions.begin(), availableVersions.end(), version)) { - availableVersions.erase(std::remove(availableVersions.begin(), availableVersions.end(), version), availableVersions.end()); - } - allFailedVersions.insert(version); - } - // refresh versions to retire based on failed reloads - requestedVersions = config.getModelVersionPolicy()->filter(availableVersions); - getVersionsToChange(config, model->getModelVersions(), requestedVersions, versionsToStart, versionsToReload, versionsToRetire); - std::shared_ptr versionsToCleanup = std::make_shared(); - std::copy_if(versionsToRetire->begin(), versionsToRetire->end(), std::back_inserter(*versionsToCleanup), [&](auto& version) { return allFailedVersions.find(version) != allFailedVersions.end(); }); - versionsToRetire->erase(std::remove_if(versionsToRetire->begin(), versionsToRetire->end(), [&](auto& version) { return allFailedVersions.find(version) != allFailedVersions.end(); }), versionsToRetire->end()); - if (versionsToRetire->size() > 0) { - auto status = model->retireVersions(versionsToRetire); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while unloading model: {}; versions; error: {}", - config.getName(), - status.string()); - return status; - } - } - if (versionsToCleanup->size() > 0) { - auto status = model->cleanupFailedLoad(versionsToCleanup); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while cleaning up model that failed to load: {}; versions; error: {}", - config.getName(), - status.string()); - return status; - } - } - - if (blocking_status.ok() && reloadNeeded) { - return StatusCode::OK_RELOADED; - } - - return blocking_status; -} - -const std::shared_ptr ModelManager::findModelByName(const std::string& name) const { - std::shared_lock lock(modelsMtx); - auto it = models.find(name); - return it != models.end() ? it->second : nullptr; -} - -Status ModelManager::getModelInstance(const std::string& modelName, - ovms::model_version_t modelVersionId, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr) { - SPDLOG_DEBUG("Requesting model: {}; version: {}.", modelName, modelVersionId); - - auto model = findModelByName(modelName); - if (model == nullptr) { - return StatusCode::MODEL_NAME_MISSING; - } - if (modelVersionId != 0) { - modelInstance = model->getModelInstanceByVersion(modelVersionId); - if (modelInstance == nullptr) { - return StatusCode::MODEL_VERSION_MISSING; - } - } else { - modelInstance = model->getDefaultModelInstance(); - if (modelInstance == nullptr) { - return StatusCode::MODEL_VERSION_MISSING; - } - } - - return modelInstance->waitForLoaded(waitForModelLoadedTimeoutMs, modelInstanceUnloadGuardPtr); -} - -const CustomNodeLibraryManager& ModelManager::getCustomNodeLibraryManager() const { - return *customNodeLibraryManager; -} - -} // namespace ovms diff --git a/src/ovms_lib/modelmanager.hpp b/src/ovms_lib/modelmanager.hpp deleted file mode 100644 index 437a2025d0..0000000000 --- a/src/ovms_lib/modelmanager.hpp +++ /dev/null @@ -1,427 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "custom_node_library_internal_manager_wrapper.hpp" -#include "customloaders.hpp" -#include "filesystem.hpp" -#include "global_sequences_viewer.hpp" -#include "model.hpp" -#include "pipeline.hpp" -#include "pipeline_factory.hpp" - -namespace ovms { - -const uint32_t DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS = 10000; -const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; - -class Config; -class IVersionReader; -class CustomNodeLibraryManager; -struct FunctorSequenceCleaner; -struct FunctorResourcesCleaner; -/** - * @brief Model manager is managing the list of model topologies enabled for serving and their versions. - */ -class ModelManager { -public: - /** - * @brief A default constructor is private - */ - ModelManager(const std::string& modelCacheDirectory = ""); - -protected: - void logPluginConfiguration(); - - Status checkStatefulFlagChange(const std::string& modelName, bool configStatefulFlag); - - std::shared_ptr getModelIfExistCreateElse(const std::string& name, const bool isStateful); - - /** - * @brief A collection of models - * - */ - std::map> models; - std::unique_ptr ieCore; - - PipelineFactory pipelineFactory; - - std::unique_ptr customNodeLibraryManager; - - std::vector> resources = {}; - - GlobalSequencesViewer globalSequencesViewer; - uint32_t waitForModelLoadedTimeoutMs; - bool watcherStarted = false; - bool cleanerStarted = false; - -private: - /** - * @brief Private copying constructor - */ - ModelManager(const ModelManager&) = delete; - - Status lastLoadConfigStatus = StatusCode::OK; - - std::string getConfigFileMD5(); - Status cleanupModelTmpFiles(ModelConfig& config); - Status reloadModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToReload, std::shared_ptr versionsFailed); - Status addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr versionsFailed); - Status loadModelsConfig(rapidjson::Document& configJson, std::vector& gatedModelConfigs); - Status tryReloadGatedModelConfigs(std::vector& gatedModelConfigs); - Status loadCustomNodeLibrariesConfig(rapidjson::Document& configJson); - Status loadPipelinesConfig(rapidjson::Document& configJson); - Status loadCustomLoadersConfig(rapidjson::Document& configJson); - - /** - * @brief creates customloader from the loader configuration - */ - Status createCustomLoader(CustomLoaderConfig& loaderConfig); - - /** - * @brief Watcher thread for monitor changes in config - */ - void watcher(std::future exitSignal); - - /** - * @brief Cleaner thread for sequence and resources cleanup - */ - void cleanerRoutine(uint32_t resourcesCleanupIntervalSec, uint32_t sequenceCleanerIntervalMinutes, std::future cleanerExitSignal); - - /** - * @brief Mutex for blocking concurrent add & remove of resources - */ - std::shared_mutex resourcesMtx; - - /** - * @brief A JSON configuration filename - */ - std::string configFilename; - - /** - * @brief A thread object used for monitoring changes in config - */ - std::thread monitor; - - /** - * @brief A thread object used for cleanup - */ - std::thread cleanerThread; - - /** - * @brief An exit trigger to notify watcher thread to exit - */ - std::promise exitTrigger; - - /** - * @brief An exit trigger to notify cleaner thread to exit - */ - std::promise cleanerExitTrigger; - - /** - * @brief A current configurations of models - * - */ - std::unordered_map servedModelConfigs; - - /** - * @brief Retires models non existing in config file - * - * @param modelsExistingInConfigFile - */ - void retireModelsRemovedFromConfigFile(const std::set& modelsExistingInConfigFile, const std::set& modelsWithInvalidConfig); - - /** - * @brief Mutex for protecting concurrent reloading config - */ - mutable std::recursive_mutex configMtx; - - /** - * Time interval between each config file check - */ - uint watcherIntervalSec = 1; - - /** - * Time interval between two consecutive sequence cleanup scans (in minutes) - */ - uint32_t sequenceCleaupIntervalMinutes = 5; - - /** - * Time interval between two consecutive resources cleanup scans (in seconds) - */ - uint32_t resourcesCleanupIntervalSec = 1; - - /** - * @brief last md5sum of configfile - */ - std::string lastConfigFileMD5; - - /** - * @brief Directory for OpenVINO to store cache files. - */ - std::string modelCacheDirectory; - -public: - /** - * @brief Mutex for blocking concurrent add & find of model - */ - mutable std::shared_mutex modelsMtx; - - /** - * @brief Gets the instance of ModelManager - */ - static ModelManager& getInstance() { - static ModelManager instance; - return instance; - } - - /** - * @brief Gets the watcher interval timestep in seconds - */ - uint getWatcherIntervalSec() { - return watcherIntervalSec; - } - - /** - * @brief Gets the cleaner resources interval timestep in seconds - */ - uint32_t getResourcesCleanupIntervalSec() { - return resourcesCleanupIntervalSec; - } - - /** - * @brief Adds new resource to watch by the cleaner thread - */ - void addResourceToCleaner(std::shared_ptr resource) { - std::unique_lock resourcesLock(resourcesMtx); - resources.emplace(resources.end(), std::move(resource)); - } - - /** - * @brief Destroy the Model Manager object - * - */ - virtual ~ModelManager(); - - /** - * @brief Gets config filename - * - * @return config filename - */ - const std::string& getConfigFilename() { - return configFilename; - } - - /** - * @brief Gets models collection - * - * @return models collection - */ - const std::map>& getModels() { - return models; - } - - /** - * @brief Starts monitoring cleanup as new thread - */ - void startCleaner(); - - const PipelineFactory& getPipelineFactory() const { - return pipelineFactory; - } - - const CustomNodeLibraryManager& getCustomNodeLibraryManager() const; - - /** - * @brief Finds model with specific name - * - * @param name of the model to search for - * - * @return pointer to Model or nullptr if not found - */ - const std::shared_ptr findModelByName(const std::string& name) const; - - Status getModelInstance(const std::string& modelName, - ovms::model_version_t modelVersionId, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr); - - const bool modelExists(const std::string& name) const { - if (findModelByName(name) == nullptr) - return false; - else - return true; - } - - /** - * @brief Finds model instance with specific name and version, returns default if version not specified - * - * @param name of the model to search for - * @param version of the model to search for or 0 if default - * - * @return pointer to ModelInstance or nullptr if not found - */ - const std::shared_ptr findModelInstance(const std::string& name, model_version_t version = 0) const { - auto model = findModelByName(name); - if (!model) { - return nullptr; - } - - if (version == 0) { - return model->getDefaultModelInstance(); - } else { - return model->getModelInstanceByVersion(version); - } - } - - template - Status createPipeline(std::unique_ptr& pipeline, - const std::string name, - const RequestType* request, - ResponseType* response) { - return pipelineFactory.create(pipeline, name, request, response, *this); - } - - const bool pipelineDefinitionExists(const std::string& name) const { - return pipelineFactory.definitionExists(name); - } - - /** - * @brief Starts model manager using provided config file - * - * @param filename - * @return status - */ - Status startFromFile(const std::string& jsonFilename); - - /** - * @brief Starts model manager using command line arguments - * - * @return Status - */ - Status startFromConfig(); - - /** - * @brief Reload model versions located in base path - * - * @param ModelConfig config - * - * @return status - */ - Status reloadModelWithVersions(ModelConfig& config); - - /** - * @brief Starts model manager using ovms::Config - * - * @return status - */ - Status start(const Config& config); - - /** - * @brief Starts monitoring as new thread - * - */ - void startWatcher(); - - /** - * @brief Gracefully finish the thread - */ - void join(); - - /** - * @brief Factory for creating a model - * - * @return std::shared_ptr - */ - virtual std::shared_ptr modelFactory(const std::string& name, const bool isStateful) { - return std::make_shared(name, isStateful, &this->globalSequencesViewer); - } - - /** - * @brief Reads available versions from given filesystem - * - * @param fs - * @param base - * @param versions - * @return Status - */ - virtual Status readAvailableVersions( - std::shared_ptr& fs, - const std::string& base, - model_versions_t& versions); - - /** - * @brief Checks what versions needs to be started, reloaded, retired based on currently served ones - * - * @param modelVersionsInstances map with currently served versions - * @param requestedVersions container with requested versions - * @param versionsToRetireIn cointainer for versions to retire - * @param versionsToReloadIn cointainer for versions to reload - * @param versionsToStartIn cointainer for versions to start - */ - static void getVersionsToChange( - const ModelConfig& newModelConfig, - const std::map>& modelVersionsInstances, - std::vector requestedVersions, - std::shared_ptr& versionsToRetireIn, - std::shared_ptr& versionsToReloadIn, - std::shared_ptr& versionsToStartIn); - - static std::shared_ptr getFilesystem(const std::string& basePath); - - /** - * @brief Check if configuration file reload is needed. - */ - Status configFileReloadNeeded(bool& isNeeded); - - /** - * @brief Reads models from configuration file - * - * @param jsonFilename configuration file - * @return Status - */ - Status loadConfig(const std::string& jsonFilename); - - /** - * @brief Updates OVMS configuration with cached configuration file. Will check for newly added model versions - */ - Status updateConfigurationWithoutConfigFile(); - - /** - * @brief Cleaner thread procedure to cleanup resources that are not used - */ - void cleanupResources(); -}; - -void cleanerRoutine(uint32_t resourcesCleanupInterval, FunctorResourcesCleaner& functorResourcesCleaner, uint32_t sequenceCleanerInterval, FunctorSequenceCleaner& functorSequenceCleaner, std::future& cleanerExitSignal); - -} // namespace ovms diff --git a/src/ovms_lib/modelversion.hpp b/src/ovms_lib/modelversion.hpp deleted file mode 100644 index fcabe46aa4..0000000000 --- a/src/ovms_lib/modelversion.hpp +++ /dev/null @@ -1,24 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -namespace ovms { - -using model_version_t = int64_t; -using model_versions_t = std::vector; -} // namespace ovms diff --git a/src/ovms_lib/modelversionstatus.cpp b/src/ovms_lib/modelversionstatus.cpp deleted file mode 100644 index 6936dc67b3..0000000000 --- a/src/ovms_lib/modelversionstatus.cpp +++ /dev/null @@ -1,122 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "modelversionstatus.hpp" - -#include -#include -#include - -#include - -// note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. - -namespace ovms { - -// those values have to match tensorflow-serving state: -static const std::unordered_map versionStatesStrings = { - {ModelVersionState::START, "START"}, - {ModelVersionState::LOADING, "LOADING"}, - {ModelVersionState::AVAILABLE, "AVAILABLE"}, - {ModelVersionState::UNLOADING, "UNLOADING"}, - {ModelVersionState::END, "END"}}; -const std::string& ModelVersionStateToString(ModelVersionState state) { - return versionStatesStrings.at(state); -} - -static const std::unordered_map versionsStatesErrors = { - {ModelVersionStatusErrorCode::OK, "OK"}, - {ModelVersionStatusErrorCode::UNKNOWN, "UNKNOWN"}, - {ModelVersionStatusErrorCode::FAILED_PRECONDITION, "FAILED_PRECONDITION"}}; - -const std::string& ModelVersionStatusErrorCodeToString(ModelVersionStatusErrorCode code) { - return versionsStatesErrors.at(code); -} - -ModelVersionStatus::ModelVersionStatus(const std::string& model_name, model_version_t version, ModelVersionState state) : - modelName(model_name), - version(version), - state(state), - errorCode(ModelVersionStatusErrorCode::OK) { - logStatus(); -} - -ModelVersionState ModelVersionStatus::getState() const { - return this->state; -} - -const std::string& ModelVersionStatus::getStateString() const { - return ModelVersionStateToString(this->state); -} - -ModelVersionStatusErrorCode ModelVersionStatus::getErrorCode() const { - return this->errorCode; -} - -const std::string& ModelVersionStatus::getErrorMsg() const { - return ModelVersionStatusErrorCodeToString(this->errorCode); -} - -bool ModelVersionStatus::willEndUnloaded() const { - return ovms::ModelVersionState::UNLOADING <= this->state; -} - -bool ModelVersionStatus::isFailedLoading() const { - return this->state == ovms::ModelVersionState::LOADING && this->errorCode == ovms::ModelVersionStatusErrorCode::UNKNOWN; -} - -void ModelVersionStatus::setLoading(ModelVersionStatusErrorCode error_code) { - SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); - state = ModelVersionState::LOADING; - errorCode = error_code; - logStatus(); -} - -void ModelVersionStatus::setAvailable(ModelVersionStatusErrorCode error_code) { - SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); - state = ModelVersionState::AVAILABLE; - errorCode = error_code; - logStatus(); -} - -void ModelVersionStatus::setUnloading(ModelVersionStatusErrorCode error_code) { - SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); - state = ModelVersionState::UNLOADING; - errorCode = error_code; - logStatus(); -} - -void ModelVersionStatus::setEnd(ModelVersionStatusErrorCode error_code) { - SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); - state = ModelVersionState::END; - errorCode = error_code; - logStatus(); -} - -void ModelVersionStatus::logStatus() { - SPDLOG_INFO("STATUS CHANGE: Version {} of model {} status change. New status: ( \"state\": \"{}\", \"error_code\": \"{}\" )", - this->version, - this->modelName, - ModelVersionStateToString(state), - ModelVersionStatusErrorCodeToString(errorCode)); -} - -void ModelVersionStatus::setState(ModelVersionState state, ModelVersionStatusErrorCode error_code) { - SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); - this->state = state; - errorCode = error_code; - logStatus(); -} -} // namespace ovms diff --git a/src/ovms_lib/modelversionstatus.hpp b/src/ovms_lib/modelversionstatus.hpp deleted file mode 100644 index 69456ef3b3..0000000000 --- a/src/ovms_lib/modelversionstatus.hpp +++ /dev/null @@ -1,105 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "modelconfig.hpp" - -// note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. - -namespace ovms { - -// those values have to match tensorflow-serving state: -enum class ModelVersionState : int { - START = 10, - LOADING = 20, - AVAILABLE = 30, - UNLOADING = 40, - END = 50 -}; - -const std::string& ModelVersionStateToString(ModelVersionState state); - -enum class ModelVersionStatusErrorCode : int { - OK = 0, - // CANCELLED = 1, - UNKNOWN = 2, - // INVALID_ARGUMENT = 3, - // DEADLINE_EXCEEDED = 4, - // NOT_FOUND = 5, - // ALREADY_EXISTS = 6, - // PERMISSION_DENIED = 7, - // UNAUTHENTICATED = 16, - // RESOURCE_EXHAUSTED = 8, - FAILED_PRECONDITION = 9, - // ABORTED = 10, - // OUT_OF_RANGE = 11, - // UNIMPLEMENTED = 12, - // INTERNAL = 13, - // UNAVAILABLE = 14, - // DATA_LOSS = 15, - // DO_NOT_USE_RESERVED_FOR_FUTURE_EXPANSION_USE_DEFAULT_IN_SWITCH_INSTEAD_ - // = 20 -}; - -const std::string& ModelVersionStatusErrorCodeToString(ModelVersionStatusErrorCode code); - -class ModelVersionStatus { - std::string modelName; - model_version_t version; - ModelVersionState state; - ModelVersionStatusErrorCode errorCode; - -public: - ModelVersionStatus() = delete; - - ModelVersionStatus(const std::string& model_name, model_version_t version, ModelVersionState state = ModelVersionState::START); - - ModelVersionState getState() const; - - const std::string& getStateString() const; - - ModelVersionStatusErrorCode getErrorCode() const; - - const std::string& getErrorMsg() const; - - /** - * @brief Check if current state is state that is either transforming to END or already in that state. - * - * @return - */ - bool willEndUnloaded() const; - - bool isFailedLoading() const; - void setLoading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); - - void setAvailable(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); - - void setUnloading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); - - void setEnd(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); - void setState(ModelVersionState state, ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); - -private: - void logStatus(); -}; - -} // namespace ovms diff --git a/src/ovms_lib/my_cc_combine.bzl b/src/ovms_lib/my_cc_combine.bzl deleted file mode 100644 index 27113911f2..0000000000 --- a/src/ovms_lib/my_cc_combine.bzl +++ /dev/null @@ -1,70 +0,0 @@ -load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") - -def _combine_impl(ctx): - cc_toolchain = find_cpp_toolchain(ctx) - - target_list = [] - for dep_target in ctx.attr.deps: - # CcInfo, InstrumentedFilesInfo, OutputGroupInfo - cc_info_linker_inputs = dep_target[CcInfo].linking_context.linker_inputs - - target_dirname_list = [] - for linker_in in cc_info_linker_inputs.to_list(): - for linker_in_lib in linker_in.libraries: - if linker_in_lib.pic_static_library != None: - target_list += [linker_in_lib.pic_static_library] - if linker_in_lib.static_library != None: - target_list += [linker_in_lib.static_library] - - output = ctx.outputs.output - if ctx.attr.genstatic: - cp_command = "" - processed_list = [] - processed_path_list = [] - for dep in target_list: - cp_command += "cp -a "+ dep.path +" "+ output.dirname + "/&&" - processed = ctx.actions.declare_file(dep.basename) - processed_list += [processed] - processed_path_list += [dep.path] - cp_command += "echo'starting to run shell'" - processed_path_list += [output.path] - - ctx.actions.run_shell( - outputs = processed_list, - inputs = target_list, - command = cp_command, - ) - - command = "cd {} && ar -x {} {}".format( - output.dirname, - "&& ar -x ".join([dep.basename for dep in target_list]), - "&& ar -rc libauto.a *.o" - ) - print("command = ", command) - ctx.actions.run_shell( - outputs = [output], - inputs = processed_list, - command = command, - ) - else: - command = "export PATH=$PATH:{} && {} -shared -fPIC -Wl,--whole-archive {} -Wl,--no-whole-archive -Wl,-soname -o {}".format ( - cc_toolchain.ld_executable, - cc_toolchain.compiler_executable, - "".join([dep.path for dep in target_list]), - output.path) - print("command = ", command) - ctx.actions.run_shell( - outputs = [output], - inputs = target_list, - command = command, - ) - -my_cc_combine = rule( - implementation = _combine_impl, - attrs = { - "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), - "genstatic": attr.bool(default = False), - "deps": attr.label_list(), - "output": attr.output() - }, -) diff --git a/src/ovms_lib/node.cpp b/src/ovms_lib/node.cpp deleted file mode 100644 index dfa6fc150a..0000000000 --- a/src/ovms_lib/node.cpp +++ /dev/null @@ -1,272 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "node.hpp" - -#include -#include -#include -#include - -#include "logging.hpp" -#include "nodesession.hpp" -#include "ov_utils.hpp" -#include "profiler.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -const uint64_t DEMULTIPLY_LIMIT = 10'000; - -namespace ovms { - -static std::string demultiplyCountSettingToString(std::optional demultiplyCount) { - if (!demultiplyCount) { - return "NA"; - } - if (demultiplyCount.value() == -1) { - return "dynamic"; - } - return std::to_string(demultiplyCount.value()); -} - -Node::Node(const std::string& nodeName, std::optional demultiplyCount, std::set gatherFromNode) : - nodeName(nodeName), - demultiplexCount(demultiplyCount), - gatherFrom(!gatherFromNode.empty() ? std::optional>(gatherFromNode) : std::nullopt) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will create node: {} with demultiply: {}, gatherFrom: {}.", - getName(), - demultiplyCountSettingToString(demultiplexCount), - std::accumulate(gatherFromNode.begin(), gatherFromNode.end(), std::string("NA"), [](const std::string& lhs, const std::string& rhs) { - if (lhs == "NA") { - return rhs; - } else { - return lhs + ", " + rhs; - } })); -} - -Status Node::fetchResults(session_key_t sessionId, SessionResults& nodeSessionOutputs) { - OVMS_PROFILE_FUNCTION(); - auto it = nodeSessions.find(sessionId); - - auto& nodeSession = it->second; - if (it == nodeSessions.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Could not find session: {} for node: {}", sessionId, getName()); - return StatusCode::UNKNOWN_ERROR; - } - auto status = fetchResults(*nodeSession, nodeSessionOutputs); - if (status.ok() && demultiplexCount) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will demultiply node: {} outputs with demultiplyCount: {}", getName(), demultiplyCountSettingToString(demultiplexCount)); - status = demultiplyOutputs(nodeSessionOutputs); - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will remove node: {} session: {}", getName(), sessionId); - nodeSessions.erase(sessionId); - return status; -} - -void Node::printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs) { - std::stringstream ss; - ss << "Links from:" << sourceNode << " to:" << nodeName << ":\n"; - for (auto& pair : pairs) { - ss << "\t" << nodeName << "[" << pair.second << "]=" << sourceNode << "[" << pair.first << "]\n"; - } - SPDLOG_DEBUG(ss.str()); -} - -Status Node::setInputs(const Node& dependency, SessionResults& sessionResults) { - OVMS_PROFILE_FUNCTION(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} set inputs from node: {}", getName(), dependency.getName()); - for (auto& [sessionKey, metadataInputsPair] : sessionResults) { - auto& [metadata, inputs] = metadataInputsPair; - auto status = this->setInputs(dependency, inputs, metadata); - if (!status.ok()) { - return status; - } - } - return StatusCode::OK; -} - -Status Node::setInputs(const Node& dependency, TensorWithSourceMap& inputs, NodeSessionMetadata& metadata) { - // mapping for dependency - keeps mapping between dependency output name and this node input name - const auto& mapping_for_dependency = this->getMappingByDependency(dependency); - NodeSession* nodeSession = getNodeSession(metadata); - if (!nodeSession) { - return StatusCode::INTERNAL_ERROR; - } - session_id_t shardId; - try { - static const std::set emptySet; - shardId = metadata.getShardId(gatherFrom.value_or(emptySet)); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to get shardId for node: {}", getName()); - return StatusCode::INTERNAL_ERROR; - } - // assign all input tensors from inputs that are required by this node for future inference - for (const auto& pair : mapping_for_dependency) { - const auto& dependency_output_name = pair.first; - const auto& current_node_input_name = pair.second; - - // possibly incorrectly constructed pipeline - required input missing from previous node - auto it = inputs.find(dependency_output_name); - if (it == inputs.end()) { - SPDLOG_LOGGER_WARN(dag_executor_logger, "node: {} error setting required input from node: {} dependency is missing output name: {}", - getName(), - dependency.getName(), - dependency_output_name); - return StatusCode::INVALID_MISSING_INPUT; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} setting required input from node: {}, input name: {}, dependency output name: {}", - getName(), - dependency.getName(), - current_node_input_name, - dependency_output_name); - auto status = nodeSession->setInput(current_node_input_name, it->second, shardId); - if (!status.ok()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "node: {} failed to set input: {}, shard: {}", getName(), current_node_input_name, shardId); - return status; - } - } - return nodeSession->notifyFinishedDependency(); -} - -NodeSession& Node::getNodeSession(const session_key_t& sessionKey) const { - auto it = nodeSessions.find(sessionKey); - if (it == nodeSessions.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to get non-existing node: {} session: {}.", getName(), sessionKey); - throw std::runtime_error("Tried to get non existing session"); - } - return *it->second; -} - -NodeSession* Node::getNodeSession(const NodeSessionMetadata& metadata) { - session_key_t sessionKey; - if (gatherFrom) { - try { - sessionKey = metadata.getSessionKey(gatherFrom.value()); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to create collapsed metadata session key for node: {}, incomming session key: {}", - getName(), metadata.getSessionKey()); - return nullptr; - } - } else { - sessionKey = metadata.getSessionKey(); - } - auto it = nodeSessions.find(sessionKey); - if (it != nodeSessions.end()) { - return it->second.get(); - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will create new session: {} for node: {}", - sessionKey, getName()); - NodeSessionMetadata newSessionMetadata; - CollapseDetails collapsingDetails; - if (gatherFrom) { - try { - std::tie(newSessionMetadata, collapsingDetails) = metadata.getCollapsedSessionMetadata(gatherFrom.value()); - } catch (const std::exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Failed to create collapsed metadata for node: {}", getName()); - return nullptr; - } - } else { - newSessionMetadata = metadata; - } - std::unique_ptr nodeSession = createNodeSession(newSessionMetadata, collapsingDetails); - auto emplacePair = nodeSessions.emplace(sessionKey, std::move(nodeSession)); - return emplacePair.first->second.get(); -} - -std::unique_ptr Node::createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails) { - return std::make_unique(metadata, getName(), previous.size(), collapsingDetails); -} - -std::vector Node::getReadySessions() const { - std::vector readySessions; - for (auto& [sessionKey, nodeSession] : nodeSessions) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Checking readiness of node: {} session: {}", getName(), nodeSession->getSessionKey()); - if (nodeSession->isReady()) { - readySessions.emplace_back(sessionKey); - } - } - return readySessions; -} - -Status Node::demultiplyOutputs(SessionResults& nodeSessionOutputs) { - OVMS_PROFILE_FUNCTION(); - if (!demultiplexCount) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} called demultiplyOutputs but node does not have demultiplexCount set", getName()); - return StatusCode::INTERNAL_ERROR; - } - auto& [metadata, tensorMap] = nodeSessionOutputs.begin()->second; - auto firstTensorShape = tensorMap.begin()->second.getActualTensor().get_shape(); - uint32_t resultsDemultiplyCount = firstTensorShape[0]; - if (firstTensorShape[0] > DEMULTIPLY_LIMIT) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} - too large dim[0] size: {} of tensor: {}. Maximum allowed is: {}", - getName(), firstTensorShape[0], tensorMap.begin()->first, DEMULTIPLY_LIMIT); - return StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will demultiply node: {} outputs to: {} shards", getName(), resultsDemultiplyCount); - std::vector newSessionMetadatas; - try { - newSessionMetadatas = std::move(metadata.generateSubsessions(getName(), resultsDemultiplyCount)); - } catch (std::exception& e) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Node: {} failed to generate subsessions due to error: {}", getName(), e.what()); - return StatusCode::INTERNAL_ERROR; - } - for (auto& [tensorName, tensorWithSource] : tensorMap) { - auto& tensor = tensorWithSource.getActualTensor(); - OVMS_PROFILE_SCOPE("Demultiply Tensor"); - auto newDims = tensor.get_shape(); - if (newDims.size() < 3) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Wrong number of dimensions: {} to demultiply. Must be at least 3", newDims.size()); - return StatusCode::PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY; - } - if ((demultiplexCount.value() != -1) && - (newDims[0] != demultiplexCount.value())) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Wrong dim[0] size: {} of tensor: {} expected: {} to demultiply", - newDims[0], tensorName, demultiplexCount.value()); - return StatusCode::PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY; - } - if (resultsDemultiplyCount == 0) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} has no results. Dynamic demultiplexer with demultiply == 0 is not supported yet.", this->getName()); - nodeSessionOutputs.erase(metadata.getSessionKey()); - return StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS; - } - - newDims.erase(newDims.begin()); - const auto step = tensor.get_byte_size() / resultsDemultiplyCount; - for (size_t i = 0; i < newSessionMetadatas.size(); ++i) { - OVMS_PROFILE_SCOPE("Create Shard"); - ov::Tensor dividedTensor; - this->createShardedTensor(dividedTensor, ovElementTypeToOvmsPrecision(tensor.get_element_type()), newDims, tensor, i, step, metadata, tensorName); - std::stringstream ss; - ss << "Node: " << getName() << " input demultiplied: " << tensorName - << "; Actual: " << TensorInfo::shapeToString(dividedTensor.get_shape()); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "{}", ss.str()); - auto sessionKey = newSessionMetadatas[i].getSessionKey(); - auto it = nodeSessionOutputs.find(sessionKey); - if (it == nodeSessionOutputs.end()) { - nodeSessionOutputs.emplace(sessionKey, SessionResult{newSessionMetadatas[i], TensorWithSourceMap{{tensorName, TensorWithSource{dividedTensor, tensor}}}}); - } else { - it->second.second.emplace(tensorName, TensorWithSource{dividedTensor, tensor}); - } - } - } - nodeSessionOutputs.erase(metadata.getSessionKey()); - return StatusCode::OK; -} - -Status Node::createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName) { - dividedTensor = createSharedTensor(tensor.get_element_type(), shape, (char*)(tensor.data()) + i * step); - return StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/node.hpp b/src/ovms_lib/node.hpp deleted file mode 100644 index 3a6cdc724b..0000000000 --- a/src/ovms_lib/node.hpp +++ /dev/null @@ -1,102 +0,0 @@ -//***************************************************************************** -// Copyright 2020,2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "aliases.hpp" -#include "nodesession.hpp" -#include "nodesessionresult.hpp" -#include "pipelineeventqueue.hpp" -#include "precision.hpp" -#include "shape.hpp" -#include "status.hpp" -#include "tensormap.hpp" - -namespace ovms { - -using TensorNames = std::vector; -using session_key_t = std::string; - -class Node { -protected: - std::string nodeName; - - std::vector> previous; - std::vector> next; - - // Tensors ready and waiting for execution - std::unordered_map> nodeSessions; - - // Input/Output name mapping and list of required inputs from previous nodes - std::unordered_map tensorNamesMapping; - - const std::optional demultiplexCount; - const std::optional> gatherFrom; - -public: - Node(const std::string& nodeName, std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}); - - virtual ~Node() = default; - - const std::string& getName() const { return this->nodeName; } - - virtual Status execute(session_key_t sessionId, PipelineEventQueue& notifyEndQueue) = 0; - Status fetchResults(session_key_t sessionId, SessionResults& nodeSessionOutputs); - -protected: - virtual Status fetchResults(NodeSession& nodeSession, SessionResults& nodeSessionOutputs) = 0; - Status demultiplyOutputs(SessionResults& nodeSessionOutputs); - virtual Status createShardedTensor(ov::Tensor& dividedTensor, Precision precision, const shape_t& shape, const ov::Tensor& tensor, size_t i, size_t step, const NodeSessionMetadata& metadata, const std::string tensorName); - -public: - Status setInputs(const Node& dependency, TensorWithSourceMap& inputs, NodeSessionMetadata& metadata); - Status setInputs(const Node& dependency, SessionResults& inputs); - - virtual void addDependency(Node& node, const Aliases& tensorNamesMapping) { - this->previous.emplace_back(node); - this->tensorNamesMapping[node.getName()] = tensorNamesMapping; - } - - virtual void addDependant(Node& node) { this->next.emplace_back(node); } - - const Aliases& getMappingByDependency(const Node& dependency) { - return tensorNamesMapping.at(dependency.getName()); - } - - std::vector getReadySessions() const; - const std::vector>& getNextNodes() { - return next; - } - virtual void release(session_key_t sessionId) {} - virtual bool tryDisarm(const session_key_t& sessionKey, const uint microseconds = 1) { return true; } - - static void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs); - -protected: - NodeSession* getNodeSession(const NodeSessionMetadata& metadata); - NodeSession& getNodeSession(const session_key_t& sessionKey) const; - virtual std::unique_ptr createNodeSession(const NodeSessionMetadata& metadata, const CollapseDetails& collapsingDetails); -}; - -} // namespace ovms diff --git a/src/ovms_lib/node_library.cpp b/src/ovms_lib/node_library.cpp deleted file mode 100644 index 495b14e306..0000000000 --- a/src/ovms_lib/node_library.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "node_library.hpp" - -namespace ovms { - -bool NodeLibrary::isValid() const { - return execute != nullptr && - getInputsInfo != nullptr && - getOutputsInfo != nullptr && - release != nullptr && - initialize != nullptr && - deinitialize != nullptr; -} - -} // namespace ovms diff --git a/src/ovms_lib/node_library.hpp b/src/ovms_lib/node_library.hpp deleted file mode 100644 index bb29d52a5e..0000000000 --- a/src/ovms_lib/node_library.hpp +++ /dev/null @@ -1,52 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "custom_node_interface.h" // NOLINT - -namespace ovms { - -typedef int (*initialize_fn)(void**, const struct CustomNodeParam*, int); -typedef int (*deinitialize_fn)(void*); -typedef int (*execute_fn)(const struct CustomNodeTensor*, int, struct CustomNodeTensor**, int*, const struct CustomNodeParam*, int, void*); -typedef int (*metadata_fn)(struct CustomNodeTensorInfo**, int*, const struct CustomNodeParam*, int, void*); -typedef int (*release_fn)(void*, void*); - -struct NodeLibrary { - initialize_fn initialize = nullptr; - deinitialize_fn deinitialize = nullptr; - execute_fn execute = nullptr; - metadata_fn getInputsInfo = nullptr; - metadata_fn getOutputsInfo = nullptr; - release_fn release = nullptr; - - std::string basePath = ""; - - bool isValid() const; - bool operator==(const NodeLibrary& other) const { - return (initialize == other.initialize) && - (deinitialize == other.deinitialize) && - (execute == other.execute) && - (getInputsInfo == other.getInputsInfo) && - (getOutputsInfo == other.getOutputsInfo) && - (release == other.release) && - (basePath == other.basePath); - } -}; - -} // namespace ovms diff --git a/src/ovms_lib/node_library_utils.cpp b/src/ovms_lib/node_library_utils.cpp deleted file mode 100644 index a1fe82d892..0000000000 --- a/src/ovms_lib/node_library_utils.cpp +++ /dev/null @@ -1,143 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "node_library_utils.hpp" - -#include - -#include "ov_utils.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -CustomNodeTensorPrecision toCustomNodeTensorPrecision(ov::element::Type_t precision) { - switch (precision) { - case ov::element::Type_t::f32: - return CustomNodeTensorPrecision::FP32; - case ov::element::Type_t::f64: - return CustomNodeTensorPrecision::FP64; - case ov::element::Type_t::i32: - return CustomNodeTensorPrecision::I32; - case ov::element::Type_t::i64: - return CustomNodeTensorPrecision::I64; - case ov::element::Type_t::i8: - return CustomNodeTensorPrecision::I8; - case ov::element::Type_t::u8: - return CustomNodeTensorPrecision::U8; - case ov::element::Type_t::f16: - return CustomNodeTensorPrecision::FP16; - case ov::element::Type_t::i16: - return CustomNodeTensorPrecision::I16; - case ov::element::Type_t::u16: - return CustomNodeTensorPrecision::U16; - default: - return CustomNodeTensorPrecision::UNSPECIFIED; - } -} - -Precision toInferenceEnginePrecision(CustomNodeTensorPrecision precision) { - static std::unordered_map precisionMap{ - {CustomNodeTensorPrecision::FP32, Precision::FP32}, - {CustomNodeTensorPrecision::FP64, Precision::FP64}, - {CustomNodeTensorPrecision::I32, Precision::I32}, - {CustomNodeTensorPrecision::I64, Precision::I64}, - {CustomNodeTensorPrecision::I8, Precision::I8}, - {CustomNodeTensorPrecision::U8, Precision::U8}, - {CustomNodeTensorPrecision::FP16, Precision::FP16}, - {CustomNodeTensorPrecision::I16, Precision::I16}, - {CustomNodeTensorPrecision::U16, Precision::U16}}; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -std::unique_ptr createCustomNodeParamArray(const std::unordered_map& paramMap) { - if (paramMap.size() == 0) { - return nullptr; - } - auto libraryParameters = std::make_unique(paramMap.size()); - int i = 0; - for (const auto& [key, value] : paramMap) { - libraryParameters[i].key = key.c_str(); - libraryParameters[i].value = value.c_str(); - i++; - } - return libraryParameters; -} - -std::unique_ptr createCustomNodeTensorArray(const TensorMap& tensorMap, const std::unordered_map& tensorsDims) { - if (tensorMap.size() == 0) { - return nullptr; - } - auto inputTensors = std::make_unique(tensorMap.size()); - int i = 0; - for (const auto& [name, tensor] : tensorMap) { - auto dimsIt = tensorsDims.find(name); - if (dimsIt == tensorsDims.end()) { - return nullptr; - } - static_assert(sizeof(size_t) == sizeof(uint64_t)); - const auto& dims = dimsIt->second; - inputTensors[i].name = static_cast(name.c_str()); - inputTensors[i].data = static_cast(tensor.data()); - inputTensors[i].dataBytes = static_cast(tensor.get_byte_size()); - inputTensors[i].dims = const_cast(dims.data()); - inputTensors[i].dimsCount = static_cast(dims.size()); - inputTensors[i].precision = toCustomNodeTensorPrecision(tensor.get_element_type()); - i++; - } - return inputTensors; -} - -Status createTensorInfoMap(struct CustomNodeTensorInfo* info, int infoCount, std::map>& out, release_fn freeCallback, void* customNodeLibraryInternalManager) { - if (info == nullptr) { - return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED; - } - if (infoCount <= 0) { - freeCallback(info, customNodeLibraryInternalManager); - return StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT; - } - // At this point it is important to not exit before we iterate over every info object. - // This is due to a fact that we need to ensure to free resources allocated by shared library using freeCallback. - for (int i = 0; i < infoCount; i++) { - if (info[i].dims == nullptr) { - continue; - } - if (info[i].dimsCount == 0) { - freeCallback(info[i].dims, customNodeLibraryInternalManager); - continue; - } - if (info[i].name == nullptr) { - continue; - } - - std::string name = std::string(info[i].name); - auto precision = toInferenceEnginePrecision(info[i].precision); - ovms::Shape shape; - for (int j = 0; j < info[i].dimsCount; ++j) { - auto dim = info[i].dims[j]; - shape.add(dim ? Dimension(dim) : Dimension::any()); - } - - freeCallback(info[i].dims, customNodeLibraryInternalManager); - out.emplace(name, std::make_shared(name, precision, std::move(shape), Layout::getUnspecifiedLayout())); - } - freeCallback(info, customNodeLibraryInternalManager); - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/node_library_utils.hpp b/src/ovms_lib/node_library_utils.hpp deleted file mode 100644 index 86c55a893b..0000000000 --- a/src/ovms_lib/node_library_utils.hpp +++ /dev/null @@ -1,40 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include "custom_node_interface.h" // NOLINT -#include "node_library.hpp" -#include "precision.hpp" -#include "shape.hpp" -#include "status.hpp" -#include "tensormap.hpp" - -namespace ovms { - -class TensorInfo; - -CustomNodeTensorPrecision toCustomNodeTensorPrecision(ov::element::Type_t precision); -Precision toInferenceEnginePrecision(CustomNodeTensorPrecision precision); -std::unique_ptr createCustomNodeParamArray(const std::unordered_map& paramMap); -std::unique_ptr createCustomNodeTensorArray(const TensorMap& tensorMap, const std::unordered_map& tensorDims); -Status createTensorInfoMap(struct CustomNodeTensorInfo* info, int infoCount, std::map>& out, release_fn freeCallback, void* customNodeLibraryInternalManager); - -} // namespace ovms diff --git a/src/ovms_lib/nodeinfo.hpp b/src/ovms_lib/nodeinfo.hpp deleted file mode 100644 index 20a4c2b3fe..0000000000 --- a/src/ovms_lib/nodeinfo.hpp +++ /dev/null @@ -1,92 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "aliases.hpp" -#include "modelversion.hpp" -#include "node_library.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -class ModelManager; -class Pipeline; - -using pipeline_connections_t = std::unordered_map>; -using parameters_t = std::unordered_map; - -enum class NodeKind { - ENTRY, - DL, - CUSTOM, - EXIT -}; - -const std::string DL_NODE_CONFIG_TYPE = "DL model"; -const std::string CUSTOM_NODE_CONFIG_TYPE = "custom"; - -Status toNodeKind(const std::string& str, NodeKind& nodeKind); - -struct DLNodeInfo { - std::string modelName; - std::optional modelVersion; -}; - -struct CustomNodeInfo { - NodeLibrary library; - parameters_t parameters; -}; - -struct NodeInfo { - NodeKind kind; - std::string nodeName; - std::string modelName; - std::optional modelVersion; - std::unordered_map outputNameAliases; - std::optional demultiplyCount; - std::set gatherFromNode; - NodeLibrary library; - parameters_t parameters; - - NodeInfo(NodeKind kind, - const std::string& nodeName, - const std::string& modelName = "", - std::optional modelVersion = std::nullopt, - std::unordered_map outputNameAliases = {}, - std::optional demultiplyCount = std::nullopt, - const std::set& gatherFromNode = {}, - const NodeLibrary& library = {}, - const parameters_t& parameters = {}) : - kind(kind), - nodeName(nodeName), - modelName(modelName), - modelVersion(modelVersion), - outputNameAliases(outputNameAliases), - demultiplyCount(demultiplyCount), - gatherFromNode(gatherFromNode), - library(library), - parameters(parameters) {} -}; -} // namespace ovms diff --git a/src/ovms_lib/nodeinputhandler.cpp b/src/ovms_lib/nodeinputhandler.cpp deleted file mode 100644 index 4ee038e6ac..0000000000 --- a/src/ovms_lib/nodeinputhandler.cpp +++ /dev/null @@ -1,60 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "nodeinputhandler.hpp" - -#include "logging.hpp" - -namespace ovms { - -NodeInputHandler::NodeInputHandler(uint32_t inputsMissingCount) : - remainingDependencies(inputsMissingCount) { -} - -Status NodeInputHandler::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Setting input: {}, shardId: {}", inputName, shardId); - if (shardId > 0) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to set input: {}, with shardId: {} >0 in NodeInputHandler.", inputName, shardId); - return StatusCode::PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER; - } - if (inputTensors.find(inputName) != inputTensors.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to set the same input: {}, shardId: {} twice for the NodeInputHandler.", inputName, shardId); - return StatusCode::PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE; - } - inputTensors.emplace(inputName, tensor.getActualTensor()); - if (tensor.hasSource()) { - sourceTensorRefs.push_back(tensor.getSourceTensor()); - } - return StatusCode::OK; -} - -void NodeInputHandler::clearInputs() { - inputTensors.clear(); - sourceTensorRefs.clear(); -} - -bool NodeInputHandler::isReady() { - if (this->isUsed) { - return false; - } - return remainingDependencies == 0; -} - -Status NodeInputHandler::notifyFinishedDependency() { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Remaining dependencies count for input handler decreased from: {} to: {}", remainingDependencies, remainingDependencies - 1); - --remainingDependencies; - return StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/nodeinputhandler.hpp b/src/ovms_lib/nodeinputhandler.hpp deleted file mode 100644 index 12665794dd..0000000000 --- a/src/ovms_lib/nodeinputhandler.hpp +++ /dev/null @@ -1,54 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include - -#include "session_id.hpp" -#include "status.hpp" -#include "tensor_utils.hpp" -#include "tensormap.hpp" - -namespace ovms { - -// This class encapsulates input tensor gathering and preprocessing before node execution. -// It is resposible for gathering multiple tensors into one (in case of demultiplexers) -// and taking care of source tensor lifetime if source tensor is present. -class NodeInputHandler { -protected: - TensorMap inputTensors; - TensorVector sourceTensorRefs; - uint32_t remainingDependencies; - bool isUsed = false; - -public: - NodeInputHandler(uint32_t inputsMissingCount); - virtual Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId); - const TensorMap& getInputs() { - isUsed = true; - return inputTensors; - } - void clearInputs(); - bool isReady(); - virtual Status notifyFinishedDependency(); - virtual ~NodeInputHandler() = default; -}; -} // namespace ovms diff --git a/src/ovms_lib/nodeoutputhandler.cpp b/src/ovms_lib/nodeoutputhandler.cpp deleted file mode 100644 index 066d0c9097..0000000000 --- a/src/ovms_lib/nodeoutputhandler.cpp +++ /dev/null @@ -1,20 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "nodeoutputhandler.hpp" - -namespace ovms { -void NodeOutputHandler::setInput(const std::string& name, ov::Tensor& tensorPtr) {} -} // namespace ovms diff --git a/src/ovms_lib/nodeoutputhandler.hpp b/src/ovms_lib/nodeoutputhandler.hpp deleted file mode 100644 index b3e2a79718..0000000000 --- a/src/ovms_lib/nodeoutputhandler.hpp +++ /dev/null @@ -1,28 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include - -namespace ovms { -class NodeOutputHandler { -public: - void setInput(const std::string& inputName, ov::Tensor& tensorPtr); -}; -} // namespace ovms diff --git a/src/ovms_lib/nodesession.cpp b/src/ovms_lib/nodesession.cpp deleted file mode 100644 index 99de8ae274..0000000000 --- a/src/ovms_lib/nodesession.cpp +++ /dev/null @@ -1,72 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "nodesession.hpp" - -#include "gathernodeinputhandler.hpp" -#include "logging.hpp" -#include "nodeinputhandler.hpp" -#include "nodeoutputhandler.hpp" -#include "tensor_utils.hpp" -#include "timer.hpp" - -namespace ovms { -NodeSession::~NodeSession() = default; - -const NodeSessionMetadata& NodeSession::getNodeSessionMetadata() const { - return this->metadata; -} - -Status NodeSession::setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId) { - return inputHandler->setInput(inputName, tensor, shardId); -} - -std::unique_ptr createNodeInputHandler(uint32_t inputsCount, const CollapseDetails& collapsingDetails) { - if (collapsingDetails.collapsedSessionNames.size() == 0) { - return std::make_unique(inputsCount); - } else { - return std::make_unique(inputsCount, collapsingDetails); - } -} - -NodeSession::NodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails) : - metadata(metadata), - sessionKey(metadata.getSessionKey()), - nodeName(nodeName), - timer(std::make_unique()), - inputHandler(createNodeInputHandler(inputsCount, collapsingDetails)), - outputHandler(std::make_unique()) {} - -bool NodeSession::isReady() const { - bool isReady = inputHandler->isReady(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "node: {} session: {} isReady: {}", getName(), getSessionKey(), isReady); - return isReady; -} - -Status NodeSession::notifyFinishedDependency() { - return this->inputHandler->notifyFinishedDependency(); -} - -Timer& NodeSession::getTimer() const { - return *this->timer; -} - -ReleaseSessionGuard::ReleaseSessionGuard(NodeSession& nodeSession) : - nodeSession(nodeSession) {} - -ReleaseSessionGuard::~ReleaseSessionGuard() { - nodeSession.release(); -} -} // namespace ovms diff --git a/src/ovms_lib/nodesession.hpp b/src/ovms_lib/nodesession.hpp deleted file mode 100644 index 7eb5028f0b..0000000000 --- a/src/ovms_lib/nodesession.hpp +++ /dev/null @@ -1,62 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include "nodesessionmetadata.hpp" -#include "status.hpp" - -namespace ovms { -struct NodeInputHandler; -struct NodeOutputHandler; -class TensorWithSource; -class Timer; - -class NodeSession { - NodeSessionMetadata metadata; - session_key_t sessionKey; - const std::string& nodeName; - -protected: - std::unique_ptr timer; - std::unique_ptr inputHandler; - std::unique_ptr outputHandler; - -public: - NodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails); - virtual ~NodeSession(); - const std::string& getName() const { return nodeName; } - Status setInput(const std::string& inputName, TensorWithSource& tensor, session_id_t shardId); - const NodeSessionMetadata& getNodeSessionMetadata() const; - const session_key_t& getSessionKey() const { return sessionKey; } - bool isReady() const; - virtual void release() {} - virtual bool tryDisarm(uint microseconds) { return true; } - Status notifyFinishedDependency(); - Timer& getTimer() const; -}; - -class ReleaseSessionGuard { - NodeSession& nodeSession; - -public: - ReleaseSessionGuard(NodeSession& nodeSession); - ~ReleaseSessionGuard(); -}; -} // namespace ovms diff --git a/src/ovms_lib/nodesessionmetadata.cpp b/src/ovms_lib/nodesessionmetadata.cpp deleted file mode 100644 index 3374229ef3..0000000000 --- a/src/ovms_lib/nodesessionmetadata.cpp +++ /dev/null @@ -1,197 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "nodesessionmetadata.hpp" - -#include -#include -#include -#include -#include - -#include "logging.hpp" - -namespace ovms { - -std::vector NodeSessionMetadata::generateSubsessions(const std::string& nodeName, session_id_t subsessionSize) const { - if (nodeName.size() == 0) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to generate subsession with empty node name"); - throw std::logic_error("Cannot generate subsession with empty parent name"); - } - if (details.find(nodeName) != details.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to generate subsession with node name: {} but it already spawned subsession.", nodeName); - throw std::logic_error("Cannot generate subsession with already used name"); - } - if (subsessionSize == 0) { - return {}; - } - std::vector metas(subsessionSize); - uint counter = 0; - for (auto& meta : metas) { - meta.details = this->details; - meta.details.insert({nodeName, {counter, subsessionSize}}); - meta.sessionsLevels = this->sessionsLevels; - meta.sessionsLevels.push_back(nodeName); - ++counter; - } - SPDLOG_LOGGER_TRACE(dag_executor_logger, "Generated subsession levels: {}", - std::accumulate(metas[0].sessionsLevels.begin(), metas[0].sessionsLevels.end(), - std::string(), [](const std::string& lhs, const std::string& rhs) { - if (lhs.empty()) { - return rhs; - } - return lhs + ", " + rhs; })); - return metas; -} - -std::string NodeSessionMetadata::getSessionKey(const std::set& ignoredNodeNames) const { - if (details.size() == 0) { - return ""; - } - if (std::any_of(ignoredNodeNames.begin(), - ignoredNodeNames.end(), - [this](auto& ignoredNodeName) { - bool notFound = (this->details.find(ignoredNodeName) == this->details.end()); - if (notFound) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to create session key ignoring subsession name: {} but it does not exist", ignoredNodeName); - } - return notFound; - })) { - throw std::logic_error("Tried to create session key ignoring non-existing subsession"); - } - std::stringstream ss; - size_t j = 0; - for (int32_t i = sessionsLevels.size() - 1; i >= 0; --i, ++j) { - if ((ignoredNodeNames.size() > 0) && - (ignoredNodeNames.size() > j) && - ((sessionsLevels.size() - ignoredNodeNames.size()) >= 0) && - (ignoredNodeNames.find(sessionsLevels[i]) == ignoredNodeNames.end())) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first", sessionsLevels[i]); - throw std::logic_error("Cannot collapse sessions not in LIFO order"); - } else { - if (j < ignoredNodeNames.size()) { - continue; - } - if (ss.tellp() > 0) { - ss << "_"; - } - ss << sessionsLevels[i] << "_" << std::get<0>(details.at(sessionsLevels[i])); - } - if (i == 0) { - break; - } - } - return ss.str(); -} - -std::pair NodeSessionMetadata::getCollapsedSessionMetadata(const std::set& ignoredNodeNames) const { - if (ignoredNodeNames.size() == 0) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse subsession with emtpy set"); - throw std::logic_error("Tried to collapse sessions with empty set"); - } - if (std::any_of( - ignoredNodeNames.begin(), - ignoredNodeNames.end(), - [this](auto& ignoredNodeName) { - bool notFound = (this->details.find(ignoredNodeName) == this->details.end()); - if (notFound) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse subsession: {} but it does not exist", ignoredNodeName); - } - return notFound; - })) { - throw std::logic_error("Tried to collapse nonexisting subsession"); - } - for (size_t i = sessionsLevels.size() - 1; i > sessionsLevels.size() - 1 - ignoredNodeNames.size(); --i) { - if (ignoredNodeNames.find(sessionsLevels[i]) == ignoredNodeNames.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first", sessionsLevels[i]); - throw std::logic_error("Cannot collapse sessions not in LIFO order"); - } - } - - NodeSessionMetadata newMeta; - std::copy_if( - std::begin(details), - std::end(details), - std::inserter(newMeta.details, newMeta.details.begin()), - [&ignoredNodeNames](auto& keyValuePair) { - return ignoredNodeNames.find(keyValuePair.first) == ignoredNodeNames.end(); - }); - CollapseDetails collapsingDetails; - for (auto& sessionLevel : sessionsLevels) { - if (ignoredNodeNames.find(sessionLevel) != ignoredNodeNames.end()) { - collapsingDetails.collapsedSessionNames.emplace_back(sessionLevel); - collapsingDetails.collapsedSessionSizes.emplace_back(getSubsessionSize(sessionLevel)); - } else { - newMeta.sessionsLevels.emplace_back(sessionLevel); - } - } - return {newMeta, std::move(collapsingDetails)}; -} - -session_id_t NodeSessionMetadata::getSubsessionSize(const std::string& subsessionName) const { - auto it = details.find(subsessionName); - if (it == details.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to get non-existing subsession: {} size", subsessionName); - throw std::logic_error("Tried to take non existing subsession size"); - } - return std::get<1>(it->second); -} - -session_id_t NodeSessionMetadata::getShardId(const std::set& collapsedNames) const { - if (collapsedNames.size() == 0) { - return 0; - } - if (collapsedNames.size() > sessionsLevels.size()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse more subsession levels than exists"); - throw std::logic_error("Tried to collapse more subsession levels than exists"); - } - for (size_t i = sessionsLevels.size() - 1; i > sessionsLevels.size() - 1 - collapsedNames.size(); --i) { - if (collapsedNames.find(sessionsLevels[i]) == collapsedNames.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Tried to collapse sessions not in LIFO order. Should collapse: {} first, but tried to: {}. SubsessionLevels: {}", - sessionsLevels[i], - std::accumulate(collapsedNames.begin(), - collapsedNames.end(), - std::string(), - [](const std::string& lhs, const std::string rhs) { - if (lhs.empty()) { - return rhs; - } - return lhs + ", " + rhs; - }), - std::accumulate(sessionsLevels.begin(), - sessionsLevels.end(), - std::string(), - [](const std::string& lhs, const std::string rhs) { - if (lhs.empty()) { - return rhs; - } - return lhs + ", " + rhs; - })); - throw std::logic_error("Cannot collapse sessions not in LIFO order"); - } - } - session_id_t multiplyFactor = 1; - session_id_t shardId = 0; - for (size_t i = 0; i < collapsedNames.size(); ++i) { - const auto& subsessionDetails = details.at(*(sessionsLevels.rbegin() + i)); - const auto& [id, sessionSize] = subsessionDetails; - shardId += multiplyFactor * id; - multiplyFactor *= sessionSize; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "getShardId calculation step shardId: {}, multiplyFactor: {}, subsessionId: {}, sessionSize: {}", - shardId, multiplyFactor, id, sessionSize); - } - return shardId; -} -} // namespace ovms diff --git a/src/ovms_lib/nodesessionmetadata.hpp b/src/ovms_lib/nodesessionmetadata.hpp deleted file mode 100644 index bf42bb3472..0000000000 --- a/src/ovms_lib/nodesessionmetadata.hpp +++ /dev/null @@ -1,48 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "session_id.hpp" - -namespace ovms { - -using session_id_t = uint32_t; -using session_key_t = std::string; - -struct CollapseDetails { - std::vector collapsedSessionNames; - std::vector collapsedSessionSizes; -}; - -class NodeSessionMetadata { - std::unordered_map> details; - std::vector sessionsLevels; - -public: - std::vector generateSubsessions(const std::string& nodeName, session_id_t subsessionSize) const; - std::string getSessionKey(const std::set& ignoredNodeNames = {}) const; - std::pair getCollapsedSessionMetadata(const std::set& ignoredNodeNames) const; - session_id_t getSubsessionSize(const std::string& subsessionName) const; - session_id_t getShardId(const std::set& collapsedNames = {}) const; -}; -} // namespace ovms diff --git a/src/ovms_lib/nodesessionresult.hpp b/src/ovms_lib/nodesessionresult.hpp deleted file mode 100644 index fdeb5f64ef..0000000000 --- a/src/ovms_lib/nodesessionresult.hpp +++ /dev/null @@ -1,29 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include "nodesessionmetadata.hpp" -#include "tensormap.hpp" - -namespace ovms { - -using SessionResult = std::pair; -using SessionResults = std::unordered_map; - -} // namespace ovms diff --git a/src/ovms_lib/nodestreamidguard.hpp b/src/ovms_lib/nodestreamidguard.hpp deleted file mode 100644 index 88b1678b75..0000000000 --- a/src/ovms_lib/nodestreamidguard.hpp +++ /dev/null @@ -1,70 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include - -#include "ovinferrequestsqueue.hpp" -#include "profiler.hpp" - -namespace ovms { - -struct NodeStreamIdGuard { - NodeStreamIdGuard(ovms::OVInferRequestsQueue& inferRequestsQueue) : - inferRequestsQueue_(inferRequestsQueue), - futureStreamId(inferRequestsQueue_.getIdleStream()) {} - - ~NodeStreamIdGuard() { - if (!disarmed) { - if (!streamId) { - SPDLOG_DEBUG("Trying to disarm stream Id that is not needed anymore..."); - streamId = futureStreamId.get(); - } - SPDLOG_DEBUG("Returning streamId: {}", streamId.value()); - inferRequestsQueue_.returnStream(streamId.value()); - } - } - - std::optional tryGetId(const uint microseconds = 1) { - OVMS_PROFILE_FUNCTION(); - if (!streamId) { - if (std::future_status::ready == futureStreamId.wait_for(std::chrono::microseconds(microseconds))) { - streamId = futureStreamId.get(); - } - } - return streamId; - } - - bool tryDisarm(const uint microseconds = 1) { - if (std::future_status::ready == futureStreamId.wait_for(std::chrono::microseconds(microseconds))) { - streamId = futureStreamId.get(); - SPDLOG_DEBUG("Returning streamId:", streamId.value()); - inferRequestsQueue_.returnStream(streamId.value()); - disarmed = true; - } - return disarmed; - } - -private: - ovms::OVInferRequestsQueue& inferRequestsQueue_; - std::future futureStreamId; - std::optional streamId = std::nullopt; - bool disarmed = false; -}; -} // namespace ovms diff --git a/src/ovms_lib/ov_utils.cpp b/src/ovms_lib/ov_utils.cpp deleted file mode 100644 index cd443c0512..0000000000 --- a/src/ovms_lib/ov_utils.cpp +++ /dev/null @@ -1,131 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "ov_utils.hpp" - -#include -#include -#include -#include -#include -#include - -#include - -#include "logging.hpp" -#include "profiler.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -// This creates tensor without data ownership. -ov::Tensor createSharedTensor(ov::element::Type_t precision, const shape_t& shape, void* data) { - auto tensor = ov::Tensor(precision, shape, data); - return tensor; -} - -Status createSharedTensor(ov::Tensor& destinationTensor, ov::element::Type_t precision, const ov::Shape& shape) { - destinationTensor = ov::Tensor(precision, shape); - return StatusCode::OK; -} - -std::string getTensorMapString(const std::map>& inputsInfo) { - std::stringstream stringStream; - for (const auto& pair : inputsInfo) { - const auto& name = pair.first; - auto inputInfo = pair.second; - auto precision = inputInfo->getPrecision(); - auto layout = inputInfo->getLayout(); - auto shape = inputInfo->getShape(); - - stringStream << "\nname: " << name - << "; mapping: " << inputInfo->getMappedName() - << "; shape: " << shape.toString() - << "; precision: " << TensorInfo::getPrecisionAsString(precision) - << "; layout: " << TensorInfo::getStringFromLayout(layout); - } - return stringStream.str(); -} - -Status tensorClone(ov::Tensor& destinationTensor, const ov::Tensor& sourceTensor) { - OVMS_PROFILE_FUNCTION(); - destinationTensor = ov::Tensor(sourceTensor.get_element_type(), sourceTensor.get_shape()); - - if (destinationTensor.get_byte_size() != sourceTensor.get_byte_size()) { - SPDLOG_ERROR("tensorClone byte size mismatch destination:{}; source:{}", - destinationTensor.get_byte_size(), - sourceTensor.get_byte_size()); - return StatusCode::OV_CLONE_TENSOR_ERROR; - } - std::memcpy(destinationTensor.data(), sourceTensor.data(), sourceTensor.get_byte_size()); - return StatusCode::OK; -} - -std::optional getLayoutFromRTMap(const ov::RTMap& rtMap) { - for (const auto& [k, v] : rtMap) { - try { - return v.as().value; - } catch (ov::Exception& e) { - } - } - return std::nullopt; -} - -void insertSupportedKeys(std::set& aggregatedPluginSupportedConfigKeys, const std::string& pluginName, const ov::Core& ieCore) { - const std::string supportedConfigKey = METRIC_KEY(SUPPORTED_CONFIG_KEYS); - try { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validating plugin: {}; configuration", pluginName); - std::vector pluginSupportedConfigKeys = ieCore.get_property(pluginName, supportedConfigKey).as>(); - std::set pluginSupportedConfigKeysSet(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); - aggregatedPluginSupportedConfigKeys.insert(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end()); - } catch (std::exception& e) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value. Error: {}", pluginName, supportedConfigKey, e.what()); - } catch (...) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Exception thrown from IE when requesting plugin: {}; key: {}; value.", pluginName, supportedConfigKey); - } -} - -Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore) { - std::set pluginSupportedConfigKeys; - std::string pluginDelimiter = ":"; - auto pluginDelimeterPos = targetDevice.find(pluginDelimiter); - if (pluginDelimeterPos != std::string::npos) { - std::string pluginName = targetDevice.substr(0, pluginDelimeterPos); - insertSupportedKeys(pluginSupportedConfigKeys, pluginName, ieCore); - char deviceDelimiter = ','; - std::stringstream ss(targetDevice.substr(pluginDelimeterPos + 1, targetDevice.length())); - std::string deviceName; - - while (getline(ss, deviceName, deviceDelimiter)) { - insertSupportedKeys(pluginSupportedConfigKeys, deviceName, ieCore); - } - } else { - insertSupportedKeys(pluginSupportedConfigKeys, targetDevice, ieCore); - } - - for (auto& config : pluginConfig) { - if (std::find(pluginSupportedConfigKeys.begin(), pluginSupportedConfigKeys.end(), config.first) == pluginSupportedConfigKeys.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config key: {} not found in supported config keys for device: {}.", config.first, targetDevice); - SPDLOG_LOGGER_INFO(modelmanager_logger, "List of supported keys for this device:"); - for (std::string supportedKey : pluginSupportedConfigKeys) { - SPDLOG_LOGGER_INFO(modelmanager_logger, "{}", supportedKey); - } - return StatusCode::MODEL_CONFIG_INVALID; - } - } - - return StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/ov_utils.hpp b/src/ovms_lib/ov_utils.hpp deleted file mode 100644 index 2f52005066..0000000000 --- a/src/ovms_lib/ov_utils.hpp +++ /dev/null @@ -1,47 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include -#include - -#include "modelconfig.hpp" -#include "status.hpp" - -namespace ovms { - -class TensorInfo; - -Status createSharedTensor(ov::Tensor& destinationTensor, ov::element::Type_t precision, const ov::Shape& shape); -/** - * Creates new tensor that copies data and owns the copy - **/ -ov::Tensor createSharedTensor(ov::element::Type_t precision, const shape_t& shape, void* data); - -std::string getTensorMapString(const std::map>& tensorMap); - -Status tensorClone(ov::Tensor& destinationTensor, const ov::Tensor& sourceTensor); - -std::optional getLayoutFromRTMap(const ov::RTMap& rtMap); - -Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore); - -} // namespace ovms diff --git a/src/ovms_lib/ovinferrequestsqueue.hpp b/src/ovms_lib/ovinferrequestsqueue.hpp deleted file mode 100644 index 9421e9f133..0000000000 --- a/src/ovms_lib/ovinferrequestsqueue.hpp +++ /dev/null @@ -1,45 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "queue.hpp" - -namespace ovms { - -class OVInferRequestsQueue : public Queue { -public: - OVInferRequestsQueue(ov::CompiledModel& compiledModel, int streamsLength) : - Queue(streamsLength) { - for (int i = 0; i < streamsLength; ++i) { - streams[i] = i; - inferRequests.push_back(compiledModel.create_infer_request()); - } - } -}; - -} // namespace ovms diff --git a/src/ovms_lib/pipeline.cpp b/src/ovms_lib/pipeline.cpp deleted file mode 100644 index 965bf386cb..0000000000 --- a/src/ovms_lib/pipeline.cpp +++ /dev/null @@ -1,270 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "pipeline.hpp" - -#include -#include -#include -#include -#include - -#include "logging.hpp" -#include "node.hpp" -#include "pipelineeventqueue.hpp" -#include "profiler.hpp" - -namespace ovms { - -using DeferredNodeSessions = std::vector, session_key_t>>; - -Pipeline::~Pipeline() = default; - -Pipeline::Pipeline(Node& entry, Node& exit, const std::string& name) : - name(name), - entry(entry), - exit(exit) {} - -void Pipeline::push(std::unique_ptr node) { - nodes.emplace_back(std::move(node)); -} -void Pipeline::connect(Node& from, Node& to, const Aliases& tensorNamesMapping) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Connecting from: {}, to: {}", from.getName(), to.getName()); - printNodeConnections(to.getName(), from.getName(), tensorNamesMapping); - from.addDependant(to); - to.addDependency(from, tensorNamesMapping); -} - -void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs) { - if (spdlog::default_logger()->level() > spdlog::level::debug) { - return; - } - std::stringstream ss; - ss << "Links from:" << sourceNode << " to:" << nodeName << ":\n"; - for (auto& pair : pairs) { - ss << "\t" << nodeName << "[" << pair.second << "]=" << sourceNode << "[" << pair.first << "]\n"; - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, ss.str()); -} - -void setFailIfNotFailEarlier(ovms::Status& earlierStatusCode, ovms::Status& newFailStatus) { - if (earlierStatusCode.ok()) { - earlierStatusCode = newFailStatus; - } -} - -#define IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE \ - if (!firstErrorStatus.ok()) { \ - if (finishedSessions.size() == startedSessions.size()) { \ - break; \ - } else { \ - continue; \ - } \ - } - -#define CHECK_AND_LOG_ERROR(NODE) \ - if (!status.ok()) { \ - setFailIfNotFailEarlier(firstErrorStatus, status); \ - SPDLOG_LOGGER_WARN(dag_executor_logger, "Executing pipeline: {} node: {} session: {} failed with ret code: {}, error message: {}", \ - getName(), NODE.getName(), sessionKey, status.getCode(), status.string()); \ - } - -Status Pipeline::execute() { - OVMS_PROFILE_FUNCTION(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Started execution of pipeline: {}", getName()); - PipelineEventQueue finishedNodeQueue; - ovms::Status firstErrorStatus{ovms::StatusCode::OK}; - std::set startedSessions; - std::set finishedSessions; - NodeSessionMetadata meta; - // entry node does not have setInputsCalled so it has no - // session created. Here is just assumption that this meta has the same key - // that the one in EntryNode::execute(); - auto entrySessionKey = meta.getSessionKey(); - startedSessions.emplace(entry.getName() + entrySessionKey); - ovms::Status status = entry.execute(entrySessionKey, finishedNodeQueue); // first node will triger first message - if (!status.ok()) { - SPDLOG_LOGGER_WARN(dag_executor_logger, "Executing pipeline: {} node: {} failed with: {}", - getName(), entry.getName(), status.string()); - return status; - } - DeferredNodeSessions deferredNodeSessions; - const uint WAIT_FOR_FINISHED_NODE_TIMEOUT_MICROSECONDS = 5000; - const uint WAIT_FOR_DEFERRED_NODE_DISARM_TIMEOUT_MICROSECONDS = 500; - // process finished session nodes and if no one is finished check if any node session with deferred execution - // has necessary resources already - while (true) { - spdlog::trace("Pipeline: {} waiting for message that node finished.", getName()); - OVMS_PROFILE_SYNC_BEGIN("PipelineEventQueue::tryPull"); - auto optionallyFinishedNode = finishedNodeQueue.tryPull(WAIT_FOR_FINISHED_NODE_TIMEOUT_MICROSECONDS); - OVMS_PROFILE_SYNC_END("PipelineEventQueue::tryPull"); - if (optionallyFinishedNode) { - OVMS_PROFILE_SCOPE_S("Processing Finished Node", "node_name", optionallyFinishedNode.value().first.get().getName().c_str()); - /* - Get results from finished node session. - */ - auto& [finishedNodeRef, sessionKey] = optionallyFinishedNode.value(); - Node& finishedNode = finishedNodeRef.get(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Pipeline: {} got message that node: {} session: {} finished.", getName(), finishedNode.getName(), sessionKey); - finishedSessions.emplace(finishedNode.getName() + sessionKey); - if (!firstErrorStatus.ok()) { - finishedNode.release(sessionKey); - } - IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE - SessionResults sessionResults; - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Fetching results of pipeline: {} node: {} session: {}", getName(), finishedNode.getName(), sessionKey); - status = finishedNode.fetchResults(sessionKey, sessionResults); - CHECK_AND_LOG_ERROR(finishedNode) - IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE - - /* - Feed next node sessions with results from currently finished node session. - */ - auto& nextNodesFromFinished = finishedNode.getNextNodes(); - for (auto& nextNode : nextNodesFromFinished) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "setting pipeline: {} node: {} session: {} outputs as inputs for node: {}", - getName(), finishedNode.getName(), sessionKey, nextNode.get().getName()); - status = nextNode.get().setInputs(finishedNode, sessionResults); - CHECK_AND_LOG_ERROR(nextNode.get()) - if (!firstErrorStatus.ok()) { - break; - } - } - - /* - Try to schedule node sessions that are following the currently finished session. - Defer next node sessions which are ready, but stream id is not ready yet. - Save defered node sessions to temporary container which will be later merged into global container. - */ - OVMS_PROFILE_SYNC_BEGIN("Try next nodes"); - DeferredNodeSessions tmpDeferredNodeSessions; - for (auto& nextNode : nextNodesFromFinished) { - auto readySessions = nextNode.get().getReadySessions(); - for (auto& sessionKey : readySessions) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Started execution of pipeline: {} node: {} session: {}", getName(), nextNode.get().getName(), sessionKey); - startedSessions.emplace(nextNode.get().getName() + sessionKey); - status = nextNode.get().execute(sessionKey, finishedNodeQueue); - if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", nextNode.get().getName(), sessionKey); - tmpDeferredNodeSessions.emplace_back(nextNode.get(), sessionKey); - status = StatusCode::OK; - } - CHECK_AND_LOG_ERROR(nextNode.get()) - if (!firstErrorStatus.ok()) { - break; - } - } - } - OVMS_PROFILE_SYNC_END("Try next nodes"); - - /* - Iterate over global container of deferred node sessions and try to schedule. - Keep in mind that newly deferred nodes are not iterated since those are in temporary container. - This is expected since newly deferred nodes were just checked for possible availability of stream ID in previous step. - */ - OVMS_PROFILE_SYNC_BEGIN("Try deferred nodes"); - for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { - // Quit trying to schedule deferred nodes since handling newly finished node has bigger priority (the node can unlock stream ID or allow scheduling next nodes) - if (finishedNodeQueue.size() > 0) { - break; - } - auto& [nodeRef, sessionKey] = *it; - auto& node = nodeRef.get(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to trigger node: {} session: {} execution", node.getName(), sessionKey); - status = node.execute(sessionKey, finishedNodeQueue); - if (status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} is ready", node.getName(), sessionKey); - it = deferredNodeSessions.erase(it); - continue; - } - it++; - if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", node.getName(), sessionKey); - status = StatusCode::OK; - } else { - CHECK_AND_LOG_ERROR(node) - } - } - OVMS_PROFILE_SYNC_END("Try deferred nodes"); - - /* - Merge temporary and global deferred node session containers. - */ - OVMS_PROFILE_SYNC_BEGIN("Merge deferred containers"); - deferredNodeSessions.insert( - deferredNodeSessions.end(), - tmpDeferredNodeSessions.begin(), - tmpDeferredNodeSessions.end()); - OVMS_PROFILE_SYNC_END("Merge deferred containers"); - - if (startedSessions.size() == finishedSessions.size()) { - break; - } - } else { - OVMS_PROFILE_SCOPE("No new finished nodes"); - // If error occurred earlier, disarm stream id guards of all deferred nodes and exit - if (!firstErrorStatus.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Will try to disarm all stream id guards of all {} deferred node sessions due to previous error in pipeline", deferredNodeSessions.size()); - if (deferredNodeSessions.size() > 0) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to disarm {} remaining deferred node sessions ...", deferredNodeSessions.size()); - for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { - auto& [nodeRef, sessionKey] = *it; - auto& node = nodeRef.get(); - if (node.tryDisarm(sessionKey, WAIT_FOR_DEFERRED_NODE_DISARM_TIMEOUT_MICROSECONDS)) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Stream id guard disarm of node {} session: {} has succeeded", node.getName(), sessionKey); - finishedSessions.emplace(node.getName() + sessionKey); - it = deferredNodeSessions.erase(it); - } else { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Cannot disarm stream id guard of node: {}, session: {} yet, will try again later", node.getName(), sessionKey); - it++; - } - } - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Disarming iteration completed, remaining deferred node sessions count: {}", deferredNodeSessions.size()); - } - // Check for deferred node queue size again to indicate if all nodes got freed - if (deferredNodeSessions.size() > 0) { - continue; - } else { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Disarming all stream id guards of deferred nodes completed, pipeline will shut down"); - IF_ERROR_OCCURRED_EARLIER_THEN_BREAK_IF_ALL_STARTED_FINISHED_CONTINUE_OTHERWISE - } - } - // else scope could be executed always however it seems most reasonable at the time to - // free blocked inferRequests from exeuction first rather than free models for reloading - OVMS_PROFILE_SYNC_BEGIN("Try deferred nodes"); - for (auto it = deferredNodeSessions.begin(); it != deferredNodeSessions.end();) { - auto& [nodeRef, sessionKey] = *it; - auto& node = nodeRef.get(); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Trying to trigger node: {} session: {} execution", node.getName(), sessionKey); - status = node.execute(sessionKey, finishedNodeQueue); - if (status.ok()) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} is ready", node.getName(), sessionKey); - it = deferredNodeSessions.erase(it); - continue; - } - it++; - if (status == StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Node: {} session: {} not ready for execution yet", node.getName(), sessionKey); - status = StatusCode::OK; - } else { - CHECK_AND_LOG_ERROR(node) - } - } - OVMS_PROFILE_SYNC_END("Try deferred nodes"); - } - } - return firstErrorStatus; -} -} // namespace ovms diff --git a/src/ovms_lib/pipeline.hpp b/src/ovms_lib/pipeline.hpp deleted file mode 100644 index 71e4cb373b..0000000000 --- a/src/ovms_lib/pipeline.hpp +++ /dev/null @@ -1,63 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include "aliases.hpp" -#include "status.hpp" - -namespace ovms { - -class Node; -template -class EntryNode; -template -class ExitNode; - -void printNodeConnections(const std::string& nodeName, const std::string& sourceNode, const Aliases& pairs); - -class Pipeline { - std::vector> nodes; - const std::string name; - Node& entry; - Node& exit; - -public: - Pipeline(Node& entry, Node& exit, const std::string& name = "default_name"); - - void push(std::unique_ptr node); - ~Pipeline(); - - Node& getEntry() const { return this->entry; } - Node& getExit() const { return this->exit; } - - static void connect(Node& from, Node& to, const Aliases& tensorNamesMapping); - - Status execute(); - const std::string& getName() const { - return name; - } - -private: - std::map prepareStatusMap() const; -}; - -} // namespace ovms diff --git a/src/ovms_lib/pipeline_factory.cpp b/src/ovms_lib/pipeline_factory.cpp deleted file mode 100644 index 73588b2260..0000000000 --- a/src/ovms_lib/pipeline_factory.cpp +++ /dev/null @@ -1,146 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "pipeline_factory.hpp" - -#include "logging.hpp" -#include "pipeline.hpp" -#include "pipelinedefinition.hpp" -#include "prediction_service_utils.hpp" - -namespace ovms { - -bool PipelineFactory::definitionExists(const std::string& name) const { - std::shared_lock lock(definitionsMtx); - return definitions.find(name) != definitions.end(); -} - -PipelineDefinition* PipelineFactory::findDefinitionByName(const std::string& name) const { - std::shared_lock lock(definitionsMtx); - auto it = definitions.find(name); - if (it == std::end(definitions)) { - return nullptr; - } else { - return it->second.get(); - } -} - -void PipelineFactory::retireOtherThan(std::set&& pipelinesInConfigFile, ModelManager& manager) { - std::for_each(definitions.begin(), - definitions.end(), - [&pipelinesInConfigFile, &manager](auto& nameDefinitionPair) { - if (pipelinesInConfigFile.find(nameDefinitionPair.second->getName()) == pipelinesInConfigFile.end() && nameDefinitionPair.second->getStateCode() != PipelineDefinitionStateCode::RETIRED) { - nameDefinitionPair.second->retire(manager); - } - }); -} - -Status PipelineFactory::createDefinition(const std::string& pipelineName, - const std::vector& nodeInfos, - const pipeline_connections_t& connections, - ModelManager& manager) { - if (definitionExists(pipelineName)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "pipeline definition: {} is already created", pipelineName); - return StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST; - } - std::unique_ptr pipelineDefinition = std::make_unique(pipelineName, nodeInfos, connections); - - pipelineDefinition->makeSubscriptions(manager); - Status validationResult = pipelineDefinition->validate(manager); - if (!validationResult.ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline definition: {} failed: {}", pipelineName, validationResult.string()); - if (validationResult == StatusCode::PIPELINE_NAME_OCCUPIED) { - pipelineDefinition->resetSubscriptions(manager); - return validationResult; - } - } else { - SPDLOG_LOGGER_INFO(modelmanager_logger, "Loading pipeline definition: {} succeeded", pipelineName); - } - - std::unique_lock lock(definitionsMtx); - definitions[pipelineName] = std::move(pipelineDefinition); - - return validationResult; -} - -template -Status PipelineFactory::createInternal(std::unique_ptr& pipeline, - const std::string& name, - const RequestType* request, - ResponseType* response, - ModelManager& manager) const { - if (!definitionExists(name)) { - SPDLOG_LOGGER_INFO(dag_executor_logger, "Pipeline with requested name: {} does not exist", name); - return StatusCode::PIPELINE_DEFINITION_NAME_MISSING; - } - std::shared_lock lock(definitionsMtx); - auto& definition = *definitions.at(name); - lock.unlock(); - return definition.create(pipeline, request, response, manager); -} -Status PipelineFactory::create(std::unique_ptr& pipeline, - const std::string& name, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, - ModelManager& manager) const { - return this->createInternal(pipeline, name, request, response, manager); -} -Status PipelineFactory::create(std::unique_ptr& pipeline, - const std::string& name, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - ModelManager& manager) const { - return this->createInternal(pipeline, name, request, response, manager); -} - -Status PipelineFactory::reloadDefinition(const std::string& pipelineName, - const std::vector&& nodeInfos, - const pipeline_connections_t&& connections, - ModelManager& manager) { - auto pd = findDefinitionByName(pipelineName); - if (pd == nullptr) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested to reload pipeline definition but it does not exist: {}", pipelineName); - return StatusCode::UNKNOWN_ERROR; - } - return pd->reload(manager, std::move(nodeInfos), std::move(connections)); -} - -Status PipelineFactory::revalidatePipelines(ModelManager& manager) { - Status firstErrorStatus = StatusCode::OK; - for (auto& [name, definition] : definitions) { - if (definition->getStatus().isRevalidationRequired()) { - auto validationResult = definition->validate(manager); - if (!validationResult.ok()) { - if (firstErrorStatus.ok()) { - firstErrorStatus = validationResult; - } - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Revalidation pipeline definition: {} failed: {}", name, validationResult.string()); - } else { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Revalidation of pipeline: {} succeeded", name); - } - } - } - return firstErrorStatus; -} -const std::vector PipelineFactory::getPipelinesNames() const { - std::vector names; - std::shared_lock lock(definitionsMtx); - names.reserve(definitions.size()); - for (auto& [name, definition] : definitions) { - names.push_back(definition->getName()); - } - return names; -} -} // namespace ovms diff --git a/src/ovms_lib/pipeline_factory.hpp b/src/ovms_lib/pipeline_factory.hpp deleted file mode 100644 index b3780b33a3..0000000000 --- a/src/ovms_lib/pipeline_factory.hpp +++ /dev/null @@ -1,84 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "kfs_grpc_inference_service.hpp" -#include "nodeinfo.hpp" -#include "status.hpp" - -namespace ovms { - -class ModelManager; -class Pipeline; -class PipelineDefinition; - -class PipelineFactory { - std::map> definitions; - mutable std::shared_mutex definitionsMtx; - -public: - Status createDefinition(const std::string& pipelineName, - const std::vector& nodeInfos, - const pipeline_connections_t& connections, - ModelManager& manager); - - bool definitionExists(const std::string& name) const; - -private: - template - Status createInternal(std::unique_ptr& pipeline, - const std::string& name, - const RequestType* request, - ResponseType* response, - ModelManager& manager) const; - -public: - Status create(std::unique_ptr& pipeline, - const std::string& name, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, - ModelManager& manager) const; - Status create(std::unique_ptr& pipeline, - const std::string& name, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - ModelManager& manager) const; - - PipelineDefinition* findDefinitionByName(const std::string& name) const; - Status reloadDefinition(const std::string& pipelineName, - const std::vector&& nodeInfos, - const pipeline_connections_t&& connections, - ModelManager& manager); - - void retireOtherThan(std::set&& pipelinesInConfigFile, ModelManager& manager); - Status revalidatePipelines(ModelManager&); - const std::vector getPipelinesNames() const; -}; - -} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinition.cpp b/src/ovms_lib/pipelinedefinition.cpp deleted file mode 100644 index 0c42bd3e83..0000000000 --- a/src/ovms_lib/pipelinedefinition.cpp +++ /dev/null @@ -1,1367 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "pipelinedefinition.hpp" - -#include -#include -#include - -#include "custom_node.hpp" -#include "dl_node.hpp" -#include "entry_node.hpp" -#include "exit_node.hpp" -#include "logging.hpp" -#include "modelmanager.hpp" -#include "node_library_utils.hpp" -#include "pipeline.hpp" -#include "pipelinedefinitionunloadguard.hpp" -#include "prediction_service_utils.hpp" - -namespace ovms { - -Status toNodeKind(const std::string& str, NodeKind& nodeKind) { - if (str == DL_NODE_CONFIG_TYPE) { - nodeKind = NodeKind::DL; - return StatusCode::OK; - } - if (str == CUSTOM_NODE_CONFIG_TYPE) { - nodeKind = NodeKind::CUSTOM; - return StatusCode::OK; - } - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Unsupported node type: {}", str); - return StatusCode::PIPELINE_NODE_WRONG_KIND_CONFIGURATION; -} - -Status PipelineDefinition::validate(ModelManager& manager) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of pipeline: {}", getName()); - ValidationResultNotifier notifier(status, loadedNotify); - auto& models = manager.getModels(); - if (std::find_if(models.begin(), models.end(), [this](auto pair) { return this->pipelineName == pair.first; }) != models.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline name: {} is already occupied by model.", pipelineName); - return StatusCode::PIPELINE_NAME_OCCUPIED; - } - - Status validationResult = initializeNodeResources(manager); - if (!validationResult.ok()) { - return validationResult; - } - validationResult = validateNodes(manager); - if (!validationResult.ok()) { - return validationResult; - } - validationResult = validateForCycles(); - if (!validationResult.ok()) { - return validationResult; - } - validationResult = validateDemultiplexerGatherNodesOrder(); - if (!validationResult.ok()) { - return validationResult; - } - std::unique_lock lock(metadataMtx); - validationResult = updateInputsInfo(manager); - if (!validationResult.ok()) { - return validationResult; - } - validationResult = updateOutputsInfo(manager); - if (!validationResult.ok()) { - return validationResult; - } - lock.unlock(); - notifier.passed = true; - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Finished validation of pipeline: {}", getName()); - SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} inputs: {}", getName(), getTensorMapString(inputsInfo)); - SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} outputs: {}", getName(), getTensorMapString(outputsInfo)); - return validationResult; -} - -Status PipelineDefinition::initializeNodeResources(ModelManager& manager) { - for (const auto& nodeInfo : nodeInfos) { - if (nodeInfo.kind == NodeKind::CUSTOM) { - void* customNodeLibraryInternalManager = nullptr; - auto params = createCustomNodeParamArray(nodeInfo.parameters); - int paramsLength = nodeInfo.parameters.size(); - if (!nodeInfo.library.isValid()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to invalid library", pipelineName, nodeInfo.nodeName); - return StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY; - } - auto status = nodeInfo.library.initialize(&customNodeLibraryInternalManager, params.get(), paramsLength); - if (status != 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Initialization of library with base path: {} failed", nodeInfo.library.basePath); - return StatusCode::NODE_LIBRARY_INITIALIZE_FAILED; - } - std::shared_ptr sharedCustomNodeLibraryInternalManager(new CNLIMWrapper{customNodeLibraryInternalManager, nodeInfo.library.deinitialize}); - manager.addResourceToCleaner(sharedCustomNodeLibraryInternalManager); - nodeResources.emplace(std::make_pair(nodeInfo.nodeName, std::move(sharedCustomNodeLibraryInternalManager))); - } - } - return StatusCode::OK; -} - -// returns NodeInfos that are in PipelineDefinition, but are not in nodeInfos(std::vector argument) -std::vector PipelineDefinition::calculateNodeInfosDiff(const std::vector& nodeInfos) { - std::vector diff; - for (const auto& nodeInfo : this->nodeInfos) { - auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), - [&nodeInfo](const auto& x) { return x.nodeName == nodeInfo.nodeName; }); - if (it == nodeInfos.end()) { - diff.push_back(nodeInfo); - } - } - return diff; -} - -void PipelineDefinition::deinitializeNodeResources(const std::vector& nodeInfosDiff) { - for (const auto& nodeInfo : nodeInfosDiff) { - if (nodeInfo.kind == NodeKind::CUSTOM) { - if (nodeResources.find(nodeInfo.nodeName) == nodeResources.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Library deinitialization of Node: {} failed. Couldn't find any initialized resources", nodeInfo.nodeName); - continue; - } - nodeResources.erase(nodeInfo.nodeName); - } - } -} - -Status PipelineDefinition::reload(ModelManager& manager, const std::vector&& nodeInfos, const pipeline_connections_t&& connections) { - // block creating new unloadGuards - this->status.handle(ReloadEvent()); - resetSubscriptions(manager); - while (requestsHandlesCounter > 0) { - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - // deinitialize all resources that are associated with nodes that are currently in PipelineDefinition, but not in nodeInfos - deinitializeNodeResources(calculateNodeInfosDiff(nodeInfos)); - this->nodeInfos = std::move(nodeInfos); - this->connections = std::move(connections); - makeSubscriptions(manager); - - return validate(manager); -} - -void PipelineDefinition::retire(ModelManager& manager) { - resetSubscriptions(manager); - this->status.handle(RetireEvent()); - while (requestsHandlesCounter > 0) { - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - // deinitalize all resources - deinitializeNodeResources(this->nodeInfos); - this->nodeResources.clear(); - this->nodeInfos.clear(); - this->connections.clear(); -} - -Status PipelineDefinition::waitForLoaded(std::unique_ptr& unloadGuard, const uint waitForLoadedTimeoutMicroseconds) { - unloadGuard = std::make_unique(*this); - - const uint waitLoadedTimestepMicroseconds = 100; - const uint waitCheckpoints = waitForLoadedTimeoutMicroseconds / waitLoadedTimestepMicroseconds; - uint waitCheckpointsCounter = waitCheckpoints; - std::mutex cvMtx; - std::unique_lock cvLock(cvMtx); - while (waitCheckpointsCounter-- != 0) { - if (status.isAvailable()) { - SPDLOG_DEBUG("Successfully waited for pipeline definition: {}", getName()); - return StatusCode::OK; - } - unloadGuard.reset(); - if (!status.canEndLoaded()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Waiting for available state for pipeline: {}, with timestep: {}us timeout: {}us check count: {}", - getName(), waitLoadedTimestepMicroseconds, waitForLoadedTimeoutMicroseconds, waitCheckpointsCounter); - loadedNotify.wait_for(cvLock, - std::chrono::microseconds(waitLoadedTimestepMicroseconds), - [this]() { - return this->status.isAvailable() || - !this->status.canEndLoaded(); - }); - unloadGuard = std::make_unique(*this); - } - if (!status.isAvailable()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Succesfully waited for pipeline definition: {}", getName()); - return StatusCode::OK; -} - -template -Status PipelineDefinition::create(std::unique_ptr& pipeline, - const RequestType* request, - ResponseType* response, - ModelManager& manager) { - std::unique_ptr unloadGuard; - Status status = waitForLoaded(unloadGuard); - if (!status.ok()) { - return status; - } - - std::unordered_map> nodes; - EntryNode* entry = nullptr; - ExitNode* exit = nullptr; - - for (const auto& info : nodeInfos) { - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Creating pipeline: {}. Adding nodeName: {}, modelName: {}", - getName(), info.nodeName, info.modelName); - switch (info.kind) { - case NodeKind::ENTRY: { - auto node = std::make_unique>(request, getInputsInfo(), info.demultiplyCount); - entry = node.get(); - nodes.emplace(info.nodeName, std::move(node)); - break; - } - case NodeKind::DL: - nodes.emplace(info.nodeName, std::make_unique( - info.nodeName, - info.modelName, - info.modelVersion, - manager, - info.outputNameAliases, - info.demultiplyCount, - info.gatherFromNode)); - break; - case NodeKind::CUSTOM: - nodes.emplace(info.nodeName, std::make_unique( - info.nodeName, - info.library, - info.parameters, - info.outputNameAliases, - info.demultiplyCount, - info.gatherFromNode, - nodeResources.at(info.nodeName))); - break; - case NodeKind::EXIT: { - auto node = std::make_unique>(response, getOutputsInfo(), info.gatherFromNode); - exit = node.get(); - nodes.emplace(info.nodeName, std::move(node)); - break; - } - default: - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Requested pipeline: {} contains unknown node kind", getName()); - throw std::invalid_argument("unknown node kind"); - } - } - for (const auto& kv : connections) { - const auto& dependantNode = nodes.at(kv.first); - for (const auto& pair : kv.second) { - const auto& dependencyNode = nodes.at(pair.first); - SPDLOG_LOGGER_DEBUG(dag_executor_logger, "Connecting pipeline: {}, from: {}, to: {}", getName(), dependencyNode->getName(), dependantNode->getName()); - Pipeline::connect(*dependencyNode, *dependantNode, pair.second); - } - } - pipeline = std::make_unique(*entry, *exit, pipelineName); - for (auto& kv : nodes) { - pipeline->push(std::move(kv.second)); - } - return status; -} - -void PipelineDefinition::resetSubscriptions(ModelManager& manager) { - for (auto& [modelName, modelVersion] : subscriptions) { - if (modelVersion) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Unsubscribing pipeline: {} from model: {}, version: {}", - getName(), modelName, modelVersion); - manager.findModelByName(modelName)->getModelInstanceByVersion(modelVersion)->unsubscribe(*this); - } else { // using default version - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Unsubscribing pipeline: {} from model: {}", - getName(), modelName); - manager.findModelByName(modelName)->unsubscribe(*this); - } - } - subscriptions.clear(); -} - -static std::string createSubscriptionErrorMessage(const std::string& pipelineName, const NodeInfo& nodeInfo) { - std::stringstream ss; - ss << "Pipeline: " << pipelineName << " Failed to make subscription to model: " << nodeInfo.modelName; - if (nodeInfo.modelVersion) { - ss << " version: " << nodeInfo.modelVersion.value(); - } - ss << " because it was missing"; - return ss.str(); -} - -void PipelineDefinition::makeSubscriptions(ModelManager& manager) { - for (auto& node : nodeInfos) { - if (node.kind == NodeKind::DL) { - if (subscriptions.find({node.modelName, node.modelVersion.value_or(0)}) != subscriptions.end()) { - continue; - } - auto model = manager.findModelByName(node.modelName); - if (nullptr == model) { - SPDLOG_LOGGER_WARN(modelmanager_logger, createSubscriptionErrorMessage(getName(), node)); - continue; - } - if (node.modelVersion) { - auto modelInstance = model->getModelInstanceByVersion(node.modelVersion.value()); - if (nullptr == modelInstance) { - SPDLOG_LOGGER_WARN(modelmanager_logger, createSubscriptionErrorMessage(getName(), node)); - continue; - } - modelInstance->subscribe(*this); - } else { - model->subscribe(*this); - } - subscriptions.insert({node.modelName, node.modelVersion.value_or(0)}); - } - } -} - -class NodeValidator { - const std::string& pipelineName; - ModelManager& manager; - const NodeInfo& dependantNodeInfo; - const pipeline_connections_t& connections; - const std::vector& nodeInfos; - std::map>& nodeResources; - const bool isMultiBatchAllowed; - - std::unique_ptr dependantModelUnloadGuard; - std::shared_ptr dependantModelInstance; - std::set remainingUnconnectedDependantInputs; - - tensor_map_t inputsInfo, outputsInfo; - tensor_map_t dependencyInputsInfo, dependencyOutputsInfo; - -public: - NodeValidator( - const std::string& pipelineName, - ModelManager& manager, - const NodeInfo& dependantNodeInfo, - const pipeline_connections_t& connections, - const std::vector& nodeInfos, - std::map>& nodeResources, - const bool isMultiBatchAllowed = true) : - pipelineName(pipelineName), - manager(manager), - dependantNodeInfo(dependantNodeInfo), - connections(connections), - nodeInfos(nodeInfos), - nodeResources(nodeResources), - isMultiBatchAllowed(isMultiBatchAllowed) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validation of pipeline: {}; node name: {}; node kind: {}", - pipelineName, - dependantNodeInfo.nodeName, - dependantNodeInfo.kind); - } - - Status fetchUnderlyingModelInstance() { - if (!manager.getModelInstance( - dependantNodeInfo.modelName, - dependantNodeInfo.modelVersion.value_or(0), - dependantModelInstance, - dependantModelUnloadGuard) - .ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing model: {}; version: {}", - pipelineName, - dependantNodeInfo.modelName, - dependantNodeInfo.modelVersion.value_or(0)); - return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL; - } - return StatusCode::OK; - } - - Status getDependencyNodeInfo(const std::string& dependencyNodeName, std::vector::const_iterator& dependencyNodeInfo) { - // Find dependency node info object. - dependencyNodeInfo = std::find_if( - std::begin(this->nodeInfos), - std::end(this->nodeInfos), - [dependencyNodeName](const NodeInfo& nodeInfo) { return nodeInfo.nodeName == dependencyNodeName; }); - if (dependencyNodeInfo == std::end(this->nodeInfos)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node (name: {}) is connected to missing dependency node (name: {})", - pipelineName, - dependantNodeInfo.nodeName, - dependencyNodeName); - return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_NODE; - } - - if (dependencyNodeInfo->kind == NodeKind::EXIT) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Exit node used as dependency node", - pipelineName); - return StatusCode::PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY; - } - - return StatusCode::OK; - } - - Status checkForForbiddenDynamicParameters() { - const auto& config = dependantModelInstance->getModelConfig(); - if (config.getBatchingMode() == Mode::AUTO || config.anyShapeSetToAuto()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node name: {} used model name: {} with batch/shape parameter set to 'auto' which is forbidden. Use dynamic shape.", - pipelineName, - dependantNodeInfo.nodeName, - dependantNodeInfo.modelName); - return StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER; - } - return StatusCode::OK; - } - - Status validateGatherNode(const NodeInfo& dependantNodeInfo) const { - for (const auto& gather : dependantNodeInfo.gatherFromNode) { - auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), [gather](const NodeInfo& nodeInfo) { return nodeInfo.nodeName == gather; }); - if (it == nodeInfos.end()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Node name: {}, have gather_from: {} which does not exist in pipeline", - pipelineName, - dependantNodeInfo.nodeName, - gather); - return StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE; - } - if (!it->demultiplyCount) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Node name: {}, have gather_from: {} which is not demultiplexer node", - pipelineName, - dependantNodeInfo.nodeName, - gather); - return StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER; - } - } - return StatusCode::OK; - } - - Status checkConnectionMappedToExistingDataSource(const NodeInfo& dependencyNodeInfo, const std::string& dataSource) { - // Check whether dependency node is configured to have required output. - if (dependencyNodeInfo.outputNameAliases.count(dataSource) == 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing dependency node: {} data item: {} for dependant node: {}", - pipelineName, - dependencyNodeInfo.nodeName, - dataSource, - dependantNodeInfo.nodeName); - return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE; - } - - // If dependency node is of type DL model, make sure there is underlying model output present. - if (dependencyNodeInfo.kind == NodeKind::DL || dependencyNodeInfo.kind == NodeKind::CUSTOM) { - // Check whether underlying model contains required output. - const auto& modelOutputName = dependencyNodeInfo.outputNameAliases.at(dataSource); - if (this->dependencyOutputsInfo.count(modelOutputName) == 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Missing output: {} of dependency node: {}; data source: {}", - pipelineName, - modelOutputName, - dependencyNodeInfo.nodeName, - dataSource); - return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT; - } - } - return StatusCode::OK; - } - - Status validateShapeWithDemultiplexer(const Shape& shape, const NodeInfo& demultiplicatorNodeInfo) const { - if (!demultiplicatorNodeInfo.demultiplyCount) { - return StatusCode::OK; - } - if (shape.size() < 3) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} demultiply cannot occur due to not enough shape dimensions: {}", - this->pipelineName, - demultiplicatorNodeInfo.nodeName, - shape.size()); - return StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY; - } - if (demultiplicatorNodeInfo.demultiplyCount.value() != -1) { - if (!shape[0].isAny()) { - auto demultiplyDimension = Dimension(demultiplicatorNodeInfo.demultiplyCount.value()); - if (!shape[0].partiallyFitsInto(demultiplyDimension)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of node: {} does not match tensor first dimension value: {}", - this->pipelineName, - demultiplicatorNodeInfo.demultiplyCount.value(), - demultiplicatorNodeInfo.nodeName, - shape[0].toString()); - return StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT; - } - } else { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is fixed while first dimenson value of node library is not: {}. This pipeline may fail at execution stage.", - this->pipelineName, - demultiplicatorNodeInfo.demultiplyCount.value(), - demultiplicatorNodeInfo.nodeName, - shape[0].toString()); - } - } else if (!shape[0].isAny()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is dynamic while first dimenson value of gather node is not: {}. This pipeline may fail at execution stage.", - this->pipelineName, - demultiplicatorNodeInfo.demultiplyCount.value(), - demultiplicatorNodeInfo.nodeName, - shape[0].toString()); - } - return StatusCode::OK; - } - - Status influenceShapeWithDemultiplexer(Shape& shape, const NodeInfo& demultiplicatorNodeInfo) { - auto result = validateShapeWithDemultiplexer(shape, demultiplicatorNodeInfo); - if (!result.ok()) { - return result; - } - shape.erase(shape.begin()); - return StatusCode::OK; - } - - bool areShapesMatching(const Shape& tensorInputShape, const Shape& tensorOutputShape) { - if (tensorInputShape.size() != tensorOutputShape.size()) { - return false; - } - - for (size_t i = 0; i < tensorInputShape.size(); i++) { - if (!tensorInputShape[i].partiallyFitsInto(tensorOutputShape[i])) { - return false; - } - } - return true; - } - - Status checkConnectionMetadataCorrectness(const NodeInfo& dependencyNodeInfo, const std::string& modelInputName, const std::string& modelOutputName) { - // If validated connection pair connects two DL model/Custom nodes, - // check if both input/output exist and its metadata (shape, precision) matches. - // Affect shape by demultiplexer/gather if applies. - const auto& tensorInput = this->inputsInfo.at(modelInputName); - const auto& tensorOutput = this->dependencyOutputsInfo.at(modelOutputName); - Shape tensorInputShape = tensorInput->getShape(); - Shape tensorOutputShape = tensorOutput->getShape(); - if (dependencyNodeInfo.demultiplyCount) { - auto result = influenceShapeWithDemultiplexer(tensorOutputShape, dependencyNodeInfo); - if (!result.ok()) { - return result; - } - } - if (dependantNodeInfo.gatherFromNode.size() == 1) { - std::vector::const_iterator demultiplicatorNode; - auto result = getDependencyNodeInfo(*dependantNodeInfo.gatherFromNode.begin(), demultiplicatorNode); - if (!result.ok()) { - return result; - } - result = influenceShapeWithDemultiplexer(tensorInputShape, *demultiplicatorNode); - if (!result.ok()) { - SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of gather_from node: {} does not match tensor first dimenson value: {} of node: {}", - this->pipelineName, - demultiplicatorNode->demultiplyCount.value(), - demultiplicatorNode->nodeName, - tensorInputShape[1].toString(), - dependencyNodeInfo.nodeName); - return result; - } - } else if (dependantNodeInfo.gatherFromNode.size() > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Manual gathering from multiple nodes is not supported in node name: {}", - this->pipelineName, - dependantNodeInfo.nodeName); - return StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED; - } - if (!areShapesMatching(tensorInputShape, tensorOutputShape)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Shape mismatch between: dependant node: {}; input: {}; shape: {} vs dependency node: {}; output: {}; shape: {}", - pipelineName, - dependantNodeInfo.nodeName, - modelInputName, - tensorInputShape.toString(), - dependencyNodeInfo.nodeName, - modelOutputName, - tensorOutputShape.toString()); - return StatusCode::INVALID_SHAPE; - } - if (tensorInput->getPrecision() != tensorOutput->getPrecision()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Precision mismatch between: dependant node: {}; input: {}; precision: {} vs dependency node: {}; output: {}; precision: {}", - pipelineName, - dependantNodeInfo.nodeName, - modelInputName, - tensorInput->getPrecisionAsString(), - dependencyNodeInfo.nodeName, - modelOutputName, - tensorOutput->getPrecisionAsString()); - return StatusCode::INVALID_PRECISION; - } - return StatusCode::OK; - } - - void prepareRemainingUnconnectedDependantInputsSet() { - // Save set of inputs which are required by underlying model/custom node of currently validated node. - // This is later used to make sure we feed each input exactly one data source. - std::transform( - this->inputsInfo.begin(), - this->inputsInfo.end(), - std::inserter( - remainingUnconnectedDependantInputs, - remainingUnconnectedDependantInputs.end()), - [](auto pair) { return pair.first; }); - } - - Status ensureAllModelInputsOfValidatedNodeHaveDataSource() { - // Make sure all model inputs of validated node is fed with some data source. - if (remainingUnconnectedDependantInputs.size() > 0) { - std::stringstream ss; - for (const auto& input : remainingUnconnectedDependantInputs) { - ss << input << ", "; - } - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} has inputs:: {} not connected to any source", - pipelineName, - dependantNodeInfo.nodeName, - ss.str()); - return StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED; - } - return StatusCode::OK; - } - - Status markInputAsConnected(const std::string& name) { - // If currently validated node is of type DL model or Custom, mark its input as connected - // by erasing from previously gathered input set. - // If such input cannot be found in the map, it means we refer - // to non existing model input or we already connected it to some other data source which is invalid. - if (this->inputsInfo.count(name) == 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} has no input with name: {}", - pipelineName, - dependantNodeInfo.nodeName, - name); - return StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT; - } - if (remainingUnconnectedDependantInputs.erase(name) == 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} input name: {} is connected to more than one data source", - pipelineName, - dependantNodeInfo.nodeName, - name); - return StatusCode::PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES; - } - return StatusCode::OK; - } - - Status validateConnection(const NodeInfo& dependencyNodeInfo, const Aliases& mapping) { - // At this point dependency node can only be either DL model node, Custom node or entry node. - // Take care when adding new node types. - std::unique_ptr dependencyModelUnloadGuard; - std::shared_ptr dependencyModelInstance; - if (dependencyNodeInfo.kind == NodeKind::DL) { - if (!manager.getModelInstance( - dependencyNodeInfo.modelName, - dependencyNodeInfo.modelVersion.value_or(0), - dependencyModelInstance, - dependencyModelUnloadGuard) - .ok()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Dependency DL model node refers to unavailable model - name: {}; version: {}", - pipelineName, - dependencyNodeInfo.modelName, - dependencyNodeInfo.modelVersion.value_or(0)); - return StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL; - } - retrieveModelNodeDependencyMetadata(dependencyModelInstance); - } - - if (dependencyNodeInfo.kind == NodeKind::CUSTOM) { - auto result = retrieveCustomNodeDependencyMetadata(dependencyNodeInfo); - if (!result.ok()) { - return result; - } - } - - for (const auto& [alias, realName] : mapping) { - if (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) { - auto result = markInputAsConnected(realName); - if (!result.ok()) { - return result; - } - } - - auto result = checkConnectionMappedToExistingDataSource(dependencyNodeInfo, alias); - if (!result.ok()) { - return result; - } - - if ( - (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) && - (dependencyNodeInfo.kind == NodeKind::DL || dependencyNodeInfo.kind == NodeKind::CUSTOM)) { - result = checkConnectionMetadataCorrectness(dependencyNodeInfo, realName, dependencyNodeInfo.outputNameAliases.at(alias)); - if (!result.ok()) { - return result; - } - } - } - - return StatusCode::OK; - } - - Status retrieveDependantMetadata() { - if (dependantNodeInfo.kind == NodeKind::DL) { - this->inputsInfo = this->dependantModelInstance->getInputsInfo(); - this->outputsInfo = this->dependantModelInstance->getOutputsInfo(); - return StatusCode::OK; - } else if (dependantNodeInfo.kind == NodeKind::CUSTOM) { - auto result = PipelineDefinition::getCustomNodeMetadata( - dependantNodeInfo, - this->inputsInfo, - dependantNodeInfo.library.getInputsInfo, - this->pipelineName, - getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); - if (!result.ok()) { - return result; - } - result = PipelineDefinition::getCustomNodeMetadata( - dependantNodeInfo, - this->outputsInfo, - dependantNodeInfo.library.getOutputsInfo, - this->pipelineName, - getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); - if (!result.ok()) { - return result; - } - } - return StatusCode::OK; - } - - void retrieveModelNodeDependencyMetadata(const std::shared_ptr& dependencyModelInstance) { - this->dependencyInputsInfo = dependencyModelInstance->getInputsInfo(); - this->dependencyOutputsInfo = dependencyModelInstance->getOutputsInfo(); - } - - Status retrieveCustomNodeDependencyMetadata(const NodeInfo& dependencyNodeInfo) { - auto result = PipelineDefinition::getCustomNodeMetadata( - dependencyNodeInfo, - this->dependencyInputsInfo, - dependencyNodeInfo.library.getInputsInfo, - this->pipelineName, - getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); - if (!result.ok()) { - return result; - } - result = PipelineDefinition::getCustomNodeMetadata( - dependencyNodeInfo, - this->dependencyOutputsInfo, - dependencyNodeInfo.library.getOutputsInfo, - this->pipelineName, - getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); - if (!result.ok()) { - return result; - } - return StatusCode::OK; - } - - Status validate() { - if (dependantNodeInfo.kind == NodeKind::DL) { - auto result = fetchUnderlyingModelInstance(); - if (!result.ok()) { - return result; - } - - result = retrieveDependantMetadata(); - if (!result.ok()) { - return result; - } - - result = checkForForbiddenDynamicParameters(); - if (!result.ok()) { - return result; - } - - prepareRemainingUnconnectedDependantInputsSet(); - } - - if (dependantNodeInfo.kind == NodeKind::CUSTOM) { - if (!dependantNodeInfo.library.isValid()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to incorrect library", pipelineName, dependantNodeInfo.nodeName); - return StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY; - } - - auto result = retrieveDependantMetadata(); - if (!result.ok()) { - return result; - } - - prepareRemainingUnconnectedDependantInputsSet(); - } - - if (dependantNodeInfo.kind == NodeKind::DL || dependantNodeInfo.kind == NodeKind::CUSTOM) { - for (const auto& [name, tensorOutput] : outputsInfo) { - auto result = validateShapeWithDemultiplexer(tensorOutput->getShape(), dependantNodeInfo); - if (!result.ok()) { - return result; - } - } - } - - if (!dependantNodeInfo.gatherFromNode.empty()) { - auto result = validateGatherNode(dependantNodeInfo); - if (!result.ok()) { - return result; - } - } - auto it = connections.find(dependantNodeInfo.nodeName); - if (it != connections.end()) { - for (const auto& [dependencyNodeName, mapping] : it->second) { - if (mapping.size() == 0) { - return StatusCode::UNKNOWN_ERROR; - } - - this->dependencyInputsInfo.clear(); - this->dependencyOutputsInfo.clear(); - std::vector::const_iterator dependencyNodeInfo; - auto result = getDependencyNodeInfo(dependencyNodeName, dependencyNodeInfo); - if (!result.ok()) { - return result; - } - - result = validateConnection(*dependencyNodeInfo, mapping); - if (!result.ok()) { - return result; - } - } - } - - return ensureAllModelInputsOfValidatedNodeHaveDataSource(); - } -}; - -Status PipelineDefinition::validateNode(ModelManager& manager, const NodeInfo& dependantNodeInfo, const bool isMultiBatchAllowed) { - NodeValidator validator(this->pipelineName, manager, dependantNodeInfo, connections, nodeInfos, nodeResources, isMultiBatchAllowed); - return validator.validate(); -} - -// Because of the way how pipeline_connections is implemented, this function is using -// transpose of PipelineDefinition graph.(Transpose contains same cycles as original graph) -Status PipelineDefinition::validateForCycles() { - std::vector visited; - std::vector parentNodes; - visited.reserve(nodeInfos.size()); - parentNodes.reserve(nodeInfos.size()); - - auto pred = [](const NodeInfo& nodeInfo) { - return nodeInfo.kind == NodeKind::EXIT; - }; - - const auto& itr = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), pred); - if (itr == nodeInfos.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} does not contain response node.", getName()); - return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; - } - std::string nodeName = itr->nodeName; - visited.push_back(nodeName); - - bool anyUnvisitedLeft = true; - while (anyUnvisitedLeft) { - bool unvisistedFound = false; - const auto& connectedToNode = connections[nodeName]; - for (const auto& node : connectedToNode) { - if (nodeName == node.first) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Node: {} is connected to itself in pipeline: {}", nodeName, getName()); - return StatusCode::PIPELINE_CYCLE_FOUND; - } - - if (std::find(visited.begin(), visited.end(), node.first) == visited.end()) { - parentNodes.push_back(nodeName); - visited.push_back(node.first); - nodeName = node.first; - unvisistedFound = true; - break; - } else { - if (std::find(parentNodes.begin(), parentNodes.end(), node.first) != parentNodes.end()) { - std::string cycleNodes; - for (auto& cycleNode : parentNodes) { - cycleNodes += cycleNode; - if (cycleNode != parentNodes.back()) { - cycleNodes += ", "; - } - } - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {}, following nodes creates cycle: {}", getName(), cycleNodes); - return StatusCode::PIPELINE_CYCLE_FOUND; - } - } - } - - if (!unvisistedFound) { - if (parentNodes.size() == 0) { - anyUnvisitedLeft = false; - if (visited.size() != nodeInfos.size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {}, there are not connected nodes", getName()); - return StatusCode::PIPELINE_CONTAINS_UNCONNECTED_NODES; - } - } else { - nodeName = parentNodes.back(); - parentNodes.pop_back(); - } - } - } - return StatusCode::OK; -} - -Status PipelineDefinition::validateDemultiplexerGatherNodesOrder() { - auto exitNode = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), [](const NodeInfo& nodeInfo) { return nodeInfo.kind == NodeKind::EXIT; }); - using gatherFromNode_t = std::set; - using demultiplyStack_t = std::vector; - std::vector> nodesToCheck{{exitNode->nodeName, {exitNode->gatherFromNode}}}; - if (exitNode->gatherFromNode.empty()) { - nodesToCheck.back().second.clear(); - } - std::map visitedNodes; - while (!nodesToCheck.empty()) { - auto [nodeName, demultiplyStack] = nodesToCheck.back(); - nodesToCheck.pop_back(); - for (auto& [connectedNodeName, aliasName] : connections[nodeName]) { - auto newDemultiplyStack(demultiplyStack); - auto& connectedNodeInfo = findNodeByName(connectedNodeName); - if (connectedNodeInfo.demultiplyCount) { - if (newDemultiplyStack.empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path that doesn't gather from demultiplexer node: {}, connection to node: {}.", getName(), connectedNodeName, nodeName); - return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; - } - auto& lastGatherSet = newDemultiplyStack.back(); - if (lastGatherSet.find(connectedNodeName) == lastGatherSet.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path where after demultiplexer node: {} there is gathering from different nodes: {}.", - getName(), - connectedNodeName, - std::accumulate(lastGatherSet.begin(), lastGatherSet.end(), std::string{}, [](const std::string& lhs, const std::string& rhs) { - if (lhs.empty()) { - return rhs; - } - return lhs + ", " + rhs; })); - return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; - } - lastGatherSet.erase(connectedNodeName); - if (lastGatherSet.empty()) { - newDemultiplyStack.pop_back(); - } - } - if (!connectedNodeInfo.gatherFromNode.empty()) { - newDemultiplyStack.emplace_back(connectedNodeInfo.gatherFromNode); - } - if (connectedNodeInfo.kind == NodeKind::ENTRY && !newDemultiplyStack.empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} exists path that gathers from nodes that are not in path: {}. Consider changing inputs of the node that gathers from mentioned demultiplexer nodes", - getName(), - std::accumulate(newDemultiplyStack.back().begin(), newDemultiplyStack.back().end(), std::string{}, [](const std::string& lhs, const std::string& rhs) { - if (lhs.empty()) { - return rhs; - } - return lhs + ", " + rhs; })); - return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; - } - auto visitedNode = std::find_if(std::begin(visitedNodes), std::end(visitedNodes), - [&connectedNodeName](const auto& visitedNode) { return visitedNode.first == connectedNodeName; }); - if (visitedNode != visitedNodes.end()) { - if (visitedNode->second != newDemultiplyStack) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "In pipeline: {} after node: {} exist paths that have different demultiply levels. Consider changing output connections of node: {}", getName(), connectedNodeName, connectedNodeName); - return StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER; - } - } else { - nodesToCheck.emplace_back(std::pair{connectedNodeName, newDemultiplyStack}); - visitedNodes.emplace(connectedNodeName, std::move(newDemultiplyStack)); - } - } - } - return StatusCode::OK; -} - -Status PipelineDefinition::validateNodes(ModelManager& manager) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Validation of pipeline definition: {} nodes started.", getName()); - - int entryNodeCount = std::count_if( - this->nodeInfos.begin(), - this->nodeInfos.end(), - [](const NodeInfo& info) { return info.kind == NodeKind::ENTRY; }); - - int exitNodeCount = std::count_if( - this->nodeInfos.begin(), - this->nodeInfos.end(), - [](const NodeInfo& info) { return info.kind == NodeKind::EXIT; }); - - if (entryNodeCount <= 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing request node", pipelineName); - return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; - } - - if (exitNodeCount <= 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing response node", pipelineName); - return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; - } - - if (entryNodeCount > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple request nodes", pipelineName); - return StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES; - } - - if (exitNodeCount > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple response nodes", pipelineName); - return StatusCode::PIPELINE_MULTIPLE_EXIT_NODES; - } - - bool isAnyNodeDynamicDemultiplexer = (std::find_if(this->nodeInfos.begin(), this->nodeInfos.end(), [](const NodeInfo& info) { - if (info.demultiplyCount) { - if (info.demultiplyCount.value() == -1) - return true; - return false; - } - return false; - }) != this->nodeInfos.end()); - int demultiplexerCount = std::count_if( - this->nodeInfos.begin(), - this->nodeInfos.end(), - [](const NodeInfo& info) { return info.demultiplyCount.has_value(); }); - if (isAnyNodeDynamicDemultiplexer && (demultiplexerCount > 1)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple demultiplexers with at least one dynamic.", pipelineName); - return StatusCode::NOT_IMPLEMENTED; - } - - const bool isMultiBatchAllowed = !std::any_of(nodeInfos.begin(), nodeInfos.end(), [](const auto& node) { return node.demultiplyCount; }); - for (const auto& node : nodeInfos) { - auto findByName = [node](const NodeInfo& nodeInfo) { - return nodeInfo.nodeName == node.nodeName; - }; - - if (std::count_if(nodeInfos.begin(), nodeInfos.end(), findByName) > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple nodes with name: {}", pipelineName, node.nodeName); - return StatusCode::PIPELINE_NODE_NAME_DUPLICATE; - } - - auto result = validateNode(manager, node, isMultiBatchAllowed); - if (!result.ok()) { - return result; - } - } - return StatusCode::OK; -} - -const tensor_map_t PipelineDefinition::getInputsInfo() const { - std::shared_lock lock(metadataMtx); - tensor_map_t copy = inputsInfo; - return copy; -} - -const tensor_map_t PipelineDefinition::getOutputsInfo() const { - std::shared_lock lock(metadataMtx); - tensor_map_t copy = outputsInfo; - return copy; -} - -std::shared_ptr applyDemultiplexerShapeForTensor(const std::shared_ptr& tensorInfo, int32_t demultiplyCount) { - return tensorInfo->createCopyWithDemultiplexerDimensionPrefix(demultiplyCount ? Dimension(demultiplyCount) : Dimension::any()); -} - -std::shared_ptr createOutputTensorInfoForPipeline(const std::string& mappedName, const std::shared_ptr& tensorInfo, const Shape& gatherShape, bool isConnectionFromDemultiplexer) { - std::shared_ptr newOwnedTensorInfo; - if (gatherShape.size() == 0) { - newOwnedTensorInfo = std::make_shared(*tensorInfo); - newOwnedTensorInfo->setMappedName(mappedName); - return newOwnedTensorInfo; - } - Shape newShape = tensorInfo->getShape(); - if (isConnectionFromDemultiplexer) { - newShape.erase(newShape.begin()); - } - newShape.insert(newShape.begin(), gatherShape.begin(), gatherShape.end()); - newOwnedTensorInfo = tensorInfo->createCopyWithNewShape(newShape); - newOwnedTensorInfo->setMappedName(mappedName); - return newOwnedTensorInfo; -} - -Status updateInputsInfoWithNodeConnection(tensor_map_t& inputsInfo, const TensorInfo& tensorInfo, const std::string& alias) { - auto newTensorInfo = std::make_shared(alias, tensorInfo.getPrecision(), tensorInfo.getShape(), tensorInfo.getLayout()); - auto it = inputsInfo.find(alias); - if (it != inputsInfo.end()) { - if (!it->second->isTensorSpecEqual(*newTensorInfo)) { - auto intersectionTensorInfo = it->second->createIntersection(*newTensorInfo); - if (intersectionTensorInfo == nullptr) { - Status status = StatusCode::PIPELINE_INPUTS_AMBIGUOUS_METADATA; - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error validating pipeline: {};\n{}\n{}", - status.string(), - it->second->asString(), - newTensorInfo->asString()); - return status; - } - inputsInfo[alias] = intersectionTensorInfo; - return StatusCode::OK; - } - } - inputsInfo[alias] = newTensorInfo; - return StatusCode::OK; -} - -template -Status updateInputsInfoWithNodeConnections(tensor_map_t& inputsInfo, const Aliases& specificDependencyMapping, Extractor extractor) { - for (const auto& [alias, realName] : specificDependencyMapping) { - auto status = updateInputsInfoWithNodeConnection(inputsInfo, extractor(realName), alias); - if (!status.ok()) { - return status; - } - } - return StatusCode::OK; -} - -Status PipelineDefinition::updateInputsInfo(const ModelManager& manager) { - // Assumptions: this can only be called on available pipeline definition. - // Add check if available when pipeline status will be implemented. - inputsInfo.clear(); - static const auto byName = [](const std::string& name) { - return [name](const NodeInfo& nodeInfo) { - return nodeInfo.nodeName == name; - }; - }; - for (const auto& [dependantNodeName, allMappings] : connections) { - const auto& dependantNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependantNodeName)); - for (const auto& [dependencyNodeName, specificDependencyMapping] : allMappings) { - const auto& dependencyNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependencyNodeName)); - if (dependencyNodeInfo->kind != NodeKind::ENTRY) { - continue; - } - - switch (dependantNodeInfo->kind) { - case NodeKind::EXIT: { - for (const auto& [alias, realName] : specificDependencyMapping) { - inputsInfo.insert({alias, TensorInfo::getUnspecifiedTensorInfo()}); - } - break; - } - case NodeKind::DL: { - auto instance = manager.findModelInstance(dependantNodeInfo->modelName, dependantNodeInfo->modelVersion.value_or(0)); - if (!instance) { - SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} inputs info fetching", dependantNodeInfo->modelName, this->getName()); - return StatusCode::MODEL_MISSING; - } - std::unique_ptr unloadGuard; - auto status = instance->waitForLoaded(0, unloadGuard); - if (!status.ok()) { - SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} inputs info fetching", instance->getName(), this->getName()); - return status; - } - status = updateInputsInfoWithNodeConnections(inputsInfo, - specificDependencyMapping, - [&instance](const std::string& realName) { - return *instance->getInputsInfo().at(realName); - }); - if (!status.ok()) { - return status; - } - break; - } - case NodeKind::CUSTOM: { - if (!dependantNodeInfo->library.isValid()) { - return StatusCode::NODE_LIBRARY_MISSING; - } - - tensor_map_t info; - auto status = getCustomNodeMetadata(*dependantNodeInfo, info, dependantNodeInfo->library.getInputsInfo, this->getName(), - getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo->nodeName))); - if (!status.ok()) { - return status; - } - - status = updateInputsInfoWithNodeConnections(inputsInfo, - specificDependencyMapping, - [&info](const std::string& realName) { - return *info.at(realName); - }); - if (!status.ok()) { - return status; - } - break; - } - default: { - // Pipeline validation does not allow connections into entry node. - SPDLOG_ERROR("Unexpected dependant node kind (name: {})", this->getName()); - return StatusCode::UNKNOWN_ERROR; - } - } - } - } - auto it = std::find_if(nodeInfos.begin(), nodeInfos.end(), [](const NodeInfo& info) { return info.kind == NodeKind::ENTRY && info.demultiplyCount; }); - if (it != nodeInfos.end()) { - int32_t demultiplyCount = it->demultiplyCount.value(); - for (auto& [inputName, inputTensorInfo] : inputsInfo) { - inputTensorInfo = applyDemultiplexerShapeForTensor(inputTensorInfo, demultiplyCount); - } - } - return StatusCode::OK; -} - -Status PipelineDefinition::populateOutputsInfoWithDLModelOutputs(const NodeInfo& dependencyNodeInfo, const ModelManager& manager, tensor_map_t& outputsInfo, const Aliases& specificDependencyMapping, const Shape& gatherShape) const { - auto instance = manager.findModelInstance(dependencyNodeInfo.modelName, dependencyNodeInfo.modelVersion.value_or(0)); - if (!instance) { - SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} outputs info fetching", dependencyNodeInfo.modelName, this->getName()); - return StatusCode::MODEL_MISSING; - } - std::unique_ptr unloadGuard; - auto status = instance->waitForLoaded(0, unloadGuard); - if (!status.ok()) { - SPDLOG_DEBUG("Model: {} was unavailable during pipeline: {} outputs info fetching", instance->getName(), this->getName()); - return status; - } - for (const auto& [alias, realName] : specificDependencyMapping) { - const auto& finalName = dependencyNodeInfo.outputNameAliases.count(alias) > 0 ? dependencyNodeInfo.outputNameAliases.at(alias) : alias; - outputsInfo[realName] = createOutputTensorInfoForPipeline(realName, instance->getOutputsInfo().at(finalName), gatherShape, dependencyNodeInfo.demultiplyCount.has_value()); - } - return StatusCode::OK; -} - -Status PipelineDefinition::populateOutputsInfoWithCustomNodeOutputs(const NodeInfo& dependencyNodeInfo, const ModelManager& manager, tensor_map_t& outputsInfo, const Aliases& specificDependencyMapping, const Shape& gatherShape) const { - if (!dependencyNodeInfo.library.isValid()) { - return StatusCode::NODE_LIBRARY_MISSING; - } - tensor_map_t info; - auto status = getCustomNodeMetadata(dependencyNodeInfo, info, dependencyNodeInfo.library.getOutputsInfo, this->getName(), - getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); - if (!status.ok()) { - return status; - } - for (const auto& [alias, realName] : specificDependencyMapping) { - const auto& finalName = dependencyNodeInfo.outputNameAliases.count(alias) > 0 ? dependencyNodeInfo.outputNameAliases.at(alias) : alias; - outputsInfo[realName] = createOutputTensorInfoForPipeline(realName, info.at(finalName), gatherShape, dependencyNodeInfo.demultiplyCount.has_value()); - } - return StatusCode::OK; -} - -Status PipelineDefinition::updateOutputsInfo(const ModelManager& manager) { - // Assumptions: this can only be called on available pipeline definition. - // Add check if available when pipeline status will be implemented. - outputsInfo.clear(); - static const auto byName = [](const std::string& name) { - return [name](const NodeInfo& nodeInfo) { - return nodeInfo.nodeName == name; - }; - }; - for (const auto& [dependantNodeName, allMappings] : connections) { - const auto& dependantNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependantNodeName)); - if (dependantNodeInfo->kind != NodeKind::EXIT) { - continue; - } - - auto gatherShape = this->getNodeGatherShape(*dependantNodeInfo); - - for (const auto& [dependencyNodeName, specificDependencyMapping] : allMappings) { - const auto& dependencyNodeInfo = std::find_if(std::begin(nodeInfos), std::end(nodeInfos), byName(dependencyNodeName)); - - switch (dependencyNodeInfo->kind) { - case NodeKind::ENTRY: { - for (const auto& [alias, realName] : specificDependencyMapping) { - outputsInfo.insert({realName, TensorInfo::getUnspecifiedTensorInfo()}); - } - break; - } - case NodeKind::DL: { - auto status = populateOutputsInfoWithDLModelOutputs( - *dependencyNodeInfo, manager, outputsInfo, specificDependencyMapping, gatherShape); - if (!status.ok()) { - return status; - } - break; - } - case NodeKind::CUSTOM: { - auto status = populateOutputsInfoWithCustomNodeOutputs( - *dependencyNodeInfo, manager, outputsInfo, specificDependencyMapping, gatherShape); - if (!status.ok()) { - return status; - } - break; - } - default: { - // Pipeline validation does not allow connections from exit node. - SPDLOG_ERROR("Unexpected dependency node kind (name: {})", this->getName()); - return StatusCode::UNKNOWN_ERROR; - } - } - } - } - return StatusCode::OK; -} - -Status PipelineDefinition::getCustomNodeMetadata(const NodeInfo& customNodeInfo, tensor_map_t& inputsInfo, metadata_fn callback, const std::string& pipelineName, void* customNodeLibraryInternalManager) { - struct CustomNodeTensorInfo* info = nullptr; - int infoCount = 0; - auto paramArray = createCustomNodeParamArray(customNodeInfo.parameters); - int paramArrayLength = customNodeInfo.parameters.size(); - int result = callback(&info, &infoCount, paramArray.get(), paramArrayLength, customNodeLibraryInternalManager); - if (result != 0) { - SPDLOG_ERROR("Metadata call to custom node: {} in pipeline: {} returned error code: {}", - customNodeInfo.nodeName, pipelineName, result); - return StatusCode::NODE_LIBRARY_METADATA_FAILED; - } - return createTensorInfoMap(info, infoCount, inputsInfo, customNodeInfo.library.release, customNodeLibraryInternalManager); -} - -const NodeInfo& PipelineDefinition::findNodeByName(const std::string& name) const { - return *std::find_if(std::begin(this->nodeInfos), std::end(this->nodeInfos), [&name](const NodeInfo& nodeInfo) { - return nodeInfo.nodeName == name; - }); -} - -Shape PipelineDefinition::getNodeGatherShape(const NodeInfo& info) const { - if (info.gatherFromNode.size() == 0) { - return {}; - } - Shape shape; - shape.reserve(info.gatherFromNode.size()); - - std::function search; - search = [this, &info, &search, &shape](const std::string& nodeName) { - if (this->connections.count(nodeName) == 0) { - return; - } - if (info.gatherFromNode.count(nodeName) > 0) { - auto someNodeInfo = this->findNodeByName(nodeName); - dimension_value_t demultiplyCount = static_cast(someNodeInfo.demultiplyCount.value_or(0)); - Dimension dim = demultiplyCount == 0 ? Dimension::any() : Dimension(demultiplyCount); - if (dim.isAny()) { - tensor_map_t nodeOutputsInfo; - if (someNodeInfo.kind == NodeKind::CUSTOM) { - auto result = PipelineDefinition::getCustomNodeMetadata( - someNodeInfo, - nodeOutputsInfo, - someNodeInfo.library.getOutputsInfo, - this->pipelineName, - getCNLIMWrapperPtr(nodeResources.at(someNodeInfo.nodeName))); - if (!result.ok()) { - SPDLOG_ERROR("Failed to read node: {} library metadata with error: {}", nodeName, result.string()); - return; - } - if (nodeOutputsInfo.size() == 0) { - SPDLOG_ERROR("Node: {} library metadata reports no outputs", nodeName); - return; - } else if (nodeOutputsInfo.begin()->second->getShape().size() < 3) { - SPDLOG_ERROR("Node: {} library metadata reports output with too small number of dimensions", nodeName); - return; - } - dim = nodeOutputsInfo.begin()->second->getShape()[0]; - } else if (someNodeInfo.kind == NodeKind::ENTRY) { - dim = Dimension::any(); - } - } - shape.emplace_back(dim); - } - - if (this->connections.at(nodeName).size() > 0) { - search(this->connections.at(nodeName).begin()->first); - } - }; - search(info.nodeName); - - if (info.gatherFromNode.size() != shape.size()) { - SPDLOG_ERROR("Pipeline: {} node: {} is misconfigured, gather shape has different number of dimensions that gather from node elements: {} vs {}", - this->getName(), info.nodeName, shape.size(), info.gatherFromNode.size()); - throw std::invalid_argument("Gather shape has different number of dimensions that gather from node elements"); - } - - std::reverse(shape.begin(), shape.end()); - return shape; -} -template Status PipelineDefinition::create( - std::unique_ptr& pipeline, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - ModelManager& manager); -template Status PipelineDefinition::create<::inference::ModelInferRequest, ::inference::ModelInferResponse>( - std::unique_ptr& pipeline, - const ::inference::ModelInferRequest* request, - ::inference::ModelInferResponse* response, - ModelManager& manager); - -} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinition.hpp b/src/ovms_lib/pipelinedefinition.hpp deleted file mode 100644 index 4d21ca8992..0000000000 --- a/src/ovms_lib/pipelinedefinition.hpp +++ /dev/null @@ -1,189 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "aliases.hpp" -#include "custom_node_library_internal_manager_wrapper.hpp" -#include "kfs_grpc_inference_service.hpp" -#include "modelversion.hpp" -#include "nodeinfo.hpp" -#include "pipelinedefinitionstatus.hpp" -#include "pipelinedefinitionunloadguard.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -class ModelManager; -class Pipeline; -class NodeValidator; - -class PipelineDefinition { - friend NodeValidator; - friend PipelineDefinitionUnloadGuard; - struct ValidationResultNotifier { - ValidationResultNotifier(PipelineDefinitionStatus& status, std::condition_variable& loadedNotify) : - status(status), - loadedNotify(loadedNotify) {} - ~ValidationResultNotifier() { - if (passed) { - status.handle(ValidationPassedEvent()); - loadedNotify.notify_all(); - } else { - status.handle(ValidationFailedEvent()); - } - } - bool passed = false; - - private: - PipelineDefinitionStatus& status; - std::condition_variable& loadedNotify; - }; - - const std::string pipelineName; - std::vector nodeInfos; - std::map> nodeResources = {}; - pipeline_connections_t connections; - -protected: - tensor_map_t inputsInfo; - tensor_map_t outputsInfo; - -private: - mutable std::shared_mutex metadataMtx; - std::atomic requestsHandlesCounter = 0; - std::shared_mutex loadMtx; - - std::condition_variable loadedNotify; - - // Pipelines are not versioned and any available definition has constant version equal 1. - static constexpr model_version_t VERSION = 1; - -protected: - PipelineDefinitionStatus status; - -private: - std::set> subscriptions; - - Status validateNode(ModelManager& manager, const NodeInfo& node, const bool isMultiBatchAllowed); - - const NodeInfo& findNodeByName(const std::string& name) const; - Shape getNodeGatherShape(const NodeInfo& info) const; - -public: - static constexpr uint64_t WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS = 10000; - PipelineDefinition(const std::string& pipelineName, - const std::vector& nodeInfos, - const pipeline_connections_t& connections) : - pipelineName(pipelineName), - nodeInfos(nodeInfos), - connections(connections), - status(this->pipelineName) {} - template - Status create(std::unique_ptr& pipeline, - const RequestType* request, - ResponseType* response, - ModelManager& manager); - -private: - template - Status createPrivate(std::unique_ptr& pipeline, - const RequestType* request, - ResponseType* response, - ModelManager& manager); - -public: - Status reload(ModelManager& manager, const std::vector&& nodeInfos, const pipeline_connections_t&& connections); - void retire(ModelManager& manager); - Status validate(ModelManager& manager); - Status validateNodes(ModelManager& manager); - Status validateForCycles(); - Status validateDemultiplexerGatherNodesOrder(); - Status initializeNodeResources(ModelManager& manager); - std::vector calculateNodeInfosDiff(const std::vector& nodeInfos); - void deinitializeNodeResources(const std::vector& nodeInfosDiff); - - const std::string& getName() const { return pipelineName; } - const PipelineDefinitionStateCode getStateCode() const { return status.getStateCode(); } - const model_version_t getVersion() const { return VERSION; } - - void notifyUsedModelChanged(const std::string& ownerDetails) { - this->status.handle(UsedModelChangedEvent(ownerDetails)); - } - - const PipelineDefinitionStatus& getStatus() const { - return this->status; - } - - const std::vector& getNodeInfos() { - return this->nodeInfos; - } - - void makeSubscriptions(ModelManager& manager); - void resetSubscriptions(ModelManager& manager); - -protected: - virtual Status updateInputsInfo(const ModelManager& manager); - virtual Status updateOutputsInfo(const ModelManager& manager); - -public: - const tensor_map_t getInputsInfo() const; - const tensor_map_t getOutputsInfo() const; - -private: - static Status getCustomNodeMetadata(const NodeInfo& customNodeInfo, tensor_map_t& inputsInfo, metadata_fn callback, const std::string& pipelineName, void* customNodeLibraryInternalManager); - - Status populateOutputsInfoWithDLModelOutputs( - const NodeInfo& dependencyNodeInfo, - const ModelManager& manager, - tensor_map_t& outputsInfo, - const Aliases& aliases, - const Shape& gatherShape) const; - - Status populateOutputsInfoWithCustomNodeOutputs( - const NodeInfo& dependencyNodeInfo, - const ModelManager& manager, - tensor_map_t& outputsInfo, - const Aliases& aliases, - const Shape& gatherShape) const; - - void increaseRequestsHandlesCount() { - ++requestsHandlesCounter; - } - - void decreaseRequestsHandlesCount() { - --requestsHandlesCounter; - } - -public: - Status waitForLoaded(std::unique_ptr& unloadGuard, const uint waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); -}; -} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionstatus.hpp b/src/ovms_lib/pipelinedefinitionstatus.hpp deleted file mode 100644 index 20421422d1..0000000000 --- a/src/ovms_lib/pipelinedefinitionstatus.hpp +++ /dev/null @@ -1,407 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include - -#include - -#include "logging.hpp" -#include "modelversionstatus.hpp" -#include "status.hpp" - -namespace ovms { - -enum class PipelineDefinitionStateCode { - BEGIN, - RELOADING, - LOADING_PRECONDITION_FAILED, - LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, - AVAILABLE_REQUIRED_REVALIDATION, - AVAILABLE, - RETIRED -}; - -inline const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code) { - static const std::unordered_map names{ - {PipelineDefinitionStateCode::BEGIN, "BEGIN"}, - {PipelineDefinitionStateCode::RELOADING, "RELOADING"}, - {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED, "LOADING_PRECONDITION_FAILED"}, - {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, - {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, - {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, - {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; - return names.at(code); -} - -template -class MachineState { -public: - MachineState(const std::string& name) : - name(name) {} - template - void handle(const Event& event) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Pipeline: {} state: {} handling: {}: {}", - name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, event.getDetails()); - try { - std::visit([this, &event](auto state) { state->handle(event).execute(*this); }, currentState); - } catch (std::logic_error& le) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} state: {} handling: {} error: {}", name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, le.what()); - throw; - } - SPDLOG_LOGGER_INFO(modelmanager_logger, "Pipeline: {} state changed to: {} after handling: {}: {}", - name, pipelineDefinitionStateCodeToString(getStateCode()), event.name, event.getDetails()); - } - - template - void changeStateTo() { - currentState = &std::get(allPossibleStates); - } - void printState() const { - std::visit([](const auto state) { state->print(); }, currentState); - } - PipelineDefinitionStateCode getStateCode() const { - while (true) { - try { - return std::visit([](const auto state) { return state->getStateCode(); }, currentState); - } catch (const std::bad_variant_access&) { - continue; - } - } - } - -private: - const std::string& name; - std::tuple allPossibleStates; - std::variant currentState{&std::get<0>(allPossibleStates)}; -}; -/** - * State in which pipeline is only defined - */ -struct BeginState; -/** - * State in which pipeline is available - */ -struct AvailableState; -/** - * State in which pipeline is available - * but there is revalidation pending since we know that one of used - * models changed - */ -struct AvailableRequiredRevalidation; -/** - * State in which pipeline is reloading - */ -struct ReloadState; -/** - * State in which pipeline is defined in config and failed validation. - */ -struct LoadingPreconditionFailedState; -/** - * State in which pipeline is defined in config, failed validation, - * but there is revalidation pending since we know that one of used - * models changed - */ -struct LoadingFailedLastValidationRequiredRevalidation; -/** - * State in which pipeline is retired - removed from config - */ -struct RetiredState; - -#define EVENT_STRUCT_WITH_NAME(x) \ - struct x { \ - static constexpr const char* name = #x; \ - x(const std::string& details = "") : \ - details(details) {} \ - const std::string& getDetails() const { \ - return details; \ - } \ - \ - private: \ - const std::string details; \ - }; - -EVENT_STRUCT_WITH_NAME(ReloadEvent); -EVENT_STRUCT_WITH_NAME(ValidationFailedEvent); -EVENT_STRUCT_WITH_NAME(ValidationPassedEvent); -EVENT_STRUCT_WITH_NAME(UsedModelChangedEvent); -EVENT_STRUCT_WITH_NAME(RetireEvent); - -template -struct StateChanger { - template - void execute(MachineState& pds) { - pds.template changeStateTo(); - } -}; - -struct StateKeeper { - template - void execute(MachineState& machine) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Keeping state"); - } -}; - -constexpr const char* INVALID_TRANSITION_MESSAGE = "Tried to conduct invalid transition."; - -struct BeginState { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::BEGIN; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } -}; - -struct ReloadState { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RELOADING; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateKeeper handle(const ReloadEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } -}; - -struct AvailableState { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateKeeper handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } -}; - -struct AvailableRequiredRevalidation { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } -}; - -struct LoadingPreconditionFailedState { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateKeeper handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } -}; - -struct LoadingFailedLastValidationRequiredRevalidation { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - return {}; - } - StateChanger handle(const RetireEvent& e) const { - return {}; - } -}; - -struct RetiredState { - static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::RETIRED; - PipelineDefinitionStateCode getStateCode() const { - return code; - } - void print() const { - SPDLOG_LOGGER_ERROR(modelmanager_logger, pipelineDefinitionStateCodeToString(getStateCode())); - } - StateChanger handle(const ReloadEvent& e) const { - return {}; - } - StateChanger handle(const ValidationPassedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateChanger handle(const ValidationFailedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const UsedModelChangedEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } - StateKeeper handle(const RetireEvent& e) const { - throw std::logic_error(INVALID_TRANSITION_MESSAGE); - return {}; - } -}; - -class PipelineDefinitionStatus : public MachineState { -public: - PipelineDefinitionStatus(const std::string& name) : - MachineState(name) {} - bool isAvailable() const { - auto state = getStateCode(); - return (state == PipelineDefinitionStateCode::AVAILABLE) || - (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); - } - bool canEndLoaded() const { - auto state = getStateCode(); - return isAvailable() || - (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || - (state == PipelineDefinitionStateCode::BEGIN) || - (state == PipelineDefinitionStateCode::RELOADING); - } - bool isRevalidationRequired() const { - auto state = getStateCode(); - return (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || - (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); - } - - std::tuple convertToModelStatus() const { - switch (getStateCode()) { - case PipelineDefinitionStateCode::BEGIN: - case PipelineDefinitionStateCode::RELOADING: - case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: - return { - ModelVersionState::LOADING, - ModelVersionStatusErrorCode::OK}; - - case PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: - return { - ModelVersionState::LOADING, - ModelVersionStatusErrorCode::FAILED_PRECONDITION}; - - case PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION: - case PipelineDefinitionStateCode::AVAILABLE: - return { - ModelVersionState::AVAILABLE, - ModelVersionStatusErrorCode::OK}; - - case PipelineDefinitionStateCode::RETIRED: - return { - ModelVersionState::END, - ModelVersionStatusErrorCode::OK}; - - default: - return {}; - } - } -}; - -} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionunloadguard.cpp b/src/ovms_lib/pipelinedefinitionunloadguard.cpp deleted file mode 100644 index fc4c6964fd..0000000000 --- a/src/ovms_lib/pipelinedefinitionunloadguard.cpp +++ /dev/null @@ -1,29 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "pipelinedefinitionunloadguard.hpp" - -#include "pipelinedefinition.hpp" - -namespace ovms { -PipelineDefinitionUnloadGuard::PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition) : - pipelineDefinition(pipelineDefinition) { - pipelineDefinition.increaseRequestsHandlesCount(); -} - -PipelineDefinitionUnloadGuard::~PipelineDefinitionUnloadGuard() { - pipelineDefinition.decreaseRequestsHandlesCount(); -} -} // namespace ovms diff --git a/src/ovms_lib/pipelinedefinitionunloadguard.hpp b/src/ovms_lib/pipelinedefinitionunloadguard.hpp deleted file mode 100644 index c36466f21b..0000000000 --- a/src/ovms_lib/pipelinedefinitionunloadguard.hpp +++ /dev/null @@ -1,30 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -namespace ovms { -class PipelineDefinition; - -class PipelineDefinitionUnloadGuard { -public: - PipelineDefinitionUnloadGuard() = delete; - PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition); - ~PipelineDefinitionUnloadGuard(); - -private: - PipelineDefinition& pipelineDefinition; -}; -} // namespace ovms diff --git a/src/ovms_lib/pipelineeventqueue.hpp b/src/ovms_lib/pipelineeventqueue.hpp deleted file mode 100644 index 85ab741d65..0000000000 --- a/src/ovms_lib/pipelineeventqueue.hpp +++ /dev/null @@ -1,28 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "threadsafequeue.hpp" - -namespace ovms { - -class Node; - -using NodeSessionKeyPair = std::pair, session_key_t>; -using PipelineEventQueue = ThreadSafeQueue; -} // namespace ovms diff --git a/src/ovms_lib/precision.cpp b/src/ovms_lib/precision.cpp deleted file mode 100644 index 46d28463f2..0000000000 --- a/src/ovms_lib/precision.cpp +++ /dev/null @@ -1,249 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "precision.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include - -namespace ovms { - -std::string toString(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::BF16, "BF16"}, - {Precision::FP64, "FP64"}, - {Precision::FP32, "FP32"}, - {Precision::FP16, "FP16"}, - {Precision::I64, "I64"}, - {Precision::I32, "I32"}, - {Precision::I16, "I16"}, - {Precision::I8, "I8"}, - {Precision::I4, "I4"}, - {Precision::U64, "U64"}, - {Precision::U32, "U32"}, - {Precision::U16, "U16"}, - {Precision::U8, "U8"}, - {Precision::U4, "U4"}, - {Precision::U1, "U1"}, - {Precision::MIXED, "MIXED"}, - {Precision::Q78, "Q78"}, - {Precision::BIN, "BIN"}, - {Precision::BOOL, "BOOL"}, - {Precision::UNDEFINED, "UNDEFINED"}, - {Precision::CUSTOM, "CUSTOM"}}; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return "UNKNOWN"; - } - return it->second; -} - -Precision fromString(const std::string& s) { - static std::unordered_map precisionMap{ - {"BF16", Precision::BF16}, - {"FP64", Precision::FP64}, - {"FP32", Precision::FP32}, - {"FP16", Precision::FP16}, - {"I64", Precision::I64}, - {"I32", Precision::I32}, - {"I16", Precision::I16}, - {"I8", Precision::I8}, - {"I4", Precision::I4}, - {"U64", Precision::U64}, - {"U32", Precision::U32}, - {"U16", Precision::U16}, - {"U8", Precision::U8}, - {"U4", Precision::U4}, - {"U1", Precision::U1}, - {"MIXED", Precision::MIXED}, - {"Q78", Precision::Q78}, - {"BIN", Precision::BIN}, - {"BOOL", Precision::BOOL}, - {"UNDEFINED", Precision::UNDEFINED}, - {"CUSTOM", Precision::CUSTOM}}; - auto it = precisionMap.find(s); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -Precision KFSPrecisionToOvmsPrecision(const KFSDataType& datatype) { - static std::unordered_map precisionMap{ - {"BOOL", Precision::BOOL}, - {"FP64", Precision::FP64}, - {"FP32", Precision::FP32}, - {"FP16", Precision::FP16}, - {"INT64", Precision::I64}, - {"INT32", Precision::I32}, - {"INT16", Precision::I16}, - {"INT8", Precision::I8}, - {"UINT64", Precision::U64}, - {"UINT32", Precision::U32}, - {"UINT16", Precision::U16}, - // {"BYTES", Precision::??}, - {"UINT8", Precision::U8}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& datatype) { - static std::unordered_map precisionMap{ - {TFSDataType::DT_FLOAT, Precision::FP32}, - {TFSDataType::DT_DOUBLE, Precision::FP64}, - {TFSDataType::DT_HALF, Precision::FP16}, - {TFSDataType::DT_INT64, Precision::I64}, - {TFSDataType::DT_INT32, Precision::I32}, - {TFSDataType::DT_INT16, Precision::I16}, - {TFSDataType::DT_INT8, Precision::I8}, - {TFSDataType::DT_UINT64, Precision::U64}, - {TFSDataType::DT_UINT16, Precision::U16}, - {TFSDataType::DT_UINT8, Precision::U8}, - {TFSDataType::DT_BOOL, Precision::BOOL}}; - auto it = precisionMap.find(datatype); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} - -size_t KFSDataTypeSize(const KFSDataType& datatype) { - static std::unordered_map datatypeSizeMap{ - {"BOOL", 1}, - {"UINT8", 1}, - {"UINT16", 2}, - {"UINT32", 4}, - {"UINT64", 8}, - {"INT8", 1}, - {"INT16", 2}, - {"INT32", 4}, - {"INT64", 8}, - {"FP16", 2}, - {"FP32", 4}, - {"FP64", 8} - // {"BYTES", }, - }; - auto it = datatypeSizeMap.find(datatype); - if (it == datatypeSizeMap.end()) { - return 0; - } - return it->second; -} - -KFSDataType ovmsPrecisionToKFSPrecision(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP64, "FP64"}, - {Precision::FP32, "FP32"}, - {Precision::FP16, "FP16"}, - {Precision::I64, "INT64"}, - {Precision::I32, "INT32"}, - {Precision::I16, "INT16"}, - {Precision::I8, "INT8"}, - {Precision::U64, "UINT64"}, - {Precision::U32, "UINT32"}, - {Precision::U16, "UINT16"}, - {Precision::U8, "UINT8"}, - {Precision::BOOL, "BOOL"}}; - // {Precision::BF16, ""}, - // {Precision::U4, ""}, - // {Precision::U1, ""}, - // {Precision::CUSTOM, ""}, - // {Precision::DYNAMIC, ""}, - // {Precision::MIXED, ""}, - // {Precision::Q78, ""}, - // {Precision::BIN, ""}, - // {Precision::I4, ""}, - // {Precision::UNDEFINED, "UNDEFINED"}}; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return "INVALID"; - } - return it->second; -} - -ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP64, ov::element::Type_t::f64}, - {Precision::FP32, ov::element::Type_t::f32}, - {Precision::FP16, ov::element::Type_t::f16}, - {Precision::I64, ov::element::Type_t::i64}, - {Precision::I32, ov::element::Type_t::i32}, - {Precision::I16, ov::element::Type_t::i16}, - {Precision::I8, ov::element::Type_t::i8}, - {Precision::I4, ov::element::Type_t::i4}, - {Precision::U64, ov::element::Type_t::u64}, - {Precision::U32, ov::element::Type_t::u32}, - {Precision::U16, ov::element::Type_t::u16}, - {Precision::U8, ov::element::Type_t::u8}, - {Precision::U4, ov::element::Type_t::u4}, - {Precision::U1, ov::element::Type_t::u1}, - {Precision::BOOL, ov::element::Type_t::boolean}, - {Precision::UNDEFINED, ov::element::Type_t::undefined}, - {Precision::DYNAMIC, ov::element::Type_t::dynamic} - // {Precision::MIXED, ov::element::Type_t::MIXED}, - // {Precision::Q78, ov::element::Type_t::Q78}, - // {Precision::BIN, ov::element::Type_t::BIN}, - // {Precision::CUSTOM, ov::element::Type_t::CUSTOM - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return ov::element::Type_t::undefined; - } - return it->second; -} - -Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f64, Precision::FP64}, - {ov::element::Type_t::f32, Precision::FP32}, - {ov::element::Type_t::f16, Precision::FP16}, - {ov::element::Type_t::bf16, Precision::BF16}, - {ov::element::Type_t::i64, Precision::I64}, - {ov::element::Type_t::i32, Precision::I32}, - {ov::element::Type_t::i16, Precision::I16}, - {ov::element::Type_t::i8, Precision::I8}, - {ov::element::Type_t::i4, Precision::I4}, - {ov::element::Type_t::u64, Precision::U64}, - {ov::element::Type_t::u32, Precision::U32}, - {ov::element::Type_t::u16, Precision::U16}, - {ov::element::Type_t::u8, Precision::U8}, - {ov::element::Type_t::u4, Precision::U4}, - {ov::element::Type_t::u1, Precision::U1}, - {ov::element::Type_t::undefined, Precision::UNDEFINED}, - {ov::element::Type_t::dynamic, Precision::DYNAMIC}, - // {ov::element::Type_t::???, Precision::MIXED}, - // {ov::element::Type_t::???, Precision::Q78}, - // {ov::element::Type_t::???, Precision::BIN}, - {ov::element::Type_t::boolean, Precision::BOOL} - // {ov::element::Type_t::CUSTOM, Precision::CUSTOM} - /* - undefined, - dynamic, -*/ - }; - auto it = precisionMap.find(type); - if (it == precisionMap.end()) { - return Precision::UNDEFINED; - } - return it->second; -} -} // namespace ovms diff --git a/src/ovms_lib/precision.hpp b/src/ovms_lib/precision.hpp deleted file mode 100644 index 9f50f7ae7d..0000000000 --- a/src/ovms_lib/precision.hpp +++ /dev/null @@ -1,70 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include - -using KFSDataType = std::string; -namespace tensorflow { -enum DataType : int; -} - -namespace ovms { - -using TFSDataType = tensorflow::DataType; - -enum class Precision { - BF16, - FP64, - FP32, - FP16, - I64, - I32, - I16, - I8, - I4, - U64, - U32, - U16, - U8, - U4, - U1, - BOOL, - CUSTOM, - UNDEFINED, - DYNAMIC, - MIXED, - Q78, - BIN -}; - -std::string toString(Precision precision); - -Precision fromString(const std::string& s); - -Precision KFSPrecisionToOvmsPrecision(const KFSDataType& s); -Precision TFSPrecisionToOvmsPrecision(const TFSDataType& s); - -size_t KFSDataTypeSize(const KFSDataType& datatype); - -KFSDataType ovmsPrecisionToKFSPrecision(Precision precision); - -ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision); - -Precision ovElementTypeToOvmsPrecision(ov::element::Type_t type); -} // namespace ovms diff --git a/src/ovms_lib/predict_request_validation_utils.cpp b/src/ovms_lib/predict_request_validation_utils.cpp deleted file mode 100644 index ee2d58a6e6..0000000000 --- a/src/ovms_lib/predict_request_validation_utils.cpp +++ /dev/null @@ -1,696 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "predict_request_validation_utils.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include -#include - -#include - -#include "kfs_grpc_inference_service.hpp" -#include "modelconfig.hpp" -#include "profiler.hpp" -#include "tfs_frontend/tfs_utils.hpp" - -namespace ovms { -namespace request_validation_utils { -template -struct RequestShapeInfo { - const RequestTensorType& tensor; - RequestShapeInfo(const RequestTensorType& tensor) : - tensor(tensor) {} - dimension_value_t getDim(size_t i); - size_t getShapeSize(); - const RequestTensorShapeType& getShape(); -}; - -using KFSRequestType = ::inference::ModelInferRequest; -using KFSInputTensorType = ::inference::ModelInferRequest_InferInputTensor; -using KFSInputTensorIteratorType = google::protobuf::internal::RepeatedPtrIterator; -using KFSShapeType = google::protobuf::RepeatedField; -using TFSRequestType = tensorflow::serving::PredictRequest; -using TFSInputTensorType = tensorflow::TensorProto; -using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; -using TFSShapeType = tensorflow::TensorShapeProto; - -template <> -dimension_value_t RequestShapeInfo::getDim(size_t i) { - return tensor.shape()[i]; -} -template <> -dimension_value_t RequestShapeInfo::getDim(size_t i) { - return tensor.tensor_shape().dim(i).size(); -} -template <> -size_t RequestShapeInfo::getShapeSize() { - return tensor.shape().size(); -} -template <> -size_t RequestShapeInfo::getShapeSize() { - return tensor.tensor_shape().dim_size(); -} -template <> -const TFSShapeType& RequestShapeInfo::getShape() { - return tensor.tensor_shape(); -} -template <> -const KFSShapeType& RequestShapeInfo::getShape() { - return tensor.shape(); -} - -template -class RequestValidator { - const RequestType& request; - const tensor_map_t& inputsInfo; - const std::string& servableName; - const model_version_t servableVersion; - const std::set& optionalAllowedInputNames; - const Mode batchingMode; - const shapes_info_map_t& shapeInfo; - - InputIterator it; - - RequestValidator() = delete; - - const std::string& getCurrentlyValidatedInputName() const; - const InputTensorType& getInputFromIt(const InputIterator& it) const; - -public: - RequestValidator( - const RequestType& request, const tensor_map_t& inputsInfo, - const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, - const Mode batchingMode, const shapes_info_map_t& shapeInfo) : - request(request), - inputsInfo(inputsInfo), - servableName(servableName), - servableVersion(servableVersion), - optionalAllowedInputNames(optionalAllowedInputNames), - batchingMode(batchingMode), - shapeInfo(shapeInfo) {} - - Status validateNumberOfInputs() const; - Status validateAndGetInput(const RequestType& request, const std::string& name, InputIterator& it, size_t& bufferId); - Status checkIfShapeValuesNegative(const InputTensorType& proto) const; - Status validateNumberOfBinaryInputShapeDimensions(const InputTensorType& proto) const; - Status checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; - Status checkBinaryBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; - Status checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; - Status validateTensorContentSize(const InputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const; - Status validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; - Status validatePrecision(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; - bool checkIfBinaryInputUsed(const InputTensorType& proto, const std::string inputName) const; - Status validateRequestCoherency() const; - Status validate(); -}; - -template <> -Status RequestValidator::validateRequestCoherency() const { - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateRequestCoherency() const { - if (!request.raw_input_contents().empty()) { - for (auto& input : request.inputs()) { - if (input.has_contents()) { - std::stringstream ss; - ss << "Passing buffers both in InferInputTensor contents and in raw_input_contents is not allowed. Detected buffer in InferInputTensor contents for input: " << input.name(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid request message - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_MESSAGE_STRUCTURE, details); - } - } - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateNumberOfInputs() const { - size_t expectedNumberOfInputs = inputsInfo.size(); - - if (optionalAllowedInputNames.size() > 0) { - auto it = request.inputs().begin(); - while (it != request.inputs().end()) { - if (optionalAllowedInputNames.find(it->name()) != optionalAllowedInputNames.end()) { - ++expectedNumberOfInputs; - } - ++it; - } - } - if (request.inputs_size() > 0 && expectedNumberOfInputs == static_cast(request.inputs_size())) { - return StatusCode::OK; - } - std::stringstream ss; - ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.inputs_size(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_INPUTS, details); -} - -template <> -const std::string& RequestValidator::getCurrentlyValidatedInputName() const { - return it->name(); -} - -template <> -const std::string& RequestValidator::getCurrentlyValidatedInputName() const { - return it->first; -} -template <> -const KFSInputTensorType& RequestValidator::getInputFromIt(const KFSInputTensorIteratorType& it) const { - return *it; -} - -template <> -const TFSInputTensorType& RequestValidator::getInputFromIt(const TFSInputTensorIteratorType& it) const { - return it->second; -} - -template <> -Status RequestValidator::validateNumberOfInputs() const { - size_t expectedNumberOfInputs = inputsInfo.size(); - for (auto optionalAllowedInputName : optionalAllowedInputNames) { - if (request.inputs().count(optionalAllowedInputName)) - expectedNumberOfInputs++; - } - if (request.inputs_size() > 0 && expectedNumberOfInputs == static_cast(request.inputs_size())) { - return StatusCode::OK; - } - std::stringstream ss; - ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.inputs_size(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_INPUTS, details); -} - -template <> -Status RequestValidator::validateAndGetInput(const TFSRequestType& request, const std::string& name, TFSInputTensorIteratorType& it, size_t& bufferId) { - it = request.inputs().find(name); - if (it != request.inputs().end()) { - return StatusCode::OK; - } - std::stringstream ss; - ss << "Required input: " << name; - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Missing input with specific name - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_MISSING_INPUT, details); -} - -template <> -Status RequestValidator::validateAndGetInput(const KFSRequestType& request, const std::string& name, KFSInputTensorIteratorType& it, size_t& bufferId) { - it = request.inputs().begin(); - bufferId = 0; - while (it != request.inputs().end()) { - if (it->name() == name) { - break; - } - ++it; - ++bufferId; - } - if (it != request.inputs().end()) { - return StatusCode::OK; - } - std::stringstream ss; - ss << "Required input: " << name; - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Missing input with specific name - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_MISSING_INPUT, details); -} - -template <> -template -Status RequestValidator::checkIfShapeValuesNegative(const InputTensorType& proto) const { - RequestShapeInfo rsi(proto); - for (size_t i = 0; i < rsi.getShapeSize(); i++) { - if (rsi.getDim(i) <= 0) { - std::stringstream ss; - ss << "Negative or zero dimension size is not acceptable: " << tensorShapeToString(rsi.getShape()) << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_SHAPE, details); - } - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const TFSInputTensorType& proto) const { - RequestShapeInfo rsi(proto); - if (rsi.getShapeSize() != 1) { - std::stringstream ss; - ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const KFSInputTensorType& proto) const { - RequestShapeInfo rsi(proto); - if (rsi.getShapeSize() != 1) { - std::stringstream ss; - ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} - -template -Status RequestValidator::checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { - RequestShapeInfo rsi(proto); - if (servableBatchSize.match(rsi.getDim(batchSizeIndex))) { - return StatusCode::OK; - } - if (batchingMode == AUTO) { - finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; - return StatusCode::OK; - } else if (shapeMode != AUTO) { - std::stringstream ss; - ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << rsi.getDim(batchSizeIndex) << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::checkBinaryBatchSizeMismatch(const TFSInputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { - RequestShapeInfo rsi(proto); - if (proto.string_val_size() <= 0) { - std::stringstream ss; - ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - if (servableBatchSize.match(rsi.getDim(0))) { - return StatusCode::OK; - } - if (batchingMode == AUTO) { - finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; - return StatusCode::OK; - } else if (shapeMode != AUTO) { - std::stringstream ss; - ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << proto.string_val_size() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - return StatusCode::OK; -} -template <> -Status RequestValidator::checkBinaryBatchSizeMismatch(const KFSInputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { - RequestShapeInfo rsi(proto); - if (proto.contents().bytes_contents_size() <= 0) { - std::stringstream ss; - ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - if (servableBatchSize.match(rsi.getDim(0))) { - return StatusCode::OK; - } - if (batchingMode == AUTO) { - finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; - return StatusCode::OK; - } else if (shapeMode != AUTO) { - std::stringstream ss; - ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << proto.contents().bytes_contents_size() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_BATCH_SIZE, details); - } - return StatusCode::OK; -} - -template -Status RequestValidator::checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { - const auto& shape = inputInfo.getShape(); - bool mismatch = false; - RequestShapeInfo rsi(proto); - if (batchingMode == AUTO) { // Skip batch dimension - for (int i = 0; i < batchSizeIndex; i++) { - if (!shape[i].match(static_cast(rsi.getDim(i)))) { - mismatch = true; - break; - } - } - for (int i = batchSizeIndex + 1; i < rsi.getShapeSize(); i++) { - if (!shape[i].match(static_cast(rsi.getDim(i)))) { - mismatch = true; - break; - } - } - } else { // Do not skip batch dimension - for (int i = 0; i < rsi.getShapeSize(); i++) { - if (!shape[i].match(static_cast(rsi.getDim(i)))) { - mismatch = true; - break; - } - } - } - if (!mismatch) { - return StatusCode::OK; - } - if (shapeMode == AUTO) { - finalStatus = StatusCode::RESHAPE_REQUIRED; - return StatusCode::OK; - } else { - std::stringstream ss; - ss << "Expected: " << inputInfo.getShape().toString() - << "; Actual: " << tensorShapeToString(rsi.getShape()) - << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid shape - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_SHAPE, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateTensorContentSize(const TFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { - /* - int8 data in request.tensor_content - uint8 data in request.tensor_content - int16 data in request.tensor_content - uint16 request.tensor_content is empty, data located in request.int_val - int32 data in request.tensor_content - uint32 data in request.tensor_content - int64 data in request.tensor_content - uint64 data in request.tensor_content - float16 request.tensor_content is empty, data located in request.half_val - float32 data in request.tensor_content - double data in request.tensor_content - - _TENSOR_CONTENT_TYPES - https://github.com/tensorflow/tensorflow/blob/903a6399aab19b549fefd0ead836af644f3d00f8/tensorflow/python/framework/tensor_util.py#L237 -*/ - - size_t expectedValueCount = 1; - for (int i = 0; i < proto.tensor_shape().dim_size(); i++) { - expectedValueCount *= proto.tensor_shape().dim(i).size(); - } - - // Network expects tensor content size or value count - if (proto.dtype() == tensorflow::DataType::DT_UINT16) { - if (proto.int_val_size() < 0 || - expectedValueCount != static_cast(proto.int_val_size())) { - std::stringstream ss; - ss << "Expected: " << expectedValueCount << "; Actual: " << proto.int_val_size() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } else if (proto.dtype() == tensorflow::DataType::DT_HALF) { - if (proto.half_val_size() < 0 || - expectedValueCount != static_cast(proto.half_val_size())) { - std::stringstream ss; - ss << "Expected: " << expectedValueCount << "; Actual: " << proto.half_val_size() << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of values in tensor proto container - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } else { - size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); - if (expectedContentSize != proto.tensor_content().size()) { - std::stringstream ss; - ss << "Expected: " << expectedContentSize << " bytes; Actual: " << proto.tensor_content().size() << " bytes; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - } - return StatusCode::OK; -} - -size_t getElementsCount(const KFSInputTensorType& proto, ovms::Precision expectedPrecision) { - switch (expectedPrecision) { - case ovms::Precision::BOOL: { - return proto.contents().bool_contents().size(); - } - /// int_contents - case ovms::Precision::I8: - case ovms::Precision::I16: - case ovms::Precision::I32: { - return proto.contents().int_contents().size(); - } - /// int64_contents - case ovms::Precision::I64: { - return proto.contents().int64_contents().size(); - } - // uint_contents - case ovms::Precision::U8: - case ovms::Precision::U16: - case ovms::Precision::U32: { - return proto.contents().uint_contents().size(); - } - // uint64_contents - case ovms::Precision::U64: { - return proto.contents().uint64_contents().size(); - } - // fp32_contents - case ovms::Precision::FP32: { - return proto.contents().fp32_contents().size(); - } - // fp64_contentes - case ovms::Precision::FP64: { - return proto.contents().fp64_contents().size(); - } - case ovms::Precision::FP16: - case ovms::Precision::U1: - case ovms::Precision::CUSTOM: - case ovms::Precision::UNDEFINED: - case ovms::Precision::DYNAMIC: - case ovms::Precision::MIXED: - case ovms::Precision::Q78: - case ovms::Precision::BIN: - default: - return 0; - } -} - -template <> -Status RequestValidator::validateTensorContentSize(const KFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { - size_t expectedValueCount = 1; - for (int i = 0; i < proto.shape().size(); i++) { - expectedValueCount *= proto.shape()[i]; - } - if (request.raw_input_contents().size()) { - size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); - if (expectedContentSize != request.raw_input_contents()[bufferId].size()) { - std::stringstream ss; - ss << "Expected: " << expectedContentSize << " bytes; Actual: " << request.raw_input_contents()[bufferId].size() << " bytes; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_CONTENT_SIZE, details); - } - } else { // buffers placed in InputTensor content - // here we should check that the elements count is equal since for some precisions there is padding - // we need to decide first which exact datatype_contents we extract that information from - size_t elementsCount = getElementsCount(proto, expectedPrecision); - if (expectedValueCount != elementsCount) { - std::stringstream ss; - ss << "Expected: " << expectedValueCount << " values; Actual: " << elementsCount << " values; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid value count of tensor proto - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_VALUE_COUNT, details); - } - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { - // Network and request must have the same number of shape dimensions, higher than 0 - const auto& shape = inputInfo.getShape(); - if (proto.tensor_shape().dim_size() <= 0 || - shape.size() != static_cast(proto.tensor_shape().dim_size())) { - std::stringstream ss; - ss << "Expected: " << shape.toString() - << "; Actual: " << tensorShapeToString(proto.tensor_shape()) - << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { - // Network and request must have the same number of shape dimensions, higher than 0 - const auto& shape = inputInfo.getShape(); - if (proto.shape().size() <= 0 || - shape.size() != static_cast(proto.shape().size())) { - std::stringstream ss; - ss << "Expected: " << shape.toString() - << "; Actual: " << tensorShapeToString(proto.shape()) - << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { - if (proto.dtype() != getPrecisionAsDataType(inputInfo.getPrecision())) { - std::stringstream ss; - ss << "Expected: " << inputInfo.getPrecisionAsString() - << "; Actual: " << getDataTypeAsString(proto.dtype()) - << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_PRECISION, details); - } - return StatusCode::OK; -} - -template <> -Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const KFSInputTensorType& proto) const { - if (proto.datatype() != inputInfo.getPrecisionAsKFSPrecision()) { - std::stringstream ss; - ss << "Expected: " << inputInfo.getPrecisionAsString() - << "; Actual: " << proto.datatype() - << "; input name: " << getCurrentlyValidatedInputName(); - const std::string details = ss.str(); - SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); - return Status(StatusCode::INVALID_PRECISION, details); - } - return StatusCode::OK; -} - -Mode getShapeMode(const shapes_info_map_t& shapeInfo, const std::string& name) { - if (shapeInfo.size() == 0) { - return Mode::FIXED; - } - auto it = shapeInfo.find(name); - if (it == shapeInfo.end()) { - it = shapeInfo.find(ANONYMOUS_INPUT_NAME); - } - if (it == shapeInfo.end()) { - return Mode::FIXED; - } - return it->second.shapeMode; -} - -template <> -bool RequestValidator::checkIfBinaryInputUsed(const TFSInputTensorType& proto, const std::string inputName) const { - if (proto.dtype() == tensorflow::DataType::DT_STRING) { - SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.string_val_size()); - return true; - } - return false; -} - -template <> -bool RequestValidator::checkIfBinaryInputUsed(const KFSInputTensorType& proto, const std::string inputName) const { - if (proto.datatype() == "BYTES") { - SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.contents().bytes_contents_size()); - return true; - } - return false; -} - -template -Status RequestValidator::validate() { - Status finalStatus = StatusCode::OK; - - auto status = validateNumberOfInputs(); - if (!status.ok()) - return status; - status = validateRequestCoherency(); - if (!status.ok()) - return status; - - size_t bufferId = 0; - for (const auto& [name, inputInfo] : inputsInfo) { - status = validateAndGetInput(request, name, it, bufferId); - if (!status.ok()) - return status; - - const auto& proto = getInputFromIt(it); - - status = checkIfShapeValuesNegative(proto); - if (!status.ok()) - return status; - auto batchIndex = inputInfo->getLayout().getBatchIndex(); - if (!batchIndex.has_value()) { - SPDLOG_DEBUG("[servable name: {} version: {}] Missing batch index in input: {} layout: {}", - servableName, servableVersion, name, inputInfo->getLayout()); - return StatusCode::INTERNAL_ERROR; - } - if (inputInfo->getShape().size() < batchIndex.value() + 1) { - SPDLOG_DEBUG("[servable name: {} version: {}] Batch index out of shape range for input: {} layout: {} shape: {}", - servableName, servableVersion, name, inputInfo->getLayout(), inputInfo->getShape().toString()); - return StatusCode::INTERNAL_ERROR; - } - const Dimension& batchSize = inputInfo->getShape()[batchIndex.value()]; - Mode shapeMode = getShapeMode(shapeInfo, name); - if (checkIfBinaryInputUsed(proto, name)) { - status = validateNumberOfBinaryInputShapeDimensions(proto); - if (!status.ok()) - return status; - - status = checkBinaryBatchSizeMismatch(proto, batchSize, finalStatus, batchingMode, shapeMode); - if (!status.ok()) - return status; - - continue; - } - - status = validatePrecision(*inputInfo, proto); - if (!status.ok()) - return status; - status = validateNumberOfShapeDimensions(*inputInfo, proto); - if (!status.ok()) - return status; - status = checkBatchSizeMismatch(proto, batchSize, batchIndex.value(), finalStatus, batchingMode, shapeMode); - if (!status.ok()) - return status; - status = checkShapeMismatch(proto, *inputInfo, batchIndex.value(), finalStatus, batchingMode, shapeMode); - if (!status.ok()) - return status; - status = validateTensorContentSize(proto, inputInfo->getPrecision(), bufferId); - if (!status.ok()) - return status; - } - return finalStatus; -} - -template <> -Status validate(const TFSRequestType& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { - OVMS_PROFILE_FUNCTION(); - return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); -} - -template <> -Status validate(const KFSRequestType& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { - OVMS_PROFILE_FUNCTION(); - return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); -} -} // namespace request_validation_utils -} // namespace ovms diff --git a/src/ovms_lib/predict_request_validation_utils.hpp b/src/ovms_lib/predict_request_validation_utils.hpp deleted file mode 100644 index 67931e3666..0000000000 --- a/src/ovms_lib/predict_request_validation_utils.hpp +++ /dev/null @@ -1,42 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include - -#include "modelversion.hpp" -#include "shape.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { -namespace request_validation_utils { - -template -Status validate( - const RequestType& request, - const tensor_map_t& inputsInfo, - const std::string& servableName, - const model_version_t servableVersion, - const std::set& optionalAllowedInputNames = {}, - const Mode batchingMode = Mode::FIXED, - const shapes_info_map_t& shapeInfo = shapes_info_map_t()); - -} // namespace request_validation_utils -} // namespace ovms diff --git a/src/ovms_lib/prediction_service.cpp b/src/ovms_lib/prediction_service.cpp deleted file mode 100644 index 0594c317df..0000000000 --- a/src/ovms_lib/prediction_service.cpp +++ /dev/null @@ -1,137 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "prediction_service.hpp" - -#include -#include -#include -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#pragma GCC diagnostic pop - -#include "get_model_metadata_impl.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "ovinferrequestsqueue.hpp" -#include "prediction_service_utils.hpp" -#include "profiler.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "status.hpp" -#include "timer.hpp" - -using grpc::ServerContext; - -using tensorflow::TensorProto; - -using tensorflow::serving::PredictionService; -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - -namespace ovms { - -PredictionServiceImpl::PredictionServiceImpl(ovms::Server& ovmsServer) : - ovmsServer(ovmsServer), - getModelMetadataImpl(ovmsServer) {} - -Status PredictionServiceImpl::getModelInstance(const PredictRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return manager.getModelInstance(request->model_spec().name(), request->model_spec().version().value(), modelInstance, modelInstanceUnloadGuardPtr); -} - -Status PredictionServiceImpl::getPipeline(const PredictRequest* request, - PredictResponse* response, - std::unique_ptr& pipelinePtr) { - OVMS_PROFILE_FUNCTION(); - auto module = this->ovmsServer.getModule(SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return StatusCode::MODEL_NOT_LOADED; - } - auto servableManagerModule = dynamic_cast(module); - // TODO if not succeed then return error - auto& manager = servableManagerModule->getServableManager(); - return manager.createPipeline(pipelinePtr, request->model_spec().name(), request, response); -} - -grpc::Status ovms::PredictionServiceImpl::Predict( - ServerContext* context, - const PredictRequest* request, - PredictResponse* response) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - timer.start("total"); - using std::chrono::microseconds; - SPDLOG_DEBUG("Processing gRPC request for model: {}; version: {}", - request->model_spec().name(), - request->model_spec().version().value()); - - std::shared_ptr modelInstance; - std::unique_ptr pipelinePtr; - - std::unique_ptr modelInstanceUnloadGuard; - auto status = getModelInstance(request, modelInstance, modelInstanceUnloadGuard); - - if (status == StatusCode::MODEL_NAME_MISSING) { - SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", request->model_spec().name()); - status = getPipeline(request, response, pipelinePtr); - } - if (!status.ok()) { - SPDLOG_INFO("Getting modelInstance or pipeline failed. {}", status.string()); - return status.grpc(); - } - - if (pipelinePtr) { - status = pipelinePtr->execute(); - } else { - status = modelInstance->infer(request, response, modelInstanceUnloadGuard); - } - - if (!status.ok()) { - return status.grpc(); - } - - timer.stop("total"); - SPDLOG_DEBUG("Total gRPC request processing time: {} ms", timer.elapsed("total") / 1000); - return grpc::Status::OK; -} - -grpc::Status PredictionServiceImpl::GetModelMetadata( - grpc::ServerContext* context, - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) { - OVMS_PROFILE_FUNCTION(); - return getModelMetadataImpl.getModelStatus(request, response).grpc(); -} - -const GetModelMetadataImpl& PredictionServiceImpl::getTFSModelMetadataImpl() const { - return this->getModelMetadataImpl; -} - -} // namespace ovms diff --git a/src/ovms_lib/prediction_service.hpp b/src/ovms_lib/prediction_service.hpp deleted file mode 100644 index 4483b79088..0000000000 --- a/src/ovms_lib/prediction_service.hpp +++ /dev/null @@ -1,64 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "get_model_metadata_impl.hpp" - -namespace ovms { -class ModelInstance; -class ModelInstanceUnloadGuard; -class ModelManager; -class Pipeline; -class Server; -class Status; - -class PredictionServiceImpl final : public tensorflow::serving::PredictionService::Service { - ovms::Server& ovmsServer; - GetModelMetadataImpl getModelMetadataImpl; - -public: - PredictionServiceImpl(ovms::Server& ovmsServer); - grpc::Status Predict( - grpc::ServerContext* context, - const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response) override; - - grpc::Status GetModelMetadata( - grpc::ServerContext* context, - const tensorflow::serving::GetModelMetadataRequest* request, - tensorflow::serving::GetModelMetadataResponse* response) override; - - const GetModelMetadataImpl& getTFSModelMetadataImpl() const; - -protected: - Status getModelInstance(const tensorflow::serving::PredictRequest* request, - std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr); - Status getPipeline(const tensorflow::serving::PredictRequest* request, - tensorflow::serving::PredictResponse* response, - std::unique_ptr& pipelinePtr); -}; - -} // namespace ovms diff --git a/src/ovms_lib/prediction_service_utils.cpp b/src/ovms_lib/prediction_service_utils.cpp deleted file mode 100644 index 2f56ca39bc..0000000000 --- a/src/ovms_lib/prediction_service_utils.cpp +++ /dev/null @@ -1,90 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "prediction_service_utils.hpp" - -#include - -#include "deserialization.hpp" -#include "executingstreamidguard.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "serialization.hpp" -#include "stringutils.hpp" -#include "timer.hpp" - -using tensorflow::serving::PredictRequest; -using tensorflow::serving::PredictResponse; - -namespace ovms { - -std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex) { - auto requestInputItr = request->inputs().begin(); - if (requestInputItr == request->inputs().end()) { - SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); - return std::nullopt; - } - auto& requestInput = requestInputItr; // assuming same batch size for all inputs - if (requestInput->shape().size() < batchSizeIndex + 1) { - SPDLOG_DEBUG("Failed to get batch size of a request. Batch size index out of shape range. Validation of request failed"); - return std::nullopt; - } - return Dimension(requestInput->shape()[batchSizeIndex]); -} - -std::map getRequestShapes(const ::inference::ModelInferRequest* request) { - std::map requestShapes; - for (auto& it : request->inputs()) { - shape_t requestShape; - std::string name = it.name(); - auto& requestInput = it; - for (int i = 0; i < requestInput.shape().size(); i++) { - requestShape.push_back(requestInput.shape()[i]); - } - requestShapes[name] = std::move(requestShape); - } - return requestShapes; -} - -std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex) { - auto requestInputItr = request->inputs().begin(); - if (requestInputItr == request->inputs().end()) { - SPDLOG_DEBUG("Failed to get batch size of a request. Validation of request failed"); - return std::nullopt; - } - auto& requestInput = requestInputItr->second; // assuming same batch size for all inputs - if (requestInput.tensor_shape().dim_size() < batchSizeIndex + 1) { - SPDLOG_DEBUG("Failed to get batch size of a request. Batch size index out of shape range. Validation of request failed"); - return std::nullopt; - } - return Dimension(requestInput.tensor_shape().dim(batchSizeIndex).size()); -} - -std::map getRequestShapes(const tensorflow::serving::PredictRequest* request) { - std::map requestShapes; - for (auto& it : request->inputs()) { - shape_t requestShape; - std::string name = it.first; - auto& requestInput = it.second; - for (int i = 0; i < requestInput.tensor_shape().dim_size(); i++) { - requestShape.push_back(requestInput.tensor_shape().dim(i).size()); - } - requestShapes[name] = std::move(requestShape); - } - return requestShapes; -} - -} // namespace ovms diff --git a/src/ovms_lib/prediction_service_utils.hpp b/src/ovms_lib/prediction_service_utils.hpp deleted file mode 100644 index 03d598f5da..0000000000 --- a/src/ovms_lib/prediction_service_utils.hpp +++ /dev/null @@ -1,38 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "kfs_grpc_inference_service.hpp" -#include "shape.hpp" - -namespace ovms { - -std::optional getRequestBatchSize(const ::inference::ModelInferRequest* request, const size_t batchSizeIndex); - -std::map getRequestShapes(const ::inference::ModelInferRequest* request); - -std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); -std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); - -} // namespace ovms diff --git a/src/ovms_lib/profiler.cpp b/src/ovms_lib/profiler.cpp deleted file mode 100644 index da6607db4a..0000000000 --- a/src/ovms_lib/profiler.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "profiler.hpp" - -#include - -namespace ovms { - -bool profiler_init(const char* file_path) { -#ifndef MTR_ENABLED - return true; -#endif - FILE* f = fopen(file_path, "wb"); - if (f == nullptr) { - return false; - } - mtr_init_from_stream(f); - return true; -} - -void profiler_shutdown() { -#ifndef MTR_ENABLED - return; -#endif - mtr_flush(); - mtr_shutdown(); -} - -Profiler::Profiler(const std::string& file_path) { - this->initialized = profiler_init(file_path.c_str()); -} - -Profiler::~Profiler() { - if (this->isInitialized()) { - profiler_shutdown(); - } -} - -bool Profiler::isInitialized() const { - return this->initialized; -} - -} // namespace ovms diff --git a/src/ovms_lib/profiler.hpp b/src/ovms_lib/profiler.hpp deleted file mode 100644 index b323c11351..0000000000 --- a/src/ovms_lib/profiler.hpp +++ /dev/null @@ -1,49 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "minitrace.h" // NOLINT - -#define OVMS_PROFILE_SCOPE(name) MTR_SCOPE("OVMS", name); -#define OVMS_PROFILE_SCOPE_S(name, vname, cstr) MTR_SCOPE_S("OVMS", name, vname, cstr); -#define OVMS_PROFILE_FUNCTION() OVMS_PROFILE_SCOPE(__PRETTY_FUNCTION__) - -#define OVMS_PROFILE_SYNC_BEGIN(name) MTR_BEGIN("OVMS", name); -#define OVMS_PROFILE_SYNC_END(name) MTR_END("OVMS", name); -#define OVMS_PROFILE_SYNC_BEGIN_S(name, vname, cstr) MTR_BEGIN_S("OVMS", name, vname, cstr); -#define OVMS_PROFILE_SYNC_END_S(name, vname, cstr) MTR_END_S("OVMS", name, vname, cstr); - -#define OVMS_PROFILE_ASYNC_BEGIN(name, id) MTR_START("OVMS", name, id); -#define OVMS_PROFILE_ASYNC_END(name, id) MTR_FINISH("OVMS", name, id); - -namespace ovms { - -bool profiler_init(const char* file_path); -void profiler_shutdown(); - -class Profiler { -public: - Profiler(const std::string& file_path); - ~Profiler(); - bool isInitialized() const; - -private: - bool initialized = false; -}; - -} // namespace ovms diff --git a/src/ovms_lib/queue.hpp b/src/ovms_lib/queue.hpp deleted file mode 100644 index 037b3ebc94..0000000000 --- a/src/ovms_lib/queue.hpp +++ /dev/null @@ -1,136 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ovms { - -template -class Queue { -public: - /** - * @brief Allocating idle stream for execution - */ - std::future getIdleStream() { - int value; - std::promise idleStreamPromise; - std::future idleStreamFuture = idleStreamPromise.get_future(); - std::unique_lock lk(front_mut); - if (streams[front_idx] < 0) { // we need to wait for any idle stream to be returned - std::unique_lock queueLock(queue_mutex); - promises.push(std::move(idleStreamPromise)); - } else { // we can give idle stream right away - value = streams[front_idx]; - streams[front_idx] = -1; // negative value indicate consumed vector index - front_idx = (front_idx + 1) % streams.size(); - lk.unlock(); - idleStreamPromise.set_value(value); - } - return idleStreamFuture; - } - - std::optional tryToGetIdleStream() { - int value; - std::unique_lock lk(front_mut); - if (streams[front_idx] < 0) { // we need to wait for any idle stream to be returned - return std::nullopt; - } else { // we can give idle stream right away - value = streams[front_idx]; - streams[front_idx] = -1; // negative value indicate consumed vector index - front_idx = (front_idx + 1) % streams.size(); - lk.unlock(); - return value; - } - } - - /** - * @brief Release stream after execution - */ - void returnStream(int streamID) { - std::unique_lock lk(queue_mutex); - if (promises.size()) { - std::promise promise = std::move(promises.front()); - promises.pop(); - lk.unlock(); - promise.set_value(streamID); - return; - } - std::uint32_t old_back = back_idx.load(); - while (!back_idx.compare_exchange_weak( - old_back, - (old_back + 1) % streams.size(), - std::memory_order_relaxed)) { - } - streams[old_back] = streamID; - } - - /** - * @brief Constructor with initialization - */ - Queue(int streamsLength) : - streams(streamsLength), - front_idx{0}, - back_idx{0} { - for (int i = 0; i < streamsLength; ++i) { - streams[i] = i; - } - } - - /** - * @brief Give InferRequest - */ - T& getInferRequest(int streamID) { - return inferRequests[streamID]; - } - -protected: - /** - * @brief Vector representing circular buffer for infer queue - */ - std::vector streams; - - /** - * @brief Index of the front of the idle streams list - */ - std::uint32_t front_idx; - - /** - * @brief Index of the back of the idle streams list - */ - std::atomic back_idx; - - /** - * @brief Vector representing OV streams and used for notification about completed inference operations - */ - std::mutex front_mut; - std::mutex queue_mutex; - /** - * - */ - std::vector inferRequests; - std::queue> promises; -}; -} // namespace ovms diff --git a/src/ovms_lib/rest_parser.cpp b/src/ovms_lib/rest_parser.cpp deleted file mode 100644 index 08a2e5149e..0000000000 --- a/src/ovms_lib/rest_parser.cpp +++ /dev/null @@ -1,468 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "rest_parser.hpp" - -#include -#include - -#include "rest_utils.hpp" -#include "tfs_frontend/tfs_utils.hpp" - -namespace ovms { - -RestParser::RestParser(const tensor_map_t& tensors) { - for (const auto& kv : tensors) { - const auto& name = kv.first; - const auto& tensor = kv.second; - tensorPrecisionMap[name] = tensor->getPrecision(); - auto& input = (*requestProto.mutable_inputs())[name]; - input.set_dtype(getPrecisionAsDataType(tensor->getPrecision())); - - auto fold = [](size_t a, const Dimension& b) { - if (b.isDynamic()) { - if (b.isAny()) { - return static_cast(0); - } else { - return static_cast(b.getMaxValue()); - } - } else { - return a * static_cast(b.getStaticValue()); - } - }; - input.mutable_tensor_content()->reserve(std::accumulate( - tensor->getShape().cbegin(), - tensor->getShape().cend(), - 1, - fold) * - DataTypeSize(getPrecisionAsDataType(tensor->getPrecision()))); - } -} - -void RestParser::removeUnusedInputs() { - auto& inputs = (*requestProto.mutable_inputs()); - auto it = inputs.begin(); - while (it != inputs.end()) { - if (!it->second.tensor_shape().dim_size()) { - SPDLOG_DEBUG("Removing {} input from proto since it's not included in the request", it->first); - it = inputs.erase(it); - } else { - it++; - } - } -} - -bool RestParser::parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - proto.set_dtype(tensorflow::DataType::DT_UINT64); - for (auto& value : doc.GetArray()) { - if (value.IsUint64()) - proto.add_uint64_val(value.GetUint64()); - else - return false; - } - return true; -} - -bool RestParser::parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - proto.set_dtype(tensorflow::DataType::DT_UINT32); - for (auto& value : doc.GetArray()) { - if (value.IsUint()) - proto.add_uint32_val(value.GetUint()); - else - return false; - } - return true; -} - -bool RestParser::parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName) { - // Special tensors are given in 1 dimentional array - if (doc.GetArray()[0].IsArray()) - return false; - - if (tensorName == "sequence_id") - return parseSequenceIdInput(doc, proto, tensorName); - else if (tensorName == "sequence_control_input") - return parseSequenceControlInput(doc, proto, tensorName); - - return false; -} - -static bool isBinary(const rapidjson::Value& value) { - if (!value.IsObject()) { - return false; - } - - if (!(value.HasMember("b64") && ((value.MemberEnd() - value.MemberBegin()) == 1))) { - return false; - } - - if (!value["b64"].IsString()) { - return false; - } - - return true; -} - -bool RestParser::parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName) { - if (isBinary(doc)) { - if (!addValue(proto, doc)) { - return false; - } - return true; - } - if (!doc.IsArray()) { - return false; - } - if (doc.GetArray().Size() == 0) { - return false; - } - if (!setDimOrValidate(proto, dim, doc.GetArray().Size())) { - return false; - } - if (tensorName == "sequence_id" || tensorName == "sequence_control_input") { - if (!parseSpecialInput(doc, proto, tensorName)) { - return false; - } - return true; - } - if (doc.GetArray()[0].IsArray()) { - for (auto& itr : doc.GetArray()) { - if (!parseArray(itr, dim + 1, proto, tensorName)) { - return false; - } - } - return true; - } else { - if (!setDTypeIfNotSet(doc.GetArray()[0], proto, tensorName)) { - return false; - } - for (auto& value : doc.GetArray()) { - if (!addValue(proto, value)) { - return false; - } - } - return true; - } - return false; -} - -bool RestParser::parseInstance(rapidjson::Value& doc) { - if (doc.GetObject().MemberCount() == 0) { - return false; - } - for (auto& itr : doc.GetObject()) { - std::string tensorName = itr.name.GetString(); - auto& proto = (*requestProto.mutable_inputs())[tensorName]; - increaseBatchSize(proto); - if (!parseArray(itr.value, 1, proto, tensorName)) { - return false; - } - } - return true; -} - -bool RestParser::isBatchSizeEqualForAllInputs() const { - int64_t size = 0; - for (const auto& kv : requestProto.inputs()) { - if (size == 0) { - size = kv.second.tensor_shape().dim(0).size(); - } else if (kv.second.tensor_shape().dim(0).size() != size) { - return false; - } - } - return true; -} - -Status RestParser::parseRowFormat(rapidjson::Value& node) { - order = Order::ROW; - if (!node.IsArray()) { - return StatusCode::REST_INSTANCES_NOT_AN_ARRAY; - } - if (node.GetArray().Size() == 0) { - return StatusCode::REST_NO_INSTANCES_FOUND; - } - if (node.GetArray()[0].IsObject() && !isBinary(node.GetArray()[0])) { - // named format - for (auto& instance : node.GetArray()) { - if (!instance.IsObject()) { - return StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT; - } - - if (!this->parseInstance(instance)) { - return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; - } - } - } else if (node.GetArray()[0].IsArray() || node.GetArray()[0].IsNumber() || isBinary(node.GetArray()[0])) { - // no named format - if (requestProto.inputs_size() != 1) { - return StatusCode::REST_INPUT_NOT_PREALLOCATED; - } - auto inputsIterator = requestProto.mutable_inputs()->begin(); - if (inputsIterator == requestProto.mutable_inputs()->end()) { - const std::string details = "Failed to parse row formatted request."; - SPDLOG_ERROR("Internal error occured: {}", details); - return Status(StatusCode::INTERNAL_ERROR, details); - } - if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { - return StatusCode::REST_COULD_NOT_PARSE_INSTANCE; - } else { - format = Format::NONAMED; - return StatusCode::OK; - } - } else { - return StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED; - } - removeUnusedInputs(); - if (!isBatchSizeEqualForAllInputs()) { - return StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER; - } - format = Format::NAMED; - return StatusCode::OK; -} - -Status RestParser::parseColumnFormat(rapidjson::Value& node) { - order = Order::COLUMN; - // no named format - if (node.IsArray()) { - if (requestProto.inputs_size() != 1) { - return StatusCode::REST_INPUT_NOT_PREALLOCATED; - } - auto inputsIterator = requestProto.mutable_inputs()->begin(); - if (inputsIterator == requestProto.mutable_inputs()->end()) { - const std::string details = "Failed to parse column formatted request."; - SPDLOG_ERROR("Internal error occured: {}", details); - return Status(StatusCode::INTERNAL_ERROR, details); - } - if (!parseArray(node, 0, inputsIterator->second, inputsIterator->first)) { - return StatusCode::REST_COULD_NOT_PARSE_INPUT; - } - format = Format::NONAMED; - return StatusCode::OK; - } - // named format - if (!node.IsObject()) { - return StatusCode::REST_INPUTS_NOT_AN_OBJECT; - } - if (node.GetObject().MemberCount() == 0) { - return StatusCode::REST_NO_INPUTS_FOUND; - } - for (auto& kv : node.GetObject()) { - std::string tensorName = kv.name.GetString(); - auto& proto = (*requestProto.mutable_inputs())[tensorName]; - if (!parseArray(kv.value, 0, proto, tensorName)) { - return StatusCode::REST_COULD_NOT_PARSE_INPUT; - } - } - removeUnusedInputs(); - format = Format::NAMED; - return StatusCode::OK; -} - -Status RestParser::parse(const char* json) { - rapidjson::Document doc; - if (doc.Parse(json).HasParseError()) { - return StatusCode::JSON_INVALID; - } - if (!doc.IsObject()) { - return StatusCode::REST_BODY_IS_NOT_AN_OBJECT; - } - auto instancesItr = doc.FindMember("instances"); - auto inputsItr = doc.FindMember("inputs"); - if (instancesItr != doc.MemberEnd() && inputsItr != doc.MemberEnd()) { - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; - } - if (instancesItr != doc.MemberEnd()) { - return parseRowFormat(instancesItr->value); - } - if (inputsItr != doc.MemberEnd()) { - return parseColumnFormat(inputsItr->value); - } - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; -} - -void RestParser::increaseBatchSize(tensorflow::TensorProto& proto) { - if (proto.tensor_shape().dim_size() < 1) { - proto.mutable_tensor_shape()->add_dim()->set_size(0); - } - proto.mutable_tensor_shape()->mutable_dim(0)->set_size(proto.tensor_shape().dim(0).size() + 1); -} - -bool RestParser::setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size) { - if (proto.tensor_shape().dim_size() > dim) { - return proto.tensor_shape().dim(dim).size() == size; - } else { - while (proto.tensor_shape().dim_size() <= dim) { - proto.mutable_tensor_shape()->add_dim()->set_size(0); - } - proto.mutable_tensor_shape()->mutable_dim(dim)->set_size(size); - return true; - } -} - -bool getB64FromValue(const rapidjson::Value& value, std::string& b64Val) { - if (!isBinary(value)) { - return false; - } - - b64Val = value["b64"].GetString(); - return true; -} - -template -bool addToTensorContent(tensorflow::TensorProto& proto, T value) { - if (sizeof(T) != DataTypeSize(proto.dtype())) { - return false; - } - proto.mutable_tensor_content()->append(reinterpret_cast(&value), sizeof(T)); - return true; -} - -template -bool addToTensorContent(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - return addToTensorContent(proto, static_cast(value.GetDouble())); - } - if (value.IsInt64()) { - return addToTensorContent(proto, static_cast(value.GetInt64())); - } - if (value.IsUint64()) { - return addToTensorContent(proto, static_cast(value.GetUint64())); - } - if (value.IsInt()) { - return addToTensorContent(proto, static_cast(value.GetInt())); - } - if (value.IsUint()) { - return addToTensorContent(proto, static_cast(value.GetUint())); - } - - return false; -} - -bool addToHalfVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - proto.add_half_val(value.GetDouble()); - return true; - } - if (value.IsInt64()) { - proto.add_half_val(value.GetInt64()); - return true; - } - if (value.IsUint64()) { - proto.add_half_val(value.GetUint64()); - return true; - } - if (value.IsInt()) { - proto.add_half_val(value.GetInt()); - return true; - } - if (value.IsUint()) { - proto.add_half_val(value.GetUint()); - return true; - } - - return false; -} - -bool addToIntVal(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (value.IsDouble()) { - proto.add_int_val(value.GetDouble()); - return true; - } - if (value.IsInt64()) { - proto.add_int_val(value.GetInt64()); - return true; - } - if (value.IsUint64()) { - proto.add_int_val(value.GetUint64()); - return true; - } - if (value.IsInt()) { - proto.add_int_val(value.GetInt()); - return true; - } - if (value.IsUint()) { - proto.add_int_val(value.GetUint()); - return true; - } - - return false; -} - -bool RestParser::addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value) { - if (isBinary(value)) { - std::string b64Val; - if (!getB64FromValue(value, b64Val)) - return false; - std::string decodedBytes; - if (decodeBase64(b64Val, decodedBytes) == StatusCode::OK) { - proto.add_string_val(decodedBytes.c_str(), decodedBytes.length()); - proto.set_dtype(tensorflow::DataType::DT_STRING); - return true; - } else { - return false; - } - } - - if (!value.IsNumber()) { - return false; - } - - switch (proto.dtype()) { - case tensorflow::DataType::DT_FLOAT: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_INT32: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_INT8: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT8: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_DOUBLE: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_HALF: - return addToHalfVal(proto, value); - case tensorflow::DataType::DT_INT16: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT16: - return addToIntVal(proto, value); - case tensorflow::DataType::DT_INT64: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT32: - return addToTensorContent(proto, value); - case tensorflow::DataType::DT_UINT64: - return addToTensorContent(proto, value); - default: - return false; - } - return false; -} - -// This is still required for parsing inputs which are not present in model/DAG. -// Such inputs are then removed from proto at the end of parsing phase. -bool RestParser::setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName) { - if (tensorPrecisionMap.count(tensorName)) - return true; - - if (value.IsInt()) - tensorPrecisionMap[tensorName] = ovms::Precision::I32; - else if (value.IsDouble()) - tensorPrecisionMap[tensorName] = ovms::Precision::FP32; - else - return false; - - proto.set_dtype(getPrecisionAsDataType(tensorPrecisionMap[tensorName])); - return true; -} - -} // namespace ovms diff --git a/src/ovms_lib/rest_parser.hpp b/src/ovms_lib/rest_parser.hpp deleted file mode 100644 index b8a0745a7c..0000000000 --- a/src/ovms_lib/rest_parser.hpp +++ /dev/null @@ -1,218 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -/** - * @brief Request order types - */ -enum class Order { - UNKNOWN, - ROW, - COLUMN -}; - -/** - * @brief Request format types - */ -enum class Format { - UNKNOWN, - NAMED, - NONAMED -}; - -/** - * @brief This class encapsulates http request body string parsing to request proto. - */ -class RestParser { - /** - * @brief Request order - */ - Order order = Order::UNKNOWN; - - /** - * @brief Request format - */ - Format format = Format::UNKNOWN; - - /** - * @brief Request proto - */ - tensorflow::serving::PredictRequest requestProto; - - /** - * @brief Request content precision - */ - std::map tensorPrecisionMap; - - void removeUnusedInputs(); - - /** - * @brief Increases batch size (0th-dimension) of tensor - */ - static void increaseBatchSize(tensorflow::TensorProto& proto); - - /** - * @brief Sets specific dimension to given size - * - * @return returns false if dimension already existed and did not match requested size, true otherwise - */ - static bool setDimOrValidate(tensorflow::TensorProto& proto, int dim, int size); - - /** - * Parses and adds rapidjson value to tensor proto depending on underlying tensor data type - */ - static bool addValue(tensorflow::TensorProto& proto, const rapidjson::Value& value); - - bool parseSequenceIdInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - bool parseSequenceControlInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - bool parseSpecialInput(rapidjson::Value& doc, tensorflow::TensorProto& proto, const std::string& tensorName); - - /** - * @brief Parses rapidjson Node for arrays or numeric values on certain level of nesting. - * - * @param doc rapidjson Node - * @param dim level of nesting - * @param input destination for parsing numeric values - * - * @return false if processing failed, true when succeeded - * - * Rapid json node expected to be passed in following structure: - * [ - * [...], - * [...], - * ... - * ] - */ - bool parseArray(rapidjson::Value& doc, int dim, tensorflow::TensorProto& proto, const std::string& tensorName); - - /** - * @brief Parses rapidjson Node for inputs in a string(name)=>array(data) format - * - * @param doc rapidjson Node - * - * @return false if processing failed, true when succeeded - * - * Rapid json node expected to be passed in following structure: - * { - * "input1": [[...], [...], ...], - * "input2": [[...], [...], ...], - * ... - * } - */ - bool parseInstance(rapidjson::Value& doc); - - /** - * @brief Checks whether all inputs have equal batch size, 0th-dimension - * - * @return true or false - */ - bool isBatchSizeEqualForAllInputs() const; - - /** - * @brief Parses row format: list of objects, each object corresponding to one batch with one or multiple inputs. - * When no named format is detected, instance is treated as array of single input batches with no name. - * - * @param node rapidjson Node - * - * @return Status indicating if processing succeeded, error code otherwise - * - * Rapid json node expected to be passed in following structure: - * [{inputs...}, {inputs...}, {inputs...}, ...] - * or: - * [no named input data batches...] - */ - Status parseRowFormat(rapidjson::Value& node); - - /** - * @brief Parses column format: object of input:batches key value pairs. - * When no named format is detected, instance is treated as array of single input batches with no name. - * - * @param node rapidjson Node - * - * @return Status indicating if processing succeeded, error code otherwise - * - * Rapid json node expected to be passed in following structure: - * {"inputA": [...], "inputB": [...], ...} - * or: - * [no named input data batches...] - */ - Status parseColumnFormat(rapidjson::Value& node); - -public: - bool setDTypeIfNotSet(const rapidjson::Value& value, tensorflow::TensorProto& proto, const std::string& tensorName); - /** - * @brief Constructor for preallocating memory for inputs beforehand. Size is calculated from tensor shape required by backend. - * - * @param tensors Tensor map with model input parameters - */ - RestParser(const tensor_map_t& tensors); - - /** - * @brief Gets parsed request proto - * - * @return proto - */ - tensorflow::serving::PredictRequest& getProto() { return requestProto; } - - /** - * @brief Gets request order - */ - Order getOrder() const { - return order; - } - - /** - * @brief Gets request format - */ - Format getFormat() const { - return format; - } - - /** - * @brief Parses http request body string - * - * @param json request string - * - * @return Status indicating error code or success - * - * JSON expected to be passed in following structure: - * { - * "signature_name": "serving_default", - * "instances": [ - * {...}, {...}, {...}, ... - * ] - * } - */ - Status parse(const char* json); -}; - -} // namespace ovms diff --git a/src/ovms_lib/rest_utils.cpp b/src/ovms_lib/rest_utils.cpp deleted file mode 100644 index d3a33fbcbc..0000000000 --- a/src/ovms_lib/rest_utils.cpp +++ /dev/null @@ -1,198 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "rest_utils.hpp" - -#include - -#include "absl/strings/escaping.h" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/util/json_tensor.h" -#pragma GCC diagnostic pop - -#include "tfs_frontend/tfs_utils.hpp" -#include "timer.hpp" - -using tensorflow::DataType; -using tensorflow::DataTypeSize; -using tensorflow::serving::JsonPredictRequestFormat; -using tensorflow::serving::MakeJsonFromTensors; -using tensorflow::serving::PredictResponse; - -namespace ovms { - -Status checkValField(const size_t& fieldSize, const size_t& expectedElementsNumber) { - if (fieldSize == 0) - return StatusCode::REST_SERIALIZE_NO_DATA; - if (fieldSize != expectedElementsNumber) - return StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE; - return StatusCode::OK; -} - -Status makeJsonFromPredictResponse( - PredictResponse& response_proto, - std::string* response_json, - Order order) { - if (order == Order::UNKNOWN) { - return StatusCode::REST_PREDICT_UNKNOWN_ORDER; - } - - Timer timer; - using std::chrono::microseconds; - - timer.start("convert"); - - for (auto& kv : *response_proto.mutable_outputs()) { - auto& tensor = kv.second; - - size_t expectedContentSize = DataTypeSize(tensor.dtype()); - for (int i = 0; i < tensor.tensor_shape().dim_size(); i++) { - expectedContentSize *= tensor.tensor_shape().dim(i).size(); - } - size_t dataTypeSize = DataTypeSize(tensor.dtype()); - size_t expectedElementsNumber = dataTypeSize > 0 ? expectedContentSize / dataTypeSize : 0; - bool seekDataInValField = false; - - if (tensor.tensor_content().size() == 0) - seekDataInValField = true; - else if (tensor.tensor_content().size() != expectedContentSize) - return StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE; - - switch (tensor.dtype()) { - case DataType::DT_FLOAT: - if (seekDataInValField) { - auto status = checkValField(tensor.float_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(float)) - tensor.add_float_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT32: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int32_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT8: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int8_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT8: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint8_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_DOUBLE: - if (seekDataInValField) { - auto status = checkValField(tensor.double_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(double)) - tensor.add_double_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT16: - if (seekDataInValField) { - auto status = checkValField(tensor.int_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int16_t)) - tensor.add_int_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_INT64: - if (seekDataInValField) { - auto status = checkValField(tensor.int64_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(int64_t)) - tensor.add_int64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT32: - if (seekDataInValField) { - auto status = checkValField(tensor.uint32_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint32_t)) - tensor.add_uint32_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - case DataType::DT_UINT64: - if (seekDataInValField) { - auto status = checkValField(tensor.uint64_val_size(), expectedElementsNumber); - if (!status.ok()) - return status; - } else { - for (size_t i = 0; i < tensor.tensor_content().size(); i += sizeof(uint64_t)) - tensor.add_uint64_val(*reinterpret_cast(tensor.mutable_tensor_content()->data() + i)); - } - break; - default: - return StatusCode::REST_UNSUPPORTED_PRECISION; - } - } - - timer.stop("convert"); - timer.start("MakeJsonFromTensors"); - - const auto& tf_status = MakeJsonFromTensors( - response_proto.outputs(), - order == Order::ROW ? JsonPredictRequestFormat::kRow : JsonPredictRequestFormat::kColumnar, - response_json); - - timer.stop("MakeJsonFromTensors"); - SPDLOG_DEBUG("tensor_content to *_val container conversion: {:.3f} ms", timer.elapsed("convert") / 1000); - SPDLOG_DEBUG("MakeJsonFromTensors call: {:.3f} ms", timer.elapsed("MakeJsonFromTensors") / 1000); - - if (!tf_status.ok()) { - SPDLOG_ERROR("Creating json from tensors failed: {}", tf_status.error_message()); - return StatusCode::REST_PROTO_TO_STRING_ERROR; - } - - return StatusCode::OK; -} - -Status decodeBase64(std::string& bytes, std::string& decodedBytes) { - auto status = Status(absl::Base64Unescape(bytes, &decodedBytes) ? StatusCode::OK : StatusCode::REST_BASE64_DECODE_ERROR); - if (!status.ok()) { - return status; - } - return StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/rest_utils.hpp b/src/ovms_lib/rest_utils.hpp deleted file mode 100644 index fbc928f9e3..0000000000 --- a/src/ovms_lib/rest_utils.hpp +++ /dev/null @@ -1,36 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "rest_parser.hpp" -#include "status.hpp" - -namespace ovms { -Status makeJsonFromPredictResponse( - tensorflow::serving::PredictResponse& response_proto, - std::string* response_json, - Order order); - -Status decodeBase64(std::string& bytes, std::string& decodedBytes); - -} // namespace ovms diff --git a/src/ovms_lib/s3filesystem.cpp b/src/ovms_lib/s3filesystem.cpp deleted file mode 100644 index b4fc5f47e3..0000000000 --- a/src/ovms_lib/s3filesystem.cpp +++ /dev/null @@ -1,552 +0,0 @@ -// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include "s3filesystem.hpp" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "logging.hpp" -#include "stringutils.hpp" - -namespace ovms { - -namespace s3 = Aws::S3; -namespace fs = std::filesystem; - -const std::string S3FileSystem::S3_URL_PREFIX = "s3://"; - -StatusCode S3FileSystem::parsePath(const std::string& path, std::string* bucket, std::string* object) { - std::smatch sm; - - if (isPathEscaped(path)) { - SPDLOG_LOGGER_ERROR(s3_logger, "Path {} escape with .. is forbidden.", path); - return StatusCode::PATH_INVALID; - } - - if (!std::regex_match(path, sm, s3_regex_)) { - int bucket_start = path.find(S3_URL_PREFIX) + S3_URL_PREFIX.size(); - int bucket_end = path.find("/", bucket_start); - - if (bucket_end > bucket_start) { - *bucket = path.substr(bucket_start, bucket_end - bucket_start); - *object = path.substr(bucket_end + 1); - } else { - *bucket = path.substr(bucket_start); - *object = ""; - } - } else { - *bucket = sm[3]; - *object = sm[4]; - } - - if (bucket->empty()) { - SPDLOG_LOGGER_ERROR(s3_logger, "No bucket name found in path {}", path); - - return StatusCode::S3_BUCKET_NOT_FOUND; - } - - return StatusCode::OK; -} - -S3FileSystem::S3FileSystem(const Aws::SDKOptions& options, const std::string& s3_path) : - options_(options), - s3_regex_(S3_URL_PREFIX + "([0-9a-zA-Z-.]+):([0-9]+)/([0-9a-z.-]+)(((/" - "[0-9a-zA-Z.-_]+)*)?)"), - proxy_regex_("^(https?)://(([^:]{1,128}):([^@]{1,256})@)?([^:/]{1,255})(:([0-9]{1,5}))?/?") { - Aws::Client::ClientConfiguration config; - Aws::Auth::AWSCredentials credentials; - - const char* profile_name = std::getenv("AWS_PROFILE"); - const char* secret_key = std::getenv("AWS_SECRET_ACCESS_KEY"); - const char* key_id = std::getenv("AWS_ACCESS_KEY_ID"); - const char* region = std::getenv("AWS_REGION"); - const char* s3_endpoint = std::getenv("S3_ENDPOINT"); - const char* http_proxy = std::getenv("http_proxy") != nullptr ? std::getenv("http_proxy") : std::getenv("HTTP_PROXY"); - const char* https_proxy = std::getenv("https_proxy") != nullptr ? std::getenv("https_proxy") : std::getenv("HTTPS_PROXY"); - const std::string default_proxy = https_proxy != nullptr ? std::string(https_proxy) : http_proxy != nullptr ? std::string(http_proxy) : ""; - - if ((secret_key != NULL) && (key_id != NULL)) { - credentials.SetAWSAccessKeyId(key_id); - credentials.SetAWSSecretKey(secret_key); - config = Aws::Client::ClientConfiguration(); - if (region != NULL) { - config.region = region; - } - } else if (profile_name) { - config = Aws::Client::ClientConfiguration(profile_name); - } else { - config = Aws::Client::ClientConfiguration("default"); - } - - std::string host_name, host_port, bucket, object; - std::smatch sm; - if (std::regex_match(s3_path, sm, s3_regex_)) { - host_name = sm[1]; - host_port = sm[2]; - bucket = sm[3]; - object = sm[4]; - - config.endpointOverride = Aws::String(host_name + ":" + host_port); - config.scheme = Aws::Http::Scheme::HTTP; - } - if (s3_endpoint != NULL) { - std::string endpoint(s3_endpoint); - if (endpoint.rfind("http://") != std::string::npos) { - endpoint = endpoint.substr(7); - } - config.endpointOverride = Aws::String(endpoint.c_str()); - config.scheme = Aws::Http::Scheme::HTTP; - } - - if (!default_proxy.empty()) { - if (std::regex_match(default_proxy, sm, proxy_regex_)) { - config.proxyHost = sm[5].str(); - config.proxyPort = std::stoi(sm[7].str()); - config.proxyScheme = sm[1].str().size() == 4 ? Aws::Http::Scheme::HTTP : Aws::Http::Scheme::HTTPS; - if (!sm[3].str().empty()) { - config.proxyUserName = sm[3].str(); - } - if (!sm[4].str().empty()) { - config.proxyPassword = sm[4].str(); - } - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Couldn't parse proxy: {}", default_proxy); - } - } - - if (profile_name) { - client_ = s3::S3Client( - config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - false); - } else if ((secret_key != NULL) && (key_id != NULL)) { - client_ = s3::S3Client( - credentials, - config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - false); - } else { - std::shared_ptr provider = std::make_shared(); - client_ = s3::S3Client( - provider, - config, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - true); - } -} - -S3FileSystem::~S3FileSystem() { - Aws::ShutdownAPI(options_); -} - -StatusCode S3FileSystem::fileExists(const std::string& path, bool* exists) { - *exists = false; - std::string bucket, object; - - auto status = parsePath(path, &bucket, &object); - if (status != StatusCode::OK) { - return status; - } - - s3::Model::HeadObjectRequest head_request; - head_request.SetBucket(bucket.c_str()); - head_request.SetKey(object.c_str()); - - auto head_object_outcome = client_.HeadObject(head_request); - if (head_object_outcome.IsSuccess()) { - *exists = true; - return StatusCode::OK; - } - - bool is_dir; - status = isDirectory(path, &is_dir); - if (status != StatusCode::OK) { - return status; - } - *exists = is_dir; - - return StatusCode::OK; -} - -StatusCode S3FileSystem::isDirectory(const std::string& path, bool* is_dir) { - *is_dir = false; - std::string bucket, object_path; - auto status = parsePath(path, &bucket, &object_path); - if (status != StatusCode::OK) { - return status; - } - - // Check if the bucket exists - s3::Model::HeadBucketRequest head_request; - head_request.WithBucket(bucket.c_str()); - - auto head_bucket_outcome = client_.HeadBucket(head_request); - if (!head_bucket_outcome.IsSuccess()) { - SPDLOG_LOGGER_ERROR(s3_logger, "Invalid or missing S3 credentials, or bucket does not exist - {}. {}", bucket, head_bucket_outcome.GetError().GetMessage()); - return StatusCode::S3_METADATA_FAIL; - } - - // Root case - bucket exists and object path is empty - if (object_path.empty()) { - *is_dir = true; - return StatusCode::OK; - } - - // List the objects in the bucket - s3::Model::ListObjectsRequest list_objects_request; - list_objects_request.SetBucket(bucket.c_str()); - list_objects_request.SetPrefix(appendSlash(object_path).c_str()); - auto list_objects_outcome = client_.ListObjects(list_objects_request); - - if (list_objects_outcome.IsSuccess()) { - *is_dir = !list_objects_outcome.GetResult().GetContents().empty(); - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to list objects with prefix {}", path); - return StatusCode::S3_FAILED_LIST_OBJECTS; - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::getDirectoryContents(const std::string& path, std::set* contents) { - // Parse bucket and dir_path - std::string bucket, dir_path, full_dir; - auto status = parsePath(path, &bucket, &dir_path); - if (status != StatusCode::OK) { - return status; - } - std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; - - // Capture the full path to facilitate content listing - full_dir = appendSlash(dir_path); - - // Issue request for objects with prefix - s3::Model::ListObjectsRequest objects_request; - objects_request.SetBucket(bucket.c_str()); - objects_request.SetPrefix(full_dir.c_str()); - auto list_objects_outcome = client_.ListObjects(objects_request); - - if (list_objects_outcome.IsSuccess()) { - Aws::Vector object_list = list_objects_outcome.GetResult().GetContents(); - for (auto const& s3_object : object_list) { - // In the case of empty directories, the directory itself will appear here - if (s3_object.GetKey().c_str() == full_dir) { - continue; - } - - // We have to make sure that subdirectory contents do not appear here - std::string name(s3_object.GetKey().c_str()); - int item_start = name.find(full_dir) + full_dir.size(); - int item_end = name.find("/", item_start); - - // Let set take care of subdirectory contents - std::string item = name.substr(item_start, item_end - item_start); - contents->insert(item); - } - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Could not list contents of directory {}", true_path); - return StatusCode::S3_INVALID_ACCESS; - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::getDirectorySubdirs(const std::string& path, std::set* subdirs) { - // Parse bucket and dir_path - std::string bucket, dir_path; - auto status = parsePath(path, &bucket, &dir_path); - if (status != StatusCode::OK) { - return status; - } - std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; - status = getDirectoryContents(true_path, subdirs); - if (status != StatusCode::OK) { - return status; - } - - // Erase non-directory entries... - for (auto iter = subdirs->begin(); iter != subdirs->end();) { - bool is_dir; - status = isDirectory(joinPath({true_path, *iter}), &is_dir); - if (status != StatusCode::OK) { - return status; - } - if (!is_dir) { - iter = subdirs->erase(iter); - } else { - ++iter; - } - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::getDirectoryFiles(const std::string& path, std::set* files) { - // Parse bucket and dir_path - std::string bucket, dir_path; - auto status = parsePath(path, &bucket, &dir_path); - if (status != StatusCode::OK) { - return status; - } - - std::string true_path = S3_URL_PREFIX + bucket + '/' + dir_path; - status = getDirectoryContents(true_path, files); - if (status != StatusCode::OK) { - return status; - } - - // Erase directory entries... - for (auto iter = files->begin(); iter != files->end();) { - bool is_dir; - status = isDirectory(joinPath({true_path, *iter}), &is_dir); - if (status != StatusCode::OK) { - return status; - } - if (is_dir) { - iter = files->erase(iter); - } else { - ++iter; - } - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::readTextFile(const std::string& path, std::string* contents) { - bool exists; - auto status = fileExists(path, &exists); - if (status != StatusCode::OK) { - return status; - } - - if (!exists) { - SPDLOG_LOGGER_ERROR(s3_logger, "File does not exist at {}", path); - return StatusCode::S3_FILE_NOT_FOUND; - } - - std::string bucket, object; - status = parsePath(path, &bucket, &object); - if (status != StatusCode::OK) { - return status; - } - - // Send a request for the objects metadata - s3::Model::GetObjectRequest object_request; - object_request.SetBucket(bucket.c_str()); - object_request.SetKey(object.c_str()); - - auto get_object_outcome = client_.GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - auto& object_result = get_object_outcome.GetResultWithOwnership().GetBody(); - - std::string data = ""; - char c = 0; - while (object_result.get(c)) { - data += c; - } - - *contents = data; - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", path); - return StatusCode::S3_FILE_INVALID; - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { - bool exists; - auto status = fileExists(path, &exists); - if (status != StatusCode::OK) { - return status; - } - if (!exists) { - SPDLOG_LOGGER_ERROR(s3_logger, "File/folder does not exist at {}", path); - return StatusCode::S3_FILE_NOT_FOUND; - } - - std::string effective_path, host_name, host_port, bucket, object; - - std::smatch sm; - if (std::regex_match(path, sm, s3_regex_)) { - host_name = sm[1]; - host_port = sm[2]; - bucket = sm[3]; - object = sm[4]; - effective_path = S3_URL_PREFIX + bucket + object; - } else { - effective_path = path; - } - - bool is_dir = false; - std::set contents, files; - status = isDirectory(effective_path, &is_dir); - if (status != StatusCode::OK) { - return status; - } - if (is_dir) { - status = getDirectoryContents(effective_path, &contents); - if (status != StatusCode::OK) { - return status; - } - - for (auto iter = contents.begin(); iter != contents.end(); ++iter) { - bool is_subdir; - std::string s3_fpath = joinPath({effective_path, *iter}); - std::string local_fpath = joinPath({local_path, *iter}); - status = isDirectory(s3_fpath, &is_subdir); - if (status != StatusCode::OK) { - return status; - } - if (is_subdir) { - // Create local mirror of sub-directories - int status = mkdir(const_cast(local_fpath.c_str()), S_IRUSR | S_IWUSR | S_IXUSR); - if (status == -1) { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to create local folder: {} {} ", local_fpath, strerror(errno)); - return StatusCode::PATH_INVALID; - } - - // Add with s3 path - std::set subdir_files; - auto s = getDirectoryFiles(s3_fpath, &subdir_files); - if (s != StatusCode::OK) { - return s; - } - for (auto itr = subdir_files.begin(); itr != subdir_files.end(); ++itr) { - files.insert(joinPath({s3_fpath, *itr})); - } - } else { - files.insert(s3_fpath); - } - } - - for (auto iter = files.begin(); iter != files.end(); ++iter) { - if (std::any_of(acceptedFiles.begin(), acceptedFiles.end(), [&iter](const std::string& x) { - return iter->size() > 0 && endsWith(*iter, x); - })) { - std::string bucket, object; - status = parsePath(*iter, &bucket, &object); - if (status != StatusCode::OK) { - return status; - } - - // Send a request for the objects metadata - s3::Model::GetObjectRequest object_request; - object_request.SetBucket(bucket.c_str()); - object_request.SetKey(object.c_str()); - - auto get_object_outcome = client_.GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - auto& retrieved_file = - get_object_outcome.GetResultWithOwnership().GetBody(); - std::string s3_removed_path = (*iter).substr(effective_path.size()); - std::string local_file_path = joinPath({local_path, s3_removed_path}); - std::ofstream output_file(local_file_path.c_str(), std::ios::binary); - output_file << retrieved_file.rdbuf(); - output_file.close(); - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", *iter); - return StatusCode::S3_FAILED_GET_OBJECT; - } - } - } - } else { - std::string bucket, object; - auto s = parsePath(effective_path, &bucket, &object); - if (s != StatusCode::OK) { - return s; - } - - // Send a request for the objects metadata - s3::Model::GetObjectRequest object_request; - object_request.SetBucket(bucket.c_str()); - object_request.SetKey(object.c_str()); - - auto get_object_outcome = client_.GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - auto& retrieved_file = get_object_outcome.GetResultWithOwnership().GetBody(); - std::ofstream output_file(local_path.c_str(), std::ios::binary); - output_file << retrieved_file.rdbuf(); - output_file.close(); - } else { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to get object at {}", effective_path); - return StatusCode::S3_FAILED_GET_OBJECT; - } - } - - return StatusCode::OK; -} - -StatusCode S3FileSystem::downloadModelVersions(const std::string& path, - std::string* local_path, - const std::vector& versions) { - auto sc = createTempPath(local_path); - if (sc != StatusCode::OK) { - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to create a temporary path {}", sc); - return sc; - } - - StatusCode result = StatusCode::OK; - for (auto& ver : versions) { - std::string versionpath = path; - if (!endsWith(versionpath, "/")) { - versionpath.append("/"); - } - versionpath.append(std::to_string(ver)); - std::string lpath = *local_path; - if (!endsWith(lpath, "/")) { - lpath.append("/"); - } - lpath.append(std::to_string(ver)); - fs::create_directory(lpath); - auto status = downloadFileFolder(versionpath, lpath); - if (status != StatusCode::OK) { - result = status; - SPDLOG_LOGGER_ERROR(s3_logger, "Failed to download model version {}", versionpath); - } - } - - return result; -} - -StatusCode S3FileSystem::deleteFileFolder(const std::string& path) { - remove(path.c_str()); - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/s3filesystem.hpp b/src/ovms_lib/s3filesystem.hpp deleted file mode 100644 index 56757c3ddf..0000000000 --- a/src/ovms_lib/s3filesystem.hpp +++ /dev/null @@ -1,155 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include -#include - -#include "filesystem.hpp" -#include "status.hpp" - -namespace ovms { - -class S3FileSystem : public FileSystem { -public: - /** - * @brief Construct a new S3FileSystem object - * - * @param options - * @param s3_path - */ - S3FileSystem(const Aws::SDKOptions& options, const std::string& s3_path); - - /** - * @brief Destroy the S3FileSystem object - * - */ - ~S3FileSystem(); - - /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ - StatusCode fileExists(const std::string& path, bool* exists) override; - - /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ - StatusCode isDirectory(const std::string& path, bool* is_dir) override; - - /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; - - /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ - StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; - - /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ - StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; - - /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ - StatusCode readTextFile(const std::string& path, std::string* contents) override; - - /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ - StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; - - /** - * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ - StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; - - /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ - StatusCode deleteFileFolder(const std::string& path) override; - - static const std::string S3_URL_PREFIX; - -private: - /** - * @brief - * - * @param path - * @param bucket - * @param object - * @return StatusCode - */ - StatusCode parsePath(const std::string& path, std::string* bucket, std::string* object); - - /** - * @brief - * - */ - Aws::SDKOptions options_; - - /** - * @brief - * - */ - Aws::S3::S3Client client_; - std::regex s3_regex_; - std::regex proxy_regex_; -}; - -} // namespace ovms diff --git a/src/ovms_lib/schema.cpp b/src/ovms_lib/schema.cpp deleted file mode 100644 index d35b8ea947..0000000000 --- a/src/ovms_lib/schema.cpp +++ /dev/null @@ -1,351 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "schema.hpp" - -#include - -#include -#include -#include -#include -#include - -namespace ovms { -const char* MODELS_CONFIG_SCHEMA = R"({ - "definitions": { - "custom_loader_config": { - "type": "object", - "required": ["config"], - "properties": { - "config": { - "type": "object", - "required": ["loader_name", "library_path"], - "properties": { - "loader_name": { - "type": "string" - }, - "library_path": { - "type": "string" - }, - "loader_config_file": { - "type": "string" - } - }, - "additionalProperties": false - }, - "additionalProperties": false - } - }, - "model_config": { - "type": "object", - "required": ["config"], - "properties": { - "config": { - "type": "object", - "required": ["name", "base_path"], - "properties": { - "name": { - "type": "string" - }, - "base_path": { - "type": "string" - }, - "batch_size": { - "type": ["integer", "string"], - "minimum": 0 - }, - "model_version_policy": { - "type": "object" - }, - "shape": { - "type": ["object", "string"] - }, - "layout": { - "type": ["object", "string"] - }, - "nireq": { - "type": "integer", - "minimum": 0 - }, - "target_device": { - "type": "string" - }, - "allow_cache": { - "type": "boolean" - }, - "plugin_config": { - "type": "object" - }, - "stateful": { - "type": "boolean" - }, - "idle_sequence_cleanup": { - "type": "boolean" - }, - "low_latency_transformation": { - "type": "boolean" - }, - "max_sequence_number": { - "type": "integer", - "minimum": 0 - }, - "custom_loader_options": { - "type": "object", - "required": ["loader_name"], - "properties": { - "loader_name": { - "type": "string" - } - }, - "minProperties": 1 - } - }, - "additionalProperties": false - }, - "additionalProperties": false - } - }, - "source_node_names": { - "type": "object", - "required": ["node_name", "data_item"], - "properties": { - "node_name": { - "type": "string" - }, - "data_item": { - "type": "string" - } - }, - "additionalProperties": false - }, - "source_node": { - "type": "object", - "additionalProperties" : { - "$ref": "#/definitions/source_node_names" - }, - "minProperties": 1, - "maxProperties": 1 - }, - "output_alias": { - "type": "object", - "required": ["data_item", "alias"], - "properties": { - "data_item": { - "type": "string" - }, - "alias": { - "type": "string" - } - }, - "additionalProperties": false - }, - "node_config": { - "type": "object", - "required": ["name", "type", "inputs", "outputs"], - "oneOf": [ - { - "properties": { "type": { "enum": ["custom"] } }, - "required": ["library_name"], - "not": { "required": ["model_name"] } - }, - { - "properties": { "type": { "enum": ["DL model"] } }, - "not": { "required": ["library_name"] }, - "required": ["model_name"] - } - ], - "properties": { - "name": { - "type": "string" - }, - "model_name": { - "type": "string" - }, - "library_name": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["DL model", "custom"] - }, - "version": { - "type": "integer", - "minimum": 1 - }, - "inputs": { - "type": "array", - "items": { - "$ref": "#/definitions/source_node" - } - }, - "outputs": { - "type": "array", - "items": { - "$ref": "#/definitions/output_alias" - } - }, - "params": { - "type": "object", - "additionalProperties": { "type": "string" } - }, - "demultiply_count": { - "type": "integer", - "minimum": -1, - "maximum": 10000 - }, - "gather_from_node": { - "type": "string" - } - }, - "additionalProperties": false - }, - "pipeline_config": { - "type": "object", - "required": ["name", "nodes", "inputs", "outputs"], - "properties": { - "name": { - "type": "string" - }, - "nodes": { - "type": "array", - "items": { - "$ref": "#/definitions/node_config" - } - }, - "inputs": { - "type": "array", - "items": { - "type": "string" - } - }, - "outputs": { - "type": "array", - "items": { - "$ref": "#/definitions/source_node" - } - }, - "demultiply_count" : { - "type": "integer", - "minimum": -1, - "maximum": 10000 - } - }, - "additionalProperties": false - }, - "custom_node_library_config": { - "type": "object", - "required": ["name", "base_path"], - "properties": { - "name": { - "type": "string" - }, - "base_path": { - "type": "string" - } - }, - "additionalProperties": false - } - }, - "type": "object", - "required": ["model_config_list"], - "properties": { - "custom_loader_config_list": { - "type": "array", - "items": { - "$ref": "#/definitions/custom_loader_config" - } - }, - "model_config_list": { - "type": "array", - "items": { - "$ref": "#/definitions/model_config" - } - }, - "pipeline_config_list": { - "type": "array", - "items": { - "$ref": "#/definitions/pipeline_config" - } - }, - "custom_node_library_config_list": { - "type": "array", - "items": { - "$ref": "#/definitions/custom_node_library_config" - } - } - }, - "additionalProperties": false -})"; - -const char* MODELS_MAPPING_INPUTS_SCHEMA = R"({ - "type": "object", - "required": [ - "inputs" - ], - "properties": { - "outputs":{ - "type": "object" - }, - "inputs":{ - "type": "object" - } - }, - "additionalProperties": false - })"; - -const char* MODELS_MAPPING_OUTPUTS_SCHEMA = R"({ - "type": "object", - "required": [ - "outputs" - ], - "properties": { - "outputs":{ - "type": "object" - }, - "inputs":{ - "type": "object" - } - }, - "additionalProperties": false - })"; - -StatusCode validateJsonAgainstSchema(rapidjson::Document& json, const char* schema) { - rapidjson::Document schemaJson; - rapidjson::ParseResult parsingSucceeded = schemaJson.Parse(schema); - if (!parsingSucceeded) { - SPDLOG_ERROR("JSON schema parse error: {}, at: {}", rapidjson::GetParseError_En(parsingSucceeded.Code()), parsingSucceeded.Offset()); - return StatusCode::JSON_INVALID; - } - rapidjson::SchemaDocument parsedSchema(schemaJson); - rapidjson::SchemaValidator validator(parsedSchema); - if (!json.Accept(validator)) { - rapidjson::StringBuffer sb; - validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); - std::string schema = sb.GetString(); - std::string keyword = validator.GetInvalidSchemaKeyword(); - sb.Clear(); - validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); - std::string key = sb.GetString(); - - SPDLOG_ERROR("Given config is invalid according to schema: {}. Keyword: {} Key: {}", schema, keyword, key); - return StatusCode::JSON_INVALID; - } - - return StatusCode::OK; -} - -} // namespace ovms diff --git a/src/ovms_lib/schema.hpp b/src/ovms_lib/schema.hpp deleted file mode 100644 index 3753314619..0000000000 --- a/src/ovms_lib/schema.hpp +++ /dev/null @@ -1,28 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#include "status.hpp" - -namespace ovms { -extern const char* MODELS_CONFIG_SCHEMA; -extern const char* MODELS_MAPPING_INPUTS_SCHEMA; -extern const char* MODELS_MAPPING_OUTPUTS_SCHEMA; - -StatusCode validateJsonAgainstSchema(rapidjson::Document& json, const char* schema); -} // namespace ovms diff --git a/src/ovms_lib/sequence.cpp b/src/ovms_lib/sequence.cpp deleted file mode 100644 index 3b3cabaf5c..0000000000 --- a/src/ovms_lib/sequence.cpp +++ /dev/null @@ -1,67 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "sequence.hpp" - -#include - -using namespace InferenceEngine; - -namespace ovms { - -const uint64_t Sequence::getId() const { - return sequenceId; -} - -const sequence_memory_state_t& Sequence::getMemoryState() const { - return memoryState; -} - -const bool Sequence::isIdle() const { - return idle; -} - -void Sequence::setIdle(bool idle) { - this->idle = idle; -} - -Status Sequence::updateMemoryState(model_memory_state_t& newState) { - for (auto&& state : newState) { - auto stateName = state.get_name(); - ov::Tensor tensor = state.get_state(); - ov::Tensor copyTensor; - auto status = tensorClone(copyTensor, tensor); - if (!status.ok()) { - return status; - } - memoryState[stateName] = copyTensor; - } - setIdle(false); - return StatusCode::OK; -} - -std::mutex& Sequence::getMutex() { - return mutex; -} - -bool Sequence::isTerminated() const { - return terminated; -} - -void Sequence::setTerminated() { - this->terminated = true; -} - -} // namespace ovms diff --git a/src/ovms_lib/sequence.hpp b/src/ovms_lib/sequence.hpp deleted file mode 100644 index 2c6157e5ed..0000000000 --- a/src/ovms_lib/sequence.hpp +++ /dev/null @@ -1,60 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "ov_utils.hpp" -#include "status.hpp" - -namespace ovms { - -using sequence_memory_state_t = std::unordered_map; -using model_memory_state_t = std::vector; - -class Sequence { -private: - uint64_t sequenceId; - sequence_memory_state_t memoryState; - std::mutex mutex; - bool terminated; - bool idle; - -public: - Sequence(uint64_t sequenceId) : - sequenceId(sequenceId), - terminated(false), - idle(false) {} - const sequence_memory_state_t& getMemoryState() const; - const uint64_t getId() const; - const bool isIdle() const; - void setIdle(bool idle = true); - // In case updateMemoryState returns non-OK status code the sequence should be dropped - Status updateMemoryState(model_memory_state_t& newState); - std::mutex& getMutex(); - bool isTerminated() const; - void setTerminated(); -}; - -} // namespace ovms diff --git a/src/ovms_lib/sequence_manager.cpp b/src/ovms_lib/sequence_manager.cpp deleted file mode 100644 index bf61396fb5..0000000000 --- a/src/ovms_lib/sequence_manager.cpp +++ /dev/null @@ -1,157 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "sequence_manager.hpp" - -#include - -#include "logging.hpp" - -namespace ovms { - -uint64_t SequenceManager::getUniqueSequenceId() { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "No sequence id has been provided on SEQUENCE_START. Seeking unique sequence id..."); - bool uniqueIdFound = false; - while (!uniqueIdFound) { - if (sequenceExists(this->sequenceIdCounter) || this->sequenceIdCounter == 0) - this->sequenceIdCounter++; - else - uniqueIdFound = true; - } - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Found unique sequence id: {}", this->sequenceIdCounter); - return this->sequenceIdCounter; -} - -const uint32_t SequenceManager::getMaxSequenceNumber() const { - return maxSequenceNumber; -} - -void SequenceManager::setMaxSequenceNumber(uint32_t maxSequenceNumber) { - this->maxSequenceNumber = maxSequenceNumber; -} - -std::mutex& SequenceManager::getMutex() { - return mutex; -} - -bool SequenceManager::sequenceExists(const uint64_t sequenceId) const { - return sequences.find(sequenceId) != sequences.end(); -} - -Status SequenceManager::removeIdleSequences() { - std::unique_lock sequenceManagerLock(mutex); - for (auto it = sequences.begin(); it != sequences.end();) { - Sequence& sequence = it->second; - // Non blocking try to get mutex - std::unique_lock sequenceLock(sequence.getMutex(), std::try_to_lock); - if (!sequence.isTerminated() && sequenceLock.owns_lock()) { - sequenceLock.unlock(); - // We hold sequence manager lock before lock and after unlock so no other thread even attempts accessing that sequence at that moment - if (sequence.isIdle()) { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "[Idle sequence cleanup] Removing sequence with id: {} on model {}, version: {}", sequence.getId(), modelName, modelVersion); - it = sequences.erase(it); - continue; - } else { - sequence.setIdle(); - } - } - ++it; - } - - return StatusCode::OK; -} - -Status SequenceManager::hasSequence(const uint64_t sequenceId) { - if (!sequenceExists(sequenceId)) - return StatusCode::SEQUENCE_MISSING; - - if (getSequence(sequenceId).isTerminated()) - return StatusCode::SEQUENCE_MISSING; - - return StatusCode::OK; -} - -Status SequenceManager::createSequence(SequenceProcessingSpec& sequenceProcessingSpec) { - if (sequences.size() >= this->maxSequenceNumber) { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Max sequence number has been reached. Could not create new sequence.", modelName, modelVersion); - return StatusCode::MAX_SEQUENCE_NUMBER_REACHED; - } - - uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); - - if (sequenceId == 0) { - uint64_t uniqueSequenceId = getUniqueSequenceId(); - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Adding new sequence with ID: {}", modelName, modelVersion, uniqueSequenceId); - sequences.emplace(uniqueSequenceId, uniqueSequenceId); - sequenceProcessingSpec.setSequenceId(uniqueSequenceId); - return StatusCode::OK; - } - - if (sequenceExists(sequenceId)) { - if (getSequence(sequenceId).isTerminated()) { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID is currently being removed", modelName, modelVersion); - return StatusCode::SEQUENCE_TERMINATED; - } - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID already exists", modelName, modelVersion); - return StatusCode::SEQUENCE_ALREADY_EXISTS; - } else { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Adding new sequence with ID: {}", modelName, modelVersion, sequenceId); - sequences.emplace(sequenceId, sequenceId); - } - return StatusCode::OK; -} - -Status SequenceManager::terminateSequence(const uint64_t sequenceId) { - auto status = hasSequence(sequenceId); - if (!status.ok()) - return status; - - getSequence(sequenceId).setTerminated(); - return StatusCode::OK; -} - -Sequence& SequenceManager::getSequence(const uint64_t sequenceId) { - return sequences.at(sequenceId); -} - -Status SequenceManager::removeSequence(const uint64_t sequenceId) { - auto it = sequences.find(sequenceId); - if (it != sequences.end()) { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} versions {} Removing sequence with ID: {}", modelName, modelVersion, sequenceId); - sequences.erase(it); - } else { - SPDLOG_LOGGER_DEBUG(sequence_manager_logger, "Model {} version {} Sequence with provided ID does not exists", modelName, modelVersion); - return StatusCode::SEQUENCE_MISSING; - } - return StatusCode::OK; -} - -Status SequenceManager::processRequestedSpec(SequenceProcessingSpec& sequenceProcessingSpec) { - const uint32_t sequenceControlInput = sequenceProcessingSpec.getSequenceControlInput(); - const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); - Status status; - - if (sequenceControlInput == SEQUENCE_START) { - status = createSequence(sequenceProcessingSpec); - } else if (sequenceControlInput == NO_CONTROL_INPUT) { - status = hasSequence(sequenceId); - } else { // sequenceControlInput == SEQUENCE_END - status = terminateSequence(sequenceId); - } - return status; -} - -} // namespace ovms diff --git a/src/ovms_lib/sequence_manager.hpp b/src/ovms_lib/sequence_manager.hpp deleted file mode 100644 index a72460c5b3..0000000000 --- a/src/ovms_lib/sequence_manager.hpp +++ /dev/null @@ -1,83 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#pragma once - -#include -#include -#include -#include - -#include "modelversion.hpp" -#include "sequence.hpp" -#include "sequence_processing_spec.hpp" -#include "status.hpp" - -namespace ovms { - -const uint32_t NO_CONTROL_INPUT = 0; -const uint32_t SEQUENCE_START = 1; -const uint32_t SEQUENCE_END = 2; - -class SequenceManager { -private: - uint32_t maxSequenceNumber; - std::string modelName; - model_version_t modelVersion; - std::mutex mutex; - -protected: - std::unordered_map sequences; - - uint64_t sequenceIdCounter; - - uint64_t getUniqueSequenceId(); - - Status hasSequence(const uint64_t sequenceId); - - Status createSequence(SequenceProcessingSpec& sequenceProcessingSpec); - - Status terminateSequence(const uint64_t sequenceId); - -public: - SequenceManager() = default; - SequenceManager(uint32_t maxSequenceNumber, std::string modelName, model_version_t modelVersion) : - maxSequenceNumber(maxSequenceNumber), - modelName(modelName), - modelVersion(modelVersion), - sequenceIdCounter(1) {} - - uint64_t getSequencesCount() { - return sequences.size(); - } - - const uint32_t getMaxSequenceNumber() const; - - void setMaxSequenceNumber(uint32_t maxSequenceNumber); - - std::mutex& getMutex(); - - bool sequenceExists(const uint64_t sequenceId) const; - - Sequence& getSequence(const uint64_t sequenceId); - - Status removeSequence(const uint64_t sequenceId); - - Status removeIdleSequences(); - - Status processRequestedSpec(SequenceProcessingSpec& sequenceProcessingSpec); -}; -} // namespace ovms diff --git a/src/ovms_lib/sequence_processing_spec.hpp b/src/ovms_lib/sequence_processing_spec.hpp deleted file mode 100644 index 782087df42..0000000000 --- a/src/ovms_lib/sequence_processing_spec.hpp +++ /dev/null @@ -1,36 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -namespace ovms { - -class SequenceProcessingSpec { - uint32_t sequenceControlInput = 0; - uint64_t sequenceId = 0; - -public: - SequenceProcessingSpec() = default; - SequenceProcessingSpec(uint32_t sequenceControlInput, uint64_t sequenceId) : - sequenceControlInput(sequenceControlInput), - sequenceId(sequenceId) {} - const uint32_t getSequenceControlInput() const { return sequenceControlInput; } - void setSequenceControlInput(uint32_t sequenceControlInput) { this->sequenceControlInput = sequenceControlInput; } - const uint64_t getSequenceId() const { return sequenceId; } - void setSequenceId(uint64_t sequenceId) { this->sequenceId = sequenceId; } -}; -} // namespace ovms diff --git a/src/ovms_lib/serialization.cpp b/src/ovms_lib/serialization.cpp deleted file mode 100644 index 686fa2e2ef..0000000000 --- a/src/ovms_lib/serialization.cpp +++ /dev/null @@ -1,245 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "serialization.hpp" - -#include "ov_utils.hpp" -#include "tfs_frontend/tfs_utils.hpp" - -namespace ovms { - -Status serializePrecision( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - if (servableOutput->getOvPrecision() != tensor.get_element_type()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in precision expected:{} vs actual:{}", - servableOutput->getName(), - TensorInfo::getPrecisionAsString(servableOutput->getPrecision()), - tensor.get_element_type().get_type_name()); - return StatusCode::INTERNAL_ERROR; - } - switch (servableOutput->getPrecision()) { - case ovms::Precision::FP32: - case ovms::Precision::I32: - case ovms::Precision::FP64: - case ovms::Precision::I8: - case ovms::Precision::U8: - case ovms::Precision::I16: // 2 byte padding [v1, v0, 0, 0, u1, u0, 0, 0, ...] - case ovms::Precision::U16: - case ovms::Precision::FP16: - case ovms::Precision::I64: - responseOutput.set_dtype(getPrecisionAsDataType(servableOutput->getPrecision())); - break; - - case ovms::Precision::Q78: - case ovms::Precision::BIN: - case ovms::Precision::BOOL: - case ovms::Precision::MIXED: - case ovms::Precision::CUSTOM: - default: { - Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; - SPDLOG_ERROR(status.string()); - return status; - } - } - return StatusCode::OK; -} - -Status serializePrecision( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - if (servableOutput->getOvPrecision() != tensor.get_element_type()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in precision expected:{} vs actual:{}", - servableOutput->getName(), - TensorInfo::getPrecisionAsString(servableOutput->getPrecision()), - tensor.get_element_type().get_type_name()); - return StatusCode::INTERNAL_ERROR; - } - switch (servableOutput->getPrecision()) { - case ovms::Precision::FP64: - case ovms::Precision::FP32: - case ovms::Precision::FP16: - case ovms::Precision::I64: - case ovms::Precision::I32: - case ovms::Precision::I16: - case ovms::Precision::I8: - case ovms::Precision::U64: - case ovms::Precision::U32: - case ovms::Precision::U16: - case ovms::Precision::U8: - case ovms::Precision::BOOL: - responseOutput.set_datatype(servableOutput->getPrecisionAsKFSPrecision()); - break; - case ovms::Precision::UNDEFINED: - case ovms::Precision::MIXED: - case ovms::Precision::Q78: - case ovms::Precision::BIN: - default: { - Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; - SPDLOG_ERROR(status.string()); - return status; - } - } - return StatusCode::OK; -} - -Status serializeShape( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - responseOutput.mutable_tensor_shape()->Clear(); - auto& effectiveNetworkOutputShape = servableOutput->getShape(); - ov::Shape actualTensorShape = tensor.get_shape(); - if (effectiveNetworkOutputShape.size() != actualTensorShape.size()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in number of dimensions expected:{} vs actual:{}", - servableOutput->getName(), effectiveNetworkOutputShape.size(), actualTensorShape.size()); - return StatusCode::INTERNAL_ERROR; - } - for (size_t i = 0; i < effectiveNetworkOutputShape.size(); ++i) { - dimension_value_t dim = actualTensorShape[i]; - if (!effectiveNetworkOutputShape[i].match(dim)) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in dimension:{} expected:{} vs actual:{}", - servableOutput->getName(), i, effectiveNetworkOutputShape[i].toString(), dim); - return StatusCode::INTERNAL_ERROR; - } - responseOutput.mutable_tensor_shape()->add_dim()->set_size(dim); - } - return StatusCode::OK; -} - -Status serializeShape( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - responseOutput.clear_shape(); - auto& effectiveNetworkOutputShape = servableOutput->getShape(); - ov::Shape actualTensorShape = tensor.get_shape(); - if (effectiveNetworkOutputShape.size() != actualTensorShape.size()) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in number of dimensions expected:{} vs actual:{}", - servableOutput->getName(), effectiveNetworkOutputShape.size(), actualTensorShape.size()); - return StatusCode::INTERNAL_ERROR; - } - for (size_t i = 0; i < effectiveNetworkOutputShape.size(); ++i) { - dimension_value_t dim = actualTensorShape[i]; - if (!effectiveNetworkOutputShape[i].match(dim)) { - SPDLOG_ERROR("Failed to serialize tensor: {}. There is difference in dimension:{} expected:{} vs actual:{}", - servableOutput->getName(), i, effectiveNetworkOutputShape[i].toString(), dim); - return StatusCode::INTERNAL_ERROR; - } - responseOutput.add_shape(dim); - } - return StatusCode::OK; -} - -void serializeContent(std::string* content, ov::Tensor& tensor) { - // We only fill if the content is not already filled. - // It can be filled in gather exit node handler. - if (content->size() == 0) { - content->assign((char*)tensor.data(), tensor.get_byte_size()); - } -} - -Status serializeTensorToTensorProto( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - auto status = serializePrecision(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - status = serializeShape(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - serializeContent(responseOutput.mutable_tensor_content(), tensor); - return StatusCode::OK; -} - -Status serializeTensorToTensorProto( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, - std::string* rawOutputContents, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor) { - auto status = serializePrecision(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - status = serializeShape(responseOutput, servableOutput, tensor); - if (!status.ok()) { - return status; - } - serializeContent(rawOutputContents, tensor); - return StatusCode::OK; -} - -template <> -Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { - try { - tensor = outputSource.get_tensor(name); - } catch (const ov::Exception& e) { - Status status = StatusCode::OV_INTERNAL_SERIALIZATION_ERROR; - SPDLOG_DEBUG("{}: {}", status.string(), e.what()); - return status; - } - return StatusCode::OK; -} - -template <> -tensorflow::TensorProto& ProtoGetter::createOutput(const std::string& name) { - return (*protoStorage->mutable_outputs())[name]; -} - -template <> -std::string* ProtoGetter::createContent(const std::string& name) { - return nullptr; -} - -template <> -::inference::ModelInferResponse::InferOutputTensor& ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createOutput(const std::string& name) { - for (int i = 0; i < protoStorage->outputs_size(); i++) { - auto& tensor = *protoStorage->mutable_outputs(i); - if (tensor.name() == name) { - return tensor; - } - } - auto* output = protoStorage->add_outputs(); - output->set_name(name); - return *output; -} - -template <> -std::string* ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&>::createContent(const std::string& name) { - for (int i = 0; i < protoStorage->outputs_size(); i++) { - auto& tensor = *protoStorage->mutable_outputs(i); - if (tensor.name() == name) { - if (protoStorage->raw_output_contents_size() <= i) { - return protoStorage->add_raw_output_contents(); - } - return protoStorage->mutable_raw_output_contents(i); - } - } - return protoStorage->add_raw_output_contents(); -} - -const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo) { - return tensorInfo.getName(); -} - -const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo) { - return first; -} -} // namespace ovms diff --git a/src/ovms_lib/serialization.hpp b/src/ovms_lib/serialization.hpp deleted file mode 100644 index 2eb5f56456..0000000000 --- a/src/ovms_lib/serialization.hpp +++ /dev/null @@ -1,126 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include - -#include -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "kfs_grpc_inference_service.hpp" -#include "profiler.hpp" -#include "status.hpp" -#include "tensorinfo.hpp" - -namespace ovms { - -template -class OutputGetter { -public: - OutputGetter(T t) : - outputSource(t) {} - Status get(const std::string& name, ov::Tensor& tensor); - -private: - T outputSource; -}; - -template -class ProtoGetter { - ProtoStorage protoStorage; - -public: - ProtoGetter(ProtoStorage protoStorage) : - protoStorage(protoStorage) {} - ProtoType createOutput(const std::string& name); - std::string* createContent(const std::string& name); -}; - -Status serializeTensorToTensorProto( - tensorflow::TensorProto& responseOutput, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor); - -Status serializeTensorToTensorProto( - ::inference::ModelInferResponse::InferOutputTensor& responseOutput, - std::string* rawOutputContents, - const std::shared_ptr& servableOutput, - ov::Tensor& tensor); - -typedef const std::string& (*outputNameChooser_t)(const std::string&, const TensorInfo&); -const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo); -const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo); - -template -Status serializePredictResponse( - OutputGetter& outputGetter, - const tensor_map_t& outputMap, - ResponseType* response, - outputNameChooser_t outputNameChooser); -// partial template specialization -template -Status serializePredictResponse( - OutputGetter& outputGetter, - const tensor_map_t& outputMap, - tensorflow::serving::PredictResponse* response, - outputNameChooser_t outputNameChooser) { - OVMS_PROFILE_FUNCTION(); - Status status; - ProtoGetter protoGetter(response); - for (const auto& [outputName, outputInfo] : outputMap) { - ov::Tensor tensor; - status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); - if (!status.ok()) { - return status; - } - auto& tensorProto = protoGetter.createOutput(outputInfo->getMappedName()); - status = serializeTensorToTensorProto(tensorProto, outputInfo, tensor); - if (!status.ok()) { - return status; - } - } - return status; -} -template -Status serializePredictResponse( - OutputGetter& outputGetter, - const tensor_map_t& outputMap, - ::inference::ModelInferResponse* response, - outputNameChooser_t outputNameChooser) { - Status status; - ProtoGetter<::inference::ModelInferResponse*, ::inference::ModelInferResponse::InferOutputTensor&> protoGetter(response); - for (const auto& [outputName, outputInfo] : outputMap) { - ov::Tensor tensor; - status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); - if (!status.ok()) { - return status; - } - auto& inferOutputTensor = protoGetter.createOutput(outputInfo->getMappedName()); - status = serializeTensorToTensorProto(inferOutputTensor, protoGetter.createContent(outputInfo->getMappedName()), outputInfo, tensor); - if (!status.ok()) { - return status; - } - } - return status; -} -} // namespace ovms diff --git a/src/ovms_lib/servablemanagermodule.cpp b/src/ovms_lib/servablemanagermodule.cpp deleted file mode 100644 index 9a7bc9f0e9..0000000000 --- a/src/ovms_lib/servablemanagermodule.cpp +++ /dev/null @@ -1,56 +0,0 @@ -//*************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "servablemanagermodule.hpp" - -#include -#include - -#include "config.hpp" -#include "logging.hpp" -#include "modelmanager.hpp" - -namespace ovms { - -ServableManagerModule::ServableManagerModule() : - servableManager(std::make_unique()) {} - -int ServableManagerModule::start(const ovms::Config& config) { - state = ModuleState::STARTED_INITIALIZE; - SPDLOG_INFO("{} starting", SERVABLE_MANAGER_MODULE_NAME); - auto status = servableManager->start(config); - if (status.ok()) { - state = ModuleState::INITIALIZED; - SPDLOG_INFO("{} started", SERVABLE_MANAGER_MODULE_NAME); - // FIXME this should be reenabled in grpcserver module when functional tests are switched to wait - // for servablemanager module start log - // #KFS_CLEANUP - SPDLOG_INFO("Server started on port {}", config.port()); - return EXIT_SUCCESS; - } - SPDLOG_ERROR("ovms::ModelManager::Start() Error: {}", status.string()); - return EXIT_FAILURE; -} -void ServableManagerModule::shutdown() { - state = ModuleState::STARTED_SHUTDOWN; - SPDLOG_INFO("{} shutting down", SERVABLE_MANAGER_MODULE_NAME); - servableManager->join(); - state = ModuleState::SHUTDOWN; - SPDLOG_INFO("{} shutdown", SERVABLE_MANAGER_MODULE_NAME); -} -ModelManager& ServableManagerModule::getServableManager() const { - return *servableManager; -} -} // namespace ovms diff --git a/src/ovms_lib/servablemanagermodule.hpp b/src/ovms_lib/servablemanagermodule.hpp deleted file mode 100644 index 77b1b2d75c..0000000000 --- a/src/ovms_lib/servablemanagermodule.hpp +++ /dev/null @@ -1,35 +0,0 @@ -//**************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include - -#include "server.hpp" - -namespace ovms { -class Config; -class ModelManager; - -class ServableManagerModule : public Module { -protected: - mutable std::unique_ptr servableManager; - -public: - ServableManagerModule(); - int start(const ovms::Config& config) override; - void shutdown() override; - ModelManager& getServableManager() const; -}; -} // namespace ovms diff --git a/src/ovms_lib/server.cpp b/src/ovms_lib/server.cpp deleted file mode 100644 index 5765cd9817..0000000000 --- a/src/ovms_lib/server.cpp +++ /dev/null @@ -1,348 +0,0 @@ -//**************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "server.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "config.hpp" -#include "grpcservermodule.hpp" -#include "http_server.hpp" -#include "kfs_grpc_inference_service.hpp" -#include "logging.hpp" -#include "model_service.hpp" -#include "modelmanager.hpp" -#include "prediction_service.hpp" -#include "profiler.hpp" -#include "servablemanagermodule.hpp" -#include "stringutils.hpp" -#include "version.hpp" - -using grpc::ServerBuilder; - -namespace ovms { -const std::string PROFILER_MODULE_NAME = "ProfilerModule"; -const std::string GRPC_SERVER_MODULE_NAME = "GRPCServerModule"; -const std::string HTTP_SERVER_MODULE_NAME = "HTTPServerModule"; -const std::string SERVABLE_MANAGER_MODULE_NAME = "ServableManagerModule"; -} // namespace ovms -using namespace ovms; - -namespace { -volatile sig_atomic_t shutdown_request = 0; -} - -Server& Server::instance() { - static Server global; - return global; -} - -void logConfig(const Config& config) { - std::string project_name(PROJECT_NAME); - std::string project_version(PROJECT_VERSION); - SPDLOG_INFO(project_name + " " + project_version); - SPDLOG_INFO("OpenVINO backend {}", OPENVINO_NAME); - SPDLOG_DEBUG("CLI parameters passed to ovms server"); - if (config.configPath().empty()) { - SPDLOG_DEBUG("model_path: {}", config.modelPath()); - SPDLOG_DEBUG("model_name: {}", config.modelName()); - SPDLOG_DEBUG("batch_size: {}", config.batchSize()); - SPDLOG_DEBUG("shape: {}", config.shape()); - SPDLOG_DEBUG("model_version_policy: {}", config.modelVersionPolicy()); - SPDLOG_DEBUG("nireq: {}", config.nireq()); - SPDLOG_DEBUG("target_device: {}", config.targetDevice()); - SPDLOG_DEBUG("plugin_config: {}", config.pluginConfig()); - SPDLOG_DEBUG("stateful: {}", config.stateful()); - SPDLOG_DEBUG("idle_sequence_cleanup: {}", config.idleSequenceCleanup()); - SPDLOG_DEBUG("max_sequence_number: {}", config.maxSequenceNumber()); - SPDLOG_DEBUG("low_latency_transformation: {}", config.lowLatencyTransformation()); - } else { - SPDLOG_DEBUG("config_path: {}", config.configPath()); - } - SPDLOG_DEBUG("gRPC port: {}", config.port()); - SPDLOG_DEBUG("REST port: {}", config.restPort()); - SPDLOG_DEBUG("gRPC bind address: {}", config.grpcBindAddress()); - SPDLOG_DEBUG("REST bind address: {}", config.restBindAddress()); - SPDLOG_DEBUG("REST workers: {}", config.restWorkers()); - SPDLOG_DEBUG("gRPC workers: {}", config.grpcWorkers()); - SPDLOG_DEBUG("gRPC channel arguments: {}", config.grpcChannelArguments()); - SPDLOG_DEBUG("log level: {}", config.logLevel()); - SPDLOG_DEBUG("log path: {}", config.logPath()); - SPDLOG_DEBUG("file system poll wait seconds: {}", config.filesystemPollWaitSeconds()); - SPDLOG_DEBUG("sequence cleaner poll wait minutes: {}", config.sequenceCleanerPollWaitMinutes()); -} - -void onInterrupt(int status) { - shutdown_request = 1; -} - -void onTerminate(int status) { - shutdown_request = 1; -} - -void onIllegal(int status) { - shutdown_request = 2; -} - -void installSignalHandlers(ovms::Server& server) { - static struct sigaction sigIntHandler; - sigIntHandler.sa_handler = onInterrupt; - sigemptyset(&sigIntHandler.sa_mask); - sigIntHandler.sa_flags = 0; - sigaction(SIGINT, &sigIntHandler, NULL); - - static struct sigaction sigTermHandler; - sigTermHandler.sa_handler = onTerminate; - sigemptyset(&sigTermHandler.sa_mask); - sigTermHandler.sa_flags = 0; - sigaction(SIGTERM, &sigTermHandler, NULL); - - static struct sigaction sigIllHandler; - sigIllHandler.sa_handler = onIllegal; - sigemptyset(&sigIllHandler.sa_mask); - sigIllHandler.sa_flags = 0; - sigaction(SIGILL, &sigIllHandler, NULL); -} - -static const int GIGABYTE = 1024 * 1024 * 1024; - -ModuleState Module::getState() const { - return state; -} - -// TODO should replace all messages like -// start REST Server with start HTTP Server -// start Server with start gRPC server -// this should be synchronized with validation tests changes - -class HTTPServerModule : public Module { - std::unique_ptr server; - Server& ovmsServer; - -public: - HTTPServerModule(ovms::Server& ovmsServer) : - ovmsServer(ovmsServer) {} - int start(const ovms::Config& config) override { - state = ModuleState::STARTED_INITIALIZE; - const std::string server_address = config.restBindAddress() + ":" + std::to_string(config.restPort()); - int workers = config.restWorkers() ? config.restWorkers() : 10; - - SPDLOG_INFO("Will start {} REST workers", workers); - server = ovms::createAndStartHttpServer(config.restBindAddress(), config.restPort(), workers, this->ovmsServer); - if (server != nullptr) { - SPDLOG_INFO("Started REST server at {}", server_address); - } else { - SPDLOG_ERROR("Failed to start REST server at " + server_address); - return EXIT_FAILURE; - } - state = ModuleState::INITIALIZED; - return EXIT_SUCCESS; - } - void shutdown() override { - if (server == nullptr) - return; - state = ModuleState::STARTED_SHUTDOWN; - server->Terminate(); - server->WaitForTermination(); - SPDLOG_INFO("Shutdown HTTP server"); - state = ModuleState::SHUTDOWN; - } -}; - -bool Server::isReady() const { - std::shared_lock lock(modulesMtx); - auto it = modules.find(SERVABLE_MANAGER_MODULE_NAME); - if (it == modules.end()) - return false; - if (ModuleState::INITIALIZED != it->second->getState()) - return false; - return true; -} - -bool Server::isLive() const { - // TODO we might want at some time start REST only/ or respond with true only if both servers started if both are requested to start. This is to be resolved especially if we implement REST API for Kserver & potentially switch to check for starting specific module - std::shared_lock lock(modulesMtx); - auto it = modules.find(GRPC_SERVER_MODULE_NAME); - if (it == modules.end()) - return false; - if (ModuleState::INITIALIZED != it->second->getState()) - return false; - return true; -} - -ModuleState Server::getModuleState(const std::string& name) const { - std::shared_lock lock(modulesMtx); - auto it = modules.find(name); - if (it == modules.end()) - return ModuleState::NOT_INITIALIZED; - return it->second->getState(); -} - -const Module* Server::getModule(const std::string& name) const { - std::shared_lock lock(modulesMtx); - auto it = modules.find(name); - if (it == modules.end()) - return nullptr; - return it->second.get(); -} - -#ifdef MTR_ENABLED -class ProfilerModule : public Module { - std::unique_ptr profiler; - -public: - ProfilerModule() = default; - int start(const Config& config) override { - state = ModuleState::STARTED_INITIALIZE; - auto profiler = std::make_unique(config.tracePath()); - if (!profiler.isInitialized()) { - SPDLOG_ERROR("Cannot open file for profiler, --trace_path: {}", config.tracePath()); - return EXIT_FAILURE; - } - state = ModuleState::INITIALIZED; - return EXIT_SUCCESS; - } - void shutdown() override { -#ifdef MTR_ENABLED - state = ModuleState::STARTED_SHUTDOWN; - profiler.reset(); - state = ModuleState::SHUTDOWN; -#endif - } -}; -#endif - -void Server::setShutdownRequest(int i) { - shutdown_request = i; -} - -Server::~Server() = default; - -std::unique_ptr Server::createModule(const std::string& name) { -#ifdef MTR_ENABLED - if (name == PROFILER_MODULE_NAME) - return std::make_unique(); -#endif - if (name == GRPC_SERVER_MODULE_NAME) - return std::make_unique(*this); - if (name == HTTP_SERVER_MODULE_NAME) - return std::make_unique(*this); - if (name == SERVABLE_MANAGER_MODULE_NAME) - return std::make_unique(); - return nullptr; -} - -int Server::startModules(ovms::Config& config) { - auto retCode = EXIT_SUCCESS; - bool inserted = false; - auto it = modules.end(); -#if MTR_ENABLED - { - auto module = this->createModule(PROFILER_MODULE_NAME); - std::unique_lock lock(modulesMtx); - std::tie(it, inserted) = this->modules.emplace(PROFILER_MODULE_NAME, std::move(module)); - } - retCode = modules.at(PROFILER_MODULE_NAME)->start(config); - if (retCode) - return retCode; -#endif - { - auto module = this->createModule(GRPC_SERVER_MODULE_NAME); - std::unique_lock lock(modulesMtx); - std::tie(it, inserted) = this->modules.emplace(GRPC_SERVER_MODULE_NAME, std::move(module)); - } - - if (!inserted) - return EXIT_FAILURE; - // if we ever decide not to start GRPC module then we need to implement HTTP responses without using grpc implementations - retCode = it->second->start(config); - if (retCode) - return retCode; - if (config.restPort() != 0) { - { - auto module = this->createModule(HTTP_SERVER_MODULE_NAME); - std::unique_lock lock(modulesMtx); - std::tie(it, inserted) = this->modules.emplace(HTTP_SERVER_MODULE_NAME, std::move(module)); - } - retCode = it->second->start(config); - if (retCode) - return retCode; - } - { - auto module = this->createModule(SERVABLE_MANAGER_MODULE_NAME); - std::unique_lock lock(modulesMtx); - std::tie(it, inserted) = this->modules.emplace(SERVABLE_MANAGER_MODULE_NAME, std::move(module)); - } - retCode = it->second->start(config); - return retCode; -} - -void Server::shutdownModules(ovms::Config& config) { - modules.at(GRPC_SERVER_MODULE_NAME)->shutdown(); - if (config.restPort() != 0) - modules.at(HTTP_SERVER_MODULE_NAME)->shutdown(); - modules.at(SERVABLE_MANAGER_MODULE_NAME)->shutdown(); -#ifdef MTR_ENABLED - modules.at(PROFILER_MODULE_NAME)->shutdown(); -#endif - // FIXME we need to be able to quickly start grpc or start it without port - // this is because the OS can have a delay between freeing up port before it can be requested and used again - modules.clear(); -} - -int Server::start(int argc, char** argv) { - ovms::Server& server = ovms::Server::instance(); - installSignalHandlers(server); - try { - auto& config = ovms::Config::instance().parse(argc, argv); - configure_logger(config.logLevel(), config.logPath()); - logConfig(config); - auto retCode = this->startModules(config); - if (retCode) - return retCode; - - while (!shutdown_request) { - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - if (shutdown_request == 2) { - SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); - } - SPDLOG_INFO("Shutting down"); - this->shutdownModules(config); - } catch (std::exception& e) { - SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); - return EXIT_FAILURE; - } catch (...) { - SPDLOG_ERROR("Unknown exception catch - will now terminate."); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; -} diff --git a/src/ovms_lib/server.hpp b/src/ovms_lib/server.hpp deleted file mode 100644 index 8b1cfb54f1..0000000000 --- a/src/ovms_lib/server.hpp +++ /dev/null @@ -1,75 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once -#include -#include -#include -#include -#include -#include - -namespace ovms { -class Config; -enum class ModuleState { - NOT_INITIALIZED, - STARTED_INITIALIZE, - INITIALIZED, - RELOADING, - STARTED_SHUTDOWN, - SHUTDOWN -}; - -class Module { -protected: - ModuleState state = ModuleState::NOT_INITIALIZED; - -public: - virtual int start(const ovms::Config& config) = 0; - virtual void shutdown() = 0; - virtual ~Module() = default; - ModuleState getState() const; -}; - -extern const std::string PROFILER_MODULE_NAME; -extern const std::string GRPC_SERVER_MODULE_NAME; -extern const std::string HTTP_SERVER_MODULE_NAME; -extern const std::string SERVABLE_MANAGER_MODULE_NAME; - -class Server { - mutable std::shared_mutex modulesMtx; - -protected: - std::unordered_map> modules; - Server() = default; - virtual std::unique_ptr createModule(const std::string& name); - -public: - static Server& instance(); - int start(int argc, char** argv); - ModuleState getModuleState(const std::string& name) const; - const Module* getModule(const std::string& name) const; - bool isReady() const; - bool isLive() const; - - void setShutdownRequest(int i); - virtual ~Server(); - - // TODO potentially to be hiden under protected and exposed only in tests by inheritance - // #KFS_CLEANUP - int startModules(ovms::Config& config); - void shutdownModules(ovms::Config& config); -}; -} // namespace ovms diff --git a/src/ovms_lib/shape.cpp b/src/ovms_lib/shape.cpp deleted file mode 100644 index b4d557d141..0000000000 --- a/src/ovms_lib/shape.cpp +++ /dev/null @@ -1,505 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "shape.hpp" - -#include -#include -#include -#include -#include - -#include "logging.hpp" -#include "stringutils.hpp" - -namespace ovms { - -Dimension::Dimension() : - Dimension(DYNAMIC_DIMENSION) { -} - -Dimension::Dimension(const ov::Dimension& dim) { - if (dim.is_static()) { - this->minimum = dim.get_length(); - this->maximum = dim.get_length(); - } else if (!dim.get_interval().has_upper_bound()) { - this->minimum = DYNAMIC_DIMENSION; - this->maximum = DYNAMIC_DIMENSION; - } else { - this->minimum = dim.get_min_length(); - this->maximum = dim.get_max_length(); - } -} - -Dimension::Dimension(dimension_value_t dim) : - Dimension(dim, dim) { -} - -Dimension::Dimension(dimension_value_t minimum, dimension_value_t maximum) { - if (minimum == -1 && maximum != -1) { - throw std::invalid_argument("Invalid range"); - } - if (minimum < -1 || maximum < -1) { - throw std::invalid_argument("Range must not be lower than -1"); - } - if (minimum > maximum) { - throw std::invalid_argument("Range maximum must be higher or equal to minimum"); - } - - this->minimum = minimum; - this->maximum = maximum; -} - -bool Dimension::isDynamic() const { - if (this->minimum != this->maximum) - return true; - if (this->minimum == DYNAMIC_DIMENSION) - return true; - return false; -} - -bool Dimension::isStatic() const { - return !this->isDynamic(); -} - -ov::Dimension Dimension::createPartialDimension() const { - if (this->isStatic()) { - return ov::Dimension(this->getStaticValue()); - } - if (this->minimum == DYNAMIC_DIMENSION) { - return ov::Dimension::dynamic(); - } - return ov::Dimension(this->minimum, this->maximum); -} - -dimension_value_t Dimension::getStaticValue() const { - if (this->isDynamic()) - throw std::invalid_argument("getStaticValue but dimension dynamic"); - return this->maximum; -} - -dimension_value_t Dimension::getMinValue() const { - if (this->isStatic()) - throw std::invalid_argument("getMinValue but dimension static"); - if (this->isAny()) - throw std::invalid_argument("getMinValue but dimension any"); - return this->minimum; -} - -dimension_value_t Dimension::getMaxValue() const { - if (this->isStatic()) - throw std::invalid_argument("getMaxValue but dimension static"); - if (this->isAny()) - throw std::invalid_argument("getMinValue but dimension any"); - return this->maximum; -} - -bool Dimension::operator==(const Dimension& rhs) const { - return this->minimum == rhs.minimum && this->maximum == rhs.maximum; -} - -bool Dimension::operator!=(const Dimension& rhs) const { - return !(this->operator==(rhs)); -} - -Status Dimension::fromString(const std::string& str, Dimension& dimOut) { - Dimension dim; - - std::string strCopy = str; - erase_spaces(strCopy); - if (strCopy.find(DIMENSION_RANGE_DELIMETER) != std::string::npos) { - // Range - if (strCopy.find_first_not_of("0123456789:") != std::string::npos) { - SPDLOG_ERROR("Parsing dimension string not a range: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - size_t delimCount = std::count(strCopy.begin(), strCopy.end(), DIMENSION_RANGE_DELIMETER); - if (delimCount != 1) { - SPDLOG_ERROR("Parsing dimension string, wrong amount of '{}' - {}; {}", DIMENSION_RANGE_DELIMETER, delimCount, strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } else { - std::vector tokens = tokenize(strCopy, DIMENSION_RANGE_DELIMETER); - if (tokens.size() == 2) { - try { - int dimNumberMin = std::stoi(tokens[0]); - int dimNumberMax = std::stoi(tokens[1]); - if (dimNumberMin > 0 && dimNumberMax > 0) { - dim = Dimension(dimNumberMin, dimNumberMax); - } else if (dimNumberMin >= dimNumberMax) { - SPDLOG_ERROR("Parsing dimension string range max must be higher than min: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } else { - SPDLOG_ERROR("Parsing dimension string range must be lager than 0: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - } catch (const std::out_of_range& e) { - SPDLOG_ERROR("Parsing dimension string out of range: {}, error: {}", strCopy, e.what()); - return StatusCode::DIM_WRONG_FORMAT; - } catch (...) { - SPDLOG_ERROR("Parsing dimension string: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - } else { - SPDLOG_ERROR("Parsing dimension string, not a number between '{}' - {}", DIMENSION_RANGE_DELIMETER, strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - } - } else { - size_t count = std::count(strCopy.begin(), strCopy.end(), '-'); - if (count > 1) { - SPDLOG_ERROR("Parsing dimension string: {}; too many '-' characters", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } else if (count == 1 && *strCopy.begin() != '-') { - SPDLOG_ERROR("Parsing dimension string: {}; invalid '-' position", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - // Number - if (strCopy.find_first_not_of("0123456789-") != std::string::npos) { - SPDLOG_ERROR("Parsing dimension string not a number: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - try { - int dimNumber = std::stoi(strCopy); - if (dimNumber == DYNAMIC_DIMENSION) { - dim = Dimension::any(); - } else if (dimNumber > 0) { - dim = Dimension(dimNumber); - } else { - SPDLOG_ERROR("Parsing dimension string out of range: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - } catch (const std::out_of_range& e) { - SPDLOG_ERROR("Parsing dimension string out of range: {}, error: {}", strCopy, e.what()); - return StatusCode::DIM_WRONG_FORMAT; - } catch (...) { - SPDLOG_ERROR("Parsing dimension string: {}", strCopy); - return StatusCode::DIM_WRONG_FORMAT; - } - } - - dimOut = dim; - return StatusCode::OK; -} - -bool Dimension::isAny() const { - return (this->maximum == DYNAMIC_DIMENSION) && - (this->minimum == DYNAMIC_DIMENSION); -} - -bool Dimension::partiallyFitsInto(const Dimension& next) const { - if (next.isAny() || isAny()) { - return true; - } - if (isStatic()) { - return next.match(getStaticValue()); - } - if (next.isStatic()) { - return this->match(next.getStaticValue()); - } - // both are dynamic - if (next.getMinValue() > getMaxValue()) { - return false; - } - if (next.getMaxValue() < getMinValue()) { - return false; - } - return true; -} - -bool Dimension::match(dimension_value_t value) const { - if (value < -1) - return false; - if (isAny()) { - return true; - } - if (isStatic()) { - return getStaticValue() == value; - } - if (value < getMinValue()) { - return false; - } - if (value > getMaxValue()) { - return false; - } - return true; -} - -std::string Dimension::toString() const { - std::stringstream ss; - - if (this->isStatic()) { - ss << this->minimum; - } else { - if (this->maximum == DYNAMIC_DIMENSION) { - ss << DYNAMIC_DIMENSION; - } else { - ss << "[" << this->minimum << "~" << this->maximum << "]"; - } - } - - return ss.str(); -} - -Dimension Dimension::any() { - return Dimension(); -} - -dimension_value_t Dimension::getLowerBound() const { - return isStatic() ? getStaticValue() : getMinValue(); -} - -dimension_value_t Dimension::getUpperBound() const { - return isStatic() ? getStaticValue() : getMaxValue(); -} - -std::optional Dimension::createIntersection(const Dimension& other) const { - if (*this == Dimension::any()) - return other; - if (other == Dimension::any()) - return *this; - auto start = std::max(this->getLowerBound(), other.getLowerBound()); - auto end = std::min(this->getUpperBound(), other.getUpperBound()); - if (end < start) - return std::nullopt; - return Dimension{start, end}; -} - -Shape::Shape() { -} - -Shape::Shape(std::initializer_list list) : - std::vector(list) {} - -Shape::Shape(const shape_t& s) { - auto status = fromFlatShape(s, *this); - if (!status.ok()) { - throw std::invalid_argument("Could not convert from flat shape"); - } -} - -Status Shape::fromFlatShape(const shape_t& shapeIn, Shape& shapeOut) { - Shape shape; - for (size_t dim : shapeIn) { - if (dim > std::numeric_limits::max()) { - return StatusCode::CANNOT_CONVERT_FLAT_SHAPE; - } else { - shape.add(Dimension{static_cast(dim)}); - } - } - shapeOut = shape; - return StatusCode::OK; -} - -Shape::Shape(const ov::PartialShape& shape) { - this->reserve(shape.size()); - for (const ov::Dimension& dim : shape) { - if (dim.is_static()) { - this->emplace_back(Dimension{dim.get_length()}); - } else if (!dim.get_interval().has_upper_bound()) { - this->emplace_back(Dimension::any()); - } else { - this->emplace_back(Dimension{dim.get_min_length(), dim.get_max_length()}); - } - } -} - -Shape& Shape::add(const Dimension& dim) { - return this->add(dim, this->size()); -} -Shape& Shape::add(const Dimension& dim, size_t pos) { - this->emplace(this->begin() + pos, dim); - return *this; -} - -ov::PartialShape Shape::createPartialShape() const { - ov::PartialShape shape; - - shape.reserve(this->size()); - for (const Dimension& dim : *this) { - if (dim.isStatic()) { - shape.push_back(ov::Dimension(dim.getStaticValue())); - } else if (dim.isAny()) { - shape.push_back(ov::Dimension::dynamic()); - } else { - shape.push_back(ov::Dimension{dim.getMinValue(), dim.getMaxValue()}); - } - } - - return shape; -} - -bool Shape::operator==(const Shape& rhs) const { - if (this->size() != rhs.size()) - return false; - - for (size_t i = 0; i < this->size(); i++) { - if ((*this)[i] != rhs[i]) { - return false; - } - } - return true; -} - -bool Shape::operator!=(const Shape& rhs) const { - return !(this->operator==(rhs)); -} - -bool Shape::match(const ov::Shape& ovShape) const { - for (size_t i = 0; i < this->size(); i++) { - if (!(*this)[i].match(ovShape[i])) { - return false; - } - } - return true; -} - -bool Shape::match(const ov::Shape& ovShape, const size_t skipPosition) const { - if (this->size() != ovShape.size()) { - return false; - } - for (size_t i = 0; i < skipPosition; i++) { - if (!(*this)[i].match(ovShape[i])) { - return false; - } - } - for (size_t i = skipPosition + 1; i < this->size(); i++) { - if (!(*this)[i].match(ovShape[i])) { - return false; - } - } - return true; -} - -std::optional Shape::createIntersection(const Shape& other) const { - if (this->size() != other.size()) - return std::nullopt; - Shape intersected; - intersected.reserve(this->size()); - for (size_t i = 0; i < this->size(); ++i) { - auto intersectedDim = (*this)[i].createIntersection(other[i]); - if (!intersectedDim.has_value()) { - return std::nullopt; - } - intersected.emplace_back(std::move(intersectedDim.value())); - } - return intersected; -} - -std::string Shape::toString() const { - std::stringstream ss; - - ss << "("; - - size_t dimensionCount = this->size(); - if (dimensionCount > 0) { - for (size_t i = 0; i < dimensionCount - 1; i++) { - ss << (*this)[i].toString() << ","; - } - ss << (*this)[dimensionCount - 1].toString(); - } - - ss << ")"; - - return ss.str(); -} - -Status Shape::fromString(const std::string& strIn, Shape& shapeOut) { - Shape shape; - std::string str = strIn; - - erase_spaces(str); - if (str.find_first_not_of("0123456789(),-:") != std::string::npos) - return StatusCode::SHAPE_WRONG_FORMAT; - - if (std::count(str.begin(), str.end(), '(') != 1) - return StatusCode::SHAPE_WRONG_FORMAT; - - if (std::count(str.begin(), str.end(), ')') != 1) - return StatusCode::SHAPE_WRONG_FORMAT; - - if (str.size() <= 2) - return StatusCode::SHAPE_WRONG_FORMAT; - - if (str.front() != '(' || str.back() != ')') - return StatusCode::SHAPE_WRONG_FORMAT; - - str.pop_back(); - str.erase(str.begin()); - - std::vector tokens = tokenize(str, ','); - - for (const std::string& token : tokens) { - size_t count = std::count(token.begin(), token.end(), '-'); - if (count > 1) { - SPDLOG_ERROR("Parsing model shape string: {}; too many '-' characters", token); - return StatusCode::SHAPE_WRONG_FORMAT; - } else if (count == 1 && !token.empty() && *token.begin() != '-') { - SPDLOG_ERROR("Parsing model shape string: {}; invalid '-' position", token); - return StatusCode::SHAPE_WRONG_FORMAT; - } - - count = std::count(token.begin(), token.end(), DIMENSION_RANGE_DELIMETER); - if (count > 1) { - SPDLOG_ERROR("Parsing model shape string: {}; too many '{}' characters", DIMENSION_RANGE_DELIMETER, token); - return StatusCode::SHAPE_WRONG_FORMAT; - } - try { - if (count == 0) { - int dimValue = std::stoi(token); - if (dimValue == DYNAMIC_DIMENSION || dimValue > 0) { - shape.add(Dimension(dimValue)); - } else { - SPDLOG_ERROR("Parsing model shape string: {}; must be {} (any) or higher than 0", token, DYNAMIC_DIMENSION); - return StatusCode::SHAPE_WRONG_FORMAT; - } - } else { - std::vector subTokens = tokenize(token, DIMENSION_RANGE_DELIMETER); - if (subTokens.size() != 2 || subTokens[0].empty() || subTokens[1].empty()) { - SPDLOG_ERROR("Parsing model shape string: {}; range must have min and max", strIn); - return StatusCode::SHAPE_WRONG_FORMAT; - } - int dimMin = std::stoi(subTokens[0]); - int dimMax = std::stoi(subTokens[1]); - if (dimMin <= 0 || dimMax <= 0) { - SPDLOG_ERROR("Parsing model shape string: {}; range must be higher than 0", token); - return StatusCode::SHAPE_WRONG_FORMAT; - } - if (dimMin >= dimMax) { - SPDLOG_ERROR("Parsing model shape string: {}; range max must be higher than min", token); - return StatusCode::SHAPE_WRONG_FORMAT; - } - shape.add(Dimension(dimMin, dimMax)); - } - } catch (const std::out_of_range& e) { - SPDLOG_ERROR("Parsing model shape string out of range: {}, error: {}", str, e.what()); - return StatusCode::SHAPE_WRONG_FORMAT; - } catch (...) { - SPDLOG_ERROR("Parsing model shape string: {}", strIn); - return StatusCode::SHAPE_WRONG_FORMAT; - } - } - - shapeOut = shape; - return StatusCode::OK; -} - -ShapeInfo::operator std::string() const { - std::stringstream ss; - ss << this->shape.toString() << " (" << (this->shapeMode == Mode::FIXED ? "fixed" : "auto") << ")"; - return ss.str(); -} - -} // namespace ovms diff --git a/src/ovms_lib/shape.hpp b/src/ovms_lib/shape.hpp deleted file mode 100644 index e8b504815a..0000000000 --- a/src/ovms_lib/shape.hpp +++ /dev/null @@ -1,128 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include - -#include "status.hpp" - -namespace ovms { - -using dimension_value_t = std::int64_t; - -constexpr dimension_value_t DYNAMIC_DIMENSION = -1; - -constexpr char DIMENSION_RANGE_DELIMETER = ':'; - -enum Mode { FIXED, - AUTO }; -using shape_t = std::vector; - -class Dimension { - dimension_value_t minimum, maximum; - -public: - Dimension(); - - Dimension(const ov::Dimension& dim); - - Dimension(dimension_value_t dim); - - Dimension(dimension_value_t minimum, dimension_value_t maximum); - - bool isStatic() const; - bool isDynamic() const; - - ov::Dimension createPartialDimension() const; - - dimension_value_t getStaticValue() const; - dimension_value_t getMinValue() const; - dimension_value_t getMaxValue() const; - - bool operator==(const Dimension& rhs) const; - bool operator!=(const Dimension& rhs) const; - - static Status fromString(const std::string& str, Dimension& dimOut); - std::string toString() const; - - static Dimension any(); - - bool match(dimension_value_t value) const; - bool partiallyFitsInto(const Dimension& value) const; - bool isAny() const; - std::optional createIntersection(const Dimension& other) const; - -private: - dimension_value_t getLowerBound() const; - dimension_value_t getUpperBound() const; -}; - -class Shape : public std::vector { -public: - Shape(); - Shape(const shape_t& s); - // Create shape out of ovms::Shape{1, 5, 100, 100} - Shape(std::initializer_list list); - - // Create ovms::Shape out of oridnary vector of dimensions. - static Status fromFlatShape(const shape_t& shapeIn, Shape& shapeOut); - - // Create ovms::Shape out of ov::PartialShape. - Shape(const ov::PartialShape& shape); - - Shape& add(const Dimension& dim, size_t pos); - Shape& add(const Dimension& dim); - - bool isStatic() const; - bool isDynamic() const; - - ov::PartialShape createPartialShape() const; - - bool operator==(const Shape& rhs) const; - bool operator!=(const Shape& rhs) const; - - bool match(const ov::Shape& rhs) const; - bool match(const ov::Shape& rhs, const size_t skipPosition) const; - std::optional createIntersection(const Shape& other) const; - - std::string toString() const; - static Status fromString(const std::string& strIn, Shape& shapeOut); -}; - -using shapes_map_t = std::unordered_map; - -struct ShapeInfo { - Mode shapeMode = FIXED; - Shape shape; - - operator std::string() const; - - bool operator==(const ShapeInfo& rhs) const { - return this->shapeMode == rhs.shapeMode && this->shape == rhs.shape; - } - - bool operator!=(const ShapeInfo& rhs) const { - return !(*this == rhs); - } -}; - -using shapes_info_map_t = std::unordered_map; - -} // namespace ovms diff --git a/src/ovms_lib/statefulmodelinstance.cpp b/src/ovms_lib/statefulmodelinstance.cpp deleted file mode 100644 index 6a71c0cb34..0000000000 --- a/src/ovms_lib/statefulmodelinstance.cpp +++ /dev/null @@ -1,324 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "statefulmodelinstance.hpp" - -#include -#include - -#include "deserialization.hpp" -#include "executingstreamidguard.hpp" -#include "logging.hpp" -#include "predict_request_validation_utils.hpp" -#include "profiler.hpp" -#include "serialization.hpp" -#include "timer.hpp" - -namespace ovms { - -const std::set StatefulModelInstance::SPECIAL_INPUT_NAMES{"sequence_id", "sequence_control_input"}; - -const Status StatefulModelInstance::extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId) { - if (!proto.tensor_shape().dim_size()) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto does not contain tensor shape information", getName(), getVersion()); - return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; - } else if (proto.tensor_shape().dim_size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid number of dimensions. Expecting shape with one dimension", getName(), getVersion()); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_id is: (1)"); - } - - if (proto.tensor_shape().dim(0).size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); - return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_id is: (1)"); - } - - if (proto.uint64_val_size() == 1) { - sequenceId = proto.uint64_val(0); - return StatusCode::OK; - } - return StatusCode::SEQUENCE_ID_BAD_TYPE; -} - -const Status StatefulModelInstance::extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput) { - if (proto.tensor_shape().dim_size() == 0) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto does not contain tensor shape information", getName(), getVersion()); - return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; - } else if (proto.tensor_shape().dim_size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid number of dimensions. Expecting shape with one dimension.", getName(), getVersion()); - return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_control_input is: (1)"); - } - - if (proto.tensor_shape().dim(0).size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); - return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_control_input is: (1)"); - } - - if (proto.uint32_val_size() == 1) { - sequenceControlInput = proto.uint32_val(0); - return StatusCode::OK; - } - return StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE; -} - -Status StatefulModelInstance::loadModel(const ModelConfig& config) { - std::lock_guard loadingLock(loadingMutex); - autoCleanupEnabled = config.getIdleSequenceCleanup(); - - Status status = ModelInstance::loadModel(config); - if (!status.ok()) - return status; - - if (autoCleanupEnabled) { - status = globalSequencesViewer->registerForCleanup(getName(), getVersion(), sequenceManager); - if (!status.ok()) - return status; - } - return StatusCode::OK; -} - -Status StatefulModelInstance::reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter) { - std::lock_guard loadingLock(loadingMutex); - Status status; - if (autoCleanupEnabled && this->status.getState() == ModelVersionState::AVAILABLE) { - status = globalSequencesViewer->unregisterFromCleanup(getName(), getVersion()); - if (!status.ok()) - return status; - } - status = ModelInstance::reloadModel(config, parameter); - if (!status.ok()) - return status; - autoCleanupEnabled = config.getIdleSequenceCleanup(); - - if (autoCleanupEnabled) { - status = globalSequencesViewer->registerForCleanup(getName(), getVersion(), sequenceManager); - if (!status.ok()) - return status; - } - return StatusCode::OK; -} - -void StatefulModelInstance::retireModel(bool isPermanent) { - std::lock_guard loadingLock(loadingMutex); - if (isPermanent && autoCleanupEnabled) { - globalSequencesViewer->unregisterFromCleanup(getName(), getVersion()); - } - ModelInstance::retireModel(isPermanent); - sequenceManager.reset(); -} - -void StatefulModelInstance::cleanupFailedLoad() { - std::lock_guard loadingLock(loadingMutex); - ModelInstance::cleanupFailedLoad(); - sequenceManager.reset(); -} - -Status StatefulModelInstance::loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter) { - performLowLatencyTransformation = config.isLowLatencyTransformationUsed(); - sequenceManager = std::make_shared(config.getMaxSequenceNumber(), config.getName(), config.getVersion()); - return ModelInstance::loadModelImpl(config, parameter); -} - -Status StatefulModelInstance::loadOVCompiledModel(const ModelConfig& config) { - if (performLowLatencyTransformation) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "[Model: {} version: {}] Performing Low Latency Transformation on the model", getName(), getVersion()); - try { - ov::pass::LowLatency2().run_on_model(model); - } catch (ov::Exception& ex) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error: {}; occurred during low latency transformation on model: {} version: {}", ex.what(), getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } catch (std::exception& ex) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error: {}; occurred during low latency transformation on model: {} version: {}", ex.what(), getName(), getVersion()); - return StatusCode::INTERNAL_ERROR; - } - } - return ModelInstance::loadOVCompiledModel(config); -} - -template <> -const Status StatefulModelInstance::validateSpecialKeys(const tensorflow::serving::PredictRequest* request, SequenceProcessingSpec& sequenceProcessingSpec) { - uint64_t sequenceId = 0; - uint32_t sequenceControlInput = 0; - Status status; - auto it = request->inputs().find("sequence_id"); - if (it != request->inputs().end()) { - status = extractSequenceId(it->second, sequenceId); - if (!status.ok()) - return status; - } - it = request->inputs().find("sequence_control_input"); - if (it != request->inputs().end()) { - status = extractSequenceControlInput(it->second, sequenceControlInput); - if (!status.ok()) - return status; - } - - if (sequenceControlInput != SEQUENCE_END && sequenceControlInput != NO_CONTROL_INPUT && sequenceControlInput != SEQUENCE_START) { - return StatusCode::INVALID_SEQUENCE_CONTROL_INPUT; - } - if ((sequenceControlInput == SEQUENCE_END || sequenceControlInput == NO_CONTROL_INPUT) && sequenceId == 0) { - return StatusCode::SEQUENCE_ID_NOT_PROVIDED; - } - - sequenceProcessingSpec.setSequenceId(sequenceId); - sequenceProcessingSpec.setSequenceControlInput(sequenceControlInput); - - return StatusCode::OK; -} - -template -const Status StatefulModelInstance::validate(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec) { - OVMS_PROFILE_FUNCTION(); - auto status = validateSpecialKeys(request, sequenceProcessingSpec); - if (!status.ok()) - return status; - - return request_validation_utils::validate( - *request, - getInputsInfo(), - getName(), - getVersion(), - SPECIAL_INPUT_NAMES, - getModelConfig().getBatchingMode(), - getModelConfig().getShapes()); -} - -Status StatefulModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - using std::chrono::microseconds; - SequenceProcessingSpec sequenceProcessingSpec; - auto status = validate(requestProto, sequenceProcessingSpec); - if (!status.ok()) - return status; - - std::unique_lock sequenceManagerLock(sequenceManager->getMutex()); - status = sequenceManager->processRequestedSpec(sequenceProcessingSpec); - if (!status.ok()) - return status; - const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); - if (!sequenceManager->sequenceExists(sequenceId)) - return StatusCode::INTERNAL_ERROR; - Sequence& sequence = sequenceManager->getSequence(sequenceId); - - std::unique_lock sequenceLock(sequence.getMutex()); - sequenceManagerLock.unlock(); - - timer.start("get infer request"); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue()); - int executingInferId = executingStreamIdGuard.getId(); - (void)executingInferId; - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop("get infer request"); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("get infer request") / 1000); - - timer.start("preprocess"); - status = preInferenceProcessing(inferRequest, sequence, sequenceProcessingSpec); - timer.stop("preprocess"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Preprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("preprocess") / 1000); - - timer.start("deserialize"); - InputSink inputSink(inferRequest); - bool isPipeline = false; - status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); - timer.stop("deserialize"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("deserialize") / 1000); - - timer.start("prediction"); - status = performInference(inferRequest); - timer.stop("prediction"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("prediction") / 1000); - - timer.start("serialize"); - OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); - timer.stop("serialize"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("serialize") / 1000); - - timer.start("postprocess"); - status = postInferenceProcessing(responseProto, inferRequest, sequence, sequenceProcessingSpec); - timer.stop("postprocess"); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Postprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed("postprocess") / 1000); - - sequenceLock.unlock(); - if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { - sequenceManagerLock.lock(); - status = sequenceManager->removeSequence(sequenceId); - if (!status.ok()) - return status; - } - - return StatusCode::OK; -} - -const Status StatefulModelInstance::preInferenceProcessing(ov::InferRequest& inferRequest, Sequence& sequence, - SequenceProcessingSpec& sequenceProcessingSpec) { - if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_START) { - // On SEQUENCE_START reset memory state of infer request to default - for (auto&& state : inferRequest.query_state()) { - state.reset(); - } - } else { - // For next requests in the sequence set infer request memory state to the last state saved by the sequence - const sequence_memory_state_t& sequenceMemoryState = sequence.getMemoryState(); - for (auto&& state : inferRequest.query_state()) { - auto stateName = state.get_name(); - if (!sequenceMemoryState.count(stateName)) - return StatusCode::INTERNAL_ERROR; - state.set_state(sequenceMemoryState.at(stateName)); - } - } - return StatusCode::OK; -} - -const Status StatefulModelInstance::postInferenceProcessing(tensorflow::serving::PredictResponse* response, - ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec) { - // Reset inferRequest states on SEQUENCE_END - if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { - spdlog::debug("Received SEQUENCE_END signal. Reseting model state and removing sequence"); - for (auto&& state : inferRequest.query_state()) { - state.reset(); - } - } else { - auto modelState = inferRequest.query_state(); - sequence.updateMemoryState(modelState); - } - - // Include sequence_id in server response - auto& tensorProto = (*response->mutable_outputs())["sequence_id"]; - tensorProto.mutable_tensor_shape()->add_dim()->set_size(1); - tensorProto.set_dtype(tensorflow::DataType::DT_UINT64); - tensorProto.add_uint64_val(sequenceProcessingSpec.getSequenceId()); - - return StatusCode::OK; -} -} // namespace ovms diff --git a/src/ovms_lib/statefulmodelinstance.hpp b/src/ovms_lib/statefulmodelinstance.hpp deleted file mode 100644 index ecb230fc41..0000000000 --- a/src/ovms_lib/statefulmodelinstance.hpp +++ /dev/null @@ -1,101 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -#include "global_sequences_viewer.hpp" -#include "modelconfig.hpp" -#include "modelinstance.hpp" -#include "sequence_manager.hpp" - -namespace ovms { - -class StatefulModelInstance : public ModelInstance { - static const std::set SPECIAL_INPUT_NAMES; - -public: - /** - * @brief A default constructor - */ - StatefulModelInstance(const std::string& name, model_version_t version, ov::Core& ieCore, GlobalSequencesViewer* globalSequencesViewer) : - ModelInstance(name, version, ieCore), - globalSequencesViewer(globalSequencesViewer) { - sequenceManager = std::make_shared(config.getMaxSequenceNumber(), name, version); - } - - const std::shared_ptr& getSequenceManager() const { - return this->sequenceManager; - } - - const Status extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId); - - const Status extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput); - /* - Performs pre inference operations: - - for SEQUENCE_START control input - reset InferRequest memory state - - for SEQUENCE_END control input or for no control input - load sequence memory state into InferRequest - - Always returns StatusCode::OK - */ - const Status preInferenceProcessing(ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec); - - /* - Performs pre inference operations: - - for SEQUENCE_START or for no control input - save InferRequest memory state in sequence memory state - - for SEQUENCE_END control input - reset InferRequest memory state - - for all requests - append sequence id to the response - - Always returns StatusCode::OK - */ - const Status postInferenceProcessing(tensorflow::serving::PredictResponse* response, - ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec); - - Status infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) override; - - Status loadModel(const ModelConfig& config) override; - - Status reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; - - void retireModel(bool isPermanent = true) override; - - void cleanupFailedLoad() override; - -protected: - std::shared_ptr sequenceManager; - - bool performLowLatencyTransformation; - - bool autoCleanupEnabled; - - GlobalSequencesViewer* globalSequencesViewer; - - template - const Status validate(const RequestType* request, SequenceProcessingSpec& processingSpec); - - Status loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; - - Status loadOVCompiledModel(const ModelConfig& config) override; - -private: - template - const Status validateSpecialKeys(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec); -}; -} // namespace ovms diff --git a/src/ovms_lib/status.cpp b/src/ovms_lib/status.cpp deleted file mode 100644 index 78cfa7279c..0000000000 --- a/src/ovms_lib/status.cpp +++ /dev/null @@ -1,426 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "status.hpp" - -namespace ovms { - -const std::unordered_map Status::statusMessageMap = { - {StatusCode::OK, ""}, - - {StatusCode::PATH_INVALID, "The provided base path is invalid or doesn't exists"}, - {StatusCode::FILE_INVALID, "File not found or cannot open"}, - {StatusCode::CONFIG_FILE_INVALID, "Configuration file not found or cannot open"}, - {StatusCode::FILESYSTEM_ERROR, "Error during filesystem operation"}, - {StatusCode::NOT_IMPLEMENTED, "Functionality not implemented"}, - {StatusCode::NO_MODEL_VERSION_AVAILABLE, "Not a single model version directory has valid numeric name"}, - {StatusCode::MODEL_NOT_LOADED, "Error while loading a model"}, - {StatusCode::JSON_INVALID, "The file is not valid json"}, - {StatusCode::JSON_SERIALIZATION_ERROR, "Data serialization to json format failed"}, - {StatusCode::MODELINSTANCE_NOT_FOUND, "ModelInstance not found"}, - {StatusCode::SHAPE_WRONG_FORMAT, "The provided shape is in wrong format"}, - {StatusCode::LAYOUT_WRONG_FORMAT, "The provided layout is in wrong format"}, - {StatusCode::DIM_WRONG_FORMAT, "The provided dimension is in wrong format"}, - {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, "Plugin config is in wrong format"}, - {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, "Model version policy is in wrong format"}, - {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, "Model version policy contains unsupported key"}, - {StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT, "Grpc channel arguments passed in wrong format"}, - {StatusCode::CONFIG_FILE_TIMESTAMP_READING_FAILED, "Error during config file timestamp reading"}, - {StatusCode::RESHAPE_ERROR, "Model could not be reshaped with requested shape"}, - {StatusCode::RESHAPE_REQUIRED, "Model needs to be reloaded with new shape"}, - {StatusCode::BATCHSIZE_CHANGE_REQUIRED, "Model needs to be reloaded with new batchsize"}, - {StatusCode::FORBIDDEN_MODEL_DYNAMIC_PARAMETER, "Value of provided parameter is forbidden"}, - {StatusCode::ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED, "Anonymous fixed shape is invalid for models with multiple inputs"}, - {StatusCode::ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED, "Anonymous fixed layout is invalid for models with multiple inputs"}, - {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, "Cannot compile model into target device"}, - {StatusCode::MODEL_MISSING, "Model with requested name and/or version is not found"}, - {StatusCode::MODEL_CONFIG_INVALID, "Model config is invalid"}, - {StatusCode::MODEL_NAME_MISSING, "Model with requested name is not found"}, - {StatusCode::MODEL_NAME_OCCUPIED, "Given model name is already occupied"}, - {StatusCode::MODEL_VERSION_MISSING, "Model with requested version is not found"}, - {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, "Model with requested version is retired"}, - {StatusCode::MODEL_VERSION_NOT_LOADED_YET, "Model with requested version is not loaded yet"}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, "Pipeline is retired"}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, "Pipeline is not loaded yet"}, - {StatusCode::MODEL_SPEC_MISSING, "model_spec missing in request"}, - {StatusCode::MODEL_VERSION_INVALID_FORMAT, "invalid model version format in request"}, - {StatusCode::INVALID_SIGNATURE_DEF, "Invalid signature name"}, - {StatusCode::CONFIG_SHAPE_IS_NOT_IN_MODEL, "Shape from config not found in model"}, - {StatusCode::CONFIG_LAYOUT_IS_NOT_IN_MODEL, "Layout from config not found in model"}, - {StatusCode::CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME, "Shape from config has real name. Use mapped name instead"}, - {StatusCode::CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME, "Layout from config has real name. Use mapped name instead"}, - {StatusCode::INVALID_NIREQ, "Nireq parameter too high"}, - {StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL, "Requested dynamic parameters but model is used in pipeline"}, - {StatusCode::PIPELINE_STREAM_ID_NOT_READY_YET, "Node is not ready for execution"}, - {StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL, "Dynamic shape and dynamic batch size are not supported for stateful models"}, - {StatusCode::REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL, "Stateful model cannot be subscribed to pipeline"}, - {StatusCode::REQUESTED_MODEL_TYPE_CHANGE, "Model type cannot be changed after it is loaded"}, - {StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER, "Stateful model config parameter used for non stateful model"}, - {StatusCode::INVALID_MAX_SEQUENCE_NUMBER, "Sequence max number parameter too high"}, - {StatusCode::CANNOT_CONVERT_FLAT_SHAPE, "Cannot convert flat shape to Shape object"}, - {StatusCode::INVALID_BATCH_DIMENSION, "Invalid batch dimension in shape"}, - {StatusCode::LAYOUT_INCOMPATIBLE_WITH_SHAPE, "Layout incompatible with given shape"}, - {StatusCode::ALLOW_CACHE_WITH_CUSTOM_LOADER, "allow_cache is set to true with custom loader usage"}, - {StatusCode::UNKNOWN_ERROR, "Unknown error"}, - - // Sequence management - {StatusCode::SEQUENCE_MISSING, "Sequence with provided ID does not exist"}, - {StatusCode::SEQUENCE_ALREADY_EXISTS, "Sequence with provided ID already exists"}, - {StatusCode::SEQUENCE_ID_NOT_PROVIDED, "Sequence ID has not been provided in request inputs"}, - {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, "Unexpected value of sequence control input"}, - {StatusCode::SEQUENCE_ID_BAD_TYPE, "Could not find sequence id in expected tensor proto field uint64_val"}, - {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, "Could not find sequence control input in expected tensor proto field uint32_val"}, - {StatusCode::SEQUENCE_TERMINATED, "Sequence last request is being processed and it's not available anymore"}, - {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, "Special input proto does not contain tensor shape information"}, - {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, "Max sequence number has been reached. Could not create new sequence."}, - - // Predict request validation - {StatusCode::INVALID_NO_OF_INPUTS, "Invalid number of inputs"}, - {StatusCode::INVALID_MISSING_INPUT, "Missing input with specific name"}, - {StatusCode::INVALID_MISSING_OUTPUT, "Missing output with specific name"}, - {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Invalid number of shape dimensions"}, - {StatusCode::INVALID_BATCH_SIZE, "Invalid input batch size"}, - {StatusCode::INVALID_SHAPE, "Invalid input shape"}, - {StatusCode::INVALID_PRECISION, "Invalid input precision"}, - {StatusCode::INVALID_VALUE_COUNT, "Invalid number of values in tensor proto container"}, - {StatusCode::INVALID_CONTENT_SIZE, "Invalid content size of tensor proto"}, - {StatusCode::INVALID_MESSAGE_STRUCTURE, "Passing buffers both in ModelInferRequest::InferInputTensor::contents and in ModelInferRequest::raw_input_contents is not allowed"}, - {StatusCode::UNSUPPORTED_LAYOUT, "Received binary image input but resource not configured to accept NHWC layout"}, - - // Deserialization - {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, "Unsupported deserialization precision"}, - {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, "Internal deserialization error"}, - - // Inference - {StatusCode::OV_INTERNAL_INFERENCE_ERROR, "Internal inference error"}, - - // Serialization - {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, "Unsupported serialization precision"}, - {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, "Internal serialization error"}, - {StatusCode::OV_CLONE_TENSOR_ERROR, "Error during tensor clone"}, - - // GetModelStatus - {StatusCode::INTERNAL_ERROR, "Internal server error"}, - - // Rest handler failure - {StatusCode::REST_NOT_FOUND, "Requested REST resource not found"}, - {StatusCode::REST_COULD_NOT_PARSE_VERSION, "Could not parse model version in request"}, - {StatusCode::REST_INVALID_URL, "Invalid request URL"}, - {StatusCode::REST_UNSUPPORTED_METHOD, "Unsupported method"}, - {StatusCode::UNKNOWN_REQUEST_COMPONENTS_TYPE, "Request components type not recognized"}, - - // Rest parser failure - {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, "Request body should be JSON object"}, - {StatusCode::REST_PREDICT_UNKNOWN_ORDER, "Invalid JSON structure. Could not detect row or column format"}, - {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, "Invalid JSON structure. Nonamed instance is not an array."}, - {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, "Invalid JSON structure. One of named instances is not a JSON object."}, - {StatusCode::REST_INPUT_NOT_PREALLOCATED, "Internal allocation error"}, - {StatusCode::REST_NO_INSTANCES_FOUND, "Invalid JSON structure. Missing instances in row format"}, - {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, "Could not detect neither named or nonamed format"}, - {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, "Could not parse instance content. Not valid ndarray detected"}, - {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, "Invalid JSON structure. Request inputs have different batch sizes"}, - {StatusCode::REST_INPUTS_NOT_AN_OBJECT, "Invalid JSON structure. One of inputs is not a JSON object."}, - {StatusCode::REST_NO_INPUTS_FOUND, "Invalid JSON structure. Missing inputs in column format"}, - {StatusCode::REST_COULD_NOT_PARSE_INPUT, "Could not parse input content. Not valid ndarray detected"}, - {StatusCode::REST_PROTO_TO_STRING_ERROR, "Response parsing to JSON error"}, - {StatusCode::REST_BASE64_DECODE_ERROR, "Decode Base64 to string error"}, - {StatusCode::REST_UNSUPPORTED_PRECISION, "Could not parse input content. Unsupported data precision detected"}, - {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, "Size of data in tensor_content does not match declared tensor shape"}, - {StatusCode::REST_SERIALIZE_VAL_FIELD_INVALID_SIZE, "Number of elements in xxx_val field does not match declared tensor shape"}, - {StatusCode::REST_SERIALIZE_NO_DATA, "No data found in tensor_content or xxx_val field matching tensor dtype"}, - - // Pipeline validation errors - {StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST, "Pipeline definition with the same name already exists"}, - {StatusCode::PIPELINE_NODE_WRONG_KIND_CONFIGURATION, "Unsupported node type"}, - {StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES, "Pipeline definition has multiple request nodes"}, - {StatusCode::PIPELINE_MULTIPLE_EXIT_NODES, "Pipeline definition has multiple response nodes"}, - {StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT, "Pipeline definition is missing request or response node"}, - {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, "Model with requested name is not found"}, - {StatusCode::PIPELINE_NODE_NAME_DUPLICATE, "Pipeline definition has multiple nodes with the same name"}, - {StatusCode::PIPELINE_CYCLE_FOUND, "Pipeline definition contains a cycle"}, - {StatusCode::PIPELINE_CONTAINS_UNCONNECTED_NODES, "Pipeline definition has unconnected nodes"}, - {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_NODE, "Pipeline definition has reference to missing node"}, - {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL, "Pipeline definition has reference to missing model"}, - {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE, "Pipeline definition has reference to missing data source"}, - {StatusCode::PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT, "Pipeline definition has reference to missing model output"}, - {StatusCode::PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT, "Pipeline definition has connection to non existing model input"}, - {StatusCode::PIPELINE_NOT_ALL_INPUTS_CONNECTED, "Pipeline definition does not have connections for all inputs of underlying models"}, - {StatusCode::PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES, "Pipeline definition has multiple connections to the same input of underlying model"}, - {StatusCode::PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY, "Pipeline definition has response node used as dependency node"}, - {StatusCode::PIPELINE_NAME_OCCUPIED, "Pipeline has the same name as model"}, - {StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY, "Pipeline refers to incorrect library"}, - {StatusCode::PIPELINE_INCONSISTENT_SHARD_DIMENSIONS, "Gathered tensor shards dimensions are different"}, - {StatusCode::PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY, "Wrong number of dimensions in a tensor to be sharded"}, - {StatusCode::PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY, "Wrong dimension size. Should match demultiply count"}, - {StatusCode::PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE, "Tried to set the same input twice for node input handler"}, - {StatusCode::PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER, "Tried to set input with shard id > 0 for ordinary input handler"}, - {StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE, "Gather node refers to not existing node"}, - {StatusCode::PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER, "Gather node refers to node that isn't demultiplexer"}, - {StatusCode::PIPELINE_NODE_GATHER_FROM_ENTRY_NODE, "Gathering from entry node is not allowed"}, - {StatusCode::PIPELINE_DEMULTIPLY_ENTRY_NODE, "Demultiplication at entry node is not allowed"}, - {StatusCode::PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT, "Demultiplication count does not match tensor first dimension"}, - {StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED, "Manual gathering from multiple nodes is not supported"}, - {StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY, "Pipeline has not enough shape dimensions to demultiply"}, - {StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY, "Too large dynamic demultiplication requested."}, - {StatusCode::PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER, "Demultiplexer and gather nodes are not in LIFO order"}, - {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, "Pipeline execution aborted due to no content from custom node"}, - {StatusCode::PIPELINE_INPUTS_AMBIGUOUS_METADATA, "Multiple nodes connected to the same pipeline input require different tensor metadata"}, - - // Storage errors - // S3 - {StatusCode::S3_BUCKET_NOT_FOUND, "S3 Bucket not found"}, - {StatusCode::S3_METADATA_FAIL, "S3 metadata failure"}, - {StatusCode::S3_FAILED_LIST_OBJECTS, "S3 Failed to list objects"}, - {StatusCode::S3_FAILED_GET_TIME, "S3 Failed to get modification time"}, - {StatusCode::S3_INVALID_ACCESS, "S3 Invalid access rights"}, - {StatusCode::S3_FILE_NOT_FOUND, "S3 File or directory not found"}, - {StatusCode::S3_FILE_INVALID, "S3 File path is invalid"}, - {StatusCode::S3_FAILED_GET_OBJECT, "S3 Failed to get object from path"}, - - // GCS - {StatusCode::GCS_BUCKET_NOT_FOUND, "GCS Bucket not found"}, - {StatusCode::GCS_METADATA_FAIL, "GCS metadata failure"}, - {StatusCode::GCS_FAILED_LIST_OBJECTS, "GCS Failed to list objects"}, - {StatusCode::GCS_FAILED_GET_TIME, "GCS Failed to list objects"}, - {StatusCode::GCS_INVALID_ACCESS, "GCS Invalid access rights"}, - {StatusCode::GCS_FILE_NOT_FOUND, "GCS File or directory not found"}, - {StatusCode::GCS_FILE_INVALID, "GCS File path is invalid"}, - {StatusCode::GCS_FAILED_GET_OBJECT, "GCS Failed to get object from path"}, - {StatusCode::GCS_INCORRECT_REQUESTED_OBJECT_TYPE, "GCS invalid object type in path"}, - - // AS - {StatusCode::AS_INVALID_PATH, "AS Invalid path"}, - {StatusCode::AS_CONTAINER_NOT_FOUND, "AS Container not found"}, - {StatusCode::AS_SHARE_NOT_FOUND, "AS Share not found"}, - {StatusCode::AS_METADATA_FAIL, "AS metadata failure"}, - {StatusCode::AS_FAILED_LIST_OBJECTS, "AS Failed to list objects"}, - {StatusCode::AS_FAILED_GET_TIME, "AS Failed to list objects"}, - {StatusCode::AS_INVALID_ACCESS, "AS Invalid access rights"}, - {StatusCode::AS_FILE_NOT_FOUND, "AS File or directory not found"}, - {StatusCode::AS_FILE_INVALID, "AS File path is invalid"}, - {StatusCode::AS_FAILED_GET_OBJECT, "AS Failed to get object from path"}, - {StatusCode::AS_INCORRECT_REQUESTED_OBJECT_TYPE, "AS invalid object type in path"}, - - // Custom Loader - {StatusCode::CUSTOM_LOADER_LIBRARY_INVALID, "Custom Loader library not found or cannot open"}, - {StatusCode::CUSTOM_LOADER_LIBRARY_LOAD_FAILED, "Cannot load the custom library"}, - {StatusCode::CUSTOM_LOADER_EXISTS, "The custom loader is already present in loaders list"}, - {StatusCode::CUSTOM_LOADER_NOT_PRESENT, "The custom loader is not present in loaders list"}, - {StatusCode::CUSTOM_LOADER_INIT_FAILED, "Custom Loader LoadInit failed"}, - {StatusCode::CUSTOM_LOADER_ERROR, "Custom Loader Generic / Unknown Error"}, - - // Custom Node - {StatusCode::NODE_LIBRARY_ALREADY_LOADED, "Custom node library is already loaded"}, - {StatusCode::NODE_LIBRARY_LOAD_FAILED_OPEN, "Custom node library failed to open"}, - {StatusCode::NODE_LIBRARY_LOAD_FAILED_SYM, "Custom node library failed to load symbol"}, - {StatusCode::NODE_LIBRARY_MISSING, "Custom node library not found"}, - {StatusCode::NODE_LIBRARY_MISSING_OUTPUT, "Custom node output is missing"}, - {StatusCode::NODE_LIBRARY_EXECUTION_FAILED, "Custom node failed during execution"}, - {StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED, "Custom node library has returned corrupted outputs handle"}, - {StatusCode::NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT, "Custom node library has produced corrupted number of outputs"}, - {StatusCode::NODE_LIBRARY_INVALID_PRECISION, "Custom node has produced tensor with unspecified precision"}, - {StatusCode::NODE_LIBRARY_INVALID_SHAPE, "Custom node has produced tensor with not matching shape"}, - {StatusCode::NODE_LIBRARY_INVALID_CONTENT_SIZE, "Custom node output has invalid content size"}, - {StatusCode::NODE_LIBRARY_METADATA_FAILED, "Custom node failed on metadata call"}, - {StatusCode::NODE_LIBRARY_OUTPUT_MISSING_NAME, "Custom node output is missing name"}, - - // Binary inputs - {StatusCode::IMAGE_PARSING_FAILED, "Image parsing failed"}, - {StatusCode::INVALID_NO_OF_CHANNELS, "Invalid number of channels in binary input"}, - {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, "Binary input images for this endpoint are required to have the same resolution"}, - {StatusCode::STRING_VAL_EMPTY, "String val is empty"}, - {StatusCode::BYTES_CONTENTS_EMPTY, "Bytes contents is empty"}, - {StatusCode::NODE_LIBRARY_INITIALIZE_FAILED, "Failure during custom node library initialization"}, - - // Model control API - {StatusCode::OK_NOT_RELOADED, "Config reload was not needed"}, - {StatusCode::OK_RELOADED, "Config reload successful"}, -}; - -const std::unordered_map Status::grpcStatusMap = { - {StatusCode::OK, grpc::StatusCode::OK}, - - {StatusCode::PATH_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::FILE_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::NO_MODEL_VERSION_AVAILABLE, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_NOT_LOADED, grpc::StatusCode::INTERNAL}, - {StatusCode::JSON_INVALID, grpc::StatusCode::INTERNAL}, - {StatusCode::MODELINSTANCE_NOT_FOUND, grpc::StatusCode::INTERNAL}, - {StatusCode::SHAPE_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, grpc::StatusCode::INTERNAL}, - {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, grpc::StatusCode::INTERNAL}, - {StatusCode::RESHAPE_ERROR, grpc::StatusCode::FAILED_PRECONDITION}, - {StatusCode::MODEL_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, grpc::StatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::NOT_FOUND}, - {StatusCode::MODEL_SPEC_MISSING, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::MODEL_VERSION_INVALID_FORMAT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SIGNATURE_DEF, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, grpc::StatusCode::ABORTED}, - {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, grpc::StatusCode::FAILED_PRECONDITION}, - - // Sequence management - {StatusCode::SEQUENCE_MISSING, grpc::StatusCode::NOT_FOUND}, - {StatusCode::SEQUENCE_ALREADY_EXISTS, grpc::StatusCode::ALREADY_EXISTS}, - {StatusCode::SEQUENCE_ID_NOT_PROVIDED, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_ID_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::SEQUENCE_TERMINATED, grpc::StatusCode::FAILED_PRECONDITION}, - {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, grpc::StatusCode::UNAVAILABLE}, - - // Predict request validation - {StatusCode::INVALID_NO_OF_INPUTS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_MISSING_INPUT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_BATCH_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_PRECISION, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_VALUE_COUNT, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_CONTENT_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::INVALID_MESSAGE_STRUCTURE, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::UNSUPPORTED_LAYOUT, grpc::StatusCode::INVALID_ARGUMENT}, - - // Deserialization - - // Should never occur - ModelInstance::validate takes care of that - {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, - {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, - - // Inference - {StatusCode::OV_INTERNAL_INFERENCE_ERROR, grpc::StatusCode::INTERNAL}, - - // Serialization - - // Should never occur - it should be validated during model loading - {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, grpc::StatusCode::INTERNAL}, - {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, grpc::StatusCode::INTERNAL}, - - // GetModelStatus - {StatusCode::INTERNAL_ERROR, grpc::StatusCode::INTERNAL}, - - // Binary input - {StatusCode::INVALID_NO_OF_CHANNELS, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::STRING_VAL_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, - {StatusCode::BYTES_CONTENTS_EMPTY, grpc::StatusCode::INVALID_ARGUMENT}, -}; - -const std::unordered_map Status::httpStatusMap = { - {StatusCode::OK, net_http::HTTPStatusCode::OK}, - {StatusCode::OK_RELOADED, net_http::HTTPStatusCode::CREATED}, - {StatusCode::OK_NOT_RELOADED, net_http::HTTPStatusCode::OK}, - - // REST handler failure - {StatusCode::REST_INVALID_URL, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_UNSUPPORTED_METHOD, net_http::HTTPStatusCode::NONE_ACC}, - - // REST parser failure - {StatusCode::REST_BODY_IS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_PREDICT_UNKNOWN_ORDER, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_NOT_AN_ARRAY, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_NAMED_INSTANCE_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INPUT_NOT_PREALLOCATED, net_http::HTTPStatusCode::ERROR}, - {StatusCode::REST_NO_INSTANCES_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_NOT_NAMED_OR_NONAMED, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_INSTANCE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INSTANCES_BATCH_SIZE_DIFFER, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_INPUTS_NOT_AN_OBJECT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_NO_INPUTS_FOUND, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_COULD_NOT_PARSE_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_PROTO_TO_STRING_ERROR, net_http::HTTPStatusCode::ERROR}, - {StatusCode::REST_UNSUPPORTED_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, net_http::HTTPStatusCode::ERROR}, - - {StatusCode::PATH_INVALID, net_http::HTTPStatusCode::ERROR}, - {StatusCode::FILE_INVALID, net_http::HTTPStatusCode::ERROR}, - {StatusCode::NO_MODEL_VERSION_AVAILABLE, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_NOT_LOADED, net_http::HTTPStatusCode::ERROR}, - {StatusCode::JSON_INVALID, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::MODELINSTANCE_NOT_FOUND, net_http::HTTPStatusCode::ERROR}, - {StatusCode::SHAPE_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::PLUGIN_CONFIG_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_VERSION_POLICY_WRONG_FORMAT, net_http::HTTPStatusCode::ERROR}, - {StatusCode::MODEL_VERSION_POLICY_UNSUPPORTED_KEY, net_http::HTTPStatusCode::ERROR}, - {StatusCode::RESHAPE_ERROR, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::MODEL_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NAME_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_VERSION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::MODEL_SPEC_MISSING, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SIGNATURE_DEF, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS, net_http::HTTPStatusCode::NO_CONTENT}, - {StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, net_http::HTTPStatusCode::PRECOND_FAILED}, - - // Sequence management - {StatusCode::SEQUENCE_MISSING, net_http::HTTPStatusCode::NOT_FOUND}, - {StatusCode::SEQUENCE_ALREADY_EXISTS, net_http::HTTPStatusCode::CONFLICT}, - {StatusCode::SEQUENCE_ID_NOT_PROVIDED, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SEQUENCE_CONTROL_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_ID_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::SEQUENCE_TERMINATED, net_http::HTTPStatusCode::PRECOND_FAILED}, - {StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::MAX_SEQUENCE_NUMBER_REACHED, net_http::HTTPStatusCode::SERVICE_UNAV}, - - // Predict request validation - {StatusCode::INVALID_NO_OF_INPUTS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_MISSING_INPUT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_BATCH_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_VALUE_COUNT, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_CONTENT_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::INVALID_MESSAGE_STRUCTURE, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::UNSUPPORTED_LAYOUT, net_http::HTTPStatusCode::BAD_REQUEST}, - - // Deserialization - - // Should never occur - ModelInstance::validate takes care of that - {StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, - {StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Inference - {StatusCode::OV_INTERNAL_INFERENCE_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Serialization - - // Should never occur - it should be validated during model loading - {StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION, net_http::HTTPStatusCode::ERROR}, - {StatusCode::OV_INTERNAL_SERIALIZATION_ERROR, net_http::HTTPStatusCode::ERROR}, - - // GetModelStatus - {StatusCode::INTERNAL_ERROR, net_http::HTTPStatusCode::ERROR}, - - // Binary input - {StatusCode::INVALID_NO_OF_CHANNELS, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH, net_http::HTTPStatusCode::BAD_REQUEST}, - {StatusCode::STRING_VAL_EMPTY, net_http::HTTPStatusCode::BAD_REQUEST}, -}; - -} // namespace ovms diff --git a/src/ovms_lib/status.hpp b/src/ovms_lib/status.hpp deleted file mode 100644 index a49b873340..0000000000 --- a/src/ovms_lib/status.hpp +++ /dev/null @@ -1,381 +0,0 @@ -//***************************************************************************** -// Copyright 2020-2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/util/net_http/server/public/response_code_enum.h" -#pragma GCC diagnostic pop - -namespace ovms { - -namespace net_http = tensorflow::serving::net_http; - -enum class StatusCode { - OK, /*!< Success */ - - PATH_INVALID, /*!< The provided path is invalid or doesn't exists */ - FILE_INVALID, /*!< File not found or cannot open */ - CONFIG_FILE_INVALID, /*!< Config file not found or cannot open */ - FILESYSTEM_ERROR, /*!< Underlaying filesystem error */ - MODEL_NOT_LOADED, - JSON_INVALID, /*!< The file/content is not valid json */ - JSON_SERIALIZATION_ERROR, /*!< Data serialization to json format failed */ - MODELINSTANCE_NOT_FOUND, - SHAPE_WRONG_FORMAT, /*!< The provided shape param is in wrong format */ - LAYOUT_WRONG_FORMAT, /*!< The provided layout param is in wrong format */ - DIM_WRONG_FORMAT, /*!< The provided dimension param is in wrong format */ - PLUGIN_CONFIG_WRONG_FORMAT, /*!< Plugin config is in wrong format */ - MODEL_VERSION_POLICY_WRONG_FORMAT, /*!< Model version policy is in wrong format */ - MODEL_VERSION_POLICY_UNSUPPORTED_KEY, /*!< Model version policy contains invalid key */ - GRPC_CHANNEL_ARG_WRONG_FORMAT, - CONFIG_FILE_TIMESTAMP_READING_FAILED, /*!< Reading config file timestamp failed */ - NO_MODEL_VERSION_AVAILABLE, /*!< No model version found in path */ - RESHAPE_ERROR, /*!< Impossible to perform reshape */ - RESHAPE_REQUIRED, /*!< Model instance needs to be reloaded with new shape */ - BATCHSIZE_CHANGE_REQUIRED, /*!< Model instance needs to be reloaded with new batch size */ - FORBIDDEN_MODEL_DYNAMIC_PARAMETER, /*!< Value of the provided param is forbidden */ - ANONYMOUS_FIXED_SHAPE_NOT_ALLOWED, /*!< Anonymous fixed shape is invalid for models with multiple inputs */ - ANONYMOUS_FIXED_LAYOUT_NOT_ALLOWED, /*!< Anonymous fixed layout is invalid for models with multiple inputs */ - CONFIG_SHAPE_IS_NOT_IN_MODEL, - CONFIG_LAYOUT_IS_NOT_IN_MODEL, - CONFIG_SHAPE_MAPPED_BUT_USED_REAL_NAME, /*!< Using old name of input/output in config shape when mapped in mapping_config.json*/ - CONFIG_LAYOUT_MAPPED_BUT_USED_REAL_NAME, /*!< Using old name of input/output in config layout when mapped in mapping_config.json*/ - CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, - REQUESTED_DYNAMIC_PARAMETERS_ON_SUBSCRIBED_MODEL, - CANNOT_CONVERT_FLAT_SHAPE, - INVALID_BATCH_DIMENSION, /*!< Invalid batch dimension in shape */ - ALLOW_CACHE_WITH_CUSTOM_LOADER, - LAYOUT_INCOMPATIBLE_WITH_SHAPE, - - // Model management - MODEL_MISSING, /*!< Model with such name and/or version does not exist */ - MODEL_CONFIG_INVALID, /*!< Model config is invalid */ - MODEL_NAME_MISSING, /*!< Model with requested name is not found */ - MODEL_NAME_OCCUPIED, /*!< Given model name is already occupied */ - MODEL_VERSION_MISSING, /*!< Model with requested version is not found */ - MODEL_VERSION_NOT_LOADED_ANYMORE, /*!< Model with requested version is retired */ - MODEL_VERSION_NOT_LOADED_YET, /*!< Model with requested version is not loaded yet */ - INVALID_NIREQ, /*!< Invalid NIREQ requested */ - REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL, /*!< Dynamic shape and dynamic batch size not supported for stateful models */ - REQUESTED_STATEFUL_PARAMETERS_ON_SUBSCRIBED_MODEL, /*!< Stateful model cannot be subscribed to pipeline */ - REQUESTED_MODEL_TYPE_CHANGE, /*!< Model type cannot be changed after it's loaded */ - INVALID_NON_STATEFUL_MODEL_PARAMETER, /*!< Stateful model config parameter used for non stateful model */ - INVALID_MAX_SEQUENCE_NUMBER, /*!< Sequence max number parameter too high */ - - // Sequence management - SEQUENCE_MISSING, /*!< Sequence with provided ID does not exist */ - SEQUENCE_ALREADY_EXISTS, /*!< Sequence with provided ID already exists */ - SEQUENCE_ID_NOT_PROVIDED, /*!< Sequence ID has not been provided in request inputs */ - SEQUENCE_ID_BAD_TYPE, /*!< Wrong sequence ID type */ - INVALID_SEQUENCE_CONTROL_INPUT, /*!< Unexpected value of sequence control input */ - SEQUENCE_CONTROL_INPUT_BAD_TYPE, /*!< Sequence control input in bad type */ - SEQUENCE_TERMINATED, /*!< Sequence last request is being processed and it's not available anymore */ - SPECIAL_INPUT_NO_TENSOR_SHAPE, /*!< Special input proto does not contain tensor shape information */ - MAX_SEQUENCE_NUMBER_REACHED, /*!< Model handles maximum number of sequences and will not accept new ones */ - - // Predict request validation - INVALID_NO_OF_INPUTS, /*!< Invalid number of inputs */ - INVALID_MISSING_INPUT, /*!< Missing one or more of inputs */ - INVALID_MISSING_OUTPUT, /*!< Missing one or more of outputs */ - INVALID_NO_OF_SHAPE_DIMENSIONS, /*!< Invalid number of shape dimensions */ - INVALID_BATCH_SIZE, /*!< Input batch size other than required */ - INVALID_SHAPE, /*!< Invalid shape dimension number or dimension value */ - INVALID_PRECISION, /*!< Invalid precision */ - INVALID_VALUE_COUNT, /*!< Invalid value count error status for uint16 and half float data types */ - INVALID_CONTENT_SIZE, /*!< Invalid content size error status for types using tensor_content() */ - INVALID_MESSAGE_STRUCTURE, /*!< Buffers can't be both in raw_input_content & input tensor content */ - - // Deserialization - OV_UNSUPPORTED_DESERIALIZATION_PRECISION, /*!< Unsupported deserialization precision, theoretically should never be returned since ModelInstance::validation checks against model precision */ - OV_INTERNAL_DESERIALIZATION_ERROR, /*!< Error occured during deserialization */ - - // Inference - OV_INTERNAL_INFERENCE_ERROR, /*!< Error occured during inference */ - - // Serialization - OV_UNSUPPORTED_SERIALIZATION_PRECISION, /*!< Unsupported serializaton precision */ - OV_INTERNAL_SERIALIZATION_ERROR, /*!< Error occurred during serialization */ - OV_CLONE_TENSOR_ERROR, /*!< Error during tensor clone */ - - // GetModelStatus - INVALID_SIGNATURE_DEF, /*!< Requested signature is not supported */ - - // Common request validation errors - MODEL_SPEC_MISSING, /*!< Request lacks model_spec */ - MODEL_VERSION_INVALID_FORMAT, - - INTERNAL_ERROR, - - UNKNOWN_ERROR, - - NOT_IMPLEMENTED, - - // S3 - S3_BUCKET_NOT_FOUND, /*!< S3 Bucket not found */ - S3_METADATA_FAIL, - S3_FAILED_LIST_OBJECTS, - S3_FAILED_GET_TIME, - S3_INVALID_ACCESS, - S3_FILE_NOT_FOUND, - S3_FILE_INVALID, - S3_FAILED_GET_OBJECT, - - // GCS - GCS_BUCKET_NOT_FOUND, - GCS_METADATA_FAIL, - GCS_FAILED_LIST_OBJECTS, - GCS_FAILED_GET_TIME, - GCS_INVALID_ACCESS, - GCS_FILE_NOT_FOUND, - GCS_FILE_INVALID, - GCS_FAILED_GET_OBJECT, - GCS_INCORRECT_REQUESTED_OBJECT_TYPE, - - // AS - AS_INVALID_PATH, - AS_CONTAINER_NOT_FOUND, - AS_SHARE_NOT_FOUND, - AS_METADATA_FAIL, - AS_FAILED_LIST_OBJECTS, - AS_FAILED_GET_TIME, - AS_INVALID_ACCESS, - AS_FILE_NOT_FOUND, - AS_FILE_INVALID, - AS_FAILED_GET_OBJECT, - AS_INCORRECT_REQUESTED_OBJECT_TYPE, - - // REST handler - REST_NOT_FOUND, /*!< Requested REST resource not found */ - REST_COULD_NOT_PARSE_VERSION, /*!< Could not parse model version in request */ - REST_INVALID_URL, /*!< Malformed REST request url */ - REST_UNSUPPORTED_METHOD, /*!< Request sent with unsupported method */ - UNKNOWN_REQUEST_COMPONENTS_TYPE, /*!< Components type not recognized */ - - // REST Parse - REST_BODY_IS_NOT_AN_OBJECT, /*!< REST body should be JSON object */ - REST_PREDICT_UNKNOWN_ORDER, /*!< Could not detect order (row/column) */ - REST_INSTANCES_NOT_AN_ARRAY, /*!< When parsing row order, instances must be an array */ - REST_NAMED_INSTANCE_NOT_AN_OBJECT, /*!< When parsing named instance it needs to be an object */ - REST_INPUT_NOT_PREALLOCATED, /*!< When parsing no named instance, exactly one input need to be preallocated */ - REST_NO_INSTANCES_FOUND, /*!< Missing instances in row order */ - REST_INSTANCES_NOT_NAMED_OR_NONAMED, /*!< Unknown instance format, neither named or nonamed */ - REST_COULD_NOT_PARSE_INSTANCE, /*!< Error while parsing instance content, not valid ndarray */ - REST_INSTANCES_BATCH_SIZE_DIFFER, /*!< In row order 0-th dimension (batch size) must be equal for all inputs */ - REST_INPUTS_NOT_AN_OBJECT, /*!< When parsing column order, inputs must be an object */ - REST_NO_INPUTS_FOUND, /*!< Missing inputs in column order */ - REST_COULD_NOT_PARSE_INPUT, /*!< Error while parsing input content, not valid ndarray */ - REST_PROTO_TO_STRING_ERROR, /*!< Error while parsing ResponseProto to JSON string */ - REST_BASE64_DECODE_ERROR, /*!< Error while decoding base64 REST binary input */ - REST_UNSUPPORTED_PRECISION, /*!< Unsupported conversion from tensor_content to _val container */ - REST_SERIALIZE_TENSOR_CONTENT_INVALID_SIZE, /*!< Size of data in tensor_content does not match declared tensor shape */ - REST_SERIALIZE_VAL_FIELD_INVALID_SIZE, /*!< Number of elements in xxx_val field does not match declared tensor shape */ - REST_SERIALIZE_NO_DATA, /*!< No data found in tensor_content or xxx_val field matching tensor dtype */ - - // Pipeline validation errors - PIPELINE_DEFINITION_ALREADY_EXIST, - PIPELINE_NODE_WRONG_KIND_CONFIGURATION, - PIPELINE_MULTIPLE_ENTRY_NODES, - PIPELINE_MULTIPLE_EXIT_NODES, - PIPELINE_MISSING_ENTRY_OR_EXIT, - PIPELINE_DEFINITION_NAME_MISSING, - PIPELINE_DEFINITION_NOT_LOADED_ANYMORE, - PIPELINE_DEFINITION_NOT_LOADED_YET, - PIPELINE_NODE_NAME_DUPLICATE, - PIPELINE_STREAM_ID_NOT_READY_YET, - PIPELINE_CYCLE_FOUND, - PIPELINE_CONTAINS_UNCONNECTED_NODES, - PIPELINE_NODE_REFERING_TO_MISSING_NODE, - PIPELINE_NODE_REFERING_TO_MISSING_MODEL, - PIPELINE_NODE_REFERING_TO_MISSING_DATA_SOURCE, - PIPELINE_NODE_REFERING_TO_MISSING_MODEL_OUTPUT, - PIPELINE_CONNECTION_TO_MISSING_MODEL_INPUT, - PIPELINE_NOT_ALL_INPUTS_CONNECTED, - PIPELINE_MODEL_INPUT_CONNECTED_TO_MULTIPLE_DATA_SOURCES, - PIPELINE_EXIT_USED_AS_NODE_DEPENDENCY, - PIPELINE_NAME_OCCUPIED, - PIPELINE_DEFINITION_INVALID_NODE_LIBRARY, - PIPELINE_INCONSISTENT_SHARD_DIMENSIONS, - PIPELINE_WRONG_NUMBER_OF_DIMENSIONS_TO_DEMULTIPLY, - PIPELINE_WRONG_DIMENSION_SIZE_TO_DEMULTIPLY, - PIPELINE_TRIED_TO_SET_THE_SAME_INPUT_TWICE, - PIPELINE_TRIED_TO_SET_INPUT_SHARD_FOR_ORDINARY_INPUT_HANDLER, - PIPELINE_NODE_GATHER_FROM_NOT_EXISTING_NODE, - PIPELINE_NODE_GATHER_FROM_NOT_DEMULTIPLEXER, - PIPELINE_NODE_GATHER_FROM_ENTRY_NODE, - PIPELINE_DEMULTIPLY_ENTRY_NODE, - PIPELINE_DEMULTIPLY_COUNT_DOES_NOT_MATCH_TENSOR_SHARD_COUNT, - PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED, - PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY, - PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY, - PIPELINE_WRONG_DEMULTIPLEXER_GATHER_NODES_ORDER, - PIPELINE_DEMULTIPLEXER_NO_RESULTS, - PIPELINE_INPUTS_AMBIGUOUS_METADATA, - - // Custom Loader - CUSTOM_LOADER_LIBRARY_INVALID, - CUSTOM_LOADER_LIBRARY_LOAD_FAILED, - CUSTOM_LOADER_EXISTS, - CUSTOM_LOADER_NOT_PRESENT, - CUSTOM_LOADER_INIT_FAILED, - CUSTOM_LOADER_ERROR, - - // Custom Node - NODE_LIBRARY_ALREADY_LOADED, - NODE_LIBRARY_LOAD_FAILED_OPEN, - NODE_LIBRARY_LOAD_FAILED_SYM, - NODE_LIBRARY_MISSING, - NODE_LIBRARY_MISSING_OUTPUT, - NODE_LIBRARY_EXECUTION_FAILED, - NODE_LIBRARY_OUTPUTS_CORRUPTED, - NODE_LIBRARY_OUTPUTS_CORRUPTED_COUNT, - NODE_LIBRARY_INVALID_PRECISION, - NODE_LIBRARY_INVALID_SHAPE, - NODE_LIBRARY_INVALID_CONTENT_SIZE, - NODE_LIBRARY_METADATA_FAILED, - NODE_LIBRARY_OUTPUT_MISSING_NAME, - NODE_LIBRARY_INITIALIZE_FAILED, - - // Binary inputs - IMAGE_PARSING_FAILED, - UNSUPPORTED_LAYOUT, - INVALID_NO_OF_CHANNELS, - BINARY_IMAGES_RESOLUTION_MISMATCH, - STRING_VAL_EMPTY, - BYTES_CONTENTS_EMPTY, - - // Model control API - OK_NOT_RELOADED, /*!< Operation succeeded but no config reload was needed */ - OK_RELOADED, /*!< Operation succeeded and config reload was needed */ - - STATUS_CODE_END -}; - -class Status { - StatusCode code; - std::unique_ptr message; - - static const std::unordered_map statusMessageMap; - static const std::unordered_map grpcStatusMap; - static const std::unordered_map httpStatusMap; - - void appendDetails(const std::string& details) { - ensureMessageAllocated(); - *this->message += " - " + details; - } - void ensureMessageAllocated() { - if (nullptr == message) { - message = std::make_unique(); - } - } - -public: - Status(StatusCode code = StatusCode::OK) : - code(code) { - if (code == StatusCode::OK) { - return; - } - auto it = statusMessageMap.find(code); - if (it != statusMessageMap.end()) - this->message = std::make_unique(it->second); - else - this->message = std::make_unique("Undefined error"); - } - - Status(StatusCode code, const std::string& details) : - Status(code) { - appendDetails(details); - } - - Status(const Status& rhs) : - code(rhs.code), - message(rhs.message != nullptr ? std::make_unique(*(rhs.message)) : nullptr) {} - - Status(Status&& rhs) = default; - - Status operator=(const Status& rhs) { - this->code = rhs.code; - this->message = (rhs.message != nullptr ? std::make_unique(*rhs.message) : nullptr); - return *this; - } - - Status& operator=(Status&&) = default; - - bool ok() const { - return (code == StatusCode::OK || code == StatusCode::OK_RELOADED || code == StatusCode::OK_NOT_RELOADED); - } - - const StatusCode getCode() const { - return this->code; - } - - bool batchSizeChangeRequired() const { - return code == StatusCode::BATCHSIZE_CHANGE_REQUIRED; - } - - bool reshapeRequired() const { - return code == StatusCode::RESHAPE_REQUIRED; - } - - bool operator==(const Status& status) const { - return this->code == status.code; - } - - bool operator!=(const Status& status) const { - return this->code != status.code; - } - - const grpc::Status grpc() const { - auto it = grpcStatusMap.find(code); - if (it != grpcStatusMap.end()) { - return grpc::Status(it->second, - this->message ? *this->message : ""); - } else { - return grpc::Status(grpc::StatusCode::UNKNOWN, "Unknown error"); - } - } - - operator grpc::Status() const { - return this->grpc(); - } - - const std::string& string() const { - return this->message ? *this->message : statusMessageMap.at(code); - } - - operator const std::string&() const { - return this->string(); - } - - const net_http::HTTPStatusCode http() const { - auto it = httpStatusMap.find(code); - if (it != httpStatusMap.end()) { - return it->second; - } else { - return net_http::HTTPStatusCode::ERROR; - } - } -}; - -} // namespace ovms diff --git a/src/ovms_lib/stringutils.cpp b/src/ovms_lib/stringutils.cpp deleted file mode 100644 index 7d7407357c..0000000000 --- a/src/ovms_lib/stringutils.cpp +++ /dev/null @@ -1,139 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** - -#include "stringutils.hpp" - -#include -#include -#include -#include -#include -#include -#include -namespace ovms { - -std::string joins(const std::vector& listOfStrings, const std::string delimiter) { - std::stringstream ss; - auto it = listOfStrings.cbegin(); - if (it == listOfStrings.end()) { - return ""; - } - for (; it != (listOfStrings.end() - 1); ++it) { - ss << *it << delimiter; - } - if (it != listOfStrings.end()) { - ss << *it; - } - return ss.str(); -} - -void ltrim(std::string& str) { - str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int c) { - return !std::isspace(c); - })); -} - -void rtrim(std::string& str) { - str.erase(std::find_if(str.rbegin(), str.rend(), [](int c) { - return !std::isspace(c); - }) - .base(), - str.end()); -} - -void trim(std::string& str) { - ltrim(str); - rtrim(str); -} - -void erase_spaces(std::string& str) { - str.erase(std::remove_if(str.begin(), str.end(), - [](char c) -> bool { - return std::isspace(c, std::locale::classic()); - }), - str.end()); -} - -std::vector tokenize(const std::string& str, const char delimiter) { - std::vector tokens; - std::string token; - std::istringstream iss(str); - while (std::getline(iss, token, delimiter)) { - tokens.push_back(token); - } - - return tokens; -} - -bool endsWith(const std::string& str, const std::string& match) { - auto it = match.begin(); - return str.size() >= match.size() && - std::all_of(std::next(str.begin(), str.size() - match.size()), str.end(), [&it](const char& c) { - return ::tolower(c) == ::tolower(*(it++)); - }); -} - -std::optional stou32(const std::string& input) { - std::string str = input; - ovms::erase_spaces(str); - - if (str.size() > 0 && str[0] == '-') { - return std::nullopt; - } - - try { - uint64_t val = std::stoul(str); - if (val > std::numeric_limits::max()) { - return std::nullopt; - } - return {static_cast(val)}; - } catch (...) { - return std::nullopt; - } -} - -std::optional stoi32(const std::string& str) { - try { - return {static_cast(std::stoi(str))}; - } catch (...) { - return std::nullopt; - } -} - -std::optional stoi64(const std::string& str) { - if (!str.size()) { - return std::nullopt; - } - bool isMinus = (str[0] == '-'); - size_t i = 0; - if (isMinus) { - i = 1; - } - for (; i < str.size(); ++i) { - if (!std::isdigit(str[i])) { - return std::nullopt; - } - } - if (str[isMinus] == '0') { - return std::nullopt; - } - try { - return std::stoll(str); - } catch (...) { - return std::nullopt; - } -} -} // namespace ovms diff --git a/src/ovms_lib/stringutils.hpp b/src/ovms_lib/stringutils.hpp deleted file mode 100644 index cc0f14a634..0000000000 --- a/src/ovms_lib/stringutils.hpp +++ /dev/null @@ -1,91 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -namespace ovms { - -std::string joins(const std::vector& listOfStrings, const std::string delimiter); - -/** - * @brief Trims the string on the left side - * - * @param std::string& - */ -void ltrim(std::string& str); - -/** - * @brief Trims the string on the right side - * - * @param str - */ -void rtrim(std::string& str); - -/** - * @brief Trims the string - * - * @param str - */ -void trim(std::string& str); - -/** - * @brief Erases all whitespace characters from string - * - * @param str - */ -void erase_spaces(std::string& str); - -/** - * @brief Tokenizes a string into a vector of tokens - * - * @param str - * @param delimiter - * @return std::vector - */ -std::vector tokenize(const std::string& str, const char delimiter); - -/** - * @brief Checks if given string ends with another one - * - * @param str - * @param match - * @return true - * @return false - */ -bool endsWith(const std::string& str, const std::string& match); - -/** - * @brief Converts string to uint32, returns 0 or specified default value if conversion failed, fails if negative number is provided - * - * @param string input - * @param default value - * @return converted value and result indicating if conversion succeeded - */ -std::optional stou32(const std::string& input); - -/** - * @brief Converts string to int32, returns 0 or specified default value if conversion failed - * - * @param string input - * @param default value - * @return converted value and result indicating if conversion succeeded - */ -std::optional stoi32(const std::string& str); - -std::optional stoi64(const std::string& str); -} // namespace ovms diff --git a/src/ovms_lib/tensor_utils.hpp b/src/ovms_lib/tensor_utils.hpp deleted file mode 100644 index f17d3edd21..0000000000 --- a/src/ovms_lib/tensor_utils.hpp +++ /dev/null @@ -1,46 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -namespace ovms { - -class TensorWithSource { -public: - TensorWithSource(const ov::Tensor& actual) : - actual(actual) {} - TensorWithSource(const ov::Tensor& actual, const ov::Tensor& source) : - actual(actual), - source(source) {} - - ov::Tensor& getActualTensor() { - return this->actual; - } - - ov::Tensor& getSourceTensor() { - return this->source; - } - - bool hasSource() const { - return (bool)this->source; - } - -private: - ov::Tensor actual, source; -}; - -} // namespace ovms diff --git a/src/ovms_lib/tensorinfo.cpp b/src/ovms_lib/tensorinfo.cpp deleted file mode 100644 index 79bafdc6a2..0000000000 --- a/src/ovms_lib/tensorinfo.cpp +++ /dev/null @@ -1,289 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "tensorinfo.hpp" - -#include -#include -#include -#include -#include - -#include "kfs_grpc_inference_service.hpp" -#include "logging.hpp" - -namespace ovms { - -// in case we change behaviour for this constructor we may need to write additional tests for TensorInfo intersection / DAGs -TensorInfo::TensorInfo(const std::string& name, - const Precision& precision, - const Shape& shape) : - name(name), - mapping(""), - precision(precision), - shape(shape), - layout(Layout::getDefaultLayout()) {} - -TensorInfo::TensorInfo(const std::string& name, - const Precision& precision, - const shape_t& shape) : - name(name), - mapping(""), - precision(precision), - shape(shape), - layout(Layout::getDefaultLayout()) { -} - -TensorInfo::TensorInfo(const std::string& name, - const ovms::Precision& precision, - const shape_t& shape, - const Layout& layout) : - name(name), - mapping(""), - precision(precision), - shape(shape), - layout(layout) { -} - -TensorInfo::TensorInfo(const std::string& name, - const ovms::Precision& precision, - const Shape& shape, - const Layout& layout) : - name(name), - mapping(""), - precision(precision), - shape(shape), - layout(layout) { -} - -TensorInfo::TensorInfo(const std::string& name, - const std::string& mapping, - const ovms::Precision& precision, - const shape_t& shape, - const Layout& layout) : - name(name), - mapping(mapping), - precision(precision), - shape(shape), - layout(layout) { -} -TensorInfo::TensorInfo(const std::string& name, - const std::string& mapping, - const ovms::Precision& precision, - const Shape& shape, - const Layout& layout) : - name(name), - mapping(mapping), - precision(precision), - shape(shape), - layout(layout) { -} -TensorInfo::TensorInfo(const std::string& name, - const std::string& mapping, - const Precision& precision, - const shape_t& shape) : - name(name), - mapping(mapping), - precision(precision), - shape(shape), - layout(Layout::getDefaultLayout()) { -} - -const std::string& TensorInfo::getName() const { - return name; -} - -const std::string& TensorInfo::getMappedName() const { - return mapping.size() == 0 ? name : mapping; -} - -void TensorInfo::setMappedName(const std::string& mappedName) { - mapping = mappedName; -} - -const Precision TensorInfo::getPrecision() const { - return precision; -} - -void TensorInfo::setPrecision(const ovms::Precision& requestedPrecision) { - precision = requestedPrecision; -} - -std::string TensorInfo::getPrecisionAsString(Precision precision) { - return toString(precision); -} - -ov::element::Type TensorInfo::getOvPrecision() const { - return ovmsPrecisionToIE2Precision(precision); -} - -std::string TensorInfo::getPrecisionAsString() const { - return getPrecisionAsString(precision); -} - -std::string TensorInfo::getPrecisionAsKFSPrecision(Precision precision) { - return ovmsPrecisionToKFSPrecision(precision); -} - -std::string TensorInfo::getPrecisionAsKFSPrecision() const { - return getPrecisionAsKFSPrecision(precision); -} - -std::string TensorInfo::getStringFromLayout(const Layout& layout) { - return layout; -} - -const Layout& TensorInfo::getLayout() const { - return layout; -} - -bool TensorInfo::isInfluencedByDemultiplexer() const { - return influencedByDemultiplexer; -} - -void TensorInfo::setShape(const Shape& shape) { - this->shape = shape; -} - -const Shape& TensorInfo::getShape() const { - return this->shape; -} - -void TensorInfo::setLayout(const Layout& layout) { - this->layout = layout; -} - -std::shared_ptr TensorInfo::createCopyWithNewShape(const Shape& shape) const { - auto copy = std::make_shared(*this); - copy->shape = shape; - copy->layout = Layout::getUnspecifiedLayout(); - return copy; -} - -std::shared_ptr TensorInfo::createCopyWithDemultiplexerDimensionPrefix(const Dimension& dim) const { - auto copy = std::make_shared(*this); - copy->influencedByDemultiplexer = true; - copy->shape.emplace(copy->shape.begin(), dim); - copy->layout = this->getLayout(); - auto batchPosition = copy->layout.find(BATCH_DIMENSION_LETTER); - if (batchPosition != std::string::npos) { - copy->layout.replace(batchPosition, 1, std::string(1, UNDEFINED_DIMENSION_CHAR)); - } - copy->layout = std::string(1, BATCH_DIMENSION_LETTER[0]) + copy->layout; - return copy; -} - -std::shared_ptr TensorInfo::createIntersection(const TensorInfo& other) { - if (this->isTensorUnspecified()) - return std::make_shared(other); - if (other.isTensorUnspecified()) - return std::make_shared(*this); - if ((this->getName() != other.getName()) || - (this->getMappedName() != other.getMappedName())) { - return nullptr; - } - Precision precision; - if (this->getPrecision() != other.getPrecision()) { - if (this->getPrecision() == Precision::UNDEFINED) { - precision = other.getPrecision(); - } else if (other.getPrecision() == Precision::UNDEFINED) { - precision = this->getPrecision(); - } else { - return nullptr; - } - } else { - precision = this->getPrecision(); - } - auto newShape = this->getShape().createIntersection(other.getShape()); - if (!newShape.has_value()) - return nullptr; - auto layout = this->getLayout().createIntersection(other.getLayout(), newShape.value().size()); - if (!layout.has_value()) - return nullptr; - return std::make_shared(this->getName(), - this->getMappedName(), - precision, - std::move(newShape.value()), - layout.value()); -} - -bool TensorInfo::isTensorSpecEqual(const TensorInfo& other) const { - return (this->getShape() == other.getShape()) && - (this->getPrecision() == other.getPrecision()) && - (this->getLayout() == other.getLayout()); -} - -bool TensorInfo::isTensorUnspecified() const { - return (this->getPrecision() == Precision::UNDEFINED) && - (this->getName() == "") && - (this->getShape() == Shape()); -} - -std::string TensorInfo::shapeToString(const shape_t& shape) { - std::ostringstream oss; - oss << "("; - size_t i = 0; - if (shape.size() > 0) { - for (; i < shape.size() - 1; i++) { - oss << shape[i] << ","; - } - oss << shape[i]; - } - oss << ")"; - - return oss.str(); -} - -std::string tensorShapeToString(const google::protobuf::RepeatedField& shape) { - std::ostringstream oss; - oss << "("; - size_t i = 0; - if (shape.size() > 0) { - for (; i < shape.size() - 1; i++) { - oss << shape[i] << ","; - } - oss << shape[i]; - } - oss << ")"; - - return oss.str(); -} - -std::shared_ptr TensorInfo::getUnspecifiedTensorInfo() { - return std::make_shared("", Precision::UNDEFINED, Shape{}); -} - -const std::optional TensorInfo::getBatchSize() const { - const auto batchIndex = this->layout.getBatchIndex(); - if (!batchIndex.has_value()) { - return std::nullopt; - } - if (getShape().size() < batchIndex.value() + 1) { - throw std::logic_error("batch outside of shape range"); - } - return getShape()[batchIndex.value()]; -} - -std::string TensorInfo::asString() const { - std::stringstream ss; - ss - << "name: " << getName() << "; " - << "mapping_name: " << getMappedName() << "; " - << "shape: " << getShape().toString() << "; " - << "precision: " << getPrecisionAsString() << "; " - << "layout: " << getStringFromLayout(getLayout()); - return ss.str(); -} -} // namespace ovms diff --git a/src/ovms_lib/tensorinfo.hpp b/src/ovms_lib/tensorinfo.hpp deleted file mode 100644 index a785d05fc9..0000000000 --- a/src/ovms_lib/tensorinfo.hpp +++ /dev/null @@ -1,238 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "layout.hpp" -#include "precision.hpp" -#include "shape.hpp" - -namespace google::protobuf { -template -class RepeatedField; -} - -namespace ovms { - -class TensorInfo; - -using tensor_map_t = std::map>; - -/** - * @brief Class containing information about the tensor - */ -class TensorInfo { -protected: - /** - * @brief Input name - */ - std::string name; - - /** - * @brief Mapping name - */ - std::string mapping; - - Precision precision; - - /** - * @brief Model input shape - */ - Shape shape; - - /** - * @brief Tensor layout - */ - Layout layout; - - /** - * @brief Information if influenced by demultiplexer - */ - bool influencedByDemultiplexer = false; - -public: - /** - * @brief Construct a new Tensor Info object - * - */ - TensorInfo() = default; - TensorInfo(const TensorInfo&) = default; - - /** - * @brief Construct a new Tensor Info object - * - * @param name - * @param precision - * @param shape - */ - TensorInfo(const std::string& name, - const Precision& precision, - const shape_t& shape); - TensorInfo(const std::string& name, - const Precision& precision, - const Shape& shape); - - /** - * @brief Construct a new Tensor Info object - * - * @param name - * @param precision - * @param shape - * @param layout - */ - TensorInfo(const std::string& name, - const Precision& precision, - const shape_t& shape, - const Layout& layout); - TensorInfo(const std::string& name, - const Precision& precision, - const Shape& shape, - const Layout& layout); - - /** - * @brief Construct a new Tensor Info object - * - * @param name - * @param precision - * @param shape - * @param layout - */ - TensorInfo(const std::string& name, - const std::string& mapping, - const Precision& precision, - const shape_t& shape, - const Layout& layout); - TensorInfo(const std::string& name, - const std::string& mapping, - const Precision& precision, - const Shape& shape, - const Layout& layout); - TensorInfo(const std::string& name, - const std::string& mapping, - const Precision& precision, - const shape_t& shape); - - /** - * @brief Get the Name object - * - * @return const std::string& - */ - const std::string& getName() const; - - /** - * @brief Get the tensor name - as in model or mapped name - * - * @return const std::string& - */ - const std::string& getMappedName() const; - void setMappedName(const std::string& mappedName); - - /** - * @brief Get the Precision object - * - * @return const InferenceEngine::Precision - */ - const Precision getPrecision() const; - - /** - * @brief Set the Precision object - * - * @return const InferenceEngine::Precision - */ - void setPrecision(const ovms::Precision& requestedPrecision); - - /** - * @brief Set the Layout object - */ - void setLayout(const Layout& layout); - - ov::element::Type getOvPrecision() const; - - /** - * @brief Get the Precision As String object - * - * @return const std::string - */ - std::string getPrecisionAsString() const; - - /** - * @brief Get the Precision As String object representing KFS precision - * - * @return const std::string - */ - std::string getPrecisionAsKFSPrecision() const; - - /** - * @brief Get the string representation of TensorInfo object - * - * @return String representation - */ - std::string asString() const; - - static std::string getPrecisionAsString(Precision precision); - - static std::string getPrecisionAsKFSPrecision(Precision precision); - - /** - * @brief Get the layout name from Layout - * - * @param Layout - * @return std::string - */ - static std::string getStringFromLayout(const Layout& layout); - - /** - * @brief Get the Layout string - * - * @return const Layout& - */ - const Layout& getLayout() const; - - /** - * @brief Gets input shape - * - * @return shape - */ - const Shape& getShape() const; - void setShape(const Shape& shape); - - bool isInfluencedByDemultiplexer() const; - - std::shared_ptr createCopyWithNewShape(const Shape& shape) const; - - std::shared_ptr createCopyWithDemultiplexerDimensionPrefix(const Dimension& dim) const; - std::shared_ptr createIntersection(const TensorInfo& other); - - bool isTensorUnspecified() const; - - bool isTensorSpecEqual(const TensorInfo& other) const; - - static std::string shapeToString(const shape_t& shape); - - static std::shared_ptr getUnspecifiedTensorInfo(); - - const std::optional getBatchSize() const; -}; -std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); - -} // namespace ovms diff --git a/src/ovms_lib/tensormap.hpp b/src/ovms_lib/tensormap.hpp deleted file mode 100644 index 830bf24e4b..0000000000 --- a/src/ovms_lib/tensormap.hpp +++ /dev/null @@ -1,33 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include - -#include - -#include "tensor_utils.hpp" - -namespace ovms { - -using TensorMap = std::unordered_map; -using TensorVector = std::vector; -using TensorWithSourceMap = std::unordered_map; - -} // namespace ovms diff --git a/src/ovms_lib/tfs_frontend/tfs_utils.cpp b/src/ovms_lib/tfs_frontend/tfs_utils.cpp deleted file mode 100644 index 31361c8416..0000000000 --- a/src/ovms_lib/tfs_frontend/tfs_utils.cpp +++ /dev/null @@ -1,98 +0,0 @@ -//***************************************************************************** -// Copyright 2021 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "tfs_utils.hpp" - -#include -#include -#include -#include -#include - -#include "../logging.hpp" - -namespace ovms { - -tensorflow::DataType getPrecisionAsDataType(Precision precision) { - static std::unordered_map precisionMap{ - {Precision::FP32, tensorflow::DataType::DT_FLOAT}, - {Precision::FP64, tensorflow::DataType::DT_DOUBLE}, - {Precision::FP16, tensorflow::DataType::DT_HALF}, - {Precision::I64, tensorflow::DataType::DT_INT64}, - {Precision::I32, tensorflow::DataType::DT_INT32}, - {Precision::I16, tensorflow::DataType::DT_INT16}, - {Precision::I8, tensorflow::DataType::DT_INT8}, - {Precision::U64, tensorflow::DataType::DT_UINT64}, - {Precision::U16, tensorflow::DataType::DT_UINT16}, - {Precision::U8, tensorflow::DataType::DT_UINT8}, - // {Precision::MIXED, tensorflow::DataType::DT_INVALID}, - // {Precision::Q78, tensorflow::DataType::DT_INVALID}, - // {Precision::BIN, tensorflow::DataType::DT_INVALID}, - {Precision::BOOL, tensorflow::DataType::DT_BOOL} - // {Precision::CUSTOM, tensorflow::DataType::DT_INVALID} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return tensorflow::DataType::DT_INVALID; - } - return it->second; -} - -std::string getDataTypeAsString(tensorflow::DataType dataType) { - switch (dataType) { - case tensorflow::DataType::DT_FLOAT: - return "FP32"; - case tensorflow::DataType::DT_DOUBLE: - return "FP64"; - case tensorflow::DataType::DT_INT32: - return "I32"; - case tensorflow::DataType::DT_INT8: - return "I8"; - case tensorflow::DataType::DT_UINT8: - return "U8"; - case tensorflow::DataType::DT_HALF: - return "FP16"; - case tensorflow::DataType::DT_INT16: - return "I16"; - case tensorflow::DataType::DT_UINT16: - return "U16"; - case tensorflow::DataType::DT_UINT64: - return "U64"; - case tensorflow::DataType::DT_INT64: - return "I64"; - case tensorflow::DataType::DT_BOOL: - return "BOOL"; - case tensorflow::DataType::DT_STRING: - return "STRING"; - default: - return "INVALID"; - } -} - -std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape) { - std::ostringstream oss; - oss << "("; - int i = 0; - if (tensorShape.dim_size() > 0) { - for (; i < tensorShape.dim_size() - 1; i++) { - oss << tensorShape.dim(i).size() << ","; - } - oss << tensorShape.dim(i).size(); - } - oss << ")"; - - return oss.str(); -} -} // namespace ovms diff --git a/src/ovms_lib/tfs_frontend/tfs_utils.hpp b/src/ovms_lib/tfs_frontend/tfs_utils.hpp deleted file mode 100644 index 578aa73892..0000000000 --- a/src/ovms_lib/tfs_frontend/tfs_utils.hpp +++ /dev/null @@ -1,33 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - -#include "../precision.hpp" - -namespace ovms { -tensorflow::DataType getPrecisionAsDataType(Precision precision); -std::string getDataTypeAsString(tensorflow::DataType dataType); -std::string tensorShapeToString(const tensorflow::TensorShapeProto& tensorShape); -// static std::string tensorShapeToString(const google::protobuf::RepeatedField& tensorShape); -} // namespace ovms diff --git a/src/ovms_lib/threadsafequeue.hpp b/src/ovms_lib/threadsafequeue.hpp deleted file mode 100644 index 2f514da08c..0000000000 --- a/src/ovms_lib/threadsafequeue.hpp +++ /dev/null @@ -1,69 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include -#include -#include - -#include - -namespace ovms { - -template -class ThreadSafeQueue { -public: - ThreadSafeQueue() {} - ~ThreadSafeQueue() {} - void push(const T& element) { - std::unique_lock lock(mtx); - queue.push(std::move(element)); - lock.unlock(); - signal.notify_one(); - } - - void push(T&& element) { - std::unique_lock lock(mtx); - queue.push(std::move(element)); - lock.unlock(); - signal.notify_one(); - } - - std::optional tryPull(const uint waitDurationMicroseconds) { - std::unique_lock lock(mtx); - if (signal.wait_for(lock, - std::chrono::microseconds(waitDurationMicroseconds), - [this]() { return queue.size() > 0; })) { - T element = std::move(queue.front()); - queue.pop(); - return std::optional{std::move(element)}; - } else { - return std::nullopt; - } - } - - size_t size() { - return queue.size(); - } - -private: - std::mutex mtx; - std::queue queue; - std::condition_variable signal; -}; -} // namespace ovms diff --git a/src/ovms_lib/timer.hpp b/src/ovms_lib/timer.hpp deleted file mode 100644 index 9a761f4b12..0000000000 --- a/src/ovms_lib/timer.hpp +++ /dev/null @@ -1,50 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#pragma once - -#include -#include -#include - -namespace ovms { - -template -struct is_chrono_duration_type : std::false_type {}; - -template -struct is_chrono_duration_type> : std::true_type {}; - -class Timer { - std::unordered_map startTimestamps; - std::unordered_map stopTimestamps; - -public: - void start(const std::string& name) { - startTimestamps[name] = std::chrono::high_resolution_clock::now(); - } - - void stop(const std::string& name) { - stopTimestamps[name] = std::chrono::high_resolution_clock::now(); - } - - template - double elapsed(const std::string& name) { - static_assert(is_chrono_duration_type::value, "Non supported type."); - return std::chrono::duration_cast(stopTimestamps[name] - startTimestamps[name]).count(); - } -}; - -} // namespace ovms diff --git a/src/ovms_lib/version.hpp b/src/ovms_lib/version.hpp deleted file mode 100644 index e9a682cdd1..0000000000 --- a/src/ovms_lib/version.hpp +++ /dev/null @@ -1,21 +0,0 @@ -//***************************************************************************** -// Copyright 2020 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#ifndef SRC_VERSION_HPP_ -#define SRC_VERSION_HPP_ -#define PROJECT_NAME "REPLACE_PROJECT_NAME" -#define PROJECT_VERSION "REPLACE_PROJECT_VERSION" -#define OPENVINO_NAME "REPLACE_OPENVINO_NAME" -#endif // SRC_VERSION_HPP_" diff --git a/src/pocapi.cpp b/src/pocapi.cpp new file mode 100644 index 0000000000..5343e42c7e --- /dev/null +++ b/src/pocapi.cpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "servablemanagermodule.hpp" +#include "server.hpp" + +using ovms::Server; + +int OVMS_Start(int argc, char** argv) { + Server& server = Server::instance(); + return server.start(argc, argv); +} + +void OVMS_Infer(char* name, float* data, float* output) { + Server& server = Server::instance(); + std::shared_ptr instance; + std::unique_ptr modelInstanceUnloadGuardPtr; + auto module = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + if (nullptr == module) { + return; + } + auto servableManagerModule = dynamic_cast(module); + auto& manager = servableManagerModule->getServableManager(); + manager.getModelInstance(name, 0, instance, modelInstanceUnloadGuardPtr); + instance->infer(data, output); +} diff --git a/src/ovms_lib/session_id.hpp b/src/pocapi.hpp similarity index 81% rename from src/ovms_lib/session_id.hpp rename to src/pocapi.hpp index e7d6282b22..bb078c9b8b 100644 --- a/src/ovms_lib/session_id.hpp +++ b/src/pocapi.hpp @@ -1,5 +1,6 @@ +#pragma once //***************************************************************************** -// Copyright 2021 Intel Corporation +// Copyright 2022 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,12 +14,5 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#pragma once - -#include - -namespace ovms { - -using session_id_t = uint32_t; -using session_key_t = std::string; -} // namespace ovms +int OVMS_Start(int argc, char** argv); +void OVMS_Infer(char* name, float* data, float* output); From 94953f4b0f086cc1a9b28d4400119e22f965c7a0 Mon Sep 17 00:00:00 2001 From: atobisze Date: Tue, 25 Oct 2022 16:30:26 +0200 Subject: [PATCH 084/130] Add header include --- src/pocapi.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pocapi.cpp b/src/pocapi.cpp index 5343e42c7e..21fdfd7a31 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -13,6 +13,8 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include "pocapi.hpp" + #include #include #include From bd6d5795201f980a910cb46f0574f5061920ac34 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 26 Oct 2022 09:45:07 +0200 Subject: [PATCH 085/130] Ovms lib (#1481) * Building so --- .bazelrc | 1 + Dockerfile.redhat | 3 ++ Dockerfile.ubuntu | 3 ++ check_coverage.bat | 2 +- src/BUILD | 81 +++++++++++++++++++++++++--------------------- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/.bazelrc b/.bazelrc index e318f855f6..84a512b46f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -73,4 +73,5 @@ build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=1 build --experimental_repo_remote_exec build --force_pic +build --experimental_cc_shared_library diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 1bf59fdac9..0cd4e8bd17 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -204,6 +204,9 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && make +# C api shared library +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libovms_shared.so + COPY ${ovms_metadata_file} metadata.json RUN ./bazel-bin/src/./ovms diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index db51698bc7..99c6407602 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -298,6 +298,9 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make +# C api shared library +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libovms_shared.so + ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json diff --git a/check_coverage.bat b/check_coverage.bat index 4543bcd83d..152592ef69 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,7 +5,7 @@ #MIN_FUNCTION_COV=87 #Rhel -MIN_LINES_COV=73.2 +MIN_LINES_COV=72.3 MIN_FUNCTION_COV=73.3 diff --git a/src/BUILD b/src/BUILD index 25e369b732..0eeeb9cc96 100644 --- a/src/BUILD +++ b/src/BUILD @@ -43,23 +43,50 @@ cc_library( ) load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") -load(":my_cc_combine.bzl", "my_cc_combine") -my_cc_combine( - name = "hello_combined", - # Here merge all the static libraries into one static library - genstatic = True, - output = "libcombined.so", - deps = [ - "//src:libovms_shared.so", - # "//src:ovms_lib", - ], +cc_shared_library( + name = "libovms_shared.so", + dynamic_deps = [], + static_deps = ["@tensorflow_serving//:__subpackages__", + "@minitrace//:__subpackages__", + "@rapidjson//:__subpackages__", + "@spdlog//:__subpackages__", + "@fmtlib//:__subpackages__", + "@cxxopts//:__subpackages__", + "@awssdk//:__subpackages__", + "@boost//:__subpackages__", + "@com_github_googleapis_google_cloud_cpp//:__subpackages__", + "@curl//:__subpackages__", + "@com_github_google_crc32c//:__subpackages__", + "@com_github_googleapis_google_cloud_cpp_common//:__subpackages__", + "@com_github_libevent_libevent//:__subpackages__", + "@com_google_protobuf//:__subpackages__", + "@com_github_tencent_rapidjson//:__subpackages__", + "@org_tensorflow//:__subpackages__", + "@com_google_absl//:__subpackages__", + "@gif//:__subpackages__", + "@libjpeg_turbo//:__subpackages__", + "@farmhash_archive//:__subpackages__", + "@fft2d//:__subpackages__", + "@highwayhash//:__subpackages__", + "@com_googlesource_code_re2//:__subpackages__", + "@snappy//:__subpackages__", + "@double_conversion//:__subpackages__", + "@nsync//:__subpackages__", + "@eigen_archive//:__subpackages__", + "@openvino//:__subpackages__", + "@com_github_jupp0r_prometheus_cpp//:__subpackages__", + "@//:__subpackages__", + "@com_github_grpc_grpc//:__subpackages__", + "@upb//:__subpackages__", + "@zlib//:__subpackages__", + ], + features = [], + roots = ["//src:ovms_lib"], ) cc_library( name = "ovms_lib", - linkstatic = True, - # linkshared = True, hdrs = ["pocapi.hpp"], srcs = [ "aliases.hpp", @@ -290,6 +317,12 @@ cc_library( visibility = [ "//visibility:public", ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ] ) cc_binary( @@ -503,7 +536,6 @@ cc_binary( ], deps = [ "//src:ovms_lib", - # "//src:libovms_shared.so", ], linkstatic = True, ) @@ -526,33 +558,10 @@ cc_binary( ], deps = [ "//src:ovms_lib", - # "//src:libovms_shared.so", ], linkstatic = False, ) -cc_binary( - name = "libovms_shared.so", - srcs = [ - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - ], - copts = [ - "-Wconversion", - "-Werror", - ], - deps = [ - "//src:ovms_lib", - #"//src/ovms_lib:ovms_lib", - ], - linkstatic = True, - linkshared = True, -) - cc_test( name = "ovms_test", linkstatic = 1, From 4a705ebaa2595b4dc412f1c04eeb90a8ca60d78a Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 2 Nov 2022 16:12:13 +0100 Subject: [PATCH 086/130] C-API Server start definitions (#1486) --- MakefileCapi | 15 ++++++++ src/my_cc_combine.bzl | 70 ----------------------------------- src/pocapi.hpp | 86 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 70 deletions(-) delete mode 100644 src/my_cc_combine.bzl diff --git a/MakefileCapi b/MakefileCapi index 9913f156b0..7d416702e1 100644 --- a/MakefileCapi +++ b/MakefileCapi @@ -1,3 +1,18 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# all: g++ -pthread --std=c++17 -I./src -L./bazel-bin/src/ -L/usr/local/lib -L/opt/intel/openvino/runtime/lib/intel64/ -fPIC src/main2.cpp -L/opt/opencv/lib/ \ -lovms_lib \ diff --git a/src/my_cc_combine.bzl b/src/my_cc_combine.bzl deleted file mode 100644 index 27113911f2..0000000000 --- a/src/my_cc_combine.bzl +++ /dev/null @@ -1,70 +0,0 @@ -load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") - -def _combine_impl(ctx): - cc_toolchain = find_cpp_toolchain(ctx) - - target_list = [] - for dep_target in ctx.attr.deps: - # CcInfo, InstrumentedFilesInfo, OutputGroupInfo - cc_info_linker_inputs = dep_target[CcInfo].linking_context.linker_inputs - - target_dirname_list = [] - for linker_in in cc_info_linker_inputs.to_list(): - for linker_in_lib in linker_in.libraries: - if linker_in_lib.pic_static_library != None: - target_list += [linker_in_lib.pic_static_library] - if linker_in_lib.static_library != None: - target_list += [linker_in_lib.static_library] - - output = ctx.outputs.output - if ctx.attr.genstatic: - cp_command = "" - processed_list = [] - processed_path_list = [] - for dep in target_list: - cp_command += "cp -a "+ dep.path +" "+ output.dirname + "/&&" - processed = ctx.actions.declare_file(dep.basename) - processed_list += [processed] - processed_path_list += [dep.path] - cp_command += "echo'starting to run shell'" - processed_path_list += [output.path] - - ctx.actions.run_shell( - outputs = processed_list, - inputs = target_list, - command = cp_command, - ) - - command = "cd {} && ar -x {} {}".format( - output.dirname, - "&& ar -x ".join([dep.basename for dep in target_list]), - "&& ar -rc libauto.a *.o" - ) - print("command = ", command) - ctx.actions.run_shell( - outputs = [output], - inputs = processed_list, - command = command, - ) - else: - command = "export PATH=$PATH:{} && {} -shared -fPIC -Wl,--whole-archive {} -Wl,--no-whole-archive -Wl,-soname -o {}".format ( - cc_toolchain.ld_executable, - cc_toolchain.compiler_executable, - "".join([dep.path for dep in target_list]), - output.path) - print("command = ", command) - ctx.actions.run_shell( - outputs = [output], - inputs = target_list, - command = command, - ) - -my_cc_combine = rule( - implementation = _combine_impl, - attrs = { - "_cc_toolchain": attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")), - "genstatic": attr.bool(default = False), - "deps": attr.label_list(), - "output": attr.output() - }, -) diff --git a/src/pocapi.hpp b/src/pocapi.hpp index bb078c9b8b..b8988e1665 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -14,5 +14,91 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include // For precise data types + +struct OVMS_Server; +struct OVMS_Status; + +struct OVMS_ServerGeneralOptions; +struct OVMS_ServerMultiModelOptions; + +typedef enum OVMSSERVER_loglevel_enum { + OVMSSERVER_LOG_TRACE, + OVMSSERVER_LOG_DEBUG, + OVMSSERVER_LOG_INFO, + OVMSSERVER_LOG_WARNING, + OVMSSERVER_LOG_ERROR +} OVMSSERVER_LogLevel; + +// TODO: The functions should return OVMS_Status or OVMS_Error + +//// +//// OVMS_ServerGeneralOptions +//// Structure for general options for both: single and multi (with config.json) management +//// +// Allocates memory for server general options and returns ptr +OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options); +// Deallocates server general options memory for given ptr +OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options); + +// --log_level +OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, + OVMSSERVER_LogLevel log_level); + +// --log_path +OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, + const char* log_path); + +// --file_system_poll_wait_seconds +OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, + uint64_t file_system_poll_wait_seconds); + +// --sequence_cleaner_poll_wait_minutes +OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, + uint64_t sequence_cleaner_poll_wait_minutes); + +// --custom_node_resources_cleaner_interval +OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerInterval(OVMS_ServerGeneralOptions* options, + uint64_t custom_node_resources_cleaner_interval); // TODO: Should include seconds or minutes in the name + +// --cache_dir +void OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, + const char* cache_dir); + +// --cpu_extension +OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, + const char* cpu_extension_path); + +//// +//// OVMS_ServerMultiModelOptions +//// Options for starting multi model server controlled by config.json file +//// +// Allocates memory for multi model server options and returns ptr +OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options); +// Deallocates options memory for given ptr +OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options); + +// --config_path +OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, + const char* config_path); + +//// +//// OVMS_Server +//// Handler for all management activities +//// +// Allocates memory for server and returns ptr +OVMS_Status* OVMS_ServerNew(OVMS_Server** server); +// Deallocates server memory for given ptr +OVMS_Status* OVMS_ServerDelete(OVMS_Server* server); + +// Start with configuration file config.json +// Return error if already started +OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, + OVMS_ServerGeneralOptions* general_options, + OVMS_ServerMultiModelOptions* multi_model_specific_options); // in fact only --config_path +// Unload all and cleanup +// TODO: Should not be possible to re-start? +OVMS_Status* OVMS_ServerStop(OVMS_Server* server); + int OVMS_Start(int argc, char** argv); void OVMS_Infer(char* name, float* data, float* output); From 3d76eb0bf118a09a5ebc1a271cdd523667672b6a Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 3 Nov 2022 11:05:20 +0100 Subject: [PATCH 087/130] Add inference part of C-API (#1490) JIRA:CVS-94367 --- src/pocapi.hpp | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/pocapi.hpp b/src/pocapi.hpp index b8988e1665..47cd0a3f95 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include #include // For precise data types struct OVMS_Server; @@ -22,6 +23,42 @@ struct OVMS_Status; struct OVMS_ServerGeneralOptions; struct OVMS_ServerMultiModelOptions; +// TODO reuse this in precision.hpp +enum DataType { + OVMS_DATATYPE_BF16, + OVMS_DATATYPE_FP64, + OVMS_DATATYPE_FP32, + OVMS_DATATYPE_FP16, + OVMS_DATATYPE_I64, + OVMS_DATATYPE_I32, + OVMS_DATATYPE_I16, + OVMS_DATATYPE_I8, + OVMS_DATATYPE_I4, + OVMS_DATATYPE_U64, + OVMS_DATATYPE_U32, + OVMS_DATATYPE_U16, + OVMS_DATATYPE_U8, + OVMS_DATATYPE_U4, + OVMS_DATATYPE_U1, + OVMS_DATATYPE_BOOL, + OVMS_DATATYPE_CUSTOM, + OVMS_DATATYPE_UNDEFINED, + OVMS_DATATYPE_DYNAMIC, + OVMS_DATATYPE_MIXED, + OVMS_DATATYPE_Q78, + OVMS_DATATYPE_BIN +}; + +enum BufferType { + OVMS_BUFFERTYPE_CPU, + OVMS_BUFFERTYPE_CPU_PINNED, + OVMS_BUFFERTYPE_GPU, + OVMS_BUFFERTYPE_HDDL, +}; + +struct OVMS_Status; +struct OVMS_InferenceRequest; +struct OVMS_InferenceResponse; typedef enum OVMSSERVER_loglevel_enum { OVMSSERVER_LOG_TRACE, OVMSSERVER_LOG_DEBUG, @@ -30,8 +67,6 @@ typedef enum OVMSSERVER_loglevel_enum { OVMSSERVER_LOG_ERROR } OVMSSERVER_LogLevel; -// TODO: The functions should return OVMS_Status or OVMS_Error - //// //// OVMS_ServerGeneralOptions //// Structure for general options for both: single and multi (with config.json) management @@ -100,5 +135,28 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, // TODO: Should not be possible to re-start? OVMS_Status* OVMS_ServerStop(OVMS_Server* server); +// OVMS_InferenceRequest +OVMS_Status* OVMS_InferenceRequestNew(char* modelName, uint32_t servableVersion); +OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); +OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, char* inputName, DataType datatype, uint64_t* shape, uint32_t dimCount); +OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, char* inputName, DataType datatype); // TODO consider no datatype & handle the parameters +// ownership of data needs to be maintained during inference +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId); +OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, char* inputName); +OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, char* inputName); // this will allow for reuse of request but with different input data +OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); +OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs +OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, char* paramaterName, DataType datatype, void* data, size_t byteSize); + +// OVMS_Inference Response +OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); +OVMS_Status* OVMS_InferenceResponseOutput(OVMS_InferenceResponse* response, uint32_t id, char* name, DataType* datatype, uint64_t* shape, uint32_t dimCount, BufferType* bufferType, uint32_t* deviceId, void** data); +OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); +OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); +OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, DataType* datatype, void** data); + +OVMS_Status* OVMS_Inference(OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); + +// POCAPI to be removed int OVMS_Start(int argc, char** argv); void OVMS_Infer(char* name, float* data, float* output); From 1de6149fdcf0e9fdfa293e0922376a5a30348371 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 4 Nov 2022 11:25:31 +0100 Subject: [PATCH 088/130] CVS-95440 C api start impl (#1509) First part of server start implementation: * separate paths for OVMS server start and C-API server start * minimum implementation for --config_path --port and --rest_port * dummy implementation for the rest * mini demo app demonstrating server start via C-API * MakefileCapi modified to run standalone mini demo app outside of OVMS * increase coverage for config death tests --- Dockerfile.redhat | 3 +- Dockerfile.ubuntu | 3 +- MakefileCapi | 44 +----- check_coverage.bat | 2 +- src/BUILD | 25 +++- src/config.cpp | 276 ++++++++++++++++++++++++++++++++++- src/config.hpp | 236 +++++++++++++----------------- src/main3.cpp | 41 ++++++ src/modelconfig.cpp | 2 +- src/poc_api_impl.cpp | 27 ++++ src/poc_api_impl.hpp | 34 +++++ src/pocapi.cpp | 93 ++++++++---- src/pocapi.hpp | 9 ++ src/server.cpp | 35 +++++ src/server.hpp | 4 + src/shape.cpp | 2 +- src/test/c_api/config.json | 8 + src/test/c_api_tests.cpp | 42 ++++++ src/test/ovmsconfig_test.cpp | 71 ++++++++- 19 files changed, 739 insertions(+), 218 deletions(-) create mode 100644 src/main3.cpp create mode 100644 src/poc_api_impl.cpp create mode 100644 src/poc_api_impl.hpp create mode 100644 src/test/c_api/config.json create mode 100644 src/test/c_api_tests.cpp diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 0cd4e8bd17..f55d409207 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -205,7 +205,8 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && make # C api shared library -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libovms_shared.so +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 COPY ${ovms_metadata_file} metadata.json diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 99c6407602..80087e2428 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -299,7 +299,8 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make # C api shared library -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:libovms_shared.so +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json diff --git a/MakefileCapi b/MakefileCapi index 7d416702e1..09b1dd489a 100644 --- a/MakefileCapi +++ b/MakefileCapi @@ -14,44 +14,6 @@ # limitations under the License. # all: - g++ -pthread --std=c++17 -I./src -L./bazel-bin/src/ -L/usr/local/lib -L/opt/intel/openvino/runtime/lib/intel64/ -fPIC src/main2.cpp -L/opt/opencv/lib/ \ - -lovms_lib \ - -lovms_shared \ - -lopenvino \ - -lopenvino_c \ - -lopenvino_gapi_preproc \ - -lopenvino \ - -lazurestorage \ - -lcpprest \ - -lboost_thread \ - -lboost_regex \ - -lboost_system \ - -lboost_log \ - -lboost_atomic \ - -lboost_chrono \ - -lboost_date_time \ - -lopenvino \ - -lopenvino_intel_hddl_plugin \ - -lopenvino_c \ - -lopenvino_ov_c \ - -lopenvino_intel_gpu_plugin \ - -lopenvino_intel_gna_plugin \ - -lopenvino_intel_cpu_plugin \ - -lopenvino_hetero_plugin \ - -lopenvino_tensorflow_fe \ - -lopenvino_paddle_frontend \ - -lopenvino_onnx_frontend \ - -lopenvino_ir_frontend \ - -lopenvino_intel_myriad_plugin \ - -lopenvino_gapi_preproc \ - -lopenvino_auto_plugin \ - -lopenvino_auto_batch_plugin \ - -lgna \ - -lstdc++fs \ - -lopencv_core \ - -lopencv_imgcodecs \ - -lopencv_imgproc \ - -lssl \ - -lcrypto \ - -ldl \ - 2>&1 | tee log; wc -l log + bazel build //src:ovms_shared + g++ src/main3.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/poc3 + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/poc3 diff --git a/check_coverage.bat b/check_coverage.bat index 152592ef69..beadd472e1 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,7 +5,7 @@ #MIN_FUNCTION_COV=87 #Rhel -MIN_LINES_COV=72.3 +MIN_LINES_COV=73.3 MIN_FUNCTION_COV=73.3 diff --git a/src/BUILD b/src/BUILD index 0eeeb9cc96..6834a77df2 100644 --- a/src/BUILD +++ b/src/BUILD @@ -45,7 +45,7 @@ cc_library( load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library") cc_shared_library( - name = "libovms_shared.so", + name = "ovms_shared", dynamic_deps = [], static_deps = ["@tensorflow_serving//:__subpackages__", "@minitrace//:__subpackages__", @@ -223,6 +223,8 @@ cc_library( "pipelineeventqueue.hpp", "pipeline_factory.cpp", "pipeline_factory.hpp", + "poc_api_impl.cpp", + "poc_api_impl.hpp", "pocapi.hpp", "pocapi.cpp", "precision.cpp", @@ -540,6 +542,26 @@ cc_binary( linkstatic = True, ) +# POC which can start OVMS with C-API built by bazel +# Standalone example for building outside of bazel (with bare g++ is inside MakefileCapi) +cc_binary( + name = "poc3", + srcs = [ + "main3.cpp", + ], + linkopts = [ + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + ], + deps = [ + "//src:ovms_lib", + ], + linkstatic = True, +) cc_binary( name = "ovms", @@ -568,6 +590,7 @@ cc_test( srcs = [ "test/azurefilesystem_test.cpp", "test/binaryutils_test.cpp", + "test/c_api_tests.cpp", "test/custom_loader_test.cpp", "test/custom_node_output_allocator_test.cpp", "test/custom_node_buffersqueue_test.cpp", diff --git a/src/config.cpp b/src/config.cpp index fdba29260d..f15aff3a4e 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -26,6 +26,7 @@ #include "logging.hpp" #include "modelconfig.hpp" +#include "poc_api_impl.hpp" #include "version.hpp" namespace ovms { @@ -37,12 +38,13 @@ const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; const std::string DEFAULT_REST_WORKERS_STRING{std::to_string(DEFAULT_REST_WORKERS)}; const uint64_t MAX_REST_WORKERS = 10'000; -uint32_t Config::maxSequenceNumber() const { +uint32_t Config::__maxSequenceNumber() const { if (!result->count("max_sequence_number")) { return DEFAULT_MAX_SEQUENCE_NUMBER; } return result->operator[]("max_sequence_number").as(); } +uint32_t Config::maxSequenceNumber() const { return _maxSequenceNumber; } Config& Config::parse(int argc, char** argv) { try { @@ -196,6 +198,40 @@ Config& Config::parse(int argc, char** argv) { exit(EX_OK); } + this->_configPath = __configPath(); + this->_port = __port(); + this->_cpuExtensionLibraryPath = __cpuExtensionLibraryPath(); + this->_grpcBindAddress = __grpcBindAddress(); + this->_restPort = __restPort(); + this->_restBindAddress = __restBindAddress(); + this->_grpcWorkers = __grpcWorkers(); + this->_restWorkers = __restWorkers(); + this->_modelName = __modelName(); + this->_modelPath = __modelPath(); + this->_batchSize = __batchSize(); + this->_shape = __shape(); + this->_layout = __layout(); + this->_modelVersionPolicy = __modelVersionPolicy(); + this->_nireq = __nireq(); + this->_targetDevice = __targetDevice(); + this->_pluginConfig = __pluginConfig(); + this->_stateful = __stateful(); + this->_metricsEnabled = __metricsEnabled(); + this->_metricsList = __metricsList(); + this->_idleSequenceCleanup = __idleSequenceCleanup(); + this->_lowLatencyTransformation = __lowLatencyTransformation(); + this->_maxSequenceNumber = __maxSequenceNumber(); + this->_logLevel = __logLevel(); + this->_logPath = __logPath(); +#ifdef MTR_ENABLED + this->_tracePath = __tracePath(); +#endif + this->_grpcChannelArguments = __grpcChannelArguments(); + this->_filesystemPollWaitSeconds = __filesystemPollWaitSeconds(); + this->_sequenceCleanerPollWaitMinutes = __sequenceCleanerPollWaitMinutes(); + this->_resourcesCleanerPollWaitSeconds = __resourcesCleanerPollWaitSeconds(); + this->_cacheDir = __cacheDir(); + validate(); } catch (const cxxopts::OptionException& e) { std::cerr << "error parsing options: " << e.what() << std::endl; @@ -205,6 +241,14 @@ Config& Config::parse(int argc, char** argv) { return instance(); } +Config& Config::parse(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { + // TODO: Implement + this->_port = go->grpcPort; + this->_restPort = go->restPort; + this->_configPath = mmo->configPath; + return instance(); +} + bool Config::check_hostname_or_ip(const std::string& input) { if (input.size() > 255) { return false; @@ -300,10 +344,6 @@ void Config::validate() { std::cerr << "grpc_bind_address has invalid format: proper hostname or IP address expected." << std::endl; exit(EX_USAGE); } - if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { - std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); - } // port and rest_port cannot be the same if (this->port() == this->restPort()) { @@ -334,4 +374,230 @@ void Config::validate() { return; } +const std::string& Config::__configPath() const { + if (result->count("config_path")) + return result->operator[]("config_path").as(); + return empty; +} + +const std::string& Config::configPath() const { return _configPath; } + +uint64_t Config::__port() const { + return result->operator[]("port").as(); +} + +uint64_t Config::port() const { return _port; } + +const std::string Config::__cpuExtensionLibraryPath() const { + if (result != nullptr && result->count("cpu_extension")) { + return result->operator[]("cpu_extension").as(); + } + return ""; +} + +const std::string Config::cpuExtensionLibraryPath() const { return _cpuExtensionLibraryPath; } + +const std::string Config::__grpcBindAddress() const { + if (result->count("grpc_bind_address")) + return result->operator[]("grpc_bind_address").as(); + return "0.0.0.0"; +} + +const std::string Config::grpcBindAddress() const { return _grpcBindAddress; } + +uint64_t Config::__restPort() const { + return result->operator[]("rest_port").as(); +} + +uint64_t Config::restPort() const { return _restPort; } + +const std::string Config::__restBindAddress() const { + if (result->count("rest_bind_address")) + return result->operator[]("rest_bind_address").as(); + return "0.0.0.0"; +} + +const std::string Config::restBindAddress() const { return _restBindAddress; } + +uint Config::__grpcWorkers() const { + return result->operator[]("grpc_workers").as(); +} + +uint Config::grpcWorkers() const { return _grpcWorkers; } + +uint Config::__restWorkers() const { + return result->operator[]("rest_workers").as(); +} + +uint Config::restWorkers() const { return _restWorkers; } + +const std::string& Config::__modelName() const { + if (result->count("model_name")) + return result->operator[]("model_name").as(); + return empty; +} + +const std::string& Config::modelName() const { return _modelName; } + +const std::string& Config::__modelPath() const { + if (result->count("model_path")) + return result->operator[]("model_path").as(); + return empty; +} + +const std::string& Config::modelPath() const { return _modelPath; } + +const std::string& Config::__batchSize() const { + if (!result->count("batch_size")) { + static const std::string d = "0"; + return d; + } + return result->operator[]("batch_size").as(); +} + +const std::string& Config::batchSize() const { return _batchSize; } + +const std::string& Config::__shape() const { + if (result->count("shape")) + return result->operator[]("shape").as(); + return empty; +} + +const std::string& Config::Config::shape() const { return _shape; } + +const std::string& Config::__layout() const { + if (result->count("layout")) + return result->operator[]("layout").as(); + return empty; +} + +const std::string& Config::layout() const { return _layout; } + +const std::string& Config::__modelVersionPolicy() const { + if (result->count("model_version_policy")) + return result->operator[]("model_version_policy").as(); + return empty; +} + +const std::string& Config::modelVersionPolicy() const { return _modelVersionPolicy; } + +uint32_t Config::__nireq() const { + if (!result->count("nireq")) { + return 0; + } + return result->operator[]("nireq").as(); +} + +uint32_t Config::nireq() const { return _nireq; } + +const std::string& Config::__targetDevice() const { + return result->operator[]("target_device").as(); +} + +const std::string& Config::targetDevice() const { return _targetDevice; } + +const std::string& Config::__pluginConfig() const { + if (result->count("plugin_config")) + return result->operator[]("plugin_config").as(); + return empty; +} +const std::string& Config::Config::pluginConfig() const { return _pluginConfig; } + +bool Config::__stateful() const { + return result->operator[]("stateful").as(); +} + +bool Config::stateful() const { return _stateful; } + +bool Config::__metricsEnabled() const { + if (!result->count("metrics_enable")) { + return false; + } + return result->operator[]("metrics_enable").as(); +} + +bool Config::metricsEnabled() const { return _metricsEnabled; } + +std::string Config::__metricsList() const { + if (!result->count("metrics_list")) { + return std::string(""); + } + return result->operator[]("metrics_list").as(); +} + +std::string Config::metricsList() const { return _metricsList; } + +bool Config::__idleSequenceCleanup() const { + return result->operator[]("idle_sequence_cleanup").as(); +} + +bool Config::idleSequenceCleanup() const { return _idleSequenceCleanup; } + +bool Config::__lowLatencyTransformation() const { + return result->operator[]("low_latency_transformation").as(); +} + +bool Config::lowLatencyTransformation() const { return _lowLatencyTransformation; } + +const std::string& Config::__logLevel() const { + if (result->count("log_level")) + return result->operator[]("log_level").as(); + return empty; +} + +const std::string& Config::logLevel() const { return _logLevel; } + +const std::string& Config::__logPath() const { + if (result->count("log_path")) + return result->operator[]("log_path").as(); + return empty; +} + +const std::string& Config::logPath() const { return _logPath; } + +#ifdef MTR_ENABLED +const std::string& Config::__tracePath() const { + if (result->count("trace_path")) + return result->operator[]("trace_path").as(); + return empty; +} + +const std::string& Config::tracePath() const { return _tracePath; } +#endif + +const std::string& Config::__grpcChannelArguments() const { + if (result->count("grpc_channel_arguments")) + return result->operator[]("grpc_channel_arguments").as(); + return empty; +} + +const std::string& Config::grpcChannelArguments() const { return _grpcChannelArguments; } + +uint Config::__filesystemPollWaitSeconds() const { + return result->operator[]("file_system_poll_wait_seconds").as(); +} + +uint Config::filesystemPollWaitSeconds() const { return _filesystemPollWaitSeconds; } + +uint32_t Config::__sequenceCleanerPollWaitMinutes() const { + return result->operator[]("sequence_cleaner_poll_wait_minutes").as(); +} + +uint32_t Config::sequenceCleanerPollWaitMinutes() const { return _sequenceCleanerPollWaitMinutes; } + +uint32_t Config::__resourcesCleanerPollWaitSeconds() const { + return result->operator[]("custom_node_resources_cleaner_interval").as(); +} + +uint32_t Config::resourcesCleanerPollWaitSeconds() const { return _resourcesCleanerPollWaitSeconds; } + +const std::string Config::__cacheDir() const { + if (result != nullptr && result->count("cache_dir")) { + return result->operator[]("cache_dir").as(); + } + return empty; +} + +const std::string Config::cacheDir() const { return _cacheDir; } + } // namespace ovms diff --git a/src/config.hpp b/src/config.hpp index 152ec1818b..9f483ffa64 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -22,6 +22,9 @@ #include namespace ovms { +class GeneralOptionsImpl; +class MultiModelOptionsImpl; + /** * @brief Provides all the configuration options from command line */ @@ -52,6 +55,41 @@ class Config { */ const std::string empty; + // new + std::string _configPath; + uint64_t _port = 9178; + std::string _cpuExtensionLibraryPath; + std::string _grpcBindAddress = "0.0.0.0"; + uint64_t _restPort = 0; + std::string _restBindAddress = "0.0.0.0"; + uint _grpcWorkers = 1; + uint _restWorkers = 8; // TODO: In OVMS this is nproc * 4 + std::string _modelName; + std::string _modelPath; + std::string _batchSize; + std::string _shape; + std::string _layout; + std::string _modelVersionPolicy; + uint32_t _nireq = 0; + std::string _targetDevice; + std::string _pluginConfig; + bool _stateful = false; + bool _metricsEnabled = false; + std::string _metricsList; + bool _idleSequenceCleanup = true; + bool _lowLatencyTransformation = false; + uint32_t _maxSequenceNumber = 500; + std::string _logLevel = "DEBUG"; + std::string _logPath; +#ifdef MTR_ENABLED + std::string _tracePath; +#endif + std::string _grpcChannelArguments; + uint _filesystemPollWaitSeconds = 1; + uint32_t _sequenceCleanerPollWaitMinutes = 5; + uint32_t _resourcesCleanerPollWaitSeconds = 1; + std::string _cacheDir; + public: /** * @brief Gets the instance of the config @@ -70,6 +108,7 @@ class Config { * @return Config& */ Config& parse(int argc, char** argv); + Config& parse(GeneralOptionsImpl*, MultiModelOptionsImpl*); /** * @brief Validate passed arguments @@ -90,237 +129,183 @@ class Config { * * @return std::string */ - const std::string& configPath() const { - if (result->count("config_path")) - return result->operator[]("config_path").as(); - return empty; - } + const std::string& __configPath() const; // TODO: Move to CLI parser when ready + const std::string& configPath() const; /** * @brief Gets the grpc port * * @return uint64_t */ - uint64_t port() const { - return result->operator[]("port").as(); - } + uint64_t __port() const; + uint64_t port() const; /** * @brief Get the gRPC network interface address to bind to * * @return const std::string */ - const std::string cpuExtensionLibraryPath() const { - if (result != nullptr && result->count("cpu_extension")) { - return result->operator[]("cpu_extension").as(); - } - return ""; - } + const std::string __cpuExtensionLibraryPath() const; // TODO: Move to CLI parser when ready + const std::string cpuExtensionLibraryPath() const; /** * @brief Get the gRPC network interface address to bind to * * @return const std::string& */ - const std::string grpcBindAddress() const { - if (result->count("grpc_bind_address")) - return result->operator[]("grpc_bind_address").as(); - return "0.0.0.0"; - } + const std::string __grpcBindAddress() const; // TODO: Move to CLI parser when ready + const std::string grpcBindAddress() const; + /** * @brief Gets the REST port * * @return uint64_t */ - uint64_t restPort() const { - return result->operator[]("rest_port").as(); - } + uint64_t __restPort() const; + uint64_t restPort() const; /** * @brief Get the rest network interface address to bind to * * @return const std::string& */ - const std::string restBindAddress() const { - if (result->count("rest_bind_address")) - return result->operator[]("rest_bind_address").as(); - return "0.0.0.0"; - } + const std::string __restBindAddress() const; // TODO: Move to CLI parser when ready + const std::string restBindAddress() const; /** * @brief Gets the gRPC workers count * * @return uint */ - uint grpcWorkers() const { - return result->operator[]("grpc_workers").as(); - } + uint __grpcWorkers() const; // TODO: Move to CLI parser when ready + uint grpcWorkers() const; /** * @brief Gets the rest workers count * * @return uint */ - uint restWorkers() const { - return result->operator[]("rest_workers").as(); - } + uint __restWorkers() const; // TODO: Move to CLI parser when ready + uint restWorkers() const; /** * @brief Get the model name * * @return const std::string& */ - const std::string& modelName() const { - if (result->count("model_name")) - return result->operator[]("model_name").as(); - return empty; - } + const std::string& __modelName() const; // TODO: Move to CLI parser when ready + const std::string& modelName() const; /** * @brief Gets the model path * * @return const std::string& */ - const std::string& modelPath() const { - if (result->count("model_path")) - return result->operator[]("model_path").as(); - return empty; - } + const std::string& __modelPath() const; // TODO: Move to CLI parser when ready + const std::string& modelPath() const; /** * @brief Gets the batch size * * @return const std::string& */ - const std::string& batchSize() const { - if (!result->count("batch_size")) { - static const std::string d = "0"; - return d; - } - return result->operator[]("batch_size").as(); - } + const std::string& __batchSize() const; // TODO: Move to CLI parser when ready + const std::string& batchSize() const; /** * @brief Get the shape * * @return const std::string& */ - const std::string& shape() const { - if (result->count("shape")) - return result->operator[]("shape").as(); - return empty; - } + const std::string& __shape() const; // TODO: Move to CLI parser when ready + const std::string& shape() const; /** * @brief Get the layout * * @return const std::string& */ - const std::string& layout() const { - if (result->count("layout")) - return result->operator[]("layout").as(); - return empty; - } + const std::string& __layout() const; // TODO: Move to CLI parser when ready + const std::string& layout() const; /** * @brief Get the shape * * @return const std::string& */ - const std::string& modelVersionPolicy() const { - if (result->count("model_version_policy")) - return result->operator[]("model_version_policy").as(); - return empty; - } + const std::string& __modelVersionPolicy() const; // TODO: Move to CLI parser when ready + const std::string& modelVersionPolicy() const; /** * @brief Get the nireq * * @return uint */ - uint32_t nireq() const { - if (!result->count("nireq")) { - return 0; - } - return result->operator[]("nireq").as(); - } + uint32_t __nireq() const; + uint32_t nireq() const; /** * @brief Get the target device * * @return const std::string& */ - const std::string& targetDevice() const { - return result->operator[]("target_device").as(); - } + const std::string& __targetDevice() const; // TODO: Move to CLI parser when ready + const std::string& targetDevice() const; /** * @brief Get the plugin config * * @return const std::string& */ - const std::string& pluginConfig() const { - if (result->count("plugin_config")) - return result->operator[]("plugin_config").as(); - return empty; - } + const std::string& __pluginConfig() const; // TODO: Move to CLI parser when ready + const std::string& pluginConfig() const; /** * @brief Get stateful flag * * @return bool */ - bool stateful() const { - return result->operator[]("stateful").as(); - } + bool __stateful() const; // TODO: Move to CLI parser when ready + bool stateful() const; /** * @brief Get metrics enabled flag * * @return bool */ - bool metricsEnabled() const { - if (!result->count("metrics_enable")) { - return false; - } - return result->operator[]("metrics_enable").as(); - } + bool __metricsEnabled() const; // TODO: Move to CLI parser when ready + bool metricsEnabled() const; /** * @brief Get metrics list * * @return std::string */ - std::string metricsList() const { - if (!result->count("metrics_list")) { - return std::string(""); - } - return result->operator[]("metrics_list").as(); - } + std::string __metricsList() const; // TODO: Move to CLI parser when ready + std::string metricsList() const; /** * @brief Get idle sequence cleanup flag * * @return uint */ - bool idleSequenceCleanup() const { - return result->operator[]("idle_sequence_cleanup").as(); - } + bool __idleSequenceCleanup() const; // TODO: Move to CLI parser when ready + bool idleSequenceCleanup() const; /** * @brief Get low latency transformation flag * * @return bool */ - bool lowLatencyTransformation() const { - return result->operator[]("low_latency_transformation").as(); - } + bool __lowLatencyTransformation() const; // TODO: Move to CLI parser when ready + bool lowLatencyTransformation() const; /** * @brief Get max number of sequences that can be processed concurrently * * @return uint */ + uint32_t __maxSequenceNumber() const; // TODO: Move to CLI parser when ready uint32_t maxSequenceNumber() const; /** @@ -328,22 +313,16 @@ class Config { * * @return const std::string& */ - const std::string& logLevel() const { - if (result->count("log_level")) - return result->operator[]("log_level").as(); - return empty; - } + const std::string& __logLevel() const; // TODO: Move to CLI parser when ready + const std::string& logLevel() const; /** * @brief Get the log path * * @return const std::string& */ - const std::string& logPath() const { - if (result->count("log_path")) - return result->operator[]("log_path").as(); - return empty; - } + const std::string& __logPath() const; // TODO: Move to CLI parser when ready + const std::string& logPath() const; #ifdef MTR_ENABLED /** @@ -351,11 +330,8 @@ class Config { * * @return const std::string& */ - const std::string& tracePath() const { - if (result->count("trace_path")) - return result->operator[]("trace_path").as(); - return empty; - } + const std::string& __tracePath() const; // TODO: Move to CLI parser when ready + const std::string& tracePath() const; #endif /** @@ -363,49 +339,39 @@ class Config { * * @return const std::string& */ - const std::string& grpcChannelArguments() const { - if (result->count("grpc_channel_arguments")) - return result->operator[]("grpc_channel_arguments").as(); - return empty; - } + const std::string& __grpcChannelArguments() const; // TODO: Move to CLI parser when ready + const std::string& grpcChannelArguments() const; /** * @brief Get the filesystem poll wait time in seconds * * @return uint */ - uint filesystemPollWaitSeconds() const { - return result->operator[]("file_system_poll_wait_seconds").as(); - } + uint __filesystemPollWaitSeconds() const; // TODO: Move to CLI parser when ready + uint filesystemPollWaitSeconds() const; /** * @brief Get the sequence cleanup poll wait time in minutes * * @return uint32_t */ - uint32_t sequenceCleanerPollWaitMinutes() const { - return result->operator[]("sequence_cleaner_poll_wait_minutes").as(); - } + uint32_t __sequenceCleanerPollWaitMinutes() const; // TODO: Move to CLI parser when ready + uint32_t sequenceCleanerPollWaitMinutes() const; /** * @brief Get the resources cleanup poll wait time in seconds * * @return uint32_t */ - uint32_t resourcesCleanerPollWaitSeconds() const { - return result->operator[]("custom_node_resources_cleaner_interval").as(); - } + uint32_t __resourcesCleanerPollWaitSeconds() const; // TODO: Move to CLI parser when ready + uint32_t resourcesCleanerPollWaitSeconds() const; /** * @brief Model cache directory * * @return const std::string& */ - const std::string cacheDir() const { - if (result != nullptr && result->count("cache_dir")) { - return result->operator[]("cache_dir").as(); - } - return ""; - } + const std::string __cacheDir() const; // TODO: Move to CLI parser when ready + const std::string cacheDir() const; }; } // namespace ovms diff --git a/src/main3.cpp b/src/main3.cpp new file mode 100644 index 0000000000..e829baaeb3 --- /dev/null +++ b/src/main3.cpp @@ -0,0 +1,41 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include + +#include "pocapi.hpp" + +int main(int argc, char** argv) { + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_Server* srv; + + OVMS_ServerGeneralOptionsNew(&go); + OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerNew(&srv); + + OVMS_ServerGeneralOptionsSetGrpcPort(go, 11337); + OVMS_ServerGeneralOptionsSetRestPort(go, 11338); + OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); + + OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + + std::cout << "Finish" << std::endl; + return 0; +} diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index 6cae53fd07..953ae82a7c 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -552,7 +552,7 @@ Status ModelConfig::parseNode(const rapidjson::Value& v) { for (auto& sh : s.value.GetArray()) { if (sh.IsUint64()) { size_t dim = sh.GetUint64(); - if (dim > std::numeric_limits::max()) { + if (dim > static_cast(std::numeric_limits::max())) { SPDLOG_WARN("There was an error parsing shape {}", s.name.GetString()); if (!firstErrorStatus.ok()) { firstErrorStatus = StatusCode::SHAPE_WRONG_FORMAT; diff --git a/src/poc_api_impl.cpp b/src/poc_api_impl.cpp new file mode 100644 index 0000000000..4d06a7cb4e --- /dev/null +++ b/src/poc_api_impl.cpp @@ -0,0 +1,27 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "poc_api_impl.hpp" + +#include "server.hpp" + +namespace ovms { + +void ServerImpl::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { + Server& server = Server::instance(); + server.start(go, mmo); +} + +} // namespace ovms diff --git a/src/poc_api_impl.hpp b/src/poc_api_impl.hpp new file mode 100644 index 0000000000..4f108a9af9 --- /dev/null +++ b/src/poc_api_impl.hpp @@ -0,0 +1,34 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include + +namespace ovms { + +struct GeneralOptionsImpl { + uint64_t grpcPort; + uint64_t restPort; +}; + +struct MultiModelOptionsImpl { + std::string configPath; +}; + +struct ServerImpl { + void start(GeneralOptionsImpl*, MultiModelOptionsImpl*); +}; + +} // namespace ovms diff --git a/src/pocapi.cpp b/src/pocapi.cpp index 21fdfd7a31..aa804d3f7f 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -15,34 +15,67 @@ //***************************************************************************** #include "pocapi.hpp" -#include -#include -#include -#include - -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" - -using ovms::Server; - -int OVMS_Start(int argc, char** argv) { - Server& server = Server::instance(); - return server.start(argc, argv); -} - -void OVMS_Infer(char* name, float* data, float* output) { - Server& server = Server::instance(); - std::shared_ptr instance; - std::unique_ptr modelInstanceUnloadGuardPtr; - auto module = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); - if (nullptr == module) { - return; - } - auto servableManagerModule = dynamic_cast(module); - auto& manager = servableManagerModule->getServableManager(); - manager.getModelInstance(name, 0, instance, modelInstanceUnloadGuardPtr); - instance->infer(data, output); +#include + +#include "poc_api_impl.hpp" + +OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { + *options = (OVMS_ServerGeneralOptions*)new ovms::GeneralOptionsImpl; + return 0; +} + +OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { + delete (ovms::GeneralOptionsImpl*)options; + return 0; +} + +OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options) { + *options = (OVMS_ServerMultiModelOptions*)new ovms::MultiModelOptionsImpl; + return 0; +} + +OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { + delete (ovms::MultiModelOptionsImpl*)options; + return 0; +} + +OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { + *server = (OVMS_Server*)new ovms::ServerImpl; + return 0; +} + +OVMS_Status* OVMS_ServerDelete(OVMS_Server* server) { + delete (ovms::ServerImpl*)server; + return 0; +} + +OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, + OVMS_ServerGeneralOptions* general_options, + OVMS_ServerMultiModelOptions* multi_model_specific_options) { + ovms::ServerImpl* srv = (ovms::ServerImpl*)server; + ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)general_options; + ovms::MultiModelOptionsImpl* mmo = (ovms::MultiModelOptionsImpl*)multi_model_specific_options; + srv->start(go, mmo); + return 0; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, + uint64_t grpcPort) { + ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)options; + go->grpcPort = grpcPort; + return 0; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, + uint64_t restPort) { + ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)options; + go->restPort = restPort; + return 0; +} + +OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, + const char* config_path) { + ovms::MultiModelOptionsImpl* mmo = (ovms::MultiModelOptionsImpl*)options; + mmo->configPath = std::string(config_path); + return 0; } diff --git a/src/pocapi.hpp b/src/pocapi.hpp index 47cd0a3f95..2997086fe1 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -59,6 +59,7 @@ enum BufferType { struct OVMS_Status; struct OVMS_InferenceRequest; struct OVMS_InferenceResponse; + typedef enum OVMSSERVER_loglevel_enum { OVMSSERVER_LOG_TRACE, OVMSSERVER_LOG_DEBUG, @@ -76,6 +77,14 @@ OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options); // Deallocates server general options memory for given ptr OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options); +// --port +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, + uint64_t grpcPort); + +// --rest_port +OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, + uint64_t restPort); + // --log_level OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, OVMSSERVER_LogLevel log_level); diff --git a/src/server.cpp b/src/server.cpp index 6afe332bd7..ff01e2ea2f 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -43,6 +43,7 @@ #include "metric_module.hpp" #include "model_service.hpp" #include "modelmanager.hpp" +#include "poc_api_impl.hpp" #include "prediction_service.hpp" #include "profiler.hpp" #include "servablemanagermodule.hpp" @@ -320,6 +321,8 @@ void Server::shutdownModules() { modules.clear(); } +// OVMS Start +// TODO: Merge with C-API Start CVS-95439 int Server::start(int argc, char** argv) { ovms::Server& server = ovms::Server::instance(); installSignalHandlers(server); @@ -348,4 +351,36 @@ int Server::start(int argc, char** argv) { } return EXIT_SUCCESS; } + +// C-API Start +// TODO: Merge with OVMS Start CVS-95439 +int Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { + ovms::Server& server = ovms::Server::instance(); + installSignalHandlers(server); + try { + auto& config = ovms::Config::instance().parse(go, mmo); + configure_logger(config.logLevel(), config.logPath()); + logConfig(config); + ModulesShutdownGuard shutdownGuard(*this); + auto retCode = this->startModules(config); + if (retCode) + return retCode; + + while (!shutdown_request) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + if (shutdown_request == 2) { + SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); + } + SPDLOG_INFO("Shutting down"); + } catch (std::exception& e) { + SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); + return EXIT_FAILURE; + } catch (...) { + SPDLOG_ERROR("Unknown exception catch - will now terminate."); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + } // namespace ovms diff --git a/src/server.hpp b/src/server.hpp index b9f3fb61d8..ea7fb8aaa8 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -25,6 +25,9 @@ namespace ovms { class Config; +class GeneralOptionsImpl; +class MultiModelOptionsImpl; + extern const std::string PROFILER_MODULE_NAME; extern const std::string GRPC_SERVER_MODULE_NAME; extern const std::string HTTP_SERVER_MODULE_NAME; @@ -42,6 +45,7 @@ class Server { public: static Server& instance(); int start(int argc, char** argv); + int start(GeneralOptionsImpl*, MultiModelOptionsImpl*); ModuleState getModuleState(const std::string& name) const; const Module* getModule(const std::string& name) const; bool isReady() const; diff --git a/src/shape.cpp b/src/shape.cpp index a63d437e7e..939640bbda 100644 --- a/src/shape.cpp +++ b/src/shape.cpp @@ -294,7 +294,7 @@ Shape::Shape(const shape_t& s) { Status Shape::fromFlatShape(const shape_t& shapeIn, Shape& shapeOut) { Shape shape; for (size_t dim : shapeIn) { - if (dim > std::numeric_limits::max()) { + if (dim > static_cast(std::numeric_limits::max())) { return StatusCode::CANNOT_CONVERT_FLAT_SHAPE; } else { shape.add(Dimension{static_cast(dim)}); diff --git a/src/test/c_api/config.json b/src/test/c_api/config.json new file mode 100644 index 0000000000..d42b9d7a42 --- /dev/null +++ b/src/test/c_api/config.json @@ -0,0 +1,8 @@ +{ + "model_config_list": [ + {"config": { + "name": "dummy", + "base_path": "/ovms/src/test/dummy", + "shape": "(30, 20)"}} + ] +} diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp new file mode 100644 index 0000000000..b1669785e6 --- /dev/null +++ b/src/test/c_api_tests.cpp @@ -0,0 +1,42 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include + +#include "../config.hpp" +#include "../poc_api_impl.hpp" + +using namespace ovms; + +class CapiConfigTest : public ::testing::Test { +protected: + void SetUp() override { + } +}; + +TEST_F(CapiConfigTest, Parse) { + GeneralOptionsImpl go; + MultiModelOptionsImpl mmo; + + go.grpcPort = 123; + go.restPort = 234; + mmo.configPath = "/path/config.json"; + + ovms::Config::instance().parse(&go, &mmo); + EXPECT_EQ(ovms::Config::instance().port(), 123); + EXPECT_EQ(ovms::Config::instance().restPort(), 234); + EXPECT_EQ(ovms::Config::instance().configPath(), "/path/config.json"); +} diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index 01e5d9713d..ce0f896b97 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -73,13 +73,19 @@ TEST_F(OvmsConfigDeathTest, emptyInput) { } TEST_F(OvmsConfigDeathTest, helpInput) { - char* n_argv[] = {"ovms", "help"}; + char* n_argv[] = {"ovms", "--help"}; int arg_count = 2; EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_OK), ""); // EXPECT_TRUE(AssertRegexMessageInOutput(std::string("config_path"))); } +TEST_F(OvmsConfigDeathTest, versionInput) { + char* n_argv[] = {"ovms", "--version"}; + int arg_count = 2; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_OK), ""); +} + TEST_F(OvmsConfigDeathTest, badInput) { char* n_argv[] = {"ovms", "--bad_option"}; int arg_count = 2; @@ -152,6 +158,30 @@ TEST_F(OvmsConfigDeathTest, negativeSamePorts) { EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "port and rest_port cannot"); } +TEST_F(OvmsConfigDeathTest, restWorkersTooLarge) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--rest_port", "8080", "--port", "8080", "--rest_workers", "100001"}; + int arg_count = 9; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "rest_workers count should be from 2 to "); +} + +TEST_F(OvmsConfigDeathTest, restWorkersDefinedRestPortUndefined) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--port", "8080", "--rest_workers", "60"}; + int arg_count = 7; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "rest_workers is set but rest_port is not set"); +} + +TEST_F(OvmsConfigDeathTest, invalidRestBindAddress) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--rest_port", "8081", "--port", "8080", "--rest_bind_address", "192.0.2"}; + int arg_count = 9; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "rest_bind_address has invalid format"); +} + +TEST_F(OvmsConfigDeathTest, invalidGrpcBindAddress) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--port", "8080", "--grpc_bind_address", "192.0.2"}; + int arg_count = 7; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "grpc_bind_address has invalid format"); +} + TEST_F(OvmsConfigDeathTest, negativeMultiParams) { char* n_argv[] = {"ovms", "--config_path", "/path1", "--batch_size", "10"}; int arg_count = 5; @@ -206,6 +236,36 @@ TEST_F(OvmsConfigDeathTest, negativeGrpcWorkersMax) { EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "grpc_workers count should be from 1"); } +TEST_F(OvmsConfigDeathTest, cpuExtensionMissingPath) { + char* n_argv[] = {"ovms", "--model_path", "/path1", "--model_name", "model", "--cpu_extension", "/wrong/dir"}; + int arg_count = 7; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "File path provided as an --cpu_extension parameter does not exists in the filesystem"); +} + +TEST_F(OvmsConfigDeathTest, nonExistingLogLevel) { + char* n_argv[] = {"ovms", "--model_path", "/path1", "--model_name", "model", "--log_level", "WRONG"}; + int arg_count = 7; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "log_level should be one of"); +} + +TEST_F(OvmsConfigDeathTest, lowLatencyUsedForNonStateful) { + char* n_argv[] = {"ovms", "--model_path", "/path1", "--model_name", "model", "--low_latency_transformation"}; + int arg_count = 6; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "require setting stateful flag for the model"); +} + +TEST_F(OvmsConfigDeathTest, maxSequenceNumberUsedForNonStateful) { + char* n_argv[] = {"ovms", "--model_path", "/path1", "--model_name", "model", "--max_sequence_number", "325"}; + int arg_count = 7; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "require setting stateful flag for the model"); +} + +TEST_F(OvmsConfigDeathTest, idleSequenceCleanupUsedForNonStateful) { + char* n_argv[] = {"ovms", "--model_path", "/path1", "--model_name", "model", "--idle_sequence_cleanup"}; + int arg_count = 6; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "require setting stateful flag for the model"); +} + TEST_F(OvmsConfigDeathTest, negativeUint64Max) { char* n_argv[] = {"ovms", "--config_path", "/path1", "--rest_port", "0xffffffffffffffff"}; int arg_count = 5; @@ -226,4 +286,13 @@ TEST_F(OvmsParamsTest, hostname_ip_regex) { EXPECT_EQ(ovms::Config::check_hostname_or_ip(too_long), false); } +TEST(OvmsConfigTest, positive) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--port", "44", "--rest_port", "45"}; + int arg_count = 7; + ovms::Config::instance().parse(arg_count, n_argv); + EXPECT_EQ(ovms::Config::instance().port(), 44); + EXPECT_EQ(ovms::Config::instance().restPort(), 45); + EXPECT_EQ(ovms::Config::instance().configPath(), "/path1"); +} + #pragma GCC diagnostic pop From 0f0c8a854d96e4c682a0793619211081633b346b Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Wed, 9 Nov 2022 11:09:34 +0100 Subject: [PATCH 089/130] C-API Request/response structures (#1511) * Cppclean fixes Fixes for other places in code as the tool has issues with headers having pure "using" directives. Since cppclean does not accept headers with using I was forced to either increase threshold for issues count or fix issues not introduced in this MR. JIRA:CVS-95297 --- cppclean.sh | 2 +- src/BUILD | 11 ++ src/buffer.cpp | 42 +++++++ src/buffer.hpp | 37 ++++++ src/http_rest_api_handler.hpp | 2 +- src/inferenceparameter.cpp | 53 ++++++++ src/inferenceparameter.hpp | 37 ++++++ src/inferencerequest.cpp | 87 +++++++++++++ src/inferencerequest.hpp | 61 +++++++++ src/inferenceresponse.cpp | 61 +++++++++ src/inferenceresponse.hpp | 49 ++++++++ src/inferencetensor.cpp | 56 +++++++++ src/inferencetensor.hpp | 46 +++++++ src/modelinstance.cpp | 1 + src/modelinstance.hpp | 3 +- src/modelmanager.hpp | 8 -- src/pipelinedefinition.cpp | 1 + src/precision.hpp | 2 +- src/status.cpp | 11 ++ src/status.hpp | 11 ++ src/test/ensemble_tests.cpp | 5 +- src/test/inferencerequest_test.cpp | 191 +++++++++++++++++++++++++++++ src/test/modelmanager_test.cpp | 110 ++++++++--------- 23 files changed, 814 insertions(+), 73 deletions(-) create mode 100644 src/buffer.cpp create mode 100644 src/buffer.hpp create mode 100644 src/inferenceparameter.cpp create mode 100644 src/inferenceparameter.hpp create mode 100644 src/inferencerequest.cpp create mode 100644 src/inferencerequest.hpp create mode 100644 src/inferenceresponse.cpp create mode 100644 src/inferenceresponse.hpp create mode 100644 src/inferencetensor.cpp create mode 100644 src/inferencetensor.hpp create mode 100644 src/test/inferencerequest_test.cpp diff --git a/cppclean.sh b/cppclean.sh index 7e2bfffe7b..0269f5bd39 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,7 +42,7 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 2 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 178 ]; then +if [ ${NO_WARNINGS} -gt 179 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 else diff --git a/src/BUILD b/src/BUILD index 6834a77df2..1223a95481 100644 --- a/src/BUILD +++ b/src/BUILD @@ -94,6 +94,8 @@ cc_library( "azurestorage.cpp", "azurefilesystem.cpp", "azurefilesystem.hpp", + "buffer.cpp", + "buffer.hpp", "cleaner_utils.cpp", "cleaner_utils.hpp", "config.cpp", @@ -155,6 +157,14 @@ cc_library( "grpc_utils.hpp", "grpcservermodule.cpp", "grpcservermodule.hpp", + "inferenceparameter.cpp", + "inferenceparameter.hpp", + "inferenceresponse.cpp", + "inferenceresponse.hpp", + "inferencerequest.cpp", + "inferencerequest.hpp", + "inferencetensor.cpp", + "inferencetensor.hpp", "kfs_frontend/kfs_grpc_inference_service.cpp", "kfs_frontend/kfs_grpc_inference_service.hpp", "kfs_frontend/kfs_utils.cpp", @@ -609,6 +619,7 @@ cc_test( "test/get_model_metadata_signature_test.cpp", "test/get_model_metadata_validation_test.cpp", "test/http_rest_api_handler_test.cpp", + "test/inferencerequest_test.cpp", "test/kfs_metadata_test.cpp", "test/kfs_rest_test.cpp", "test/layout_test.cpp", diff --git a/src/buffer.cpp b/src/buffer.cpp new file mode 100644 index 0000000000..b08b895cc6 --- /dev/null +++ b/src/buffer.cpp @@ -0,0 +1,42 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "buffer.hpp" + +#include + +#include "logging.hpp" + +namespace ovms { +Buffer::Buffer(const void* pptr, size_t byteSize, BufferType bufferType, std::optional bufferDeviceId, bool createCopy) : + ptr(createCopy ? nullptr : pptr), + byteSize(byteSize), + bufferType(bufferType), + bufferDeviceId(bufferDeviceId) { + // TODO in later stages + // it can be advantageous to have custom made release fn especially with buffers passed by external users + if (!createCopy) + return; + ownedCopy = std::make_unique(byteSize); + std::memcpy(ownedCopy.get(), pptr, byteSize); +} +const void* Buffer::data() const { + return (ptr != nullptr) ? ptr : ownedCopy.get(); +} +size_t Buffer::getByteSize() const { + return byteSize; +} +Buffer::~Buffer() = default; +} // namespace ovms diff --git a/src/buffer.hpp b/src/buffer.hpp new file mode 100644 index 0000000000..2921638d49 --- /dev/null +++ b/src/buffer.hpp @@ -0,0 +1,37 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include "pocapi.hpp" +namespace ovms { + +class Buffer { + const void* ptr; + size_t byteSize; + BufferType bufferType; + std::optional bufferDeviceId; + std::unique_ptr ownedCopy = nullptr; + +public: + Buffer(const void* ptr, size_t byteSize, BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional bufferDeviceId = std::nullopt, bool createCopy = false); + ~Buffer(); + const void* data() const; + size_t getByteSize() const; +}; + +} // namespace ovms diff --git a/src/http_rest_api_handler.hpp b/src/http_rest_api_handler.hpp index c404e94a0a..d232ae950b 100644 --- a/src/http_rest_api_handler.hpp +++ b/src/http_rest_api_handler.hpp @@ -28,6 +28,7 @@ #pragma GCC diagnostic pop #include "rest_parser.hpp" +#include "status.hpp" namespace ovms { class ServableMetricReporter; @@ -35,7 +36,6 @@ class KFSInferenceServiceImpl; class GetModelMetadataImpl; class Server; class ModelManager; -class Status; enum RequestType { Predict, GetModelStatus, diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp new file mode 100644 index 0000000000..5044aa9b34 --- /dev/null +++ b/src/inferenceparameter.cpp @@ -0,0 +1,53 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "inferenceparameter.hpp" + +#include + +#include "pocapi.hpp" +namespace ovms { +// TODO should we own our own copy of value? +// +static size_t DataTypeToByteSize(DataType datatype) { + switch (datatype) { + case OVMS_DATATYPE_FP32: + case OVMS_DATATYPE_I32: + case OVMS_DATATYPE_U32: + return 4; + default: + throw std::invalid_argument("Unsupported"); + return 0; + } +} +InferenceParameter::InferenceParameter(const char* name, DataType datatype, const void* data) : + name(name), + datatype(datatype), + data(reinterpret_cast(data), DataTypeToByteSize(datatype)) { +} +// InferenceParameter::InferenceParameter(const char* name, DataType datatype, const void* data, size_t byteSize); +const std::string& InferenceParameter::getName() const { + return this->name; +} +DataType InferenceParameter::getDataType() const { + return this->datatype; +} +size_t InferenceParameter::getByteSize() const { + return this->data.size(); +} +const void* InferenceParameter::getData() const { + return reinterpret_cast(this->data.c_str()); +} +} // namespace ovms diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp new file mode 100644 index 0000000000..204dab5168 --- /dev/null +++ b/src/inferenceparameter.hpp @@ -0,0 +1,37 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include + +#include "pocapi.hpp" + +namespace ovms { + +// TODO should we own our own copy of value? +class InferenceParameter { + const std::string name; + DataType datatype; + const std::string data; + +public: + InferenceParameter(const char* name, DataType datatype, const void* data); + InferenceParameter(const char* name, DataType datatype, const void* data, size_t byteSize); + const std::string& getName() const; + DataType getDataType() const; + size_t getByteSize() const; + const void* getData() const; +}; +} // namespace ovms diff --git a/src/inferencerequest.cpp b/src/inferencerequest.cpp new file mode 100644 index 0000000000..e58c9dd545 --- /dev/null +++ b/src/inferencerequest.cpp @@ -0,0 +1,87 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "inferencerequest.hpp" + +#include "status.hpp" +namespace ovms { + +InferenceRequest::InferenceRequest(const char* servableName, model_version_t servableVersion) : + servableName(servableName), + servableVersion(servableVersion) { +} + +const std::string& InferenceRequest::getServableName() const { + return this->servableName; +} +model_version_t InferenceRequest::getServableVersion() const { + return this->servableVersion; +} +Status InferenceRequest::addInput(const char* name, DataType datatype, const size_t* shape, size_t dimCount) { + auto [it, emplaced] = inputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); + return emplaced ? StatusCode::OK : StatusCode::DOUBLE_TENSOR_INSERT; +} +Status InferenceRequest::setInputBuffer(const char* name, const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId) { + auto it = inputs.find(name); + if (it == inputs.end()) { + return StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER; + } + return it->second.setBuffer(addr, byteSize, bufferType, deviceId); +} +Status InferenceRequest::removeInputBuffer(const char* name) { + auto it = inputs.find(name); + if (it == inputs.end()) { + return StatusCode::NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER; + } + return it->second.removeBuffer(); +} +Status InferenceRequest::removeAllInputs() { + inputs.clear(); + return StatusCode::OK; +} +Status InferenceRequest::getInput(const char* name, const InferenceTensor** tensor) const { + auto it = inputs.find(name); + if (it == inputs.end()) { + *tensor = nullptr; + return StatusCode::NONEXISTENT_TENSOR; + } + *tensor = &it->second; + return StatusCode::OK; +} +Status InferenceRequest::removeInput(const char* name) { + auto count = inputs.erase(name); + if (count) { + return StatusCode::OK; + } + return StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL; +} +Status InferenceRequest::addParameter(const char* parameterName, DataType datatype, const void* data) { + auto [it, emplaced] = parameters.emplace(parameterName, InferenceParameter{parameterName, datatype, data}); + return emplaced ? StatusCode::OK : StatusCode::DOUBLE_PARAMETER_INSERT; +} +Status InferenceRequest::removeParameter(const char* name) { + auto count = parameters.erase(name); + if (count) { + return StatusCode::OK; + } + return StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL; +} +const InferenceParameter* InferenceRequest::getParameter(const char* name) const { + auto it = parameters.find(name); + if (it != parameters.end()) + return &it->second; + return nullptr; +} +} // namespace ovms diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp new file mode 100644 index 0000000000..9686355c85 --- /dev/null +++ b/src/inferencerequest.hpp @@ -0,0 +1,61 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include "inferenceparameter.hpp" +#include "inferencetensor.hpp" +#include "modelversion.hpp" +#include "pocapi.hpp" + +namespace ovms { + +class Status; + +class InferenceRequest { + const std::string servableName; + const model_version_t servableVersion; + std::unordered_map parameters; + std::unordered_map inputs; + +public: + InferenceRequest(const char* modelName, model_version_t modelVersion); + Status addInput(const char* name, DataType datatype, const size_t* shape, size_t dimCount); + Status getInput(const char* name, const InferenceTensor** tensor) const; + Status removeInput(const char* name); + Status removeAllInputs(); + + Status setInputBuffer(const char* name, const void* addr, size_t byteSize, BufferType, std::optional deviceId); + Status removeInputBuffer(const char* name); + Status addParameter(const char* parameterName, DataType datatype, const void* data); + Status removeParameter(const char* parameterName); + const InferenceParameter* getParameter(const char* name) const; + + const std::string& getServableName() const; + model_version_t getServableVersion() const; + + Status setId(); + Status getId(); + + Status setPriority(); + Status getPriority(); + + Status setTimeoutMicorseconds(uint64_t microseconds); + InferenceParameter* getInferenceParameter(const char* name); + InferenceTensor* getTensor(const char* name); +}; +} // namespace ovms diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp new file mode 100644 index 0000000000..08d48c20f3 --- /dev/null +++ b/src/inferenceresponse.cpp @@ -0,0 +1,61 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "inferenceresponse.hpp" + +#include +#include + +#include "inferenceparameter.hpp" +#include "inferencetensor.hpp" +#include "modelversion.hpp" +#include "status.hpp" + +namespace ovms { +InferenceResponse::InferenceResponse(const std::string& servableName, model_version_t servableVersion) : + servableName(servableName), + servableVersion(servableVersion) {} +const std::string& InferenceResponse::getServableName() const { + return this->servableName; +} +model_version_t InferenceResponse::getServableVersion() const { + return this->servableVersion; +} +Status InferenceResponse::addOutput(const std::string& name, DataType datatype, const size_t* shape, size_t dimCount) { + // TODO insert tensor with wrong shape/datatype/name/dimcount + // TODO reuse infer response/request + auto [it, emplaced] = outputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); + return emplaced ? StatusCode::OK : StatusCode::DOUBLE_TENSOR_INSERT; +} +Status InferenceResponse::getOutput(const char* name, InferenceTensor** tensor) { + auto it = outputs.find(name); + if (outputs.end() == it) { + *tensor = nullptr; + return StatusCode::NONEXISTENT_TENSOR; + } + *tensor = &it->second; + return StatusCode::OK; +} +Status InferenceResponse::addParameter(const char* parameterName, DataType datatype, const void* data) { + auto [it, emplaced] = parameters.emplace(parameterName, InferenceParameter{parameterName, datatype, data}); + return emplaced ? StatusCode::OK : StatusCode::DOUBLE_PARAMETER_INSERT; +} +const InferenceParameter* InferenceResponse::getParameter(const char* name) const { + auto it = parameters.find(name); + if (it != parameters.end()) + return &it->second; + return nullptr; +} +} // namespace ovms diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp new file mode 100644 index 0000000000..fa3b2f58cb --- /dev/null +++ b/src/inferenceresponse.hpp @@ -0,0 +1,49 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include "inferenceparameter.hpp" +#include "inferencetensor.hpp" +#include "modelversion.hpp" +#include "pocapi.hpp" + +namespace ovms { + +class Status; + +class InferenceResponse { + const std::string& servableName; + const model_version_t servableVersion; + std::unordered_map parameters; + std::unordered_map outputs; + +public: + InferenceResponse(const std::string& servableName, model_version_t servableVersion); + Status addOutput(const std::string& name, DataType datatype, const size_t* shape, size_t dimCount); + Status getOutput(const char* name, InferenceTensor** tensor); + Status addParameter(const char* parameterName, DataType datatype, const void* data); + const InferenceParameter* getParameter(const char* name) const; + + const std::string& getServableName() const; + model_version_t getServableVersion() const; + + Status setId(); + Status getId(); + InferenceParameter* getInferenceParameter(const char* name); +}; +} // namespace ovms diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp new file mode 100644 index 0000000000..d35195429a --- /dev/null +++ b/src/inferencetensor.cpp @@ -0,0 +1,56 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "inferencetensor.hpp" + +#include + +#include "buffer.hpp" +#include "pocapi.hpp" +#include "status.hpp" + +namespace ovms { +InferenceTensor::~InferenceTensor() = default; +InferenceTensor::InferenceTensor(InferenceTensor&& rhs) : + datatype(std::move(rhs.datatype)), + shape(std::move(rhs.shape)), + buffer(std::move(rhs.buffer)) {} +InferenceTensor::InferenceTensor(DataType datatype, const size_t* shape, size_t dimCount) : + datatype(datatype), + shape(shape, shape + dimCount) {} +Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy) { + if (nullptr != buffer) { + return StatusCode::DOUBLE_BUFFER_SET; + } + buffer = std::make_unique(addr, byteSize, bufferType, deviceId, createCopy); + return StatusCode::OK; +} +DataType InferenceTensor::getDataType() const { + return this->datatype; +} +const shape_t& InferenceTensor::getShape() const { + return this->shape; +} +const Buffer* const InferenceTensor::getBuffer() const { + return this->buffer.get(); +} +Status InferenceTensor::removeBuffer() { + if (nullptr != this->buffer) { + this->buffer.reset(); + return StatusCode::OK; + } + return StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL; +} +} // namespace ovms diff --git a/src/inferencetensor.hpp b/src/inferencetensor.hpp new file mode 100644 index 0000000000..f08306f08f --- /dev/null +++ b/src/inferencetensor.hpp @@ -0,0 +1,46 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "pocapi.hpp" +#include "shape.hpp" + +namespace ovms { +class Buffer; +class Status; + +class InferenceTensor { + const DataType datatype; + shape_t shape; + std::unique_ptr buffer; + +public: + InferenceTensor(DataType datatype, const size_t* shape, size_t dimCount); + ~InferenceTensor(); + InferenceTensor(InferenceTensor&&); + InferenceTensor(const InferenceTensor&) = delete; + InferenceTensor& operator=(const InferenceTensor&) = delete; + InferenceTensor& operator=(const InferenceTensor&&); + Status setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy = false); + Status removeBuffer(); + DataType getDataType() const; + const shape_t& getShape() const; + const Buffer* const getBuffer() const; +}; +} // namespace ovms diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 6a434c5667..c52f1db98a 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -72,6 +72,7 @@ const int DEFAULT_OV_STREAMS = std::thread::hardware_concurrency() / 4; const uint UNLOAD_AVAILABILITY_CHECKING_INTERVAL_MILLISECONDS = 10; +ModelInstance::~ModelInstance() = default; ModelInstance::ModelInstance(const std::string& name, model_version_t version, ov::Core& ieCore, MetricRegistry* registry, const MetricConfig* metricConfig) : ieCore(ieCore), name(name), diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index 3c0c56f9e4..17c150971d 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -42,7 +42,6 @@ namespace ovms { class MetricRegistry; -class ModelConfig; class ModelInstanceUnloadGuard; class PipelineDefinition; class Status; @@ -327,7 +326,7 @@ class ModelInstance { /** * @brief Destroy the Model Instance object */ - virtual ~ModelInstance() = default; + virtual ~ModelInstance(); /** * @brief Increases predict requests usage count diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index c84c3b3754..8acc069be3 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -211,14 +211,6 @@ class ModelManager { */ mutable std::shared_mutex modelsMtx; - /** - * @brief Gets the instance of ModelManager - */ - static ModelManager& getInstance() { - static ModelManager instance; - return instance; - } - /** * @brief Gets the watcher interval timestep in seconds */ diff --git a/src/pipelinedefinition.cpp b/src/pipelinedefinition.cpp index 444aa45dac..6c6d455ec9 100644 --- a/src/pipelinedefinition.cpp +++ b/src/pipelinedefinition.cpp @@ -27,6 +27,7 @@ #include "logging.hpp" #include "model_metric_reporter.hpp" #include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" #include "node_library_utils.hpp" #include "nodeinfo.hpp" diff --git a/src/precision.hpp b/src/precision.hpp index 3b6c834caf..d2cabf98a5 100644 --- a/src/precision.hpp +++ b/src/precision.hpp @@ -43,7 +43,7 @@ enum class Precision { DYNAMIC, MIXED, Q78, - BIN + BIN // TODO remove BIN,Q78, DYNAMIC, CUSTOM, MIXED }; const std::string& toString(Precision precision); diff --git a/src/status.cpp b/src/status.cpp index d756da8e0b..f500292c64 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -259,5 +259,16 @@ const std::unordered_map Status::statusMess {StatusCode::INVALID_METRICS_ENDPOINT, "Metrics config endpoint path is invalid"}, {StatusCode::INVALID_METRICS_FAMILY_NAME, "Invalid name in metrics_list"}, {StatusCode::METRICS_REST_PORT_MISSING, "Missing rest_port parameter in server CLI"}, + + // C-API + {StatusCode::DOUBLE_BUFFER_SET, "Cannot set buffer more than once to the same tensor"}, + {StatusCode::DOUBLE_TENSOR_INSERT, "Cannot insert more than one tensor with the same name"}, + {StatusCode::DOUBLE_PARAMETER_INSERT, "Cannot insert more than one parameter with the same name"}, + {StatusCode::NONEXISTENT_TENSOR, "Tried to get nonexisting tensor"}, + {StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER, "Tried to set buffer for nonexisting tensor"}, + {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, "Tried to remove buffer for nonexisting tensor"}, + {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL, "Tried to remove nonexisting tensor"}, + {StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL, "Tried to remove nonexisting buffer"}, + {StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL, "Tried to remove nonexisting parameter"}, }; } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 3ab452b81f..6c7a26a7f3 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -271,6 +271,17 @@ enum class StatusCode { INVALID_METRICS_FAMILY_NAME, METRICS_REST_PORT_MISSING, + // C-API + DOUBLE_BUFFER_SET, + DOUBLE_TENSOR_INSERT, + DOUBLE_PARAMETER_INSERT, + NONEXISTENT_TENSOR, + NONEXISTENT_TENSOR_FOR_SET_BUFFER, + NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, + NONEXISTENT_TENSOR_FOR_REMOVAL, + NONEXISTENT_BUFFER_FOR_REMOVAL, + NONEXISTENT_PARAMETER_FOR_REMOVAL, + STATUS_CODE_END }; diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp index 2d1cbf635c..8f5ecdff0e 100644 --- a/src/test/ensemble_tests.cpp +++ b/src/test/ensemble_tests.cpp @@ -1751,7 +1751,7 @@ TEST_F(EnsembleFlowTest, FailInDLNodeExecuteInputsMissingInput) { class DLNodeFailInFetch : public DLNode { public: - DLNodeFailInFetch(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, ModelManager& modelManager = ModelManager::getInstance()) : + DLNodeFailInFetch(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, ModelManager& modelManager) : DLNode(nodeName, modelName, modelVersion, modelManager, {}) {} ovms::Status fetchResults(NodeSession& nodeSession, SessionResults& sessionResults) override { // no release is called as in dl_node.cpp when on error path @@ -2621,12 +2621,13 @@ TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_EntryMissing) { } TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_DefinitionMissing) { + ConstructorEnabledModelManager manager; PipelineFactory factory; PredictRequest request; PredictResponse response; std::unique_ptr pipeline; - EXPECT_EQ(factory.create(pipeline, "pipeline", &request, &response, ovms::ModelManager::getInstance()), StatusCode::PIPELINE_DEFINITION_NAME_MISSING); + EXPECT_EQ(factory.create(pipeline, "pipeline", &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NAME_MISSING); } TEST_F(EnsembleFlowTest, PipelineFactoryWrongConfiguration_NodeNameDuplicate) { diff --git a/src/test/inferencerequest_test.cpp b/src/test/inferencerequest_test.cpp new file mode 100644 index 0000000000..d7ccaa4f55 --- /dev/null +++ b/src/test/inferencerequest_test.cpp @@ -0,0 +1,191 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include + +#include +#include + +#include "../buffer.hpp" +#include "../inferenceparameter.hpp" +#include "../inferencerequest.hpp" +#include "../inferenceresponse.hpp" +#include "../logging.hpp" +#include "../shape.hpp" +#include "../status.hpp" + +using testing::ElementsAre; + +using ovms::Buffer; +using ovms::InferenceParameter; +using ovms::InferenceRequest; +using ovms::InferenceResponse; +using ovms::InferenceTensor; +using ovms::Shape; +using ovms::Status; +using ovms::StatusCode; + +namespace { +const std::string MODEL_NAME{"SomeModelName"}; +const uint64_t MODEL_VERSION{42}; +const std::string PARAMETER_NAME{"SEQUENCE_ID"}; // TODO check if in ovms there is such constant +const DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; + +const uint32_t PARAMETER_VALUE{13}; +const uint32_t PRIORITY{7}; +const uint64_t REQUEST_ID{3}; + +const std::string INPUT_NAME{"NOT_RANDOM_NAME"}; +const ovms::shape_t INPUT_SHAPE{1, 3, 220, 230}; +const std::array INPUT_DATA{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; +constexpr size_t INPUT_DATA_BYTESIZE{INPUT_DATA.size() * sizeof(float)}; +const DataType DATATYPE{OVMS_DATATYPE_FP32}; +} // namespace + +TEST(InferenceParameter, CreateParameter) { + InferenceParameter parameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); +} + +TEST(InferenceRequest, CreateInferenceRequest) { + InferenceRequest request(MODEL_NAME.c_str(), MODEL_VERSION); + EXPECT_EQ(request.getServableName(), MODEL_NAME); + EXPECT_EQ(request.getServableVersion(), MODEL_VERSION); + + auto status = request.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + + // add parameter + const InferenceParameter* parameter = request.getParameter(PARAMETER_NAME.c_str()); + ASSERT_NE(parameter, nullptr); + EXPECT_EQ(parameter->getName(), PARAMETER_NAME); + EXPECT_EQ(parameter->getDataType(), PARAMETER_DATATYPE); + EXPECT_EQ(*(reinterpret_cast(parameter->getData())), PARAMETER_VALUE); + // add same parameter second time should fail + status = request.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); + ASSERT_EQ(status, StatusCode::DOUBLE_PARAMETER_INSERT) << status.string(); + + // add input + status = request.addInput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // add same input second time should fail + status = request.addInput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); + ASSERT_EQ(status, StatusCode::DOUBLE_TENSOR_INSERT) << status.string(); + + // set input buffer + status = request.setInputBuffer(INPUT_NAME.c_str(), INPUT_DATA.data(), INPUT_DATA.size() * sizeof(float), OVMS_BUFFERTYPE_CPU, std::nullopt); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // set buffer second time should fail + status = request.setInputBuffer(INPUT_NAME.c_str(), INPUT_DATA.data(), INPUT_DATA.size() * sizeof(float), OVMS_BUFFERTYPE_CPU, std::nullopt); + ASSERT_EQ(status, StatusCode::DOUBLE_BUFFER_SET) << status.string(); + + // get input & buffer + const InferenceTensor* tensor; + status = request.getInput(INPUT_NAME.c_str(), &tensor); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + ASSERT_NE(nullptr, tensor); + EXPECT_EQ(tensor->getDataType(), DATATYPE); + EXPECT_TRUE(Shape(tensor->getShape()).match(INPUT_SHAPE)); + const Buffer* buffer = tensor->getBuffer(); + ASSERT_NE(nullptr, buffer); + ASSERT_NE(nullptr, buffer->data()); + ASSERT_EQ(buffer->data(), INPUT_DATA.data()); + ASSERT_EQ(buffer->getByteSize(), INPUT_DATA_BYTESIZE); + // save data 2nd time should fail + // compare data content with what was saved + auto res = std::memcmp(buffer->data(), reinterpret_cast(INPUT_DATA.data()), INPUT_DATA_BYTESIZE); + EXPECT_EQ(0, res) << res; + + // remove input buffer + status = request.removeInputBuffer(INPUT_NAME.c_str()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + ASSERT_EQ(nullptr, tensor->getBuffer()); + // remove buffer twice + status = request.removeInputBuffer(INPUT_NAME.c_str()); + ASSERT_EQ(status, StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL) << status.string(); + + // remove input + status = request.removeInput(INPUT_NAME.c_str()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // verify removing all inputs + status = request.addInput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + status = request.removeAllInputs(); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + + // verify removing parameter + status = request.removeParameter(PARAMETER_NAME.c_str()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + ASSERT_EQ(nullptr, request.getParameter(PARAMETER_NAME.c_str())); +} +TEST(InferenceResponse, CreateAndReadData) { + // create response + InferenceResponse response{MODEL_NAME, MODEL_VERSION}; + EXPECT_EQ(response.getServableName(), MODEL_NAME); + EXPECT_EQ(response.getServableVersion(), MODEL_VERSION); + // add output + auto status = response.addOutput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // add 2nd output with the same name should fail + status = response.addOutput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); + ASSERT_EQ(status, StatusCode::DOUBLE_TENSOR_INSERT) << status.string(); + // get nonexistent output + InferenceTensor* tensor = nullptr; + status = response.getOutput("SOME_NOT_RANDOM_NAME", &tensor); + ASSERT_EQ(status, StatusCode::NONEXISTENT_TENSOR) << status.string(); + // get output + status = response.getOutput(INPUT_NAME.c_str(), &tensor); + ASSERT_NE(nullptr, tensor); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // compare datatype + ASSERT_EQ(tensor->getDataType(), DATATYPE); + // compare shape + auto shape = tensor->getShape(); + ASSERT_THAT(shape, ElementsAre(1, 3, 220, 230)); + + // save data into output (it should have it's own copy in contrast to request) + bool createCopy = true; + status = tensor->setBuffer(INPUT_DATA.data(), INPUT_DATA_BYTESIZE, OVMS_BUFFERTYPE_CPU, std::nullopt, createCopy); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + // savind data into output twice should fail + std::array RANDOM_DATA{10., 9, 8, 7, 6, 5, 4, 3, 2, 1}; + status = tensor->setBuffer(RANDOM_DATA.data(), INPUT_DATA_BYTESIZE, OVMS_BUFFERTYPE_CPU, std::nullopt, createCopy); + ASSERT_EQ(status, StatusCode::DOUBLE_BUFFER_SET) << status.string(); + + const Buffer* buffer = tensor->getBuffer(); + ASSERT_NE(nullptr, buffer); + ASSERT_NE(nullptr, buffer->data()); + ASSERT_NE(buffer->data(), INPUT_DATA.data()); + ASSERT_EQ(buffer->getByteSize(), INPUT_DATA_BYTESIZE); + // save data 2nd time should fail + // compare data content with what was saved + auto res = std::memcmp(buffer->data(), reinterpret_cast(INPUT_DATA.data()), INPUT_DATA_BYTESIZE); + EXPECT_EQ(0, res) << res; + + // verify parameter handling + status = response.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + + // add parameter + const InferenceParameter* parameter = response.getParameter(PARAMETER_NAME.c_str()); + ASSERT_NE(parameter, nullptr); + EXPECT_EQ(parameter->getName(), PARAMETER_NAME); + EXPECT_EQ(parameter->getDataType(), PARAMETER_DATATYPE); + EXPECT_EQ(*(reinterpret_cast(parameter->getData())), PARAMETER_VALUE); + // add same parameter second time should fail + status = response.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); + ASSERT_EQ(status, StatusCode::DOUBLE_PARAMETER_INSERT) << status.string(); +} +// TODO logging diff --git a/src/test/modelmanager_test.cpp b/src/test/modelmanager_test.cpp index 386287bd24..81b36a6bca 100644 --- a/src/test/modelmanager_test.cpp +++ b/src/test/modelmanager_test.cpp @@ -112,17 +112,20 @@ class MockModelManager : public ovms::ModelManager { } }; -TEST(ModelManager, ConfigParseNoModels) { +class ModelManager : public ::testing::Test { +protected: + ConstructorEnabledModelManager fixtureManager; +}; + +TEST_F(ModelManager, ConfigParseNoModels) { std::string configFile = createConfigFileWithContent("{ \"model_config_list\": [ ] }\n"); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::OK); } -TEST(ModelManager, WrongConfigFile) { +TEST_F(ModelManager, WrongConfigFile) { std::string configFile = "123/tmp/not_a_valid_file_name"; - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::CONFIG_FILE_INVALID); } @@ -146,7 +149,6 @@ class ModelManagerMetricsTest : public TestWithTempDir { int arg_count = 7; ovms::Config::instance().parse(arg_count, n_argv); - // Prepare manager modelPath = directoryPath + "/dummy/"; configFilePath = directoryPath + "/ovms_config2.json"; } @@ -172,7 +174,6 @@ class ModelManagerMetricsTestNoPort : public TestWithTempDir { int arg_count = 5; ovms::Config::instance().parse(arg_count, n_argv); - // Prepare manager modelPath = directoryPath + "/dummy/"; configFilePath = directoryPath + "/ovms_config.json"; } @@ -279,28 +280,25 @@ TEST_F(ModelManagerMetricsTestNoPort, RestPortMissingWithMetrics) { EXPECT_EQ(status, ovms::StatusCode::METRICS_REST_PORT_MISSING); } -TEST(ModelManager, ConfigParseEmpty) { +TEST_F(ModelManager, ConfigParseEmpty) { std::string configFile = createConfigFileWithContent("\n"); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::JSON_INVALID); } -TEST(ModelManager, ConfigNotAJson) { +TEST_F(ModelManager, ConfigNotAJson) { std::string configFile = createConfigFileWithContent("abcdfgh"); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::JSON_INVALID); } -TEST(ModelManager, ConfigParseEmptyJson) { +TEST_F(ModelManager, ConfigParseEmptyJson) { std::string configFile = createConfigFileWithContent("{}\n"); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::JSON_INVALID); } -TEST(ModelManager, ConfigParseNodeConfigWithoutNameKey) { +TEST_F(ModelManager, ConfigParseNodeConfigWithoutNameKey) { const char* configWithoutNameKey = R"({ "model_config_list": [ { @@ -311,12 +309,11 @@ TEST(ModelManager, ConfigParseNodeConfigWithoutNameKey) { })"; std::string configFile = createConfigFileWithContent(configWithoutNameKey); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::JSON_INVALID); } -TEST(ModelManager, ConfigParseNodeConfigWihoutBasePathKey) { +TEST_F(ModelManager, ConfigParseNodeConfigWihoutBasePathKey) { const char* configWithoutBasePathKey = R"({ "model_config_list": [ { @@ -327,12 +324,11 @@ TEST(ModelManager, ConfigParseNodeConfigWihoutBasePathKey) { })"; std::string configFile = createConfigFileWithContent(configWithoutBasePathKey); - ovms::ModelManager& manager = ovms::ModelManager::getInstance(); - auto status = manager.startFromFile(configFile); + auto status = fixtureManager.startFromFile(configFile); EXPECT_EQ(status, ovms::StatusCode::JSON_INVALID); } -TEST(ModelManager, parseConfigWhenPipelineDefinitionMatchSchema) { +TEST_F(ModelManager, parseConfigWhenPipelineDefinitionMatchSchema) { const char* configWithPipelineDefinitionMatchSchema = R"({ "model_config_list": [ { @@ -391,7 +387,7 @@ void setupModelsDirs() { std::filesystem::create_directory("/tmp/models/dummy2"); } -TEST(ModelManager, configRelodNotNeededManyThreads) { +TEST(ModelManagerWatcher, configRelodNotNeededManyThreads) { std::string configFile = "/tmp/config.json"; modelMock = std::make_shared(); @@ -421,7 +417,7 @@ TEST(ModelManager, configRelodNotNeededManyThreads) { modelMock.reset(); } -TEST(ModelManager, configRelodNeededManyThreads) { +TEST(ModelManagerWatcher, configRelodNeededManyThreads) { std::string configFile = "/tmp/config.json"; modelMock = std::make_shared(); @@ -457,7 +453,7 @@ TEST(ModelManager, configRelodNeededManyThreads) { modelMock.reset(); } -TEST(ModelManager, configReloadNeededChange) { +TEST(ModelManagerWatcher, configReloadNeededChange) { std::string configFile = "/tmp/config.json"; modelMock = std::make_shared(); @@ -480,7 +476,7 @@ TEST(ModelManager, configReloadNeededChange) { modelMock.reset(); } -TEST(ModelManager, loadConfigManyThreads) { +TEST(ModelManagerWatcher, loadConfigManyThreads) { std::string configFile = "/tmp/config.json"; modelMock = std::make_shared(); @@ -509,7 +505,7 @@ TEST(ModelManager, loadConfigManyThreads) { modelMock.reset(); } -TEST(ModelManager, configReloadNeededBeforeConfigLoad) { +TEST(ModelManagerWatcher, configReloadNeededBeforeConfigLoad) { std::string configFile = "/tmp/config.json"; modelMock = std::make_shared(); @@ -537,7 +533,7 @@ TEST(ModelManager, configReloadNeededBeforeConfigLoad) { modelMock.reset(); } -TEST(ModelManager, parseConfigWhenOnlyPipelineDefinitionProvided) { +TEST(ModelManagerWatcher, parseConfigWhenOnlyPipelineDefinitionProvided) { const char* configWithOnlyPipelineDefinitionProvided = R"({ "pipeline_config_list": { @@ -569,7 +565,7 @@ TEST(ModelManager, parseConfigWhenOnlyPipelineDefinitionProvided) { modelMock.reset(); } -TEST(ModelManager, ReadsVersionsFromDisk) { +TEST_F(ModelManager, ReadsVersionsFromDisk) { const std::string path = "/tmp/test_model/"; for (auto i : {1, 5, 8, 10}) { @@ -580,35 +576,35 @@ TEST(ModelManager, ReadsVersionsFromDisk) { ovms::model_versions_t versions; std::shared_ptr fs = std::make_shared(); - auto status = ovms::ModelManager::getInstance().readAvailableVersions(fs, path, versions); + auto status = fixtureManager.readAvailableVersions(fs, path, versions); EXPECT_EQ(status, ovms::StatusCode::OK); EXPECT_THAT(versions, ::testing::UnorderedElementsAre(1, 5, 8, 10)); } -TEST(ModelManager, PathEscapeError1) { +TEST_F(ModelManager, PathEscapeError1) { const std::string path = "/tmp/../test_model/"; ovms::model_versions_t versions; std::shared_ptr fs = std::make_shared(); - auto status = ovms::ModelManager::getInstance().readAvailableVersions(fs, path, versions); + auto status = fixtureManager.readAvailableVersions(fs, path, versions); EXPECT_EQ(status, ovms::StatusCode::PATH_INVALID); } -TEST(ModelManager, PathEscapeError2) { +TEST_F(ModelManager, PathEscapeError2) { const std::string path = "../tmp/test_model/"; ovms::model_versions_t versions; std::shared_ptr fs = std::make_shared(); - auto status = ovms::ModelManager::getInstance().readAvailableVersions(fs, path, versions); + auto status = fixtureManager.readAvailableVersions(fs, path, versions); EXPECT_EQ(status, ovms::StatusCode::PATH_INVALID); } -TEST(ModelManager, ReadVersionsInvalidPath) { +TEST_F(ModelManager, ReadVersionsInvalidPath) { const std::string path = "/tmp/inexisting_path/8bt4kv"; try { @@ -618,11 +614,11 @@ TEST(ModelManager, ReadVersionsInvalidPath) { std::vector versions; std::shared_ptr fs = std::make_shared(); - auto status = ovms::ModelManager::getInstance().readAvailableVersions(fs, path, versions); + auto status = fixtureManager.readAvailableVersions(fs, path, versions); EXPECT_EQ(status, ovms::StatusCode::PATH_INVALID); } -TEST(ModelManager, StartFromFile) { +TEST(ModelManagerWatcher, StartFromFile) { std::filesystem::create_directories(model_1_path); std::filesystem::create_directories(model_2_path); std::string fileToReload = "/tmp/ovms_config_file1.json"; @@ -639,7 +635,7 @@ TEST(ModelManager, StartFromFile) { modelMock.reset(); } -TEST(ModelManager, StartFromFileWhenModelFilesMissing) { +TEST(ModelManagerWatcher, StartFromFileWhenModelFilesMissing) { std::filesystem::create_directories(model_1_path); std::string fileToReload = "/tmp/ovms_config_file1.json"; createConfigFileWithContent(config_1_model, fileToReload); @@ -650,7 +646,7 @@ TEST(ModelManager, StartFromFileWhenModelFilesMissing) { manager.join(); } -TEST(ModelManager, ConfigReloadingShouldAddNewModel) { +TEST(ModelManagerWatcher, ConfigReloadingShouldAddNewModel) { std::filesystem::create_directories(model_1_path); std::filesystem::create_directories(model_2_path); std::string fileToReload = "/tmp/ovms_config_file2.json"; @@ -695,7 +691,7 @@ struct CNLIMWrapperMock : public ovms::CNLIMWrapper { } }; -TEST(ModelManager, ConfigReloadShouldCleanupResources) { +TEST(ModelManagerCleaner, ConfigReloadShouldCleanupResources) { ResourcesAccessModelManager manager; manager.startCleaner(); ASSERT_EQ(manager.getResourcesSize(), 0); @@ -866,30 +862,28 @@ TEST_F(ModelManagerCleanerThread, CleanerShouldCleanupResourcesAndSequenceWhenIn std::this_thread::sleep_for(std::chrono::milliseconds(waitTime)); } -TEST(ModelManager, ConfigReloadingWithWrongInputName) { - ConstructorEnabledModelManager manager; +TEST_F(ModelManager, ConfigReloadingWithWrongInputName) { ovms::ModelConfig config; config.parseShapeParameter("{\"wrong_input_name\": \"(1,3,224,224)\"}"); config.setBasePath("/ovms/src/test/dummy"); - auto status = manager.reloadModelWithVersions(config); + auto status = fixtureManager.reloadModelWithVersions(config); ASSERT_EQ(status, ovms::StatusCode::CONFIG_SHAPE_IS_NOT_IN_MODEL); } -TEST(ModelManager, ConfigReloadingStatefulDynamic) { - ConstructorEnabledModelManager manager; +TEST_F(ModelManager, ConfigReloadingStatefulDynamic) { auto config = DUMMY_MODEL_CONFIG; - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); + ASSERT_EQ(fixtureManager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); config.setStateful(true); config.setBatchingMode(ovms::Mode::AUTO); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL); + ASSERT_EQ(fixtureManager.reloadModelWithVersions(config), ovms::StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL); config.setBatchingMode(ovms::Mode::FIXED); config.setShapes({{"A", {ovms::Mode::AUTO, {1, 3, 224, 224}}}}); - ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL); + ASSERT_EQ(fixtureManager.reloadModelWithVersions(config), ovms::StatusCode::REQUESTED_DYNAMIC_PARAMETERS_ON_STATEFUL_MODEL); } -TEST(ModelManager, ConfigReloadingNonStateful) { +TEST_F(ModelManager, ConfigReloadingNonStateful) { ConstructorEnabledModelManager manager; auto config = DUMMY_MODEL_CONFIG; ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); @@ -899,7 +893,7 @@ TEST(ModelManager, ConfigReloadingNonStateful) { ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::INVALID_NON_STATEFUL_MODEL_PARAMETER); } -TEST(ModelManager, ConfigReloadingStatelessToStateful) { +TEST_F(ModelManager, ConfigReloadingStatelessToStateful) { ConstructorEnabledModelManager manager; auto config = DUMMY_MODEL_CONFIG; ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); @@ -908,7 +902,7 @@ TEST(ModelManager, ConfigReloadingStatelessToStateful) { ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::REQUESTED_MODEL_TYPE_CHANGE); } -TEST(ModelManager, ConfigReloadingStatefulToStateless) { +TEST_F(ModelManager, ConfigReloadingStatefulToStateless) { ConstructorEnabledModelManager manager; auto config = DUMMY_MODEL_CONFIG; config.setStateful(true); @@ -950,7 +944,7 @@ class DummyModelDirectoryStructure { } }; -TEST(ModelManager, HandlingInvalidLastVersion) { +TEST_F(ModelManager, HandlingInvalidLastVersion) { DummyModelDirectoryStructure modelDirectory("HandlingInvalidLastVersion"); bool validVersion = true; // valid version 1 and 2, invalid 3 @@ -1005,7 +999,7 @@ TEST(ModelManager, HandlingInvalidLastVersion) { ASSERT_EQ(modelInstance2->getStatus().getErrorCode(), ovms::ModelVersionStatusErrorCode::OK); } -TEST(ModelManager, InitialFailedLoadingVersionSavesModelVersionWithProperStatus) { +TEST_F(ModelManager, InitialFailedLoadingVersionSavesModelVersionWithProperStatus) { DummyModelDirectoryStructure modelDirectory("InitialFailedLoadingVersionSavesModelVersionWithProperStatus"); bool validVersion = true; modelDirectory.addVersion(1, !validVersion); @@ -1024,7 +1018,7 @@ TEST(ModelManager, InitialFailedLoadingVersionSavesModelVersionWithProperStatus) ASSERT_EQ(versionIt->second.getStatus().getState(), ovms::ModelVersionState::LOADING); } -TEST(ModelManager, ModelVersionFailedReloadReportsFailedStatus) { +TEST_F(ModelManager, ModelVersionFailedReloadReportsFailedStatus) { DummyModelDirectoryStructure modelDirectory("ModelVersionFailedReloadReportsFailedStatus"); bool validVersion = true; modelDirectory.addVersion(1, validVersion); @@ -1039,7 +1033,7 @@ TEST(ModelManager, ModelVersionFailedReloadReportsFailedStatus) { ASSERT_EQ(manager.reloadModelWithVersions(config), ovms::StatusCode::CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE); } -TEST(ModelManager, ConfigReloadingWithTwoModelsWithTheSameName) { +TEST_F(ModelManager, ConfigReloadingWithTwoModelsWithTheSameName) { const char* configWithTwoSameNames = R"({ "model_config_list": [ { @@ -1093,7 +1087,7 @@ class MockModelManagerWithModelInstancesJustChangingStates : public ovms::ModelM std::vector toRegister; }; -TEST(ModelManager, ConfigReloadingShouldRetireModelInstancesOfModelRemovedFromJson) { +TEST_F(ModelManager, ConfigReloadingShouldRetireModelInstancesOfModelRemovedFromJson) { std::filesystem::create_directories(model_1_path); std::filesystem::create_directories(model_2_path); std::string fileToReload = "/tmp/ovms_config_file2.json"; From 739d66ae6b742896c13cfa741fcf29b279dac0d5 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Tue, 15 Nov 2022 14:47:34 +0100 Subject: [PATCH 090/130] C-API request deserialization (#1521) * Cppclean lower threshold * Increase coverage threshold TODO: * binary inputs JIRA:CVS-95304 --- check_coverage.bat | 8 +- cppclean.sh | 7 +- src/BUILD | 2 + src/deserialization.cpp | 14 +++- src/deserialization.hpp | 93 +++++++++++++++++++++- src/pocapiinternal.cpp | 119 ++++++++++++++++++++++++++++ src/pocapiinternal.hpp | 23 ++++++ src/precision.hpp | 3 +- src/test/deserialization_tests.cpp | 122 ++++++++++++++++++++++++++++- src/test/test_utils.hpp | 52 ++++++++++-- 10 files changed, 421 insertions(+), 22 deletions(-) create mode 100644 src/pocapiinternal.cpp create mode 100644 src/pocapiinternal.hpp diff --git a/check_coverage.bat b/check_coverage.bat index beadd472e1..82f1c862f1 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -1,12 +1,12 @@ #!/bin/bash #Ubuntu -#MIN_LINES_COV=74.9 -#MIN_FUNCTION_COV=87 +#MIN_LINES_COV=75.2 +#MIN_FUNCTION_COV=87.4 #Rhel -MIN_LINES_COV=73.3 -MIN_FUNCTION_COV=73.3 +MIN_LINES_COV=73.5 +MIN_FUNCTION_COV=74.3 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` diff --git a/cppclean.sh b/cppclean.sh index 0269f5bd39..c8fdebad8c 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -45,13 +45,10 @@ fi if [ ${NO_WARNINGS} -gt 179 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 -else - exit 0; fi -if [ ${NO_WARNINGS_TEST} -gt 131 ]; then +if [ ${NO_WARNINGS_TEST} -gt 124 ]; then echo "Failed due to higher than allowed number of issues in test code: ${NO_WARNINGS_TEST}" + cat ${CPPCLEAN_RESULTS_FILE_TEST} exit 1 -else - exit 0; fi exit 0 diff --git a/src/BUILD b/src/BUILD index 1223a95481..d83f549f51 100644 --- a/src/BUILD +++ b/src/BUILD @@ -236,6 +236,8 @@ cc_library( "poc_api_impl.cpp", "poc_api_impl.hpp", "pocapi.hpp", + "pocapiinternal.cpp", + "pocapiinternal.hpp", "pocapi.cpp", "precision.cpp", "precision.hpp", diff --git a/src/deserialization.cpp b/src/deserialization.cpp index 5f8ae1c682..7b0092dee2 100644 --- a/src/deserialization.cpp +++ b/src/deserialization.cpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2021 Intel Corporation +// Copyright 2021-2022 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ //***************************************************************************** #include "deserialization.hpp" +#include "buffer.hpp" + namespace ovms { template <> @@ -38,6 +40,16 @@ Status InputSink::give(const std::string& name, ov::Tensor& t return status; } +ov::Tensor makeTensor(const InferenceTensor& requestInput, + const std::shared_ptr& tensorInfo) { + OVMS_PROFILE_FUNCTION(); + ov::Shape shape; + for (const auto& dim : requestInput.getShape()) { + shape.push_back(dim); + } + ov::element::Type_t precision = tensorInfo->getOvPrecision(); + return ov::Tensor(precision, shape, const_cast(reinterpret_cast(requestInput.getBuffer()->data()))); +} ov::Tensor makeTensor(const tensorflow::TensorProto& requestInput, const std::shared_ptr& tensorInfo) { diff --git a/src/deserialization.hpp b/src/deserialization.hpp index 444a5d30bd..136f1fa733 100644 --- a/src/deserialization.hpp +++ b/src/deserialization.hpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2020 Intel Corporation +// Copyright 2020-2022 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -30,6 +30,8 @@ #pragma GCC diagnostic pop #include "binaryutils.hpp" +#include "inferencerequest.hpp" +#include "inferencetensor.hpp" #include "profiler.hpp" #include "status.hpp" #include "tensorinfo.hpp" @@ -45,6 +47,9 @@ ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, ov::Tensor makeTensor(const ::KFSRequest::InferInputTensor& requestInput, const std::shared_ptr& tensorInfo); +ov::Tensor makeTensor(const InferenceTensor& requestInput, + const std::shared_ptr& tensorInfo); + class ConcreteTensorProtoDeserializator { public: static ov::Tensor deserializeTensorProto( @@ -210,6 +215,36 @@ class ConcreteTensorProtoDeserializator { } } + static ov::Tensor deserializeTensorProto( + const InferenceTensor& requestInput, + const std::shared_ptr& tensorInfo) { + OVMS_PROFILE_FUNCTION(); + switch (tensorInfo->getPrecision()) { + case ovms::Precision::FP64: + case ovms::Precision::FP32: + case ovms::Precision::FP16: + case ovms::Precision::I64: + case ovms::Precision::I32: + case ovms::Precision::I16: + case ovms::Precision::I8: + case ovms::Precision::U64: + case ovms::Precision::U32: + case ovms::Precision::U16: + case ovms::Precision::BOOL: + case ovms::Precision::U1: + case ovms::Precision::U8: { + return makeTensor(requestInput, tensorInfo); + } + case ovms::Precision::CUSTOM: + case ovms::Precision::UNDEFINED: + case ovms::Precision::DYNAMIC: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + default: + return ov::Tensor(); + } + } static ov::Tensor deserializeTensorProto( const tensorflow::TensorProto& requestInput, const std::shared_ptr& tensorInfo) { @@ -277,6 +312,13 @@ ov::Tensor deserializeTensorProto( return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo, buffer); } +template +ov::Tensor deserializeTensorProto( + const InferenceTensor& requestInput, + const std::shared_ptr& tensorInfo) { + return TensorProtoDeserializator::deserializeTensorProto(requestInput, tensorInfo); +} + template class InputSink { Requester requester; @@ -404,4 +446,53 @@ Status deserializePredictRequest( } return status; } +template +Status deserializePredictRequest( + const InferenceRequest& request, + const tensor_map_t& inputMap, + Sink& inputSink, bool isPipeline) { + OVMS_PROFILE_FUNCTION(); + Status status; + for (const auto& [name, tensorInfo] : inputMap) { + try { + const InferenceTensor* requestInputPtr{nullptr}; + auto status = request.getInput(name.c_str(), &requestInputPtr); + if (!status.ok() || requestInputPtr == nullptr) { + SPDLOG_DEBUG("Failed to deserialize request. Validation of request failed"); + return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); + } + ov::Tensor tensor; + // TODO binary input handling + /* if (requestInputPtr->getDataType() == "BYTES") { + SPDLOG_DEBUG("Request contains binary input: {}", name); + return StatusCode::NOT_IMPLEMENTED; + } else { */ + tensor = deserializeTensorProto(*requestInputPtr, tensorInfo); + if (!tensor) { + status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; + SPDLOG_DEBUG(status.string()); + return status; + } + + const std::string ovTensorName = isPipeline ? name : tensorInfo->getName(); + status = inputSink.give(ovTensorName, tensor); + if (!status.ok()) { + SPDLOG_DEBUG("Feeding input:{} to inference performer failed:{}", ovTensorName, status.string()); + return status; + } + // OV implementation the ov::Exception is not + // a base class for all other exceptions thrown from OV. + // OV can throw exceptions derived from std::logic_error. + } catch (const ov::Exception& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } catch (std::logic_error& e) { + status = StatusCode::OV_INTERNAL_DESERIALIZATION_ERROR; + SPDLOG_DEBUG("{}: {}", status.string(), e.what()); + return status; + } + } + return status; +} } // namespace ovms diff --git a/src/pocapiinternal.cpp b/src/pocapiinternal.cpp new file mode 100644 index 0000000000..47c4c95020 --- /dev/null +++ b/src/pocapiinternal.cpp @@ -0,0 +1,119 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pocapiinternal.hpp" + +namespace ovms { +DataType getPrecisionAsOVMSDataType(Precision precision) { + switch (precision) { + case Precision::BF16: + return OVMS_DATATYPE_BF16; + case Precision::FP64: + return OVMS_DATATYPE_FP64; + case Precision::FP32: + return OVMS_DATATYPE_FP32; + case Precision::FP16: + return OVMS_DATATYPE_FP16; + case Precision::I64: + return OVMS_DATATYPE_I64; + case Precision::I32: + return OVMS_DATATYPE_I32; + case Precision::I16: + return OVMS_DATATYPE_I16; + case Precision::I8: + return OVMS_DATATYPE_I8; + case Precision::I4: + return OVMS_DATATYPE_I4; + case Precision::U64: + return OVMS_DATATYPE_U64; + case Precision::U32: + return OVMS_DATATYPE_U32; + case Precision::U16: + return OVMS_DATATYPE_U16; + case Precision::U8: + return OVMS_DATATYPE_U8; + case Precision::U4: + return OVMS_DATATYPE_U4; + case Precision::U1: + return OVMS_DATATYPE_U1; + case Precision::BOOL: + return OVMS_DATATYPE_BOOL; + case Precision::CUSTOM: + return OVMS_DATATYPE_CUSTOM; + case Precision::UNDEFINED: + return OVMS_DATATYPE_UNDEFINED; + case Precision::DYNAMIC: + return OVMS_DATATYPE_DYNAMIC; + case Precision::MIXED: + return OVMS_DATATYPE_MIXED; + case Precision::Q78: + return OVMS_DATATYPE_Q78; + case Precision::BIN: + return OVMS_DATATYPE_BIN; + default: + return OVMS_DATATYPE_UNDEFINED; + } +} +Precision getOVMSDataTypeAsPrecision(DataType datatype) { + switch (datatype) { + case OVMS_DATATYPE_BF16: + return Precision::BF16; + case OVMS_DATATYPE_FP64: + return Precision::FP64; + case OVMS_DATATYPE_FP32: + return Precision::FP32; + case OVMS_DATATYPE_FP16: + return Precision::FP16; + case OVMS_DATATYPE_I64: + return Precision::I64; + case OVMS_DATATYPE_I32: + return Precision::I32; + case OVMS_DATATYPE_I16: + return Precision::I16; + case OVMS_DATATYPE_I8: + return Precision::I8; + case OVMS_DATATYPE_I4: + return Precision::I4; + case OVMS_DATATYPE_U64: + return Precision::U64; + case OVMS_DATATYPE_U32: + return Precision::U32; + case OVMS_DATATYPE_U16: + return Precision::U16; + case OVMS_DATATYPE_U8: + return Precision::U8; + case OVMS_DATATYPE_U4: + return Precision::U4; + case OVMS_DATATYPE_U1: + return Precision::U1; + case OVMS_DATATYPE_BOOL: + return Precision::BOOL; + case OVMS_DATATYPE_CUSTOM: + return Precision::CUSTOM; + case OVMS_DATATYPE_UNDEFINED: + return Precision::UNDEFINED; + case OVMS_DATATYPE_DYNAMIC: + return Precision::DYNAMIC; + case OVMS_DATATYPE_MIXED: + return Precision::MIXED; + case OVMS_DATATYPE_Q78: + return Precision::Q78; + case OVMS_DATATYPE_BIN: + return Precision::BIN; + default: + return Precision::UNDEFINED; + } +} +} // namespace ovms diff --git a/src/pocapiinternal.hpp b/src/pocapiinternal.hpp new file mode 100644 index 0000000000..a90af3e3ac --- /dev/null +++ b/src/pocapiinternal.hpp @@ -0,0 +1,23 @@ +#pragma once +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "pocapi.hpp" +#include "precision.hpp" + +namespace ovms { +DataType getPrecisionAsOVMSDataType(Precision precision); +Precision getOVMSDataTypeAsPrecision(DataType datatype); +} // namespace ovms diff --git a/src/precision.hpp b/src/precision.hpp index d2cabf98a5..f5e5272dec 100644 --- a/src/precision.hpp +++ b/src/precision.hpp @@ -43,7 +43,8 @@ enum class Precision { DYNAMIC, MIXED, Q78, - BIN // TODO remove BIN,Q78, DYNAMIC, CUSTOM, MIXED + BIN, // TODO remove BIN,Q78, DYNAMIC, CUSTOM, MIXED + PRECISION_END }; const std::string& toString(Precision precision); diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index 26e99ac605..b394535f71 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2020 Intel Corporation +// Copyright 2020-2022 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -28,8 +28,12 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "../buffer.hpp" #include "../deserialization.hpp" +#include "../inferencerequest.hpp" +#include "../inferencetensor.hpp" #include "../kfs_frontend/kfs_utils.hpp" +#include "../pocapiinternal.hpp" #include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" @@ -43,6 +47,7 @@ using TFPredictResponse = tensorflow::serving::PredictResponse; using namespace ovms; using testing::_; +using testing::ElementsAre; using testing::NiceMock; using testing::Throw; @@ -64,7 +69,7 @@ class TensorflowGRPCPredict : public ::testing::TestWithParam { tensorMap[tensorName] = std::make_shared( tensorName, precision, - shape_t{1, 3}, + shape_t{1, DUMMY_MODEL_INPUT_SIZE}, Layout{"NC"}); SetUpTensorProto(getPrecisionAsDataType(precision)); } @@ -74,7 +79,7 @@ class TensorflowGRPCPredict : public ::testing::TestWithParam { tensorShape->Clear(); tensorShape->add_dim()->set_size(1); tensorShape->add_dim()->set_size(DUMMY_MODEL_INPUT_SIZE); - *(tensorProto.mutable_tensor_content()) = std::string(1 * DUMMY_MODEL_INPUT_SIZE, '1'); + *(tensorProto.mutable_tensor_content()) = std::string(1 * sizeof(float) * DUMMY_MODEL_INPUT_SIZE, '1'); } TFTensorProto tensorProto; const char* tensorName = DUMMY_MODEL_INPUT_NAME; @@ -85,6 +90,81 @@ class TensorflowGRPCPredict : public ::testing::TestWithParam { class DeserializeTFTensorProto : public TensorflowGRPCPredict {}; class DeserializeTFTensorProtoNegative : public TensorflowGRPCPredict {}; +class CAPIPredict : public ::testing::TestWithParam { +protected: + void SetUp() override { + tensorMap[tensorName] = std::make_shared( + tensorName, + PRECISION, + shape_t{1, DUMMY_MODEL_INPUT_SIZE}, + Layout{"NC"}); + SetUpTensorProto(getPrecisionAsOVMSDataType(PRECISION)); + } + void SetUpTensorProto(DataType dataType) { + std::array shape{1, DUMMY_MODEL_INPUT_SIZE}; + tensorCapi = std::make_unique(dataType, + shape.data(), + shape.size()); + bool createCopy{true}; + const std::string data(1 * sizeof(float) * DUMMY_MODEL_INPUT_SIZE, '1'); + auto status = tensorCapi->setBuffer(data.data(), data.size(), OVMS_BUFFERTYPE_CPU, std::nullopt, createCopy); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + const Buffer* buffer{nullptr}; + buffer = tensorCapi->getBuffer(); + ASSERT_NE(buffer, nullptr); + } + ovms::Precision PRECISION = ovms::Precision::FP32; + std::unique_ptr tensorCapi; + const char* tensorName = DUMMY_MODEL_INPUT_NAME; + ovms::tensor_map_t tensorMap; + bool isPipeline = false; +}; + +class CAPIPredictRequest : public CAPIPredict { +protected: + InferenceRequest request{"dummy", 1}; + static const std::string DATA; + static constexpr std::array SHAPE{1, 10}; + void SetUp() { + CAPIPredict::SetUp(); + request.addInput(DUMMY_MODEL_INPUT_NAME, + OVMS_DATATYPE_FP32, + SHAPE.data(), + SHAPE.size()); + request.setInputBuffer(DUMMY_MODEL_INPUT_NAME, + DATA.data(), + sizeof(float) * DUMMY_MODEL_INPUT_SIZE, + OVMS_BUFFERTYPE_CPU, + std::nullopt); + } + void TearDown() { + } +}; +const std::string CAPIPredictRequest::DATA{1 * sizeof(float) * DUMMY_MODEL_INPUT_SIZE, '1'}; +// dispose connection to CAPI Predict +TEST_F(CAPIPredictRequest, ShouldSuccessForSupportedPrecision) { + ov::Core ieCore; + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); + ov::InferRequest inferRequest = compiledModel.create_infer_request(); + InputSink inputSink(inferRequest); + auto status = deserializePredictRequest(request, tensorMap, inputSink, isPipeline); + EXPECT_TRUE(status.ok()); + auto tensor = inferRequest.get_tensor(CAPIPredictRequest::tensorName); + EXPECT_EQ(PRECISION, ovElementTypeToOvmsPrecision(tensor.get_element_type())); + auto shape = tensor.get_shape(); + EXPECT_THAT(shape, ElementsAre(1, DUMMY_MODEL_INPUT_SIZE)); + const InferenceTensor* requestTensor{nullptr}; + status = request.getInput(DUMMY_MODEL_INPUT_NAME, &requestTensor); + ASSERT_NE(nullptr, requestTensor); + const Buffer* buffer = requestTensor->getBuffer(); + EXPECT_EQ(tensor.data(), buffer->data()); + EXPECT_EQ(tensor.get_byte_size(), buffer->getByteSize()); +} + +class DeserializeCAPITensor : public CAPIPredict {}; +class DeserializeCAPITensorProtoNegative : public CAPIPredict {}; + class GRPCPredictRequest : public TensorflowGRPCPredict { public: void SetUp() { @@ -193,6 +273,15 @@ TEST_P(DeserializeTFTensorProtoNegative, ShouldReturnNullptrForPrecision) { << " should return nullptr"; } +TEST_P(DeserializeCAPITensorProtoNegative, ShouldReturnNullptrForPrecision) { + ovms::Precision testedPrecision = GetParam(); + tensorMap[tensorName]->setPrecision(testedPrecision); + ov::Tensor tensor = deserializeTensorProto(*tensorCapi, tensorMap[tensorName]); + EXPECT_FALSE((bool)tensor) << "Unsupported OVMS precision:" + << toString(testedPrecision) + << " should return nullptr"; +} + TEST_P(DeserializeTFTensorProto, ShouldReturnValidTensor) { ovms::Precision testedPrecision = GetParam(); SetUpTensorProto(getPrecisionAsDataType(testedPrecision)); @@ -201,6 +290,17 @@ TEST_P(DeserializeTFTensorProto, ShouldReturnValidTensor) { EXPECT_TRUE((bool)tensor) << "Supported OVMS precision:" << toString(testedPrecision) << " should return valid tensor ptr"; + EXPECT_EQ(ovElementTypeToOvmsPrecision(tensor.get_element_type()), testedPrecision); +} +TEST_P(DeserializeCAPITensor, ShouldReturnValidTensor) { + ovms::Precision testedPrecision = GetParam(); + SetUpTensorProto(getPrecisionAsOVMSDataType(testedPrecision)); + tensorMap[tensorName]->setPrecision(testedPrecision); + ov::Tensor tensor = deserializeTensorProto(*tensorCapi, tensorMap[tensorName]); + EXPECT_TRUE((bool)tensor) << "Supported OVMS precision:" + << toString(testedPrecision) + << " should return valid tensor ptr"; + EXPECT_EQ(ovElementTypeToOvmsPrecision(tensor.get_element_type()), testedPrecision); } class KserveGRPCPredict : public ::testing::TestWithParam> { @@ -338,6 +438,7 @@ TEST_P(DeserializeKFSTensorProto, ShouldReturnValidTensor) { EXPECT_TRUE((bool)tensor) << "Supported OVMS precision:" << toString(testedPrecision) << " should return valid tensor ptr"; + EXPECT_EQ(ovElementTypeToOvmsPrecision(tensor.get_element_type()), testedPrecision); } TEST_F(KserveGRPCPredict, ShouldReturnValidTensor) { @@ -480,6 +581,14 @@ INSTANTIATE_TEST_SUITE_P( return toString(info.param); }); +INSTANTIATE_TEST_SUITE_P( + Test, + DeserializeCAPITensorProtoNegative, + ::testing::ValuesIn(UNSUPPORTED_CAPI_INPUT_PRECISIONS), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); + INSTANTIATE_TEST_SUITE_P( Test, DeserializeTFTensorProto, @@ -487,6 +596,13 @@ INSTANTIATE_TEST_SUITE_P( [](const ::testing::TestParamInfo& info) { return toString(info.param); }); +INSTANTIATE_TEST_SUITE_P( + Test, + DeserializeCAPITensor, + ::testing::ValuesIn(SUPPORTED_CAPI_INPUT_PRECISIONS), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); std::vector> DeserializeKFSTensorProtoNegativeParams = cartesianProduct(UNSUPPORTED_KFS_INPUT_PRECISIONS, {true, false}); diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 64bc9fdbb7..3aa3c42ad1 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -349,12 +349,12 @@ void read4x4RgbJpg(size_t& filesize, std::unique_ptr& image_bytes); void readImage(const std::string& path, size_t& filesize, std::unique_ptr& image_bytes); static const std::vector SUPPORTED_INPUT_PRECISIONS{ - // ovms::Precision::UNSPECIFIED, + // ovms::Precision::UNDEFINED, // ovms::Precision::MIXED, ovms::Precision::FP64, ovms::Precision::FP32, ovms::Precision::FP16, - // InferenceEngine::Precision::Q78, + // ovms::Precision::Q78, ovms::Precision::I16, ovms::Precision::U8, ovms::Precision::I8, @@ -384,13 +384,52 @@ static const std::vector UNSUPPORTED_INPUT_PRECISIONS{ // ovms::Precision::CUSTOM) }; +static const std::vector SUPPORTED_CAPI_INPUT_PRECISIONS{ + // ovms::Precision::UNDEFINED, + // ovms::Precision::MIXED, + ovms::Precision::FP64, + ovms::Precision::FP32, + ovms::Precision::FP16, + // ovms::Precision::Q78, + ovms::Precision::I16, + ovms::Precision::U8, + ovms::Precision::U1, + ovms::Precision::I8, + ovms::Precision::U16, + ovms::Precision::I32, + ovms::Precision::I64, + ovms::Precision::U32, + ovms::Precision::U64, + // ovms::Precision::BIN, + ovms::Precision::BOOL + // ovms::Precision::CUSTOM) +}; +static const std::vector UNSUPPORTED_CAPI_INPUT_PRECISIONS{ + ovms::Precision::UNDEFINED, + ovms::Precision::MIXED, + // ovms::Precision::FP64, + // ovms::Precision::FP32, + // ovms::Precision::FP16, + ovms::Precision::Q78, + // ovms::Precision::I16, + // ovms::Precision::U8, + // ovms::Precision::U1, + // vms::Precision::I8, + // ovms::Precision::U16, + // ovms::Precision::I32, + // ovms::Precision::I64, + // ovms::Precision::U32, + // ovms::Precision::U64, + ovms::Precision::BIN, + // ovms::Precision::BOOL + ovms::Precision::CUSTOM}; static const std::vector SUPPORTED_KFS_INPUT_PRECISIONS{ - // ovms::Precision::UNSPECIFIED, + // ovms::Precision::UNDEFINED, // ovms::Precision::MIXED, ovms::Precision::FP64, ovms::Precision::FP32, ovms::Precision::FP16, - // InferenceEngine::Precision::Q78, + // ovms::Precision::Q78, ovms::Precision::I16, ovms::Precision::U8, ovms::Precision::I8, @@ -421,11 +460,10 @@ static const std::vector UNSUPPORTED_KFS_INPUT_PRECISIONS{ // ovms::Precision::U64, ovms::Precision::BIN, // ovms::Precision::BOOL - // ovms::Precision::CUSTOM) -}; + ovms::Precision::CUSTOM}; static const std::vector SUPPORTED_KFS_INPUT_PRECISIONS_TENSORINPUTCONTENT{ - // ovms::Precision::UNSPECIFIED, + // ovms::Precision::UNDEFINED, // ovms::Precision::MIXED, ovms::Precision::FP64, ovms::Precision::FP32, From cd737693b361750e429711bd651599a940d1c330 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 15 Nov 2022 16:50:49 +0100 Subject: [PATCH 091/130] CVS-95302 C api serialization (#1516) --- check_coverage.bat | 2 +- cppclean.sh | 2 +- src/pocapi.hpp | 3 +- src/precision.cpp | 1 + src/serialization.cpp | 6 - src/serialization.hpp | 88 ++++++++++++ src/test/prediction_service_test.cpp | 3 + src/test/serialization_tests.cpp | 205 ++++++++++++++++++++++++++- 8 files changed, 296 insertions(+), 14 deletions(-) diff --git a/check_coverage.bat b/check_coverage.bat index 82f1c862f1..b26c44b157 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -6,7 +6,7 @@ #Rhel MIN_LINES_COV=73.5 -MIN_FUNCTION_COV=74.3 +MIN_FUNCTION_COV=74.4 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` diff --git a/cppclean.sh b/cppclean.sh index c8fdebad8c..be065a22f2 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,7 +42,7 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 2 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 179 ]; then +if [ ${NO_WARNINGS} -gt 180 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi diff --git a/src/pocapi.hpp b/src/pocapi.hpp index 2997086fe1..ec8e1a12b2 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -46,7 +46,8 @@ enum DataType { OVMS_DATATYPE_DYNAMIC, OVMS_DATATYPE_MIXED, OVMS_DATATYPE_Q78, - OVMS_DATATYPE_BIN + OVMS_DATATYPE_BIN, + OVMS_DATATYPE_END }; enum BufferType { diff --git a/src/precision.cpp b/src/precision.cpp index ea68b9b358..23d537735e 100644 --- a/src/precision.cpp +++ b/src/precision.cpp @@ -102,6 +102,7 @@ ov::element::Type_t ovmsPrecisionToIE2Precision(Precision precision) { {Precision::U4, ov::element::Type_t::u4}, {Precision::U1, ov::element::Type_t::u1}, {Precision::BOOL, ov::element::Type_t::boolean}, + {Precision::BF16, ov::element::Type_t::bf16}, {Precision::UNDEFINED, ov::element::Type_t::undefined}, {Precision::DYNAMIC, ov::element::Type_t::dynamic} // {Precision::MIXED, ov::element::Type_t::MIXED}, diff --git a/src/serialization.cpp b/src/serialization.cpp index f8667f452d..2c8435e77c 100644 --- a/src/serialization.cpp +++ b/src/serialization.cpp @@ -263,12 +263,6 @@ tensorflow::TensorProto& ProtoGettermutable_outputs())[name]; } -template <> -std::string* ProtoGetter::createContent(const std::string& name) { - OVMS_PROFILE_FUNCTION(); - return nullptr; -} - template <> ::KFSResponse::InferOutputTensor& ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&>::createOutput(const std::string& name) { OVMS_PROFILE_FUNCTION(); diff --git a/src/serialization.hpp b/src/serialization.hpp index dae3ae7537..10d52bc270 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -27,12 +27,15 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "inferenceresponse.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" +#include "pocapiinternal.hpp" #include "profiler.hpp" #include "status.hpp" #include "tensorinfo.hpp" namespace ovms { +class InferenceTensor; template class OutputGetter { @@ -72,6 +75,11 @@ Status serializeTensorToTensorProtoRaw( const std::shared_ptr& servableOutput, ov::Tensor& tensor); +Status serializeTensorToTensorProto( + InferenceTensor& responseOutput, + const std::shared_ptr& servableOutput, + ov::Tensor& tensor); + typedef const std::string& (*outputNameChooser_t)(const std::string&, const TensorInfo&); const std::string& getTensorInfoName(const std::string& first, const TensorInfo& tensorInfo); const std::string& getOutputMapKeyName(const std::string& first, const TensorInfo& tensorInfo); @@ -130,4 +138,84 @@ Status serializePredictResponse( } return status; } +template +Status serializePredictResponse( + OutputGetter& outputGetter, + const tensor_map_t& outputMap, + InferenceResponse* response, + outputNameChooser_t outputNameChooser) { + OVMS_PROFILE_FUNCTION(); + Status status; + ProtoGetter protoGetter(response); + for (const auto& [outputName, outputInfo] : outputMap) { + ov::Tensor tensor; + status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); + if (!status.ok()) { + return status; + } + auto servableMetaPrecision = outputInfo->getPrecision(); + auto actualPrecision = ovElementTypeToOvmsPrecision(tensor.get_element_type()); + if (servableMetaPrecision != actualPrecision) { + return StatusCode::INTERNAL_ERROR; + } + if (!outputInfo->getShape().match(tensor.get_shape())) { + return StatusCode::INTERNAL_ERROR; + } + switch (servableMetaPrecision) { + case ovms::Precision::FP64: + case ovms::Precision::FP32: + case ovms::Precision::FP16: + case ovms::Precision::I64: + case ovms::Precision::I32: + case ovms::Precision::I16: + case ovms::Precision::I8: + case ovms::Precision::U64: + case ovms::Precision::U32: + case ovms::Precision::U16: + case ovms::Precision::U8: + break; + case ovms::Precision::BF16: + case ovms::Precision::U4: + case ovms::Precision::U1: + case ovms::Precision::BOOL: // ? + case ovms::Precision::CUSTOM: + case ovms::Precision::UNDEFINED: + case ovms::Precision::DYNAMIC: + case ovms::Precision::MIXED: + case ovms::Precision::Q78: + case ovms::Precision::BIN: + default: { + Status status = StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION; + SPDLOG_ERROR(status.string()); + return status; + } + } + InferenceTensor* outputTensor{nullptr}; + // Mapped name for single model result serialization: possible mapping_config.json setting + // For DAG: setting in pipeline output configuration + status = response->addOutput( + outputInfo->getMappedName(), + getPrecisionAsOVMSDataType(actualPrecision), + tensor.get_shape().data(), + tensor.get_shape().size()); + if (!status.ok()) { + SPDLOG_ERROR("Cannot serialize output with name:{} for servable name:{}; version:{}; error: duplicate output name", + outputName, response->getServableName(), response->getServableVersion()); + return StatusCode::INTERNAL_ERROR; + } + status = response->getOutput(outputInfo->getMappedName().c_str(), &outputTensor); + if (!status.ok()) { + SPDLOG_ERROR("Cannot serialize output with name:{} for servable name:{}; version:{}; error: cannot find inserted input", + outputName, response->getServableName(), response->getServableVersion()); + return StatusCode::INTERNAL_ERROR; + } + outputTensor->setBuffer( + tensor.data(), + tensor.get_byte_size(), + OVMS_BUFFERTYPE_CPU, + std::nullopt, + true); + } + return StatusCode::OK; +} } // namespace ovms diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index c95de52ffa..03e41d5e21 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -42,6 +42,9 @@ using testing::Eq; using ovms::StatusCode; +// TODO: These tests test both TFS and KFS for prediction, +// but output is always serialized to TFS, therefore we only test TFS serialization here. +// TODO: Add C-API tests when predict/infer becomes ready. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnarrowing" void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest, const std::string& outputName, const ovms::tensor_map_t& outputsInfo) { diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index 9177303e55..8f3ce4d8b0 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -14,6 +14,7 @@ // limitations under the License. //***************************************************************************** +#include #include #include #include @@ -28,6 +29,9 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "../buffer.hpp" +#include "../inferenceresponse.hpp" +#include "../inferencetensor.hpp" #include "../serialization.hpp" #include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" @@ -42,6 +46,7 @@ using TFPredictResponse = tensorflow::serving::PredictResponse; using namespace ovms; using testing::_; +using testing::ElementsAre; using testing::NiceMock; using testing::Throw; @@ -121,6 +126,46 @@ const std::vector UNSUPPORTED_KFS_OUTPUT_PRECISIONS{ // ovms::Precision::CUSTOM) }; +const std::vector SUPPORTED_CAPI_OUTPUT_PRECISIONS{ + // ovms::Precision::BF16, + ovms::Precision::FP64, + ovms::Precision::FP32, + ovms::Precision::FP16, + ovms::Precision::I64, + ovms::Precision::I32, + ovms::Precision::I16, + ovms::Precision::I8, + // ovms::Precision::I4, + ovms::Precision::U64, + ovms::Precision::U32, + ovms::Precision::U16, + ovms::Precision::U8, + // ovms::Precision::U4, + // ovms::Precision::U1, + // ovms::Precision::BOOL, + // ovms::Precision::UNDEFINED, +}; + +const std::vector UNSUPPORTED_CAPI_OUTPUT_PRECISIONS{ + ovms::Precision::BF16, + // ovms::Precision::FP64, + // ovms::Precision::FP32, + // ovms::Precision::FP16, + // ovms::Precision::I64, + // ovms::Precision::I32, + // ovms::Precision::I16, + // ovms::Precision::I8, + ovms::Precision::I4, + // ovms::Precision::U64, + // ovms::Precision::U32, + // ovms::Precision::U16, + // ovms::Precision::U8, + ovms::Precision::U4, + ovms::Precision::U1, + ovms::Precision::BOOL, + // ovms::Precision::UNDEFINED, // Cannot create ov tensor with such precision +}; + class TensorflowGRPCPredict : public ::testing::TestWithParam { protected: void SetUp() override { @@ -212,7 +257,7 @@ TEST_P(SerializeTFTensorProto, SerializeTensorProtoShouldSucceedForPrecision) { class SerializeTFTensorProtoNegative : public SerializeTFTensorProto {}; -TEST_P(SerializeTFTensorProtoNegative, SerializeTensorProtoShouldSucceedForPrecision) { +TEST_P(SerializeTFTensorProtoNegative, SerializeTensorProtoShouldFailForPrecision) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); TFTensorProto responseOutput; @@ -233,7 +278,7 @@ TEST(SerializeTFGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; std::shared_ptr tensorInfo = std::make_shared( - DUMMY_MODEL_INPUT_NAME, + DUMMY_MODEL_OUTPUT_NAME, ovms::Precision::FP32, ovms::Shape{1, 10}, Layout{"NC"}); @@ -417,7 +462,7 @@ TEST_P(SerializeKFSInferOutputTensor, SerializeTensorProtoShouldSucceedForPrecis class SerializeKFSInferOutputTensorNegative : public SerializeKFSInferOutputTensor {}; -TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoRawShouldFailedForPrecision) { +TEST_P(SerializeKFSInferOutputTensorNegative, SerializeTensorProtoShouldFailForPrecision) { ovms::Precision testedPrecision = GetParam(); auto inputs = getInputs(testedPrecision); KFSResponse response; @@ -457,7 +502,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { ov::InferRequest inferRequest = compiledModel.create_infer_request(); ovms::tensor_map_t tenMap; std::shared_ptr tensorInfo = std::make_shared( - DUMMY_MODEL_INPUT_NAME, + DUMMY_MODEL_OUTPUT_NAME, ovms::Precision::FP32, ovms::Shape{1, 10}, Layout{"NC"}); @@ -467,7 +512,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { OutputGetter outputGetter(inferRequest); auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); ASSERT_TRUE(status.ok()); - EXPECT_EQ(DUMMY_MODEL_INPUT_NAME, response.outputs(0).name()); + EXPECT_EQ(DUMMY_MODEL_OUTPUT_NAME, response.outputs(0).name()); EXPECT_EQ("FP32", response.outputs(0).datatype()); EXPECT_EQ(1, response.outputs(0).shape(0)); EXPECT_EQ(10, response.outputs(0).shape(1)); @@ -541,3 +586,153 @@ INSTANTIATE_TEST_SUITE_P( [](const ::testing::TestParamInfo& info) { return toString(info.param); }); + +// C-API + +class CApiSerialization : public ::testing::TestWithParam { +protected: + tensor_map_t prepareInputs(ovms::Precision precision, ovms::Shape shape = ovms::Shape{1, 10}) { + tensor_map_t ret; + std::shared_ptr servableOutput = + std::make_shared(std::string(DUMMY_MODEL_OUTPUT_NAME), precision, shape, Layout{"NC"}); + ret[DUMMY_MODEL_OUTPUT_NAME] = servableOutput; + return ret; + } + InferenceResponse response{"dummy", 1}; +}; + +TEST(SerializeCApiTensorSingle, NegativeMismatchBetweenTensorInfoAndTensorPrecision) { + InferenceResponse response{"dummy", 1}; + ov::Core ieCore; + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); + ov::InferRequest inferRequest = compiledModel.create_infer_request(); + ovms::tensor_map_t tenMap; + std::shared_ptr tensorInfo = std::make_shared( + DUMMY_MODEL_OUTPUT_NAME, + ovms::Precision::I32, // wrong precision + ovms::Shape{1, 10}, + Layout{"NC"}); + tenMap[DUMMY_MODEL_OUTPUT_NAME] = tensorInfo; + ov::Tensor tensor(ov::element::Type_t::f32, ov::Shape{1, 10}); + float data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.10}; + std::memcpy(tensor.data(), data, tensor.get_byte_size()); + inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); + OutputGetter outputGetter(inferRequest); + auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); +} + +TEST(SerializeCApiTensorSingle, NegativeMismatchBetweenTensorInfoAndTensorShape) { + InferenceResponse response{"dummy", 1}; + ov::Core ieCore; + std::shared_ptr model = ieCore.read_model(std::filesystem::current_path().u8string() + "/src/test/dummy/1/dummy.xml"); + ov::CompiledModel compiledModel = ieCore.compile_model(model, "CPU"); + ov::InferRequest inferRequest = compiledModel.create_infer_request(); + ovms::tensor_map_t tenMap; + std::shared_ptr tensorInfo = std::make_shared( + DUMMY_MODEL_OUTPUT_NAME, + ovms::Precision::FP32, + ovms::Shape{1, 8}, // wrong shape + Layout{"NC"}); + tenMap[DUMMY_MODEL_OUTPUT_NAME] = tensorInfo; + ov::Tensor tensor(ov::element::Type_t::f32, ov::Shape{1, 10}); + float data[] = {1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9, 10.10}; + std::memcpy(tensor.data(), data, tensor.get_byte_size()); + inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); + OutputGetter outputGetter(inferRequest); + auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); +} + +class SerializeCApiTensorPositive : public CApiSerialization {}; + +struct MockedTensorProvider { + ov::Tensor& tensor; + MockedTensorProvider(ov::Tensor& tensor) : + tensor(tensor) {} +}; +template <> +Status OutputGetter::get(const std::string& name, ov::Tensor& tensor) { + tensor = outputSource.tensor; + return StatusCode::OK; +} + +TEST_P(SerializeCApiTensorPositive, SerializeTensorShouldSucceedForPrecision) { + ovms::Precision testedPrecision = GetParam(); + ov::Tensor tensor(ovmsPrecisionToIE2Precision(testedPrecision), ov::Shape{1, 10}); + MockedTensorProvider provider(tensor); + OutputGetter outputGetter(provider); + + auto inputs = prepareInputs(testedPrecision); + auto status = serializePredictResponse(outputGetter, + inputs, + &response, + getTensorInfoName); + EXPECT_TRUE(status.ok()) + << "Supported OV serialization precision" + << toString(testedPrecision) + << "should succeed"; +} + +INSTANTIATE_TEST_SUITE_P( + Test, + SerializeCApiTensorPositive, + ::testing::ValuesIn(SUPPORTED_CAPI_OUTPUT_PRECISIONS), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); + +class SerializeCApiTensorNegative : public CApiSerialization {}; + +TEST_P(SerializeCApiTensorNegative, SerializeTensorShouldFailForPrecision) { + ovms::Precision testedPrecision = GetParam(); + ov::Tensor tensor(ovmsPrecisionToIE2Precision(testedPrecision), ov::Shape{1, 10}); + MockedTensorProvider provider(tensor); + OutputGetter outputGetter(provider); + + auto inputs = prepareInputs(testedPrecision); + auto status = serializePredictResponse(outputGetter, + inputs, + &response, + getTensorInfoName); + EXPECT_EQ(status, ovms::StatusCode::OV_UNSUPPORTED_SERIALIZATION_PRECISION) + << "Unsupported OV serialization precision " + << toString(testedPrecision) + << " should fail"; +} + +INSTANTIATE_TEST_SUITE_P( + Test, + SerializeCApiTensorNegative, + ::testing::ValuesIn(UNSUPPORTED_CAPI_OUTPUT_PRECISIONS), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); + +TEST_F(CApiSerialization, ValidSerialization) { + constexpr size_t NUMBER_OF_ELEMENTS = 3; + std::array data = {3.0, 2.0, 1.0}; + shape_t shape{1, NUMBER_OF_ELEMENTS, 1, 1}; + ov::Tensor tensor(ov::element::f32, shape); + std::memcpy(tensor.data(), data.data(), sizeof(float) * NUMBER_OF_ELEMENTS); + MockedTensorProvider provider(tensor); + OutputGetter outputGetter(provider); + + auto inputs = prepareInputs(ovms::Precision::FP32, shape); + auto status = serializePredictResponse(outputGetter, + inputs, + &response, + getTensorInfoName); + ASSERT_EQ(status.getCode(), ovms::StatusCode::OK); + InferenceTensor* responseOutput{nullptr}; + ASSERT_EQ(response.getOutput(DUMMY_MODEL_OUTPUT_NAME, &responseOutput), StatusCode::OK); + ASSERT_NE(responseOutput, nullptr); + EXPECT_EQ(responseOutput->getDataType(), OVMS_DATATYPE_FP32); + EXPECT_THAT(responseOutput->getShape(), ElementsAre(1, NUMBER_OF_ELEMENTS, 1, 1)); + const auto* buffer = responseOutput->getBuffer(); + ASSERT_NE(buffer, nullptr); + ASSERT_NE(buffer->data(), nullptr); + EXPECT_EQ(buffer->getByteSize(), tensor.get_byte_size()); + EXPECT_EQ(std::memcmp(tensor.data(), buffer->data(), sizeof(float) * NUMBER_OF_ELEMENTS), 0); +} From 69717017fb3fb7eb023c373730398e25102f0e67 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 22 Nov 2022 15:09:57 +0100 Subject: [PATCH 092/130] CVS-95439 CLI parser decoupled from ovms::Config (#1527) CLI parser decoupled from ovms::Config class. This enables C-API and OVMS to use ovms::Config for internal behavior and use CLI parser to be used within OVMS. - server option validation has been moved from cli parser to ovms::Config to support both - C-API and OVMS --- check_coverage.bat | 4 +- cppclean.sh | 4 +- src/BUILD | 2 + src/cli_parser.cpp | 269 +++++++++++++++++ src/cli_parser.hpp | 39 +++ src/config.cpp | 570 ++++++----------------------------- src/config.hpp | 91 +----- src/main.cpp | 3 +- src/main3.cpp | 8 +- src/poc_api_impl.cpp | 4 +- src/poc_api_impl.hpp | 39 ++- src/pocapi.cpp | 5 +- src/server.cpp | 40 +-- src/test/ovmsconfig_test.cpp | 181 ++++++++++- 14 files changed, 646 insertions(+), 613 deletions(-) create mode 100644 src/cli_parser.cpp create mode 100644 src/cli_parser.hpp diff --git a/check_coverage.bat b/check_coverage.bat index b26c44b157..1be541a27b 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,8 +5,8 @@ #MIN_FUNCTION_COV=87.4 #Rhel -MIN_LINES_COV=73.5 -MIN_FUNCTION_COV=74.4 +MIN_LINES_COV=73.7 +MIN_FUNCTION_COV=74.1 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` diff --git a/cppclean.sh b/cppclean.sh index be065a22f2..582c8525a8 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -38,11 +38,11 @@ if [ ${NO_WARNINGS_DIRECT} -gt 14 ]; then echo "Failed probably due to not using static keyword with functions definitions: ${NO_WARNINGS_DIRECT}"; exit 1; fi -if [ ${NO_WARNINGS_NOTUSED} -gt 2 ]; then +if [ ${NO_WARNINGS_NOTUSED} -gt 3 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 180 ]; then +if [ ${NO_WARNINGS} -gt 181 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi diff --git a/src/BUILD b/src/BUILD index d83f549f51..c155c576bd 100644 --- a/src/BUILD +++ b/src/BUILD @@ -98,6 +98,8 @@ cc_library( "buffer.hpp", "cleaner_utils.cpp", "cleaner_utils.hpp", + "cli_parser.cpp", + "cli_parser.hpp", "config.cpp", "config.hpp", "custom_node.cpp", diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp new file mode 100644 index 0000000000..4188785079 --- /dev/null +++ b/src/cli_parser.cpp @@ -0,0 +1,269 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "cli_parser.hpp" + +#include +#include + +#include + +#include "poc_api_impl.hpp" +#include "version.hpp" + +namespace ovms { + +void CLIParser::parse(int argc, char** argv) { + try { + options = std::make_unique(argv[0], "OpenVINO Model Server"); + + // clang-format off + options->add_options() + ("h, help", + "Show this help message and exit") + ("version", + "Show binary version") + ("port", + "gRPC server port", + cxxopts::value()->default_value("9178"), + "PORT") + ("grpc_bind_address", + "Network interface address to bind to for the gRPC API", + cxxopts::value()->default_value("0.0.0.0"), + "GRPC_BIND_ADDRESS") + ("rest_port", + "REST server port, the REST server will not be started if rest_port is blank or set to 0", + cxxopts::value()->default_value("0"), + "REST_PORT") + ("rest_bind_address", + "Network interface address to bind to for the REST API", + cxxopts::value()->default_value("0.0.0.0"), + "REST_BIND_ADDRESS") + ("grpc_workers", + "Number of gRPC servers. Default 1. Increase for multi client, high throughput scenarios", + cxxopts::value()->default_value("1"), + "GRPC_WORKERS") + ("rest_workers", + "Number of worker threads in REST server - has no effect if rest_port is not set. Default value depends on number of CPUs. ", + cxxopts::value(), + "REST_WORKERS") + ("log_level", + "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", + cxxopts::value()->default_value("INFO"), "LOG_LEVEL") + ("log_path", + "Optional path to the log file", + cxxopts::value(), "LOG_PATH") +#ifdef MTR_ENABLED + ("trace_path", + "Path to the trace file", + cxxopts::value(), "TRACE_PATH") +#endif + ("grpc_channel_arguments", + "A comma separated list of arguments to be passed to the grpc server. (e.g. grpc.max_connection_age_ms=2000)", + cxxopts::value(), "GRPC_CHANNEL_ARGUMENTS") + ("file_system_poll_wait_seconds", + "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", + cxxopts::value()->default_value("1"), + "FILE_SYSTEM_POLL_WAIT_SECONDS") + ("sequence_cleaner_poll_wait_minutes", + "Time interval between two consecutive sequence cleanup scans. Default is 5. Zero value disables sequence cleaner.", + cxxopts::value()->default_value("5"), + "SEQUENCE_CLEANER_POLL_WAIT_MINUTES") + ("custom_node_resources_cleaner_interval", + "Time interval between two consecutive resources cleanup scans. Default is 1. Must be greater than 0.", + cxxopts::value()->default_value("1"), + "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL") + ("cache_dir", + "Overrides model cache directory. By default cache files are saved into /opt/cache if the directory is present. When enabled, first model load will produce cache files.", + cxxopts::value(), + "CACHE_DIR") + ("cpu_extension", + "A path to shared library containing custom CPU layer implementation. Default: empty.", + cxxopts::value()->default_value(""), + "CPU_EXTENSION"); + + options->add_options("multi model") + ("config_path", + "Absolute path to json configuration file", + cxxopts::value(), "CONFIG_PATH"); + + options->add_options("single model") + ("model_name", + "Name of the model", + cxxopts::value(), + "MODEL_NAME") + ("model_path", + "Absolute path to model, as in tf serving", + cxxopts::value(), + "MODEL_PATH") + ("batch_size", + "Resets models batchsize, int value or auto. This parameter will be ignored if shape is set", + cxxopts::value(), + "BATCH_SIZE") + ("shape", + "Resets models shape (model must support reshaping). If set, batch_size parameter is ignored", + cxxopts::value(), + "SHAPE") + ("layout", + "Resets model layout.", + cxxopts::value(), + "LAYOUT") + ("model_version_policy", + "Model version policy", + cxxopts::value(), + "MODEL_VERSION_POLICY") + ("nireq", + "Size of inference request queue for model executions. Recommended to be >= parallel executions. Default value calculated by OpenVINO based on available resources. Request for 0 is treated as request for default value", + cxxopts::value(), + "NIREQ") + ("target_device", + "Target device to run the inference", + cxxopts::value()->default_value("CPU"), + "TARGET_DEVICE") + ("plugin_config", + "A dictionary of plugin configuration keys and their values, eg \"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"1\\\"}\". Default throughput streams for CPU and GPU are calculated by OpenVINO", + cxxopts::value(), + "PLUGIN_CONFIG") + ("stateful", + "Flag indicating model is stateful", + cxxopts::value()->default_value("false"), + "STATEFUL") + ("metrics_enable", + "Flag enabling metrics endpoint on rest_port.", + cxxopts::value()->default_value("false"), + "METRICS") + ("metrics_list", + "Comma separated list of metrics. If unset, only default metrics will be enabled. Default metrics: ovms_requests_success, ovms_requests_fail, ovms_request_time_us, ovms_streams, ovms_inference_time_us, ovms_wait_for_infer_req_time_us. When set, only the listed metrics will be enabled. Optional metrics: ovms_infer_req_queue_size, ovms_infer_req_active.", + cxxopts::value()->default_value(""), + "METRICS_LIST") + ("idle_sequence_cleanup", + "Flag indicating if model is subject to sequence cleaner scans", + cxxopts::value()->default_value("true"), + "IDLE_SEQUENCE_CLEANUP") + ("low_latency_transformation", + "Flag indicating that Model Server should perform low latency transformation on that model", + cxxopts::value()->default_value("false"), + "LOW_LATENCY_TRANSFORMATION") + ("max_sequence_number", + "Determines how many sequences can be processed concurrently by one model instance. When that value is reached, attempt to start a new sequence will result in error.", + cxxopts::value(), + "MAX_SEQUENCE_NUMBER"); + + result = std::make_unique(options->parse(argc, argv)); + + if (result->count("version")) { + std::string project_name(PROJECT_NAME); + std::string project_version(PROJECT_VERSION); + std::cout << project_name + " " + project_version << std::endl; + std::cout << "OpenVINO backend " << OPENVINO_NAME << std::endl; + exit(EX_OK); + } + + if (result->count("help") || result->arguments().size() == 0) { + std::cout << options->help({"", "multi model", "single model"}) << std::endl; + exit(EX_OK); + } + } catch (const cxxopts::OptionException& e) { + std::cerr << "error parsing options: " << e.what() << std::endl; + exit(EX_USAGE); + } +} + +void CLIParser::prepare(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { + go->grpcPort = result->operator[]("port").as(); + go->restPort = result->operator[]("rest_port").as(); + + if (result->count("model_name")) + mmo->modelName = result->operator[]("model_name").as(); + if (result->count("model_path")) + mmo->modelPath = result->operator[]("model_path").as(); + + if (result->count("max_sequence_number")) + mmo->maxSequenceNumber = result->operator[]("max_sequence_number").as(); + + if (result != nullptr && result->count("cpu_extension")) { + go->cpuExtensionLibraryPath = result->operator[]("cpu_extension").as(); + } + + if (result->count("grpc_bind_address")) + go->grpcBindAddress = result->operator[]("grpc_bind_address").as(); + + if (result->count("rest_bind_address")) + go->restBindAddress = result->operator[]("rest_bind_address").as(); + + go->grpcWorkers = result->operator[]("grpc_workers").as(); + + if (result->count("rest_workers")) + go->restWorkers = result->operator[]("rest_workers").as(); + + if (result->count("batch_size")) + mmo->batchSize = result->operator[]("batch_size").as(); + + if (result->count("shape")) + mmo->shape = result->operator[]("shape").as(); + + if (result->count("layout")) + mmo->layout = result->operator[]("layout").as(); + + if (result->count("model_version_policy")) + mmo->modelVersionPolicy = result->operator[]("model_version_policy").as(); + + if (result->count("nireq")) + mmo->nireq = result->operator[]("nireq").as(); + + if (result->count("target_device")) + mmo->targetDevice = result->operator[]("target_device").as(); + + if (result->count("plugin_config")) + mmo->pluginConfig = result->operator[]("plugin_config").as(); + + if (result->count("stateful")) + mmo->stateful = result->operator[]("stateful").as(); + + go->metricsEnabled = result->operator[]("metrics_enable").as(); + go->metricsList = result->operator[]("metrics_list").as(); + + if (result->count("idle_sequence_cleanup")) + mmo->idleSequenceCleanup = result->operator[]("idle_sequence_cleanup").as(); + + if (result->count("low_latency_transformation")) + mmo->lowLatencyTransformation = result->operator[]("low_latency_transformation").as(); + + if (result->count("log_level")) + go->logLevel = result->operator[]("log_level").as(); + if (result->count("log_path")) + go->logPath = result->operator[]("log_path").as(); + +#ifdef MTR_ENABLED + if (result->count("trace_path")) + go->tracePath = result->operator[]("trace_path").as(); +#endif + + if (result->count("grpc_channel_arguments")) + go->grpcChannelArguments = result->operator[]("grpc_channel_arguments").as(); + + go->filesystemPollWaitSeconds = result->operator[]("file_system_poll_wait_seconds").as(); + go->sequenceCleanerPollWaitMinutes = result->operator[]("sequence_cleaner_poll_wait_minutes").as(); + go->resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval").as(); + + if (result != nullptr && result->count("cache_dir")) { + go->cacheDir = result->operator[]("cache_dir").as(); + } + + if (result->count("config_path")) + mmo->configPath = result->operator[]("config_path").as(); +} + +} // namespace ovms diff --git a/src/cli_parser.hpp b/src/cli_parser.hpp new file mode 100644 index 0000000000..fb54d76eff --- /dev/null +++ b/src/cli_parser.hpp @@ -0,0 +1,39 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +#include + +namespace ovms { + +struct GeneralOptionsImpl; +struct SingleModelOptionsImpl; +struct MultiModelOptionsImpl; + +class CLIParser { + std::unique_ptr options; + std::unique_ptr result; + +public: + CLIParser() = default; + void parse(int argc, char** argv); + + void prepare(GeneralOptionsImpl*, MultiModelOptionsImpl*); +}; + +} // namespace ovms diff --git a/src/config.cpp b/src/config.cpp index f15aff3a4e..695c62b021 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -15,19 +15,16 @@ //***************************************************************************** #include "config.hpp" -#include #include #include #include #include -#include #include -#include "logging.hpp" +#include "cli_parser.hpp" #include "modelconfig.hpp" #include "poc_api_impl.hpp" -#include "version.hpp" namespace ovms { @@ -35,218 +32,23 @@ const uint AVAILABLE_CORES = std::thread::hardware_concurrency(); const uint MAX_PORT_NUMBER = std::numeric_limits::max(); const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; -const std::string DEFAULT_REST_WORKERS_STRING{std::to_string(DEFAULT_REST_WORKERS)}; const uint64_t MAX_REST_WORKERS = 10'000; -uint32_t Config::__maxSequenceNumber() const { - if (!result->count("max_sequence_number")) { - return DEFAULT_MAX_SEQUENCE_NUMBER; - } - return result->operator[]("max_sequence_number").as(); -} -uint32_t Config::maxSequenceNumber() const { return _maxSequenceNumber; } - Config& Config::parse(int argc, char** argv) { - try { - options = std::make_unique(argv[0], "OpenVINO Model Server"); - - // clang-format off - options->add_options() - ("h, help", - "Show this help message and exit") - ("version", - "Show binary version") - ("port", - "gRPC server port", - cxxopts::value()->default_value("9178"), - "PORT") - ("grpc_bind_address", - "Network interface address to bind to for the gRPC API", - cxxopts::value()->default_value("0.0.0.0"), - "GRPC_BIND_ADDRESS") - ("rest_port", - "REST server port, the REST server will not be started if rest_port is blank or set to 0", - cxxopts::value()->default_value("0"), - "REST_PORT") - ("rest_bind_address", - "Network interface address to bind to for the REST API", - cxxopts::value()->default_value("0.0.0.0"), - "REST_BIND_ADDRESS") - ("grpc_workers", - "Number of gRPC servers. Default 1. Increase for multi client, high throughput scenarios", - cxxopts::value()->default_value("1"), - "GRPC_WORKERS") - ("rest_workers", - "Number of worker threads in REST server - has no effect if rest_port is not set. Default value depends on number of CPUs. ", - cxxopts::value()->default_value(DEFAULT_REST_WORKERS_STRING.c_str()), - "REST_WORKERS") - ("log_level", - "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", - cxxopts::value()->default_value("INFO"), "LOG_LEVEL") - ("log_path", - "Optional path to the log file", - cxxopts::value(), "LOG_PATH") -#ifdef MTR_ENABLED - ("trace_path", - "Path to the trace file", - cxxopts::value(), "TRACE_PATH") -#endif - ("grpc_channel_arguments", - "A comma separated list of arguments to be passed to the grpc server. (e.g. grpc.max_connection_age_ms=2000)", - cxxopts::value(), "GRPC_CHANNEL_ARGUMENTS") - ("file_system_poll_wait_seconds", - "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", - cxxopts::value()->default_value("1"), - "FILE_SYSTEM_POLL_WAIT_SECONDS") - ("sequence_cleaner_poll_wait_minutes", - "Time interval between two consecutive sequence cleanup scans. Default is 5. Zero value disables sequence cleaner.", - cxxopts::value()->default_value("5"), - "SEQUENCE_CLEANER_POLL_WAIT_MINUTES") - ("custom_node_resources_cleaner_interval", - "Time interval between two consecutive resources cleanup scans. Default is 1. Must be greater than 0.", - cxxopts::value()->default_value("1"), - "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL") - ("cache_dir", - "Overrides model cache directory. By default cache files are saved into /opt/cache if the directory is present. When enabled, first model load will produce cache files.", - cxxopts::value(), - "CACHE_DIR") - ("cpu_extension", - "A path to shared library containing custom CPU layer implementation. Default: empty.", - cxxopts::value()->default_value(""), - "CPU_EXTENSION"); - options->add_options("multi model") - ("config_path", - "Absolute path to json configuration file", - cxxopts::value(), "CONFIG_PATH"); - - options->add_options("single model") - ("model_name", - "Name of the model", - cxxopts::value(), - "MODEL_NAME") - ("model_path", - "Absolute path to model, as in tf serving", - cxxopts::value(), - "MODEL_PATH") - ("batch_size", - "Resets models batchsize, int value or auto. This parameter will be ignored if shape is set", - cxxopts::value(), - "BATCH_SIZE") - ("shape", - "Resets models shape (model must support reshaping). If set, batch_size parameter is ignored", - cxxopts::value(), - "SHAPE") - ("layout", - "Resets model layout.", - cxxopts::value(), - "LAYOUT") - ("model_version_policy", - "Model version policy", - cxxopts::value(), - "MODEL_VERSION_POLICY") - ("nireq", - "Size of inference request queue for model executions. Recommended to be >= parallel executions. Default value calculated by OpenVINO based on available resources. Request for 0 is treated as request for default value", - cxxopts::value(), - "NIREQ") - ("target_device", - "Target device to run the inference", - cxxopts::value()->default_value("CPU"), - "TARGET_DEVICE") - ("plugin_config", - "A dictionary of plugin configuration keys and their values, eg \"{\\\"PERFORMANCE_HINT\\\": \\\"LATENCY\\\"}\". Default is PERFORMANCE_HINT set to THROUGHPUT mode. ", - cxxopts::value(), - "PLUGIN_CONFIG") - ("stateful", - "Flag indicating model is stateful", - cxxopts::value()->default_value("false"), - "STATEFUL") - ("metrics_enable", - "Flag enabling metrics endpoint on rest_port.", - cxxopts::value()->default_value("false"), - "METRICS") - ("metrics_list", - "Comma separated list of metrics. If unset, only default metrics will be enabled. Default metrics: ovms_requests_success, ovms_requests_fail, ovms_request_time_us, ovms_streams, ovms_inference_time_us, ovms_wait_for_infer_req_time_us. When set, only the listed metrics will be enabled. Optional metrics: ovms_infer_req_queue_size, ovms_infer_req_active.", - cxxopts::value()->default_value(""), - "METRICS_LIST") - ("idle_sequence_cleanup", - "Flag indicating if model is subject to sequence cleaner scans", - cxxopts::value()->default_value("true"), - "IDLE_SEQUENCE_CLEANUP") - ("low_latency_transformation", - "Flag indicating that Model Server should perform low latency transformation on that model", - cxxopts::value()->default_value("false"), - "LOW_LATENCY_TRANSFORMATION") - ("max_sequence_number", - "Determines how many sequences can be processed concurrently by one model instance. When that value is reached, attempt to start a new sequence will result in error.", - cxxopts::value(), - "MAX_SEQUENCE_NUMBER"); - - // clang-format on - - result = std::make_unique(options->parse(argc, argv)); - - if (result->count("version")) { - std::string project_name(PROJECT_NAME); - std::string project_version(PROJECT_VERSION); - std::cout << project_name + " " + project_version << std::endl; - std::cout << "OpenVINO backend " << OPENVINO_NAME << std::endl; - exit(EX_OK); - } - - if (result->count("help") || result->arguments().size() == 0) { - std::cout << options->help({"", "multi model", "single model"}) << std::endl; - exit(EX_OK); - } - - this->_configPath = __configPath(); - this->_port = __port(); - this->_cpuExtensionLibraryPath = __cpuExtensionLibraryPath(); - this->_grpcBindAddress = __grpcBindAddress(); - this->_restPort = __restPort(); - this->_restBindAddress = __restBindAddress(); - this->_grpcWorkers = __grpcWorkers(); - this->_restWorkers = __restWorkers(); - this->_modelName = __modelName(); - this->_modelPath = __modelPath(); - this->_batchSize = __batchSize(); - this->_shape = __shape(); - this->_layout = __layout(); - this->_modelVersionPolicy = __modelVersionPolicy(); - this->_nireq = __nireq(); - this->_targetDevice = __targetDevice(); - this->_pluginConfig = __pluginConfig(); - this->_stateful = __stateful(); - this->_metricsEnabled = __metricsEnabled(); - this->_metricsList = __metricsList(); - this->_idleSequenceCleanup = __idleSequenceCleanup(); - this->_lowLatencyTransformation = __lowLatencyTransformation(); - this->_maxSequenceNumber = __maxSequenceNumber(); - this->_logLevel = __logLevel(); - this->_logPath = __logPath(); -#ifdef MTR_ENABLED - this->_tracePath = __tracePath(); -#endif - this->_grpcChannelArguments = __grpcChannelArguments(); - this->_filesystemPollWaitSeconds = __filesystemPollWaitSeconds(); - this->_sequenceCleanerPollWaitMinutes = __sequenceCleanerPollWaitMinutes(); - this->_resourcesCleanerPollWaitSeconds = __resourcesCleanerPollWaitSeconds(); - this->_cacheDir = __cacheDir(); - - validate(); - } catch (const cxxopts::OptionException& e) { - std::cerr << "error parsing options: " << e.what() << std::endl; + ovms::CLIParser p; + ovms::GeneralOptionsImpl go; + ovms::MultiModelOptionsImpl mmo; + p.parse(argc, argv); + p.prepare(&go, &mmo); + if (!this->parse(&go, &mmo)) exit(EX_USAGE); - } - - return instance(); + return *this; } -Config& Config::parse(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - // TODO: Implement - this->_port = go->grpcPort; - this->_restPort = go->restPort; - this->_configPath = mmo->configPath; - return instance(); +bool Config::parse(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { + this->go = *go; + this->mmo = *mmo; + return validate(); } bool Config::check_hostname_or_ip(const std::string& input) { @@ -271,333 +73,143 @@ bool Config::check_hostname_or_ip(const std::string& input) { } } -void Config::validate() { - // cannot set both config path & model_name/model_path - if (result->count("config_path") && (result->count("model_name") || result->count("model_path"))) { +bool Config::validate() { + if (!configPath().empty() && (!modelName().empty() || !modelPath().empty())) { std::cerr << "Use either config_path or model_path with model_name" << std::endl; - exit(EX_USAGE); + return false; } - if (!result->count("config_path") && !(result->count("model_name") && result->count("model_path"))) { + if (configPath().empty() && !(!modelName().empty() && !modelPath().empty())) { std::cerr << "Use config_path or model_path with model_name" << std::endl; - exit(EX_USAGE); + return false; } - if (result->count("config_path") && (result->count("batch_size") || result->count("shape") || - result->count("nireq") || result->count("model_version_policy") || result->count("target_device") || - result->count("plugin_config"))) { + if (!configPath().empty() && (!this->mmo.batchSize.empty() || !shape().empty() || + nireq() != 0 || !modelVersionPolicy().empty() || !this->mmo.targetDevice.empty() || + !pluginConfig().empty())) { std::cerr << "Model parameters in CLI are exclusive with the config file" << std::endl; - exit(EX_USAGE); + return false; } // check grpc_workers value - if (result->count("grpc_workers") && ((this->grpcWorkers() > AVAILABLE_CORES) || (this->grpcWorkers() < 1))) { + if (((grpcWorkers() > AVAILABLE_CORES) || (grpcWorkers() < 1))) { std::cerr << "grpc_workers count should be from 1 to CPU core count : " << AVAILABLE_CORES << std::endl; - exit(EX_USAGE); + return false; } // check rest_workers value - if (result->count("rest_workers") && ((this->restWorkers() > MAX_REST_WORKERS) || (this->restWorkers() < 2))) { + if (((restWorkers() > MAX_REST_WORKERS) || (restWorkers() < 2))) { std::cerr << "rest_workers count should be from 2 to " << MAX_REST_WORKERS << std::endl; - exit(EX_USAGE); + return false; } - if (result->count("rest_workers") && (this->restWorkers() != DEFAULT_REST_WORKERS) && this->restPort() == 0) { + if (this->go.restWorkers.has_value() && restPort() == 0) { std::cerr << "rest_workers is set but rest_port is not set. rest_port is required to start rest servers" << std::endl; - exit(EX_USAGE); + return false; } - // check docker ports - if (result->count("port") && ((this->port() > MAX_PORT_NUMBER) || (this->port() < 0))) { + if (port() && ((port() > MAX_PORT_NUMBER) || (port() < 0))) { std::cerr << "port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); + return false; } - if (result->count("rest_port") && ((this->restPort() > MAX_PORT_NUMBER) || (this->restPort() < 0))) { + + if (restPort() != 0 && ((restPort() > MAX_PORT_NUMBER) || (restPort() < 0))) { std::cerr << "rest_port number out of range from 0 to " << MAX_PORT_NUMBER << std::endl; - exit(EX_USAGE); + return false; } // metrics on rest port - if (result->count("metrics_enable") && !result->count("rest_port")) { + if (metricsEnabled() && restPort() == 0) { std::cerr << "rest_port setting is missing, metrics are enabled on rest port" << std::endl; - exit(EX_USAGE); + return false; } // metrics on rest port - if ((result->count("metrics_enable") || result->count("metrics_list")) && result->count("config_path")) { + if ((metricsEnabled() || !metricsList().empty()) && !configPath().empty()) { std::cerr << "metrics_enable or metrics_list and config_path cant be used together. Use json config file to enable metrics when using config_path." << std::endl; - exit(EX_USAGE); + return false; } // metrics_list without metrics_enable - if (!result->count("metrics_enable") && result->count("metrics_list")) { + if (!metricsEnabled() && !metricsList().empty()) { std::cerr << "metrics_enable setting is missing, required when metrics_list is provided" << std::endl; - exit(EX_USAGE); + return false; } // check bind addresses: - if (result->count("rest_bind_address") && check_hostname_or_ip(this->restBindAddress()) == false) { + if (!restBindAddress().empty() && check_hostname_or_ip(restBindAddress()) == false) { std::cerr << "rest_bind_address has invalid format: proper hostname or IP address expected." << std::endl; - exit(EX_USAGE); + return false; } - if (result->count("grpc_bind_address") && check_hostname_or_ip(this->grpcBindAddress()) == false) { + if (!grpcBindAddress().empty() && check_hostname_or_ip(grpcBindAddress()) == false) { std::cerr << "grpc_bind_address has invalid format: proper hostname or IP address expected." << std::endl; - exit(EX_USAGE); + return false; } // port and rest_port cannot be the same - if (this->port() == this->restPort()) { + if (port() == restPort()) { std::cerr << "port and rest_port cannot have the same values" << std::endl; - exit(EX_USAGE); + return false; } // check cpu_extension path: - if (result->count("cpu_extension") && !std::filesystem::exists(this->cpuExtensionLibraryPath())) { + if (!cpuExtensionLibraryPath().empty() && !std::filesystem::exists(cpuExtensionLibraryPath())) { std::cerr << "File path provided as an --cpu_extension parameter does not exists in the filesystem: " << this->cpuExtensionLibraryPath() << std::endl; - exit(EX_USAGE); + return false; } // check log_level values - if (result->count("log_level")) { - std::vector v({"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"}); - if (std::find(v.begin(), v.end(), this->logLevel()) == v.end()) { - std::cerr << "log_level should be one of: TRACE, DEBUG, INFO, WARNING, ERROR" << std::endl; - exit(EX_USAGE); - } + std::vector v({"TRACE", "DEBUG", "INFO", "WARNING", "ERROR"}); + if (std::find(v.begin(), v.end(), logLevel()) == v.end()) { + std::cerr << "log_level should be one of: TRACE, DEBUG, INFO, WARNING, ERROR" << std::endl; + return false; } - // check stateful flags: - if ((result->count("low_latency_transformation") || result->count("max_sequence_number") || result->count("idle_sequence_cleanup")) && !result->count("stateful")) { + if ((this->mmo.lowLatencyTransformation.has_value() || this->mmo.maxSequenceNumber.has_value() || this->mmo.idleSequenceCleanup.has_value()) && !stateful()) { std::cerr << "Setting low_latency_transformation, max_sequence_number and idle_sequence_cleanup require setting stateful flag for the model." << std::endl; - exit(EX_USAGE); - } - return; -} - -const std::string& Config::__configPath() const { - if (result->count("config_path")) - return result->operator[]("config_path").as(); - return empty; -} - -const std::string& Config::configPath() const { return _configPath; } - -uint64_t Config::__port() const { - return result->operator[]("port").as(); -} - -uint64_t Config::port() const { return _port; } - -const std::string Config::__cpuExtensionLibraryPath() const { - if (result != nullptr && result->count("cpu_extension")) { - return result->operator[]("cpu_extension").as(); - } - return ""; -} - -const std::string Config::cpuExtensionLibraryPath() const { return _cpuExtensionLibraryPath; } - -const std::string Config::__grpcBindAddress() const { - if (result->count("grpc_bind_address")) - return result->operator[]("grpc_bind_address").as(); - return "0.0.0.0"; -} - -const std::string Config::grpcBindAddress() const { return _grpcBindAddress; } - -uint64_t Config::__restPort() const { - return result->operator[]("rest_port").as(); -} - -uint64_t Config::restPort() const { return _restPort; } - -const std::string Config::__restBindAddress() const { - if (result->count("rest_bind_address")) - return result->operator[]("rest_bind_address").as(); - return "0.0.0.0"; -} - -const std::string Config::restBindAddress() const { return _restBindAddress; } - -uint Config::__grpcWorkers() const { - return result->operator[]("grpc_workers").as(); -} - -uint Config::grpcWorkers() const { return _grpcWorkers; } - -uint Config::__restWorkers() const { - return result->operator[]("rest_workers").as(); -} - -uint Config::restWorkers() const { return _restWorkers; } - -const std::string& Config::__modelName() const { - if (result->count("model_name")) - return result->operator[]("model_name").as(); - return empty; -} - -const std::string& Config::modelName() const { return _modelName; } - -const std::string& Config::__modelPath() const { - if (result->count("model_path")) - return result->operator[]("model_path").as(); - return empty; -} - -const std::string& Config::modelPath() const { return _modelPath; } - -const std::string& Config::__batchSize() const { - if (!result->count("batch_size")) { - static const std::string d = "0"; - return d; - } - return result->operator[]("batch_size").as(); -} - -const std::string& Config::batchSize() const { return _batchSize; } - -const std::string& Config::__shape() const { - if (result->count("shape")) - return result->operator[]("shape").as(); - return empty; -} - -const std::string& Config::Config::shape() const { return _shape; } - -const std::string& Config::__layout() const { - if (result->count("layout")) - return result->operator[]("layout").as(); - return empty; -} - -const std::string& Config::layout() const { return _layout; } - -const std::string& Config::__modelVersionPolicy() const { - if (result->count("model_version_policy")) - return result->operator[]("model_version_policy").as(); - return empty; -} - -const std::string& Config::modelVersionPolicy() const { return _modelVersionPolicy; } - -uint32_t Config::__nireq() const { - if (!result->count("nireq")) { - return 0; - } - return result->operator[]("nireq").as(); -} - -uint32_t Config::nireq() const { return _nireq; } - -const std::string& Config::__targetDevice() const { - return result->operator[]("target_device").as(); -} - -const std::string& Config::targetDevice() const { return _targetDevice; } - -const std::string& Config::__pluginConfig() const { - if (result->count("plugin_config")) - return result->operator[]("plugin_config").as(); - return empty; -} -const std::string& Config::Config::pluginConfig() const { return _pluginConfig; } - -bool Config::__stateful() const { - return result->operator[]("stateful").as(); -} - -bool Config::stateful() const { return _stateful; } - -bool Config::__metricsEnabled() const { - if (!result->count("metrics_enable")) { return false; } - return result->operator[]("metrics_enable").as(); -} - -bool Config::metricsEnabled() const { return _metricsEnabled; } - -std::string Config::__metricsList() const { - if (!result->count("metrics_list")) { - return std::string(""); - } - return result->operator[]("metrics_list").as(); -} - -std::string Config::metricsList() const { return _metricsList; } - -bool Config::__idleSequenceCleanup() const { - return result->operator[]("idle_sequence_cleanup").as(); -} - -bool Config::idleSequenceCleanup() const { return _idleSequenceCleanup; } - -bool Config::__lowLatencyTransformation() const { - return result->operator[]("low_latency_transformation").as(); -} - -bool Config::lowLatencyTransformation() const { return _lowLatencyTransformation; } - -const std::string& Config::__logLevel() const { - if (result->count("log_level")) - return result->operator[]("log_level").as(); - return empty; -} - -const std::string& Config::logLevel() const { return _logLevel; } - -const std::string& Config::__logPath() const { - if (result->count("log_path")) - return result->operator[]("log_path").as(); - return empty; -} - -const std::string& Config::logPath() const { return _logPath; } - + return true; +} + +const std::string& Config::configPath() const { return this->mmo.configPath; } +uint64_t Config::port() const { return this->go.grpcPort; } +const std::string Config::cpuExtensionLibraryPath() const { return this->go.cpuExtensionLibraryPath; } +const std::string Config::grpcBindAddress() const { return this->go.grpcBindAddress; } +uint64_t Config::restPort() const { return this->go.restPort; } +const std::string Config::restBindAddress() const { return this->go.restBindAddress; } +uint Config::grpcWorkers() const { return this->go.grpcWorkers; } +uint Config::restWorkers() const { return this->go.restWorkers.value_or(DEFAULT_REST_WORKERS); } +const std::string& Config::modelName() const { return this->mmo.modelName; } +const std::string& Config::modelPath() const { return this->mmo.modelPath; } +const std::string& Config::batchSize() const { + static const std::string defaultBatch = "0"; + return this->mmo.batchSize.empty() ? defaultBatch : this->mmo.batchSize; +} +const std::string& Config::Config::shape() const { return this->mmo.shape; } +const std::string& Config::layout() const { return this->mmo.layout; } +const std::string& Config::modelVersionPolicy() const { return this->mmo.modelVersionPolicy; } +uint32_t Config::nireq() const { return this->mmo.nireq; } +const std::string& Config::targetDevice() const { + static const std::string defaultTargetDevice = "CPU"; + return this->mmo.targetDevice.empty() ? defaultTargetDevice : this->mmo.targetDevice; +} +const std::string& Config::Config::pluginConfig() const { return this->mmo.pluginConfig; } +bool Config::stateful() const { return this->mmo.stateful.value_or(false); } +bool Config::metricsEnabled() const { return this->go.metricsEnabled; } +std::string Config::metricsList() const { return this->go.metricsList; } +bool Config::idleSequenceCleanup() const { return this->mmo.idleSequenceCleanup.value_or(true); } +uint32_t Config::maxSequenceNumber() const { return this->mmo.maxSequenceNumber.value_or(DEFAULT_MAX_SEQUENCE_NUMBER); } +bool Config::lowLatencyTransformation() const { return this->mmo.lowLatencyTransformation.value_or(false); } +const std::string& Config::logLevel() const { return this->go.logLevel; } +const std::string& Config::logPath() const { return this->go.logPath; } #ifdef MTR_ENABLED -const std::string& Config::__tracePath() const { - if (result->count("trace_path")) - return result->operator[]("trace_path").as(); - return empty; -} - -const std::string& Config::tracePath() const { return _tracePath; } +const std::string& Config::tracePath() const { return this->go.tracePath; } #endif - -const std::string& Config::__grpcChannelArguments() const { - if (result->count("grpc_channel_arguments")) - return result->operator[]("grpc_channel_arguments").as(); - return empty; -} - -const std::string& Config::grpcChannelArguments() const { return _grpcChannelArguments; } - -uint Config::__filesystemPollWaitSeconds() const { - return result->operator[]("file_system_poll_wait_seconds").as(); -} - -uint Config::filesystemPollWaitSeconds() const { return _filesystemPollWaitSeconds; } - -uint32_t Config::__sequenceCleanerPollWaitMinutes() const { - return result->operator[]("sequence_cleaner_poll_wait_minutes").as(); -} - -uint32_t Config::sequenceCleanerPollWaitMinutes() const { return _sequenceCleanerPollWaitMinutes; } - -uint32_t Config::__resourcesCleanerPollWaitSeconds() const { - return result->operator[]("custom_node_resources_cleaner_interval").as(); -} - -uint32_t Config::resourcesCleanerPollWaitSeconds() const { return _resourcesCleanerPollWaitSeconds; } - -const std::string Config::__cacheDir() const { - if (result != nullptr && result->count("cache_dir")) { - return result->operator[]("cache_dir").as(); - } - return empty; -} - -const std::string Config::cacheDir() const { return _cacheDir; } +const std::string& Config::grpcChannelArguments() const { return this->go.grpcChannelArguments; } +uint Config::filesystemPollWaitSeconds() const { return this->go.filesystemPollWaitSeconds; } +uint32_t Config::sequenceCleanerPollWaitMinutes() const { return this->go.sequenceCleanerPollWaitMinutes; } +uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->go.resourcesCleanerPollWaitSeconds; } +const std::string Config::cacheDir() const { return this->go.cacheDir; } } // namespace ovms diff --git a/src/config.hpp b/src/config.hpp index 9f483ffa64..69fb74c5e1 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -15,80 +15,36 @@ //***************************************************************************** #pragma once -#include +#include #include -#include -#include +#include "poc_api_impl.hpp" namespace ovms { -class GeneralOptionsImpl; -class MultiModelOptionsImpl; /** * @brief Provides all the configuration options from command line */ class Config { -private: +protected: /** * @brief A default constructor is private */ Config() = default; +private: /** * @brief Private copying constructor */ Config(const Config&) = delete; - /** - * @brief cxxopts options dictionary definition - */ - std::unique_ptr options; - - /** - * @brief cxxopts contains parsed parameters - */ - std::unique_ptr result; - /** * @brief */ const std::string empty; - // new - std::string _configPath; - uint64_t _port = 9178; - std::string _cpuExtensionLibraryPath; - std::string _grpcBindAddress = "0.0.0.0"; - uint64_t _restPort = 0; - std::string _restBindAddress = "0.0.0.0"; - uint _grpcWorkers = 1; - uint _restWorkers = 8; // TODO: In OVMS this is nproc * 4 - std::string _modelName; - std::string _modelPath; - std::string _batchSize; - std::string _shape; - std::string _layout; - std::string _modelVersionPolicy; - uint32_t _nireq = 0; - std::string _targetDevice; - std::string _pluginConfig; - bool _stateful = false; - bool _metricsEnabled = false; - std::string _metricsList; - bool _idleSequenceCleanup = true; - bool _lowLatencyTransformation = false; - uint32_t _maxSequenceNumber = 500; - std::string _logLevel = "DEBUG"; - std::string _logPath; -#ifdef MTR_ENABLED - std::string _tracePath; -#endif - std::string _grpcChannelArguments; - uint _filesystemPollWaitSeconds = 1; - uint32_t _sequenceCleanerPollWaitMinutes = 5; - uint32_t _resourcesCleanerPollWaitSeconds = 1; - std::string _cacheDir; + GeneralOptionsImpl go; + MultiModelOptionsImpl mmo; public: /** @@ -108,14 +64,14 @@ class Config { * @return Config& */ Config& parse(int argc, char** argv); - Config& parse(GeneralOptionsImpl*, MultiModelOptionsImpl*); + bool parse(GeneralOptionsImpl*, MultiModelOptionsImpl*); /** * @brief Validate passed arguments * * @return void */ - void validate(); + bool validate(); /** * @brief checks if input is a proper hostname or IP address value @@ -129,7 +85,6 @@ class Config { * * @return std::string */ - const std::string& __configPath() const; // TODO: Move to CLI parser when ready const std::string& configPath() const; /** @@ -137,7 +92,6 @@ class Config { * * @return uint64_t */ - uint64_t __port() const; uint64_t port() const; /** @@ -145,7 +99,6 @@ class Config { * * @return const std::string */ - const std::string __cpuExtensionLibraryPath() const; // TODO: Move to CLI parser when ready const std::string cpuExtensionLibraryPath() const; /** @@ -153,7 +106,6 @@ class Config { * * @return const std::string& */ - const std::string __grpcBindAddress() const; // TODO: Move to CLI parser when ready const std::string grpcBindAddress() const; /** @@ -161,7 +113,6 @@ class Config { * * @return uint64_t */ - uint64_t __restPort() const; uint64_t restPort() const; /** @@ -169,7 +120,6 @@ class Config { * * @return const std::string& */ - const std::string __restBindAddress() const; // TODO: Move to CLI parser when ready const std::string restBindAddress() const; /** @@ -177,7 +127,6 @@ class Config { * * @return uint */ - uint __grpcWorkers() const; // TODO: Move to CLI parser when ready uint grpcWorkers() const; /** @@ -185,7 +134,6 @@ class Config { * * @return uint */ - uint __restWorkers() const; // TODO: Move to CLI parser when ready uint restWorkers() const; /** @@ -193,7 +141,6 @@ class Config { * * @return const std::string& */ - const std::string& __modelName() const; // TODO: Move to CLI parser when ready const std::string& modelName() const; /** @@ -201,7 +148,6 @@ class Config { * * @return const std::string& */ - const std::string& __modelPath() const; // TODO: Move to CLI parser when ready const std::string& modelPath() const; /** @@ -209,7 +155,6 @@ class Config { * * @return const std::string& */ - const std::string& __batchSize() const; // TODO: Move to CLI parser when ready const std::string& batchSize() const; /** @@ -217,7 +162,6 @@ class Config { * * @return const std::string& */ - const std::string& __shape() const; // TODO: Move to CLI parser when ready const std::string& shape() const; /** @@ -225,7 +169,6 @@ class Config { * * @return const std::string& */ - const std::string& __layout() const; // TODO: Move to CLI parser when ready const std::string& layout() const; /** @@ -233,7 +176,6 @@ class Config { * * @return const std::string& */ - const std::string& __modelVersionPolicy() const; // TODO: Move to CLI parser when ready const std::string& modelVersionPolicy() const; /** @@ -241,7 +183,6 @@ class Config { * * @return uint */ - uint32_t __nireq() const; uint32_t nireq() const; /** @@ -249,7 +190,6 @@ class Config { * * @return const std::string& */ - const std::string& __targetDevice() const; // TODO: Move to CLI parser when ready const std::string& targetDevice() const; /** @@ -257,7 +197,6 @@ class Config { * * @return const std::string& */ - const std::string& __pluginConfig() const; // TODO: Move to CLI parser when ready const std::string& pluginConfig() const; /** @@ -265,7 +204,6 @@ class Config { * * @return bool */ - bool __stateful() const; // TODO: Move to CLI parser when ready bool stateful() const; /** @@ -273,7 +211,6 @@ class Config { * * @return bool */ - bool __metricsEnabled() const; // TODO: Move to CLI parser when ready bool metricsEnabled() const; /** @@ -281,7 +218,6 @@ class Config { * * @return std::string */ - std::string __metricsList() const; // TODO: Move to CLI parser when ready std::string metricsList() const; /** @@ -289,7 +225,6 @@ class Config { * * @return uint */ - bool __idleSequenceCleanup() const; // TODO: Move to CLI parser when ready bool idleSequenceCleanup() const; /** @@ -297,7 +232,6 @@ class Config { * * @return bool */ - bool __lowLatencyTransformation() const; // TODO: Move to CLI parser when ready bool lowLatencyTransformation() const; /** @@ -305,7 +239,6 @@ class Config { * * @return uint */ - uint32_t __maxSequenceNumber() const; // TODO: Move to CLI parser when ready uint32_t maxSequenceNumber() const; /** @@ -313,7 +246,6 @@ class Config { * * @return const std::string& */ - const std::string& __logLevel() const; // TODO: Move to CLI parser when ready const std::string& logLevel() const; /** @@ -321,7 +253,6 @@ class Config { * * @return const std::string& */ - const std::string& __logPath() const; // TODO: Move to CLI parser when ready const std::string& logPath() const; #ifdef MTR_ENABLED @@ -330,7 +261,6 @@ class Config { * * @return const std::string& */ - const std::string& __tracePath() const; // TODO: Move to CLI parser when ready const std::string& tracePath() const; #endif @@ -339,7 +269,6 @@ class Config { * * @return const std::string& */ - const std::string& __grpcChannelArguments() const; // TODO: Move to CLI parser when ready const std::string& grpcChannelArguments() const; /** @@ -347,7 +276,6 @@ class Config { * * @return uint */ - uint __filesystemPollWaitSeconds() const; // TODO: Move to CLI parser when ready uint filesystemPollWaitSeconds() const; /** @@ -355,7 +283,6 @@ class Config { * * @return uint32_t */ - uint32_t __sequenceCleanerPollWaitMinutes() const; // TODO: Move to CLI parser when ready uint32_t sequenceCleanerPollWaitMinutes() const; /** @@ -363,7 +290,6 @@ class Config { * * @return uint32_t */ - uint32_t __resourcesCleanerPollWaitSeconds() const; // TODO: Move to CLI parser when ready uint32_t resourcesCleanerPollWaitSeconds() const; /** @@ -371,7 +297,6 @@ class Config { * * @return const std::string& */ - const std::string __cacheDir() const; // TODO: Move to CLI parser when ready const std::string cacheDir() const; }; } // namespace ovms diff --git a/src/main.cpp b/src/main.cpp index 3f3d9b8c58..a79b8325ec 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2018-2020 Intel Corporation +// Copyright 2018-2022 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** - #include "server.hpp" using ovms::Server; diff --git a/src/main3.cpp b/src/main3.cpp index e829baaeb3..bd0ee9b354 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -30,12 +30,16 @@ int main(int argc, char** argv) { OVMS_ServerGeneralOptionsSetRestPort(go, 11338); OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); - OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); OVMS_ServerDelete(srv); OVMS_ServerMultiModelOptionsDelete(mmo); OVMS_ServerGeneralOptionsDelete(go); - std::cout << "Finish" << std::endl; + if (res == 0) { + std::cout << "Finish with success" << std::endl; + } else { + std::cout << "Finish with fail" << std::endl; + } return 0; } diff --git a/src/poc_api_impl.cpp b/src/poc_api_impl.cpp index 4d06a7cb4e..7411448cc4 100644 --- a/src/poc_api_impl.cpp +++ b/src/poc_api_impl.cpp @@ -19,9 +19,9 @@ namespace ovms { -void ServerImpl::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { +int ServerImpl::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { Server& server = Server::instance(); - server.start(go, mmo); + return server.start(go, mmo); } } // namespace ovms diff --git a/src/poc_api_impl.hpp b/src/poc_api_impl.hpp index 4f108a9af9..a42559306e 100644 --- a/src/poc_api_impl.hpp +++ b/src/poc_api_impl.hpp @@ -14,21 +14,54 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include +#include #include namespace ovms { struct GeneralOptionsImpl { - uint64_t grpcPort; - uint64_t restPort; + uint64_t grpcPort = 9178; + uint64_t restPort = 0; + std::string grpcBindAddress = "0.0.0.0"; + std::string restBindAddress = "0.0.0.0"; + uint grpcWorkers = 1; + std::optional restWorkers; + bool metricsEnabled = false; + std::string metricsList; + std::string cpuExtensionLibraryPath; + std::string logLevel = "INFO"; + std::string logPath; +#ifdef MTR_ENABLED + std::string tracePath; +#endif + std::string grpcChannelArguments; + uint filesystemPollWaitSeconds = 1; + uint32_t sequenceCleanerPollWaitMinutes = 5; + uint32_t resourcesCleanerPollWaitSeconds = 1; + std::string cacheDir; }; struct MultiModelOptionsImpl { + std::string modelName; + std::string modelPath; + std::string batchSize; + std::string shape; + std::string layout; + std::string modelVersionPolicy; + uint32_t nireq = 0; + std::string targetDevice; + std::string pluginConfig; + std::optional stateful; + std::optional lowLatencyTransformation; + std::optional maxSequenceNumber; + std::optional idleSequenceCleanup; + std::string configPath; }; struct ServerImpl { - void start(GeneralOptionsImpl*, MultiModelOptionsImpl*); + int start(GeneralOptionsImpl*, MultiModelOptionsImpl*); }; } // namespace ovms diff --git a/src/pocapi.cpp b/src/pocapi.cpp index aa804d3f7f..3f6f0ce507 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include "pocapi.hpp" +#include #include #include "poc_api_impl.hpp" @@ -55,8 +56,8 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, ovms::ServerImpl* srv = (ovms::ServerImpl*)server; ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)general_options; ovms::MultiModelOptionsImpl* mmo = (ovms::MultiModelOptionsImpl*)multi_model_specific_options; - srv->start(go, mmo); - return 0; + std::int64_t res = srv->start(go, mmo); + return (OVMS_Status*)res; // TODO: Return proper OVMS_Status instead of a raw status code } OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, diff --git a/src/server.cpp b/src/server.cpp index ff01e2ea2f..fb35a7f4fe 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -32,8 +32,10 @@ #include #include #include +#include #include +#include "cli_parser.hpp" #include "config.hpp" #include "grpcservermodule.hpp" #include "http_server.hpp" @@ -322,43 +324,23 @@ void Server::shutdownModules() { } // OVMS Start -// TODO: Merge with C-API Start CVS-95439 int Server::start(int argc, char** argv) { - ovms::Server& server = ovms::Server::instance(); - installSignalHandlers(server); - try { - auto& config = ovms::Config::instance().parse(argc, argv); - configure_logger(config.logLevel(), config.logPath()); - logConfig(config); - ModulesShutdownGuard shutdownGuard(*this); - auto retCode = this->startModules(config); - if (retCode) - return retCode; - - while (!shutdown_request) { - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - if (shutdown_request == 2) { - SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); - } - SPDLOG_INFO("Shutting down"); - } catch (std::exception& e) { - SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); - return EXIT_FAILURE; - } catch (...) { - SPDLOG_ERROR("Unknown exception catch - will now terminate."); - return EXIT_FAILURE; - } - return EXIT_SUCCESS; + ovms::CLIParser parser; + ovms::GeneralOptionsImpl go; + ovms::MultiModelOptionsImpl mmo; + parser.parse(argc, argv); + parser.prepare(&go, &mmo); + return start(&go, &mmo); } // C-API Start -// TODO: Merge with OVMS Start CVS-95439 int Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { ovms::Server& server = ovms::Server::instance(); installSignalHandlers(server); try { - auto& config = ovms::Config::instance().parse(go, mmo); + auto& config = ovms::Config::instance(); + if (!config.parse(go, mmo)) + return EX_USAGE; configure_logger(config.logLevel(), config.logPath()); logConfig(config); ModulesShutdownGuard shutdownGuard(*this); diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index ce0f896b97..3667920766 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -98,6 +98,42 @@ TEST_F(OvmsConfigDeathTest, negativeTwoParams) { EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Use either config_path or model_path"); } +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithBatchSize) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--batch_size", "5"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithShape) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--shape", "(1,2)"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithNireq) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--nireq", "3"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithModelVersionPolicy) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--model_version_policy", "policy"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithTargetDevice) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--target_device", "GPU"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + +TEST_F(OvmsConfigDeathTest, negativeConfigPathWithPluginConfig) { + char* n_argv[] = {"ovms", "--config_path", "/path1", "--plugin_config", "setting"}; + int arg_count = 5; + EXPECT_EXIT(ovms::Config::instance().parse(arg_count, n_argv), ::testing::ExitedWithCode(EX_USAGE), "Model parameters in CLI are exclusive with the config file"); +} + TEST_F(OvmsConfigDeathTest, negativeMissingPathAndName) { char* n_argv[] = {"ovms", "--rest_port", "8080"}; int arg_count = 3; @@ -286,13 +322,144 @@ TEST_F(OvmsParamsTest, hostname_ip_regex) { EXPECT_EQ(ovms::Config::check_hostname_or_ip(too_long), false); } -TEST(OvmsConfigTest, positive) { - char* n_argv[] = {"ovms", "--config_path", "/path1", "--port", "44", "--rest_port", "45"}; - int arg_count = 7; - ovms::Config::instance().parse(arg_count, n_argv); - EXPECT_EQ(ovms::Config::instance().port(), 44); - EXPECT_EQ(ovms::Config::instance().restPort(), 45); - EXPECT_EQ(ovms::Config::instance().configPath(), "/path1"); +class MockedConfig : public ovms::Config { +public: + MockedConfig() {} +}; + +TEST(OvmsConfigTest, positiveMulti) { + char* n_argv[] = {"ovms", + "--port", "44", + "--grpc_workers", "2", + "--grpc_bind_address", "1.1.1.1", + "--rest_port", "45", + "--rest_workers", "46", + "--rest_bind_address", "2.2.2.2", + "--grpc_channel_arguments", "grpc_channel_args", + "--file_system_poll_wait_seconds", "2", + "--sequence_cleaner_poll_wait_minutes", "7", + "--custom_node_resources_cleaner_interval", "8", + "--cpu_extension", "/ovms", + "--cache_dir", "/tmp/model_cache", + "--log_path", "/tmp/log_path", + "--log_level", "ERROR", + + "--config_path", "/config.json"}; + int arg_count = 31; + MockedConfig config; + config.parse(arg_count, n_argv); + + EXPECT_EQ(config.port(), 44); + EXPECT_EQ(config.grpcWorkers(), 2); + EXPECT_EQ(config.grpcBindAddress(), "1.1.1.1"); + EXPECT_EQ(config.restPort(), 45); + EXPECT_EQ(config.restWorkers(), 46); + EXPECT_EQ(config.restBindAddress(), "2.2.2.2"); + EXPECT_EQ(config.grpcChannelArguments(), "grpc_channel_args"); + EXPECT_EQ(config.filesystemPollWaitSeconds(), 2); + EXPECT_EQ(config.sequenceCleanerPollWaitMinutes(), 7); + EXPECT_EQ(config.resourcesCleanerPollWaitSeconds(), 8); + EXPECT_EQ(config.cpuExtensionLibraryPath(), "/ovms"); + EXPECT_EQ(config.cacheDir(), "/tmp/model_cache"); + EXPECT_EQ(config.logPath(), "/tmp/log_path"); + EXPECT_EQ(config.logLevel(), "ERROR"); + + EXPECT_EQ(config.configPath(), "/config.json"); +} + +TEST(OvmsConfigTest, positiveSingle) { + char* n_argv[] = { + "ovms", + "--port", + "44", + "--grpc_workers", + "2", + "--grpc_bind_address", + "1.1.1.1", + "--rest_port", + "45", + "--rest_workers", + "46", + "--rest_bind_address", + "2.2.2.2", + "--grpc_channel_arguments", + "grpc_channel_args", + "--file_system_poll_wait_seconds", + "2", + "--sequence_cleaner_poll_wait_minutes", + "7", + "--custom_node_resources_cleaner_interval", + "8", + "--cpu_extension", + "/ovms", + "--cache_dir", + "/tmp/model_cache", + "--log_path", + "/tmp/log_path", + "--log_level", + "ERROR", + + "--model_name", + "model", + "--model_path", + "/path", + "--batch_size", + "(3:5)", + "--shape", + "(3:5,5:6)", + "--layout", + "nchw:nhwc", + "--model_version_policy", + "setting", + "--nireq", + "2", + "--target_device", + "GPU", + "--plugin_config", + "pluginsetting", + "--stateful", + "--metrics_enable", + "--metrics_list", + "ovms_streams,ovms_other", + "--idle_sequence_cleanup=false", + "--low_latency_transformation", + "--max_sequence_number", + "52", + }; + int arg_count = 55; + MockedConfig config; + config.parse(arg_count, n_argv); + + EXPECT_EQ(config.port(), 44); + EXPECT_EQ(config.grpcWorkers(), 2); + EXPECT_EQ(config.grpcBindAddress(), "1.1.1.1"); + EXPECT_EQ(config.restPort(), 45); + EXPECT_EQ(config.restWorkers(), 46); + EXPECT_EQ(config.restBindAddress(), "2.2.2.2"); + EXPECT_EQ(config.grpcChannelArguments(), "grpc_channel_args"); + EXPECT_EQ(config.filesystemPollWaitSeconds(), 2); + EXPECT_EQ(config.sequenceCleanerPollWaitMinutes(), 7); + EXPECT_EQ(config.resourcesCleanerPollWaitSeconds(), 8); + EXPECT_EQ(config.cpuExtensionLibraryPath(), "/ovms"); + EXPECT_EQ(config.cacheDir(), "/tmp/model_cache"); + EXPECT_EQ(config.logPath(), "/tmp/log_path"); + EXPECT_EQ(config.logLevel(), "ERROR"); + + EXPECT_EQ(config.modelPath(), "/path"); + EXPECT_EQ(config.modelName(), "model"); + EXPECT_EQ(config.batchSize(), "(3:5)"); + EXPECT_EQ(config.shape(), "(3:5,5:6)"); + EXPECT_EQ(config.layout(), "nchw:nhwc"); + EXPECT_EQ(config.modelVersionPolicy(), "setting"); + EXPECT_EQ(config.nireq(), 2); + EXPECT_EQ(config.targetDevice(), "GPU"); + EXPECT_EQ(config.pluginConfig(), "pluginsetting"); + EXPECT_EQ(config.stateful(), true); + EXPECT_EQ(config.metricsEnabled(), true); + EXPECT_EQ(config.metricsList(), "ovms_streams,ovms_other"); + EXPECT_EQ(config.idleSequenceCleanup(), false); + EXPECT_EQ(config.lowLatencyTransformation(), true); + EXPECT_EQ(config.maxSequenceNumber(), 52); } #pragma GCC diagnostic pop From 026fb1e93c2323b6cc387502d5feca991d422b43 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 25 Nov 2022 11:06:56 +0100 Subject: [PATCH 093/130] Request validation (#1530) * Request validation --- check_coverage.bat | 1 + cppclean.sh | 4 +- src/BUILD | 3 + src/buffer.cpp | 6 + src/buffer.hpp | 2 + src/capi_frontend/capi_utils.cpp | 28 + src/capi_frontend/capi_utils.hpp | 23 + src/grpc_utils.cpp | 2 + src/http_server.cpp | 2 + src/inferenceparameter.cpp | 37 +- src/inferenceparameter.hpp | 10 +- src/inferencerequest.cpp | 7 +- src/inferencerequest.hpp | 5 +- src/inferenceresponse.cpp | 4 +- src/inferenceresponse.hpp | 4 +- src/inferencetensor.cpp | 5 +- src/inferencetensor.hpp | 6 +- src/main3.cpp | 1 + src/modelinstance.cpp | 1 + src/pocapi.hpp | 15 +- src/pocapiinternal.cpp | 4 +- src/pocapiinternal.hpp | 4 +- src/predict_request_validation_utils.cpp | 211 +++++- src/serialization.hpp | 2 +- src/server.cpp | 1 - src/statefulmodelinstance.hpp | 2 - src/status.cpp | 2 + src/status.hpp | 2 + src/test/capi_predict_validation_test.cpp | 829 ++++++++++++++++++++++ src/test/deserialization_tests.cpp | 3 +- src/test/inferencerequest_test.cpp | 4 +- src/test/predict_validation_test.cpp | 31 +- src/test/test_utils.cpp | 36 + src/test/test_utils.hpp | 68 ++ 34 files changed, 1283 insertions(+), 82 deletions(-) create mode 100644 src/capi_frontend/capi_utils.cpp create mode 100644 src/capi_frontend/capi_utils.hpp create mode 100644 src/test/capi_predict_validation_test.cpp diff --git a/check_coverage.bat b/check_coverage.bat index 1be541a27b..1847dc8387 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -9,6 +9,7 @@ MIN_LINES_COV=73.7 MIN_FUNCTION_COV=74.1 + LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` FUNC_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | tail -n 1` diff --git a/cppclean.sh b/cppclean.sh index 582c8525a8..0b3b11ffd5 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,11 +42,11 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 3 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 181 ]; then +if [ ${NO_WARNINGS} -gt 178 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi -if [ ${NO_WARNINGS_TEST} -gt 124 ]; then +if [ ${NO_WARNINGS_TEST} -gt 128 ]; then echo "Failed due to higher than allowed number of issues in test code: ${NO_WARNINGS_TEST}" cat ${CPPCLEAN_RESULTS_FILE_TEST} exit 1 diff --git a/src/BUILD b/src/BUILD index c155c576bd..c2cf9b13c5 100644 --- a/src/BUILD +++ b/src/BUILD @@ -96,6 +96,8 @@ cc_library( "azurefilesystem.hpp", "buffer.cpp", "buffer.hpp", + "capi_frontend/capi_utils.cpp", + "capi_frontend/capi_utils.hpp", "cleaner_utils.cpp", "cleaner_utils.hpp", "cli_parser.cpp", @@ -646,6 +648,7 @@ cc_test( "test/ovinferrequestqueue_test.cpp", "test/ov_utils_test.cpp", "test/pipelinedefinitionstatus_test.cpp", + "test/capi_predict_validation_test.cpp", "test/predict_validation_test.cpp", "test/prediction_service_test.cpp", "test/tfs_rest_parser_row_test.cpp", diff --git a/src/buffer.cpp b/src/buffer.cpp index b08b895cc6..074eac8912 100644 --- a/src/buffer.cpp +++ b/src/buffer.cpp @@ -38,5 +38,11 @@ const void* Buffer::data() const { size_t Buffer::getByteSize() const { return byteSize; } +const BufferType Buffer::getBufferType() const { + return bufferType; +} +const std::optional& Buffer::getDeviceId() const { + return bufferDeviceId; +} Buffer::~Buffer() = default; } // namespace ovms diff --git a/src/buffer.hpp b/src/buffer.hpp index 2921638d49..c28838acf1 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -31,6 +31,8 @@ class Buffer { Buffer(const void* ptr, size_t byteSize, BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional bufferDeviceId = std::nullopt, bool createCopy = false); ~Buffer(); const void* data() const; + const BufferType getBufferType() const; + const std::optional& getDeviceId() const; size_t getByteSize() const; }; diff --git a/src/capi_frontend/capi_utils.cpp b/src/capi_frontend/capi_utils.cpp new file mode 100644 index 0000000000..db89d15e7b --- /dev/null +++ b/src/capi_frontend/capi_utils.cpp @@ -0,0 +1,28 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "capi_utils.hpp" + +#include + +#include "../shape.hpp" + +namespace ovms { + +std::string tensorShapeToString(const Shape& shape) { + return shape.toString(); +} + +} // namespace ovms diff --git a/src/capi_frontend/capi_utils.hpp b/src/capi_frontend/capi_utils.hpp new file mode 100644 index 0000000000..d30158d2aa --- /dev/null +++ b/src/capi_frontend/capi_utils.hpp @@ -0,0 +1,23 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include + +namespace ovms { +class Shape; +std::string tensorShapeToString(const Shape& tensorShape); + +} // namespace ovms diff --git a/src/grpc_utils.cpp b/src/grpc_utils.cpp index b05fa8d159..d6ea5942ec 100644 --- a/src/grpc_utils.cpp +++ b/src/grpc_utils.cpp @@ -68,6 +68,8 @@ const grpc::Status grpc(const Status& status) { {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, grpc::StatusCode::INVALID_ARGUMENT}, {StatusCode::INVALID_BATCH_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, {StatusCode::INVALID_SHAPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_BUFFER_TYPE, grpc::StatusCode::INVALID_ARGUMENT}, + {StatusCode::INVALID_DEVICE_ID, grpc::StatusCode::INVALID_ARGUMENT}, {StatusCode::INVALID_PRECISION, grpc::StatusCode::INVALID_ARGUMENT}, {StatusCode::INVALID_VALUE_COUNT, grpc::StatusCode::INVALID_ARGUMENT}, {StatusCode::INVALID_CONTENT_SIZE, grpc::StatusCode::INVALID_ARGUMENT}, diff --git a/src/http_server.cpp b/src/http_server.cpp index c1f838b265..d6e6a1890f 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -110,6 +110,8 @@ const net_http::HTTPStatusCode http(const ovms::Status& status) { {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, net_http::HTTPStatusCode::BAD_REQUEST}, {StatusCode::INVALID_BATCH_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, {StatusCode::INVALID_SHAPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_BUFFER_TYPE, net_http::HTTPStatusCode::BAD_REQUEST}, + {StatusCode::INVALID_DEVICE_ID, net_http::HTTPStatusCode::BAD_REQUEST}, {StatusCode::INVALID_PRECISION, net_http::HTTPStatusCode::BAD_REQUEST}, {StatusCode::INVALID_VALUE_COUNT, net_http::HTTPStatusCode::BAD_REQUEST}, {StatusCode::INVALID_CONTENT_SIZE, net_http::HTTPStatusCode::BAD_REQUEST}, diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index 5044aa9b34..6219afcb98 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** + #include "inferenceparameter.hpp" #include @@ -21,18 +22,34 @@ namespace ovms { // TODO should we own our own copy of value? // -static size_t DataTypeToByteSize(DataType datatype) { - switch (datatype) { - case OVMS_DATATYPE_FP32: - case OVMS_DATATYPE_I32: - case OVMS_DATATYPE_U32: - return 4; - default: - throw std::invalid_argument("Unsupported"); +size_t DataTypeToByteSize(OVMS_DataType datatype) { + static std::unordered_map datatypeSizeMap{ + {OVMS_DATATYPE_BOOL, 1}, + {OVMS_DATATYPE_U1, 1}, + {OVMS_DATATYPE_U4, 1}, + {OVMS_DATATYPE_U8, 1}, + {OVMS_DATATYPE_U16, 2}, + {OVMS_DATATYPE_U32, 4}, + {OVMS_DATATYPE_U64, 8}, + {OVMS_DATATYPE_I4, 1}, + {OVMS_DATATYPE_I8, 1}, + {OVMS_DATATYPE_I16, 2}, + {OVMS_DATATYPE_I32, 4}, + {OVMS_DATATYPE_I64, 8}, + {OVMS_DATATYPE_FP16, 2}, + {OVMS_DATATYPE_FP32, 4}, + {OVMS_DATATYPE_FP64, 8}, + {OVMS_DATATYPE_BF16, 2}, + // {"BYTES", }, + }; + auto it = datatypeSizeMap.find(datatype); + if (it == datatypeSizeMap.end()) { return 0; } + return it->second; } -InferenceParameter::InferenceParameter(const char* name, DataType datatype, const void* data) : + +InferenceParameter::InferenceParameter(const char* name, OVMS_DataType datatype, const void* data) : name(name), datatype(datatype), data(reinterpret_cast(data), DataTypeToByteSize(datatype)) { @@ -41,7 +58,7 @@ InferenceParameter::InferenceParameter(const char* name, DataType datatype, cons const std::string& InferenceParameter::getName() const { return this->name; } -DataType InferenceParameter::getDataType() const { +OVMS_DataType InferenceParameter::getDataType() const { return this->datatype; } size_t InferenceParameter::getByteSize() const { diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp index 204dab5168..a5695ebbc7 100644 --- a/src/inferenceparameter.hpp +++ b/src/inferenceparameter.hpp @@ -15,22 +15,24 @@ // limitations under the License. //***************************************************************************** #include +#include #include "pocapi.hpp" namespace ovms { +size_t DataTypeToByteSize(OVMS_DataType datatype); // TODO should we own our own copy of value? class InferenceParameter { const std::string name; - DataType datatype; + OVMS_DataType datatype; const std::string data; public: - InferenceParameter(const char* name, DataType datatype, const void* data); - InferenceParameter(const char* name, DataType datatype, const void* data, size_t byteSize); + InferenceParameter(const char* name, OVMS_DataType datatype, const void* data); + InferenceParameter(const char* name, OVMS_DataType datatype, const void* data, size_t byteSize); const std::string& getName() const; - DataType getDataType() const; + OVMS_DataType getDataType() const; size_t getByteSize() const; const void* getData() const; }; diff --git a/src/inferencerequest.cpp b/src/inferencerequest.cpp index e58c9dd545..a11a2df5ce 100644 --- a/src/inferencerequest.cpp +++ b/src/inferencerequest.cpp @@ -29,7 +29,7 @@ const std::string& InferenceRequest::getServableName() const { model_version_t InferenceRequest::getServableVersion() const { return this->servableVersion; } -Status InferenceRequest::addInput(const char* name, DataType datatype, const size_t* shape, size_t dimCount) { +Status InferenceRequest::addInput(const char* name, OVMS_DataType datatype, const size_t* shape, size_t dimCount) { auto [it, emplaced] = inputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); return emplaced ? StatusCode::OK : StatusCode::DOUBLE_TENSOR_INSERT; } @@ -60,6 +60,9 @@ Status InferenceRequest::getInput(const char* name, const InferenceTensor** tens *tensor = &it->second; return StatusCode::OK; } +uint64_t InferenceRequest::getInputsSize() const { + return inputs.size(); +} Status InferenceRequest::removeInput(const char* name) { auto count = inputs.erase(name); if (count) { @@ -67,7 +70,7 @@ Status InferenceRequest::removeInput(const char* name) { } return StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL; } -Status InferenceRequest::addParameter(const char* parameterName, DataType datatype, const void* data) { +Status InferenceRequest::addParameter(const char* parameterName, OVMS_DataType datatype, const void* data) { auto [it, emplaced] = parameters.emplace(parameterName, InferenceParameter{parameterName, datatype, data}); return emplaced ? StatusCode::OK : StatusCode::DOUBLE_PARAMETER_INSERT; } diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp index 9686355c85..c00839e822 100644 --- a/src/inferencerequest.hpp +++ b/src/inferencerequest.hpp @@ -34,14 +34,15 @@ class InferenceRequest { public: InferenceRequest(const char* modelName, model_version_t modelVersion); - Status addInput(const char* name, DataType datatype, const size_t* shape, size_t dimCount); + Status addInput(const char* name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); Status getInput(const char* name, const InferenceTensor** tensor) const; + uint64_t getInputsSize() const; Status removeInput(const char* name); Status removeAllInputs(); Status setInputBuffer(const char* name, const void* addr, size_t byteSize, BufferType, std::optional deviceId); Status removeInputBuffer(const char* name); - Status addParameter(const char* parameterName, DataType datatype, const void* data); + Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); Status removeParameter(const char* parameterName); const InferenceParameter* getParameter(const char* name) const; diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp index 08d48c20f3..4f5241f229 100644 --- a/src/inferenceresponse.cpp +++ b/src/inferenceresponse.cpp @@ -33,7 +33,7 @@ const std::string& InferenceResponse::getServableName() const { model_version_t InferenceResponse::getServableVersion() const { return this->servableVersion; } -Status InferenceResponse::addOutput(const std::string& name, DataType datatype, const size_t* shape, size_t dimCount) { +Status InferenceResponse::addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount) { // TODO insert tensor with wrong shape/datatype/name/dimcount // TODO reuse infer response/request auto [it, emplaced] = outputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); @@ -48,7 +48,7 @@ Status InferenceResponse::getOutput(const char* name, InferenceTensor** tensor) *tensor = &it->second; return StatusCode::OK; } -Status InferenceResponse::addParameter(const char* parameterName, DataType datatype, const void* data) { +Status InferenceResponse::addParameter(const char* parameterName, OVMS_DataType datatype, const void* data) { auto [it, emplaced] = parameters.emplace(parameterName, InferenceParameter{parameterName, datatype, data}); return emplaced ? StatusCode::OK : StatusCode::DOUBLE_PARAMETER_INSERT; } diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index fa3b2f58cb..c15e4d522d 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -34,9 +34,9 @@ class InferenceResponse { public: InferenceResponse(const std::string& servableName, model_version_t servableVersion); - Status addOutput(const std::string& name, DataType datatype, const size_t* shape, size_t dimCount); + Status addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); Status getOutput(const char* name, InferenceTensor** tensor); - Status addParameter(const char* parameterName, DataType datatype, const void* data); + Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); const InferenceParameter* getParameter(const char* name) const; const std::string& getServableName() const; diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp index d35195429a..eb86903487 100644 --- a/src/inferencetensor.cpp +++ b/src/inferencetensor.cpp @@ -27,7 +27,7 @@ InferenceTensor::InferenceTensor(InferenceTensor&& rhs) : datatype(std::move(rhs.datatype)), shape(std::move(rhs.shape)), buffer(std::move(rhs.buffer)) {} -InferenceTensor::InferenceTensor(DataType datatype, const size_t* shape, size_t dimCount) : +InferenceTensor::InferenceTensor(OVMS_DataType datatype, const size_t* shape, size_t dimCount) : datatype(datatype), shape(shape, shape + dimCount) {} Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy) { @@ -37,7 +37,8 @@ Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, BufferType buffer = std::make_unique(addr, byteSize, bufferType, deviceId, createCopy); return StatusCode::OK; } -DataType InferenceTensor::getDataType() const { + +OVMS_DataType InferenceTensor::getDataType() const { return this->datatype; } const shape_t& InferenceTensor::getShape() const { diff --git a/src/inferencetensor.hpp b/src/inferencetensor.hpp index f08306f08f..113ff63d5c 100644 --- a/src/inferencetensor.hpp +++ b/src/inferencetensor.hpp @@ -26,12 +26,12 @@ class Buffer; class Status; class InferenceTensor { - const DataType datatype; + const OVMS_DataType datatype; shape_t shape; std::unique_ptr buffer; public: - InferenceTensor(DataType datatype, const size_t* shape, size_t dimCount); + InferenceTensor(OVMS_DataType datatype, const size_t* shape, size_t dimCount); ~InferenceTensor(); InferenceTensor(InferenceTensor&&); InferenceTensor(const InferenceTensor&) = delete; @@ -39,7 +39,7 @@ class InferenceTensor { InferenceTensor& operator=(const InferenceTensor&&); Status setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy = false); Status removeBuffer(); - DataType getDataType() const; + OVMS_DataType getDataType() const; const shape_t& getShape() const; const Buffer* const getBuffer() const; }; diff --git a/src/main3.cpp b/src/main3.cpp index bd0ee9b354..98c5a3f412 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -41,5 +41,6 @@ int main(int argc, char** argv) { } else { std::cout << "Finish with fail" << std::endl; } + return 0; } diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index c52f1db98a..0db306e237 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1116,6 +1116,7 @@ const Status ModelInstance::validate(const RequestType* request) { getModelConfig().getShapes()); } +template const Status ModelInstance::validate(const InferenceRequest* request); template const Status ModelInstance::validate(const ::KFSRequest* request); template const Status ModelInstance::validate(const tensorflow::serving::PredictRequest* request); diff --git a/src/pocapi.hpp b/src/pocapi.hpp index ec8e1a12b2..a3777f19de 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -24,7 +24,7 @@ struct OVMS_ServerGeneralOptions; struct OVMS_ServerMultiModelOptions; // TODO reuse this in precision.hpp -enum DataType { +enum OVMS_DataType { OVMS_DATATYPE_BF16, OVMS_DATATYPE_FP64, OVMS_DATATYPE_FP32, @@ -148,22 +148,25 @@ OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest OVMS_Status* OVMS_InferenceRequestNew(char* modelName, uint32_t servableVersion); OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); -OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, char* inputName, DataType datatype, uint64_t* shape, uint32_t dimCount); -OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, char* inputName, DataType datatype); // TODO consider no datatype & handle the parameters + +OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, char* inputName, OVMS_DataType datatype, uint64_t* shape, uint32_t dimCount); +OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters + // ownership of data needs to be maintained during inference OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId); OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, char* inputName); OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, char* inputName); // this will allow for reuse of request but with different input data OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs -OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, char* paramaterName, DataType datatype, void* data, size_t byteSize); + +OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, char* paramaterName, OVMS_DataType datatype, void* data, size_t byteSize); // OVMS_Inference Response OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseOutput(OVMS_InferenceResponse* response, uint32_t id, char* name, DataType* datatype, uint64_t* shape, uint32_t dimCount, BufferType* bufferType, uint32_t* deviceId, void** data); +OVMS_Status* OVMS_InferenceResponseOutput(OVMS_InferenceResponse* response, uint32_t id, char* name, OVMS_DataType* datatype, uint64_t* shape, uint32_t dimCount, BufferType* bufferType, uint32_t* deviceId, void** data); OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, DataType* datatype, void** data); +OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, void** data); OVMS_Status* OVMS_Inference(OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); diff --git a/src/pocapiinternal.cpp b/src/pocapiinternal.cpp index 47c4c95020..6137983d41 100644 --- a/src/pocapiinternal.cpp +++ b/src/pocapiinternal.cpp @@ -16,7 +16,7 @@ #include "pocapiinternal.hpp" namespace ovms { -DataType getPrecisionAsOVMSDataType(Precision precision) { +OVMS_DataType getPrecisionAsOVMSDataType(Precision precision) { switch (precision) { case Precision::BF16: return OVMS_DATATYPE_BF16; @@ -66,7 +66,7 @@ DataType getPrecisionAsOVMSDataType(Precision precision) { return OVMS_DATATYPE_UNDEFINED; } } -Precision getOVMSDataTypeAsPrecision(DataType datatype) { +Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype) { switch (datatype) { case OVMS_DATATYPE_BF16: return Precision::BF16; diff --git a/src/pocapiinternal.hpp b/src/pocapiinternal.hpp index a90af3e3ac..e0303221ac 100644 --- a/src/pocapiinternal.hpp +++ b/src/pocapiinternal.hpp @@ -18,6 +18,6 @@ #include "precision.hpp" namespace ovms { -DataType getPrecisionAsOVMSDataType(Precision precision); -Precision getOVMSDataTypeAsPrecision(DataType datatype); +OVMS_DataType getPrecisionAsOVMSDataType(Precision precision); +Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype); } // namespace ovms diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index 3377afdeb2..6ea3d8e206 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -23,9 +23,14 @@ #include +#include "buffer.hpp" +#include "capi_frontend/capi_utils.hpp" +#include "inferencerequest.hpp" +#include "inferencetensor.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "kfs_frontend/kfs_utils.hpp" #include "modelconfig.hpp" +#include "pocapiinternal.hpp" #include "profiler.hpp" #include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" @@ -56,6 +61,10 @@ dimension_value_t RequestShapeInfo::getDim(siz return tensor.tensor_shape().dim(i).size(); } template <> +dimension_value_t RequestShapeInfo::getDim(size_t i) { + return tensor.getShape()[i]; +} +template <> size_t RequestShapeInfo::getShapeSize() { return tensor.shape().size(); } @@ -64,6 +73,10 @@ size_t RequestShapeInfo::getShapeSize() { return tensor.tensor_shape().dim_size(); } template <> +size_t RequestShapeInfo::getShapeSize() { + return tensor.getShape().size(); +} +template <> const TFSShapeType& RequestShapeInfo::getShape() { return tensor.tensor_shape(); } @@ -71,7 +84,10 @@ template <> const KFSShapeType& RequestShapeInfo::getShape() { return tensor.shape(); } - +template <> +const shape_t& RequestShapeInfo::getShape() { + return tensor.getShape(); +} template class RequestValidator { const RequestType& request; @@ -86,6 +102,8 @@ class RequestValidator { RequestValidator() = delete; + const std::string* currentlyValidatedName; + const std::string& getCurrentlyValidatedInputName() const; const InputTensorType& getInputFromIt(const InputIterator& it) const; @@ -102,6 +120,7 @@ class RequestValidator { batchingMode(batchingMode), shapeInfo(shapeInfo) {} + Status validateInferenceTensorBufferType(const InferenceTensor& it) const; Status validateNumberOfInputs() const; Status validateAndGetInput(const RequestType& request, const std::string& name, InputIterator& it, size_t& bufferId); Status checkIfShapeValuesNegative(const InputTensorType& proto) const; @@ -109,7 +128,7 @@ class RequestValidator { Status checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; Status checkBinaryBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; Status checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const; - Status validateTensorContentSize(const InputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const; + Status validateTensorContent(const InputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const; Status validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; Status validatePrecision(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; bool checkIfBinaryInputUsed(const InputTensorType& proto, const std::string inputName) const; @@ -138,6 +157,11 @@ Status RequestValidator +Status RequestValidator::validateRequestCoherency() const { + return StatusCode::OK; +} + template <> Status RequestValidator::validateNumberOfInputs() const { size_t expectedNumberOfInputs = inputsInfo.size(); @@ -163,22 +187,29 @@ Status RequestValidator const std::string& RequestValidator::getCurrentlyValidatedInputName() const { - return it->name(); + return *currentlyValidatedName; } - template <> const std::string& RequestValidator::getCurrentlyValidatedInputName() const { - return it->first; + return *currentlyValidatedName; } +template <> +const std::string& RequestValidator::getCurrentlyValidatedInputName() const { + return *currentlyValidatedName; +} + template <> const KFSTensorInputProto& RequestValidator::getInputFromIt(const KFSInputTensorIteratorType& it) const { return *it; } - template <> const TFSInputTensorType& RequestValidator::getInputFromIt(const TFSInputTensorIteratorType& it) const { return it->second; } +template <> +const InferenceTensor& RequestValidator::getInputFromIt(const InferenceTensor* const& it) const { + return *it; +} template <> Status RequestValidator::validateNumberOfInputs() const { @@ -196,20 +227,33 @@ Status RequestValidator +Status RequestValidator::validateNumberOfInputs() const { + size_t expectedNumberOfInputs = inputsInfo.size(); + if (request.getInputsSize() > 0 && expectedNumberOfInputs == static_cast(request.getInputsSize())) { + return StatusCode::OK; + } + std::stringstream ss; + ss << "Expected: " << expectedNumberOfInputs << "; Actual: " << request.getInputsSize(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of inputs - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_INPUTS, details); +} template <> Status RequestValidator::validateAndGetInput(const TFSRequestType& request, const std::string& name, TFSInputTensorIteratorType& it, size_t& bufferId) { it = request.inputs().find(name); if (it != request.inputs().end()) { + currentlyValidatedName = &name; return StatusCode::OK; } + currentlyValidatedName = nullptr; std::stringstream ss; ss << "Required input: " << name; const std::string details = ss.str(); SPDLOG_DEBUG("[servable name: {} version: {}] Missing input with specific name - {}", servableName, servableVersion, details); return Status(StatusCode::INVALID_MISSING_INPUT, details); } - template <> Status RequestValidator::validateAndGetInput(const KFSRequest& request, const std::string& name, KFSInputTensorIteratorType& it, size_t& bufferId) { it = request.inputs().begin(); @@ -222,8 +266,25 @@ Status RequestValidator +Status RequestValidator::validateAndGetInput(const InferenceRequest& request, const std::string& name, const InferenceTensor*& it, size_t& bufferId) { + if (request.getInput(name.c_str(), &it) != StatusCode::NONEXISTENT_TENSOR) { + currentlyValidatedName = &name; + return StatusCode::OK; + } + + currentlyValidatedName = nullptr; std::stringstream ss; ss << "Required input: " << name; const std::string details = ss.str(); @@ -259,7 +320,6 @@ Status RequestValidator Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const KFSTensorInputProto& proto) const { RequestShapeInfo rsi(proto); @@ -272,6 +332,18 @@ Status RequestValidator +Status RequestValidator::validateNumberOfBinaryInputShapeDimensions(const InferenceTensor& tensor) const { + RequestShapeInfo rsi(tensor); + if (rsi.getShapeSize() != 1) { + std::stringstream ss; + ss << "Expected number of binary input shape dimensions: 1; Actual: " << rsi.getShapeSize() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} template Status RequestValidator::checkBatchSizeMismatch(const InputTensorType& proto, const Dimension& servableBatchSize, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { @@ -342,6 +414,31 @@ Status RequestValidator +Status RequestValidator::checkBinaryBatchSizeMismatch(const InferenceTensor& tensor, const Dimension& servableBatchSize, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { + RequestShapeInfo rsi(tensor); + if (rsi.getDim(0) <= 0) { + std::stringstream ss; + ss << "Batch size must be positive; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + if (servableBatchSize.match(rsi.getDim(0))) { + return StatusCode::OK; + } + if (batchingMode == AUTO) { + finalStatus = StatusCode::BATCHSIZE_CHANGE_REQUIRED; + return StatusCode::OK; + } else if (shapeMode != AUTO) { + std::stringstream ss; + ss << "Expected: " << servableBatchSize.toString() << "; Actual: " << tensor.getBuffer()->getByteSize() << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid batch size - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BATCH_SIZE, details); + } + return StatusCode::OK; +} template Status RequestValidator::checkShapeMismatch(const InputTensorType& proto, const ovms::TensorInfo& inputInfo, const size_t batchSizeIndex, Status& finalStatus, Mode batchingMode, Mode shapeMode) const { @@ -387,8 +484,41 @@ Status RequestValidator:: return StatusCode::OK; } +template +Status RequestValidator::validateInferenceTensorBufferType(const InferenceTensor& it) const { + const Buffer* buffer = it.getBuffer(); + const BufferType bufType = buffer->getBufferType(); + if (bufType < BufferType::OVMS_BUFFERTYPE_CPU || bufType > BufferType::OVMS_BUFFERTYPE_HDDL) { + std::stringstream ss; + ss << "Required input "; + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Has invalid buffer type for input with specific name - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BUFFER_TYPE, details); + + } else { + // Remove this when other buffer types are supported + if (bufType != BufferType::OVMS_BUFFERTYPE_CPU) { + std::stringstream ss; + ss << "Required input "; + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Has invalid buffer type for input with specific name - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_BUFFER_TYPE, details); + } + } + + if (buffer->getBufferType() == BufferType::OVMS_BUFFERTYPE_CPU && buffer->getDeviceId() != std::nullopt && buffer->getDeviceId() != 0) { + std::stringstream ss; + ss << "Required input "; + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Has invalid device id for buffer, input with specific name - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_DEVICE_ID, details); + } + + return StatusCode::OK; +} + template <> -Status RequestValidator::validateTensorContentSize(const TFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { +Status RequestValidator::validateTensorContent(const TFSInputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const { /* int8 data in request.tensor_content uint8 data in request.tensor_content @@ -490,7 +620,7 @@ static size_t getElementsCount(const KFSTensorInputProto& proto, ovms::Precision } template <> -Status RequestValidator::validateTensorContentSize(const KFSTensorInputProto& proto, ovms::Precision expectedPrecision, size_t bufferId) const { +Status RequestValidator::validateTensorContent(const KFSTensorInputProto& proto, ovms::Precision expectedPrecision, size_t bufferId) const { size_t expectedValueCount = 1; for (int i = 0; i < proto.shape().size(); i++) { expectedValueCount *= proto.shape()[i]; @@ -518,6 +648,23 @@ Status RequestValidator +Status RequestValidator::validateTensorContent(const InferenceTensor& tensor, ovms::Precision expectedPrecision, size_t bufferId) const { + size_t expectedValueCount = 1; + for (size_t i = 0; i < tensor.getShape().size(); i++) { + expectedValueCount *= tensor.getShape()[i]; + } + size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); + if (expectedContentSize != tensor.getBuffer()->getByteSize()) { + std::stringstream ss; + ss << "Expected: " << expectedContentSize << " bytes; Actual: " << tensor.getBuffer()->getByteSize() << " bytes; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_CONTENT_SIZE, details); + } + + return validateInferenceTensorBufferType(tensor); +} template <> Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { @@ -552,6 +699,22 @@ Status RequestValidator +Status RequestValidator::validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const InferenceTensor& tensor) const { + // Network and request must have the same number of shape dimensions, higher than 0 + const auto& shape = inputInfo.getShape(); + if (tensor.getShape().size() <= 0 || + shape.size() != static_cast(tensor.getShape().size())) { + std::stringstream ss; + ss << "Expected: " << shape.toString() + << "; Actual: " << tensorShapeToString(tensor.getShape()) + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid number of shape dimensions - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, details); + } + return StatusCode::OK; +} template <> Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const TFSInputTensorType& proto) const { @@ -566,7 +729,6 @@ Status RequestValidator Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const KFSTensorInputProto& proto) const { if (proto.datatype() != ovmsPrecisionToKFSPrecision(inputInfo.getPrecision())) { @@ -580,6 +742,19 @@ Status RequestValidator +Status RequestValidator::validatePrecision(const ovms::TensorInfo& inputInfo, const InferenceTensor& tensor) const { + if (tensor.getDataType() != getPrecisionAsOVMSDataType(inputInfo.getPrecision())) { + std::stringstream ss; + ss << "Expected: " << inputInfo.getPrecisionAsString() + << "; Actual: " << tensor.getDataType() + << "; input name: " << getCurrentlyValidatedInputName(); + const std::string details = ss.str(); + SPDLOG_DEBUG("[servable name: {} version: {}] Invalid precision - {}", servableName, servableVersion, details); + return Status(StatusCode::INVALID_PRECISION, details); + } + return StatusCode::OK; +} static Mode getShapeMode(const shapes_info_map_t& shapeInfo, const std::string& name) { if (shapeInfo.size() == 0) { @@ -603,7 +778,6 @@ bool RequestValidator bool RequestValidator::checkIfBinaryInputUsed(const KFSTensorInputProto& proto, const std::string inputName) const { if (proto.datatype() == "BYTES") { @@ -612,6 +786,11 @@ bool RequestValidator +bool RequestValidator::checkIfBinaryInputUsed(const InferenceTensor& tensor, const std::string inputName) const { + // TODO no strig no bytes currently, will implement one of those types with binary input. + return false; +} template Status RequestValidator::validate() { @@ -672,7 +851,7 @@ Status RequestValidator:: status = checkShapeMismatch(proto, *inputInfo, batchIndex.value(), finalStatus, batchingMode, shapeMode); if (!status.ok()) return status; - status = validateTensorContentSize(proto, inputInfo->getPrecision(), bufferId); + status = validateTensorContent(proto, inputInfo->getPrecision(), bufferId); if (!status.ok()) return status; } @@ -690,5 +869,11 @@ Status validate(const KFSRequest& request, const tensor_map_t& inputsInfo, const OVMS_PROFILE_FUNCTION(); return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); } + +template <> +Status validate(const InferenceRequest& request, const tensor_map_t& inputsInfo, const std::string& servableName, const model_version_t servableVersion, const std::set& optionalAllowedInputNames, const Mode batchingMode, const shapes_info_map_t& shapeInfo) { + OVMS_PROFILE_FUNCTION(); + return RequestValidator(request, inputsInfo, servableName, servableVersion, optionalAllowedInputNames, batchingMode, shapeInfo).validate(); +} } // namespace request_validation_utils } // namespace ovms diff --git a/src/serialization.hpp b/src/serialization.hpp index 10d52bc270..c5248fde1b 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -28,6 +28,7 @@ #pragma GCC diagnostic pop #include "inferenceresponse.hpp" +#include "inferencetensor.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "pocapiinternal.hpp" #include "profiler.hpp" @@ -35,7 +36,6 @@ #include "tensorinfo.hpp" namespace ovms { -class InferenceTensor; template class OutputGetter { diff --git a/src/server.cpp b/src/server.cpp index fb35a7f4fe..0a97dec2b5 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -364,5 +364,4 @@ int Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { } return EXIT_SUCCESS; } - } // namespace ovms diff --git a/src/statefulmodelinstance.hpp b/src/statefulmodelinstance.hpp index 4f1e3ddbbb..f6d4c30a13 100644 --- a/src/statefulmodelinstance.hpp +++ b/src/statefulmodelinstance.hpp @@ -24,8 +24,6 @@ #include "sequence_manager.hpp" namespace ovms { -class MetricRegistry; -class MetricConfig; class ModelConfig; class StatefulModelInstance : public ModelInstance { static const std::set SPECIAL_INPUT_NAMES; diff --git a/src/status.cpp b/src/status.cpp index f500292c64..504894d792 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -94,6 +94,8 @@ const std::unordered_map Status::statusMess {StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Invalid number of shape dimensions"}, {StatusCode::INVALID_BATCH_SIZE, "Invalid input batch size"}, {StatusCode::INVALID_SHAPE, "Invalid input shape"}, + {StatusCode::INVALID_BUFFER_TYPE, "Invalid input buffer type"}, + {StatusCode::INVALID_DEVICE_ID, "Invalid input buffer device id"}, {StatusCode::INVALID_PRECISION, "Invalid input precision"}, {StatusCode::INVALID_VALUE_COUNT, "Invalid number of values in tensor proto container"}, {StatusCode::INVALID_CONTENT_SIZE, "Invalid content size of tensor proto"}, diff --git a/src/status.hpp b/src/status.hpp index 6c7a26a7f3..9abfdefe2b 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -97,6 +97,8 @@ enum class StatusCode { INVALID_VALUE_COUNT, /*!< Invalid value count error status for uint16 and half float data types */ INVALID_CONTENT_SIZE, /*!< Invalid content size error status for types using tensor_content() */ INVALID_MESSAGE_STRUCTURE, /*!< Buffers can't be both in raw_input_content & input tensor content */ + INVALID_BUFFER_TYPE, /*!< Invalid buffer type */ + INVALID_DEVICE_ID, /*!< Invalid buffer device id */ // Deserialization OV_UNSUPPORTED_DESERIALIZATION_PRECISION, /*!< Unsupported deserialization precision, theoretically should never be returned since ModelInstance::validation checks against model precision */ diff --git a/src/test/capi_predict_validation_test.cpp b/src/test/capi_predict_validation_test.cpp new file mode 100644 index 0000000000..f78ae6c286 --- /dev/null +++ b/src/test/capi_predict_validation_test.cpp @@ -0,0 +1,829 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include + +#include +#include + +#include "../buffer.hpp" +#include "../inferencerequest.hpp" +#include "../modelconfig.hpp" +#include "../predict_request_validation_utils.hpp" +#include "test_utils.hpp" + +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::ReturnRef; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" + +class CAPIPredictValidation : public ::testing::Test { +protected: + std::unique_ptr ieCore; + std::unique_ptr> instance; + ovms::InferenceRequest request{"model_name", 1}; + ovms::ModelConfig modelConfig{"model_name", "model_path"}; + ovms::tensor_map_t servableInputs; + bool createCopy{false}; + uint32_t decrementBufferSize{0}; + std::vector requestData{10000000}; + + void SetUp() override { + ieCore = std::make_unique(); + instance = std::make_unique>(*ieCore); + std::iota(requestData.begin(), requestData.end(), 1.0); + + servableInputs = ovms::tensor_map_t({ + {"Input_FP32_1_224_224_3_NHWC", + std::make_shared("Input_FP32_1_3_224_224_NHWC", ovms::Precision::FP32, ovms::shape_t{1, 224, 224, 3}, ovms::Layout{"NHWC"})}, + {"Input_U8_1_3_62_62_NCHW", + std::make_shared("Input_U8_1_3_62_62_NCHW", ovms::Precision::U8, ovms::shape_t{1, 3, 62, 62}, ovms::Layout{"NCHW"})}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::make_shared("Input_I64_1_6_128_128_16_NCDHW", ovms::Precision::I64, ovms::shape_t{1, 6, 128, 128, 16}, ovms::Layout{"NCDHW"})}, + {"Input_U16_1_2_8_4_NCHW", + std::make_shared("Input_U16_1_2_8_4_NCHW", ovms::Precision::U16, ovms::shape_t{1, 2, 8, 4}, ovms::Layout{"NCHW"})}, + }); + + ON_CALL(*instance, getInputsInfo()).WillByDefault(ReturnRef(servableInputs)); + ON_CALL(*instance, getBatchSize()).WillByDefault(Return(1)); + ON_CALL(*instance, getModelConfig()).WillByDefault(ReturnRef(modelConfig)); + + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + } +}; + +TEST_F(CAPIPredictValidation, ValidRequest) { + auto status = instance->mockValidate(&request); + EXPECT_TRUE(status.ok()) << status.string(); +} + +TEST_F(CAPIPredictValidation, InvalidPrecision) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, static_cast(99)}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_PRECISION) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestNotEnoughInputs) { + request.removeInput("Input_U16_1_2_8_4_NCHW"); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_INPUTS) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestTooManyInputs) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_INPUTS) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestMissingInputName) { + preparePredictRequest(request, + {{"BadInput_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_MISSING_INPUT); +} + +TEST_F(CAPIPredictValidation, RequestWrongInputName) { + request.removeInput("Input_U16_1_2_8_4_NCHW"); + preparePredictRequest(request, + {{"BADInput_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_INPUTS) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestTooManyShapeDimensions) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestNotEnoughShapeDimensions) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62, 5}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16, 6}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4, 5}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongBatchSize) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{2, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{2, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{2, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{2, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); // dim(0) is batch size + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongBatchSizeAuto) { + modelConfig.setBatchingParams("auto"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{2, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{2, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{2, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{2, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongAndCorrectBatchSizeAuto) { + modelConfig.setBatchingParams("auto"); + + // First is incorrect, second is correct + preparePredictRequest(request, {{"im_data", {{3, 3, 800, 1344}, ovms::Precision::FP32}}, {"im_info", {{1, 3}, ovms::Precision::FP32}}}, requestData); + + servableInputs.clear(); + servableInputs = ovms::tensor_map_t{ + {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, + {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{1, 3}, ovms::Layout{"NC"})}, + }; + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED); + + preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, {"im_info", {{3, 3}, ovms::Precision::FP32}}}, requestData); + + status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongAndCorrectShapeAuto) { + modelConfig.parseShapeParameter("auto"); + preparePredictRequest(request, {{"im_data", {{1, 3, 900, 1344}, ovms::Precision::FP32}}, {"im_info", {{1, 3}, ovms::Precision::FP32}}}, requestData); + + // First is incorrect, second is correct + servableInputs.clear(); + servableInputs = ovms::tensor_map_t{ + {"im_data", std::make_shared("im_data", ovms::Precision::FP32, ovms::shape_t{1, 3, 800, 1344}, ovms::Layout{"NCHW"})}, + {"im_info", std::make_shared("im_info", ovms::Precision::FP32, ovms::shape_t{1, 3}, ovms::Layout{"NC"})}, + }; + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED) << status.string(); + + // First is correct, second is incorrect + preparePredictRequest(request, {{"im_data", {{1, 3, 800, 1344}, ovms::Precision::FP32}}, {"im_info", {{1, 6}, ovms::Precision::FP32}}}, requestData); + + status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED) << status.string(); +} +TEST_F(CAPIPredictValidation, RequestValidBatchSizeAuto) { + modelConfig.setBatchingParams("auto"); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValues) { + modelConfig.setBatchingParams("auto"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 17}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesTwoInputsOneWrong) { // one input fails validation, request denied + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 17}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesAuto) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 61, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesAutoTwoInputs) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\", \"Input_U16_1_2_8_4_NCHW\": \"auto\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 61, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 2, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesAutoNoNamedInput) { + modelConfig.parseShapeParameter("auto"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 214, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 2, 61, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 1, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 3, 2, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesAutoFirstDim) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"auto\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{2, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED); +} + +TEST_F(CAPIPredictValidation, RequestValidShapeValuesTwoInputsFixed) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\", \"Input_U16_1_2_8_4_NCHW\": \"(1,2,8,4)\"}"); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesFixed) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 4, 63, 63}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongShapeValuesFixedFirstDim) { + modelConfig.parseShapeParameter("{\"Input_U8_1_3_62_62_NCHW\": \"(1,3,62,62)\"}"); + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{2, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorrectContentSize) { + decrementBufferSize = 1; + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorrectContentSizeZero) { + decrementBufferSize = 602112; + + servableInputs = ovms::tensor_map_t({{"Input_FP32_1_224_224_3_NHWC", + std::make_shared("Input_FP32_1_3_224_224_NHWC", ovms::Precision::FP32, ovms::shape_t{1, 224, 224, 3}, ovms::Layout{"NHWC"})}}); + ON_CALL(*instance, getInputsInfo()).WillByDefault(ReturnRef(servableInputs)); + + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}}, + requestData, decrementBufferSize); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorrectBufferType) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, static_cast(999)); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestNegativeBufferType) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, static_cast(-22)); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorectDeviceId) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_CPU, 1); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_DEVICE_ID) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorectBufferType) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_GPU); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestCorectDeviceId) { + GTEST_SKIP() << "Enable when Other buffer types are supported"; + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_GPU, 1); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestNotNullDeviceId) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_CPU, 1); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_DEVICE_ID) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorrectContentSizeBatchAuto) { + modelConfig.setBatchingParams("auto"); + decrementBufferSize = 1; + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestIncorrectContentSizeShapeAuto) { + modelConfig.parseShapeParameter("auto"); + decrementBufferSize = 1; + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::U8}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData, decrementBufferSize); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); +} + +class CAPIPredictValidationInputTensorContent : public ::testing::TestWithParam { +protected: + std::unique_ptr ieCore; + std::unique_ptr> instance; + ovms::InferenceRequest request{"model_name", 1}; + + ovms::ModelConfig modelConfig{"model_name", "model_path"}; + ovms::tensor_map_t servableInputs; + std::vector requestData{10000000}; + + void SetUp() override { + ieCore = std::make_unique(); + instance = std::make_unique>(*ieCore); + std::iota(requestData.begin(), requestData.end(), 1.0); + } +}; + +TEST_P(CAPIPredictValidationInputTensorContent, RequestCorrectContentSizeInputTensorContent) { + ovms::Precision testedPrecision = GetParam(); + const std::string inputName = "someName"; + servableInputs = ovms::tensor_map_t({ + {inputName, + std::make_shared(inputName, testedPrecision, ovms::shape_t{1, 224, 224, 3}, ovms::Layout{"NHWC"})}, + }); + ON_CALL(*instance, getInputsInfo()).WillByDefault(ReturnRef(servableInputs)); + ON_CALL(*instance, getBatchSize()).WillByDefault(Return(1)); + ON_CALL(*instance, getModelConfig()).WillByDefault(ReturnRef(modelConfig)); + preparePredictRequest(request, + {{inputName, + std::tuple{{1, 224, 224, 3}, testedPrecision}}}, + requestData, // data, + false); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); +} + +TEST_F(CAPIPredictValidation, RequestWrongPrecision) { + preparePredictRequest(request, + {{"Input_FP32_1_224_224_3_NHWC", + std::tuple{{1, 224, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_1_3_62_62_NCHW", + std::tuple{{1, 3, 62, 62}, ovms::Precision::Q78}}, + {"Input_I64_1_6_128_128_16_NCDHW", + std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, + {"Input_U16_1_2_8_4_NCHW", + std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_PRECISION) << status.string(); +} + +class CAPIPredictValidationArbitraryBatchPosition : public CAPIPredictValidation { +protected: + void SetUp() override { + CAPIPredictValidation::SetUp(); + + servableInputs = ovms::tensor_map_t({ + {"Input_FP32_224_224_3_1_HWCN", + std::make_shared("Input_FP32_224_224_3_1_HWCN", ovms::Precision::FP32, ovms::shape_t{224, 224, 3, 1}, ovms::Layout{"HWCN"})}, + {"Input_U8_3_1_128_CNH", + std::make_shared("Input_U8_3_1_128_CNH", ovms::Precision::U8, ovms::shape_t{3, 1, 128}, ovms::Layout{"CNH"})}, + }); + + preparePredictRequest(request, + { + {"Input_FP32_224_224_3_1_HWCN", + std::tuple{{224, 224, 3, 1}, ovms::Precision::FP32}}, + {"Input_U8_3_1_128_CNH", + std::tuple{{3, 1, 128}, ovms::Precision::U8}}, + }, + requestData); + } +}; + +TEST_F(CAPIPredictValidationArbitraryBatchPosition, Valid) { + auto status = instance->mockValidate(&request); + EXPECT_TRUE(status.ok()); +} + +TEST_F(CAPIPredictValidationArbitraryBatchPosition, RequestWrongBatchSize) { + // Edit fourth dimension (N), expect validator to report wrong batch size instead of wrong shape. + preparePredictRequest(request, + { + {"Input_FP32_224_224_3_1_HWCN", + std::tuple{{224, 224, 3, 10}, ovms::Precision::FP32}}, + {"Input_U8_3_1_128_CNH", + std::tuple{{3, 1, 128}, ovms::Precision::U8}}, + }, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidationArbitraryBatchPosition, RequestWrongBatchSizeAuto) { + modelConfig.setBatchingParams("auto"); + // Edit fourth dimension (N), expect validator to report batch size change request instead of reshape request. + preparePredictRequest(request, + { + {"Input_FP32_224_224_3_1_HWCN", + std::tuple{{224, 224, 3, 10}, ovms::Precision::FP32}}, + {"Input_U8_3_1_128_CNH", + std::tuple{{3, 1, 128}, ovms::Precision::U8}}, + }, + requestData); + + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::BATCHSIZE_CHANGE_REQUIRED) << status.string(); +} + +TEST_F(CAPIPredictValidationArbitraryBatchPosition, RequestWrongShapeValues) { + // Edit first dimension (H), expect validator to report wrong shape instead of wrong batch size. + preparePredictRequest(request, + { + {"Input_FP32_224_224_3_1_HWCN", + std::tuple{{221, 224, 3, 1}, ovms::Precision::FP32}}, + {"Input_U8_3_1_128_CNH", + std::tuple{{3, 1, 128}, ovms::Precision::U8}}, + }, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidationArbitraryBatchPosition, RequestWrongShapeValuesAuto) { + modelConfig.parseShapeParameter("auto"); + // Edit first dimension (H), expect validator to report reshape request instead of requesting batch size change. + preparePredictRequest(request, + { + {"Input_FP32_224_224_3_1_HWCN", + std::tuple{{10, 224, 3, 1}, ovms::Precision::FP32}}, + {"Input_U8_3_1_128_CNH", + std::tuple{{3, 1, 128}, ovms::Precision::U8}}, + }, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::RESHAPE_REQUIRED) << status.string(); +} + +class CAPIPredictValidationDynamicModel : public CAPIPredictValidation { +protected: + void SetUp() override { + CAPIPredictValidation::SetUp(); + + servableInputs = ovms::tensor_map_t({{"Input_FP32_any_224:512_224:512_3_NHWC", + std::make_shared("Input_FP32_any_224:512_224:512_3_NHWC", ovms::Precision::FP32, ovms::Shape{ovms::Dimension::any(), {224, 512}, {224, 512}, 3}, ovms::Layout{"NHWC"})}, + {"Input_U8_100:200_any_CN", + std::make_shared("Input_U8_100:200_any_CN", ovms::Precision::U8, ovms::Shape{{100, 200}, ovms::Dimension::any()}, ovms::Layout{"CN"})}}); + + ON_CALL(*instance, getBatchSize()).WillByDefault(Return(ovms::Dimension::any())); + + const ovms::dimension_value_t requestBatchSize = 16; + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{requestBatchSize, 300, 320, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{101, requestBatchSize}, ovms::Precision::U8}}, + }, + requestData); + } +}; + +TEST_F(CAPIPredictValidationDynamicModel, ValidRequest) { + auto status = instance->mockValidate(&request); + EXPECT_TRUE(status.ok()); +} + +TEST_F(CAPIPredictValidationDynamicModel, RequestBatchNotInRangeFirstPosition) { + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{16, 300, 320, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{101, 16}, ovms::Precision::U8}}, + }, + requestData); + + servableInputs["Input_FP32_any_224:512_224:512_3_NHWC"] = std::make_shared("Input_FP32_any_224:512_224:512_3_NHWC", ovms::Precision::FP32, ovms::Shape{{1, 5}, {224, 512}, {224, 512}, 3}, ovms::Layout{"NHWC"}); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); +} + +TEST_F(CAPIPredictValidationDynamicModel, RequestDimensionNotInRangeFirstPosition) { + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{16, 300, 320, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{98, 1}, ovms::Precision::U8}}, + }, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidationDynamicModel, RequestBatchNotInRangeSecondPosition) { + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{16, 300, 320, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{100, 98}, ovms::Precision::U8}}, + }, + requestData); + servableInputs["Input_U8_100:200_any_CN"] = std::make_shared("Input_U8_100:200_any_CN", ovms::Precision::U8, ovms::Shape{{100, 200}, {1, 5}}, ovms::Layout{"CN"}); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE) << status.string(); +} + +TEST_F(CAPIPredictValidationDynamicModel, RequestDimensionNotInRangeSecondPosition) { + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{1, 223, 224, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{101, 16}, ovms::Precision::U8}}, + }, + requestData); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_SHAPE) << status.string(); +} + +TEST_F(CAPIPredictValidationDynamicModel, RequestDimensionInRangeWrongTensorContent) { + decrementBufferSize = 1; + preparePredictRequest(request, + { + {"Input_FP32_any_224:512_224:512_3_NHWC", + std::tuple{{16, 300, 320, 3}, ovms::Precision::FP32}}, + {"Input_U8_100:200_any_CN", + std::tuple{{101, 16}, ovms::Precision::U8}}, + }, + requestData, decrementBufferSize); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); +} + +// TODO: Add request parameters name validation tests + +class CAPIPredictValidationPrecision : public ::testing::TestWithParam { +protected: + std::vector requestData{10000000}; + + void SetUp() override { + std::iota(requestData.begin(), requestData.end(), 1.0); + auto precision = ovms::Precision::FP32; + mockedInputsInfo[tensorName] = std::make_shared(tensorName, precision, ovms::shape_t{1, DUMMY_MODEL_INPUT_SIZE}, ovms::Layout{"NC"}); + } + ovms::InferenceRequest request{"model_name", 1}; + + const char* tensorName = DUMMY_MODEL_INPUT_NAME; + ovms::tensor_map_t mockedInputsInfo; +}; + +TEST_P(CAPIPredictValidationPrecision, ValidPrecisions) { + ovms::Precision testedPrecision = GetParam(); + mockedInputsInfo[tensorName]->setPrecision(testedPrecision); + preparePredictRequest(request, + { + {tensorName, + std::tuple{{1, DUMMY_MODEL_INPUT_SIZE}, testedPrecision}}, + }, + requestData); + auto status = ovms::request_validation_utils::validate(request, mockedInputsInfo, "dummy", ovms::model_version_t{1}); + EXPECT_EQ(status, ovms::StatusCode::OK) << "Precision validation failed:" + << toString(testedPrecision) + << " should pass validation"; +} + +INSTANTIATE_TEST_SUITE_P( + Test, + CAPIPredictValidationPrecision, + ::testing::ValuesIn(SUPPORTED_CAPI_INPUT_PRECISIONS), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); + +INSTANTIATE_TEST_SUITE_P( + Test, + CAPIPredictValidationInputTensorContent, + ::testing::ValuesIn(SUPPORTED_CAPI_INPUT_PRECISIONS_TENSORINPUTCONTENT), + [](const ::testing::TestParamInfo& info) { + return toString(info.param); + }); + +#pragma GCC diagnostic pop diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index b394535f71..9cea22f66e 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -100,7 +100,8 @@ class CAPIPredict : public ::testing::TestWithParam { Layout{"NC"}); SetUpTensorProto(getPrecisionAsOVMSDataType(PRECISION)); } - void SetUpTensorProto(DataType dataType) { + + void SetUpTensorProto(OVMS_DataType dataType) { std::array shape{1, DUMMY_MODEL_INPUT_SIZE}; tensorCapi = std::make_unique(dataType, shape.data(), diff --git a/src/test/inferencerequest_test.cpp b/src/test/inferencerequest_test.cpp index d7ccaa4f55..c92ff0efd1 100644 --- a/src/test/inferencerequest_test.cpp +++ b/src/test/inferencerequest_test.cpp @@ -42,7 +42,7 @@ namespace { const std::string MODEL_NAME{"SomeModelName"}; const uint64_t MODEL_VERSION{42}; const std::string PARAMETER_NAME{"SEQUENCE_ID"}; // TODO check if in ovms there is such constant -const DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; +const OVMS_DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; const uint32_t PARAMETER_VALUE{13}; const uint32_t PRIORITY{7}; @@ -52,7 +52,7 @@ const std::string INPUT_NAME{"NOT_RANDOM_NAME"}; const ovms::shape_t INPUT_SHAPE{1, 3, 220, 230}; const std::array INPUT_DATA{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; constexpr size_t INPUT_DATA_BYTESIZE{INPUT_DATA.size() * sizeof(float)}; -const DataType DATATYPE{OVMS_DATATYPE_FP32}; +const OVMS_DataType DATATYPE{OVMS_DATATYPE_FP32}; } // namespace TEST(InferenceParameter, CreateParameter) { diff --git a/src/test/predict_validation_test.cpp b/src/test/predict_validation_test.cpp index 7889867dfe..5f49391e78 100644 --- a/src/test/predict_validation_test.cpp +++ b/src/test/predict_validation_test.cpp @@ -32,32 +32,17 @@ using ::testing::ReturnRef; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" -class MockModelInstance : public ovms::ModelInstance { -public: - MockModelInstance(ov::Core& ieCore) : - ModelInstance("UNUSED_NAME", 42, ieCore) {} - MOCK_METHOD(const ovms::tensor_map_t&, getInputsInfo, (), (const, override)); - MOCK_METHOD(ovms::Dimension, getBatchSize, (), (const, override)); - MOCK_METHOD(const ovms::ModelConfig&, getModelConfig, (), (const, override)); - const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { - return validate(request); - } - const ovms::Status mockValidate(const ::KFSRequest* request) { - return validate(request); - } -}; - class TfsPredictValidation : public ::testing::Test { protected: std::unique_ptr ieCore; - std::unique_ptr> instance; + std::unique_ptr> instance; tensorflow::serving::PredictRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; void SetUp() override { ieCore = std::make_unique(); - instance = std::make_unique>(*ieCore); + instance = std::make_unique>(*ieCore); servableInputs = ovms::tensor_map_t({ {"Input_FP32_1_224_224_3_NHWC", @@ -678,14 +663,14 @@ INSTANTIATE_TEST_SUITE_P( class KFSPredictValidation : public ::testing::Test { protected: std::unique_ptr ieCore; - std::unique_ptr> instance; + std::unique_ptr> instance; ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; void SetUp() override { ieCore = std::make_unique(); - instance = std::make_unique>(*ieCore); + instance = std::make_unique>(*ieCore); servableInputs = ovms::tensor_map_t({ {"Input_FP32_1_224_224_3_NHWC", @@ -1002,14 +987,14 @@ TEST_F(KFSPredictValidation, RequestIncorrectContentSizeShapeAuto) { class KFSPredictValidationInputTensorContent : public ::testing::TestWithParam { protected: std::unique_ptr ieCore; - std::unique_ptr> instance; + std::unique_ptr> instance; ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; void SetUp() override { ieCore = std::make_unique(); - instance = std::make_unique>(*ieCore); + instance = std::make_unique>(*ieCore); } }; @@ -1058,14 +1043,14 @@ TEST_P(KFSPredictValidationInputTensorContent, RequestCorrectContentSizeInputTen class KFSPredictValidationInputTensorContentNegative : public ::testing::Test { protected: std::unique_ptr ieCore; - std::unique_ptr> instance; + std::unique_ptr> instance; ::KFSRequest request; ovms::ModelConfig modelConfig{"model_name", "model_path"}; ovms::tensor_map_t servableInputs; void SetUp() override { ieCore = std::make_unique(); - instance = std::make_unique>(*ieCore); + instance = std::make_unique>(*ieCore); servableInputs = ovms::tensor_map_t({ {"Input_FP32_1_224_224_3_NHWC", diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 6845ff9b34..801a6bcbff 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -17,7 +17,10 @@ #include +#include "../capi_frontend/capi_utils.hpp" +#include "../inferenceparameter.hpp" #include "../kfs_frontend/kfs_utils.hpp" +#include "../pocapiinternal.hpp" #include "../prediction_service_utils.hpp" #include "../tensorinfo.hpp" #include "../tfs_frontend/tfs_utils.hpp" @@ -35,6 +38,13 @@ void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, c } } +void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs, const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { + request.removeAllInputs(); + for (auto const& it : requestInputs) { + prepareCAPIInferInputTensor(request, it.first, it.second, data, decrementBufferSize, bufferType, deviceId); + } +} + void preparePredictRequest(tensorflow::serving::PredictRequest& request, inputs_info_t requestInputs, const std::vector& data) { request.mutable_inputs()->clear(); for (auto const& it : requestInputs) { @@ -374,6 +384,32 @@ void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, data, putBufferInInputTensorContent); } +void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, + const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { + auto [shape, type] = inputInfo; + prepareCAPIInferInputTensor(request, name, + {shape, getPrecisionAsOVMSDataType(type)}, + data, decrementBufferSize, bufferType, deviceId); +} + +void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, + const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { + auto [shape, datatype] = inputInfo; + size_t elementsCount = 1; + for (auto const& dim : shape) { + ASSERT_GE(dim, 0); + elementsCount *= dim; + } + + request.addInput(name.c_str(), datatype, shape.data(), shape.size()); + + size_t dataSize = elementsCount * ovms::DataTypeToByteSize(datatype); + if (decrementBufferSize) + dataSize -= decrementBufferSize; + + request.setInputBuffer(name.c_str(), data.data(), dataSize, bufferType, deviceId); +} + void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data, bool putBufferInInputTensorContent) { auto it = request.mutable_inputs()->begin(); diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 3aa3c42ad1..81ec3d1cb7 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -35,8 +35,10 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop #include "../execution_context.hpp" +#include "../inferencerequest.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../metric_registry.hpp" +#include "../modelinstance.hpp" #include "../modelmanager.hpp" #include "../node_library.hpp" #include "../tensorinfo.hpp" @@ -158,8 +160,16 @@ void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::tuple& inputInfo, const std::vector& data = {}, bool putBufferInInputTensorContent = false); +void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, + const std::vector& data, uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); +void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, + const std::vector& data, uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); + void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, const std::vector& data = {}, bool putBufferInInputTensorContent = false); +void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs, const std::vector& data, + uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); + void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize); @@ -298,6 +308,24 @@ class ConstructorEnabledModelManager : public ovms::ModelManager { } }; +class MockedMetadataModelIns : public ovms::ModelInstance { +public: + MockedMetadataModelIns(ov::Core& ieCore) : + ModelInstance("UNUSED_NAME", 42, ieCore) {} + MOCK_METHOD(const ovms::tensor_map_t&, getInputsInfo, (), (const, override)); + MOCK_METHOD(ovms::Dimension, getBatchSize, (), (const, override)); + MOCK_METHOD(const ovms::ModelConfig&, getModelConfig, (), (const, override)); + const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { + return validate(request); + } + const ovms::Status mockValidate(const ::KFSRequest* request) { + return validate(request); + } + const ovms::Status mockValidate(const ovms::InferenceRequest* request) { + return validate(request); + } +}; + class ResourcesAccessModelManager : public ConstructorEnabledModelManager { public: int getResourcesSize() { @@ -502,4 +530,44 @@ static const std::vector UNSUPPORTED_KFS_INPUT_PRECISIONS_TENSO // ovms::Precision::CUSTOM) }; +static const std::vector SUPPORTED_CAPI_INPUT_PRECISIONS_TENSORINPUTCONTENT{ + // ovms::Precision::UNDEFINED, + // ovms::Precision::MIXED, + ovms::Precision::FP64, + ovms::Precision::FP32, + // ovms::Precision::FP16, + // ovms::Precision::Q78, + ovms::Precision::I16, + ovms::Precision::U8, + ovms::Precision::I8, + ovms::Precision::U16, + ovms::Precision::I32, + ovms::Precision::I64, + ovms::Precision::U32, + ovms::Precision::U64, + // ovms::Precision::BIN, + ovms::Precision::BOOL + // ovms::Precision::CUSTOM) +}; + +static const std::vector UNSUPPORTED_CAPI_INPUT_PRECISIONS_TENSORINPUTCONTENT{ + ovms::Precision::UNDEFINED, + ovms::Precision::MIXED, + // ovms::Precision::FP64, + // ovms::Precision::FP32, + ovms::Precision::FP16, + ovms::Precision::Q78, + // ovms::Precision::I16, + // ovms::Precision::U8, + // ovms::Precision::I8, + // ovms::Precision::U16, + // ovms::Precision::I32, + // ovms::Precision::I64, + // ovms::Precision::U32, + // ovms::Precision::U64, + ovms::Precision::BIN, + // ovms::Precision::BOOL + // ovms::Precision::CUSTOM) +}; + void randomizePort(std::string& port); From 14418226563ea8eb39591068e78c1f2037530711 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Mon, 28 Nov 2022 17:37:15 +0100 Subject: [PATCH 094/130] C-API cpp integration without infer (#1545) * Change coverage threshold Increase for lines Decrease for functions. For some reason lambda in InferenceResponse::addOutput is not counted event tough it is clearly being used as stated by coverage tool. No other uncovered functions were added. * Review 2 --- check_coverage.bat | 4 +- src/buffer.cpp | 11 +- src/buffer.hpp | 2 +- src/inferenceparameter.cpp | 6 +- src/inferenceresponse.cpp | 62 ++++++-- src/inferenceresponse.hpp | 14 +- src/inferencetensor.cpp | 2 +- src/pocapi.cpp | 158 +++++++++++++++++++++ src/pocapi.hpp | 24 ++-- src/serialization.hpp | 6 +- src/status.hpp | 2 +- src/test/c_api_tests.cpp | 196 ++++++++++++++++++++++++++ src/test/inferencerequest_test.cpp | 16 ++- src/test/ovinferrequestqueue_test.cpp | 2 +- src/test/serialization_tests.cpp | 9 +- 15 files changed, 464 insertions(+), 50 deletions(-) diff --git a/check_coverage.bat b/check_coverage.bat index 1847dc8387..9783dcae6a 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,8 +5,8 @@ #MIN_FUNCTION_COV=87.4 #Rhel -MIN_LINES_COV=73.7 -MIN_FUNCTION_COV=74.1 +MIN_LINES_COV=73.8 +MIN_FUNCTION_COV=74.0 diff --git a/src/buffer.cpp b/src/buffer.cpp index 074eac8912..fce2841f8f 100644 --- a/src/buffer.cpp +++ b/src/buffer.cpp @@ -17,8 +17,6 @@ #include -#include "logging.hpp" - namespace ovms { Buffer::Buffer(const void* pptr, size_t byteSize, BufferType bufferType, std::optional bufferDeviceId, bool createCopy) : ptr(createCopy ? nullptr : pptr), @@ -32,17 +30,22 @@ Buffer::Buffer(const void* pptr, size_t byteSize, BufferType bufferType, std::op ownedCopy = std::make_unique(byteSize); std::memcpy(ownedCopy.get(), pptr, byteSize); } + const void* Buffer::data() const { return (ptr != nullptr) ? ptr : ownedCopy.get(); } + size_t Buffer::getByteSize() const { return byteSize; } -const BufferType Buffer::getBufferType() const { - return bufferType; + +BufferType Buffer::getBufferType() const { + return this->bufferType; } + const std::optional& Buffer::getDeviceId() const { return bufferDeviceId; } + Buffer::~Buffer() = default; } // namespace ovms diff --git a/src/buffer.hpp b/src/buffer.hpp index c28838acf1..007765230a 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -31,7 +31,7 @@ class Buffer { Buffer(const void* ptr, size_t byteSize, BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional bufferDeviceId = std::nullopt, bool createCopy = false); ~Buffer(); const void* data() const; - const BufferType getBufferType() const; + BufferType getBufferType() const; const std::optional& getDeviceId() const; size_t getByteSize() const; }; diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index 6219afcb98..ebb4f5351f 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -40,7 +40,7 @@ size_t DataTypeToByteSize(OVMS_DataType datatype) { {OVMS_DATATYPE_FP32, 4}, {OVMS_DATATYPE_FP64, 8}, {OVMS_DATATYPE_BF16, 2}, - // {"BYTES", }, + // {"BYTES", }, TODO }; auto it = datatypeSizeMap.find(datatype); if (it == datatypeSizeMap.end()) { @@ -54,16 +54,18 @@ InferenceParameter::InferenceParameter(const char* name, OVMS_DataType datatype, datatype(datatype), data(reinterpret_cast(data), DataTypeToByteSize(datatype)) { } -// InferenceParameter::InferenceParameter(const char* name, DataType datatype, const void* data, size_t byteSize); + const std::string& InferenceParameter::getName() const { return this->name; } OVMS_DataType InferenceParameter::getDataType() const { return this->datatype; } + size_t InferenceParameter::getByteSize() const { return this->data.size(); } + const void* InferenceParameter::getData() const { return reinterpret_cast(this->data.c_str()); } diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp index 4f5241f229..f7d88353a3 100644 --- a/src/inferenceresponse.cpp +++ b/src/inferenceresponse.cpp @@ -15,8 +15,10 @@ //***************************************************************************** #include "inferenceresponse.hpp" +#include #include #include +#include #include "inferenceparameter.hpp" #include "inferencetensor.hpp" @@ -30,32 +32,62 @@ InferenceResponse::InferenceResponse(const std::string& servableName, model_vers const std::string& InferenceResponse::getServableName() const { return this->servableName; } + model_version_t InferenceResponse::getServableVersion() const { return this->servableVersion; } + Status InferenceResponse::addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount) { // TODO insert tensor with wrong shape/datatype/name/dimcount - // TODO reuse infer response/request - auto [it, emplaced] = outputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); - return emplaced ? StatusCode::OK : StatusCode::DOUBLE_TENSOR_INSERT; + auto it = std::find_if(outputs.begin(), + outputs.end(), + [&name](const std::pair& pair) { + return name == pair.first; + }); + if (outputs.end() != it) { + return StatusCode::DOUBLE_TENSOR_INSERT; + } + + auto pair = std::pair(name, InferenceTensor{datatype, shape, dimCount}); + outputs.push_back(std::move(pair)); + return StatusCode::OK; } -Status InferenceResponse::getOutput(const char* name, InferenceTensor** tensor) { - auto it = outputs.find(name); - if (outputs.end() == it) { + +Status InferenceResponse::getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor) { + if (outputs.size() <= id) { *tensor = nullptr; return StatusCode::NONEXISTENT_TENSOR; } - *tensor = &it->second; + *name = &(outputs[id].first); + *tensor = &(outputs[id].second); return StatusCode::OK; } + Status InferenceResponse::addParameter(const char* parameterName, OVMS_DataType datatype, const void* data) { - auto [it, emplaced] = parameters.emplace(parameterName, InferenceParameter{parameterName, datatype, data}); - return emplaced ? StatusCode::OK : StatusCode::DOUBLE_PARAMETER_INSERT; -} -const InferenceParameter* InferenceResponse::getParameter(const char* name) const { - auto it = parameters.find(name); - if (it != parameters.end()) - return &it->second; - return nullptr; + auto it = std::find_if(parameters.begin(), + parameters.end(), + [¶meterName](const InferenceParameter& parameter) { + return parameterName == parameter.getName(); + }); + if (parameters.end() != it) { + return StatusCode::DOUBLE_PARAMETER_INSERT; + } + parameters.emplace_back(parameterName, datatype, data); + return StatusCode::OK; +} + +const InferenceParameter* InferenceResponse::getParameter(uint32_t id) const { + if (id >= parameters.size()) { + return nullptr; + } + return ¶meters[id]; +} + +uint32_t InferenceResponse::getOutputCount() const { + return this->outputs.size(); +} + +uint32_t InferenceResponse::getParameterCount() const { + return this->parameters.size(); } } // namespace ovms diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index c15e4d522d..42d21e1814 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -15,7 +15,8 @@ // limitations under the License. //***************************************************************************** #include -#include +#include +#include #include "inferenceparameter.hpp" #include "inferencetensor.hpp" @@ -29,18 +30,21 @@ class Status; class InferenceResponse { const std::string& servableName; const model_version_t servableVersion; - std::unordered_map parameters; - std::unordered_map outputs; + std::vector parameters; // TODO after benchmark app verify if additional map wouldn't be better + std::vector> outputs; // TODO after benchmark app verify if additional map wouldn't be better public: InferenceResponse(const std::string& servableName, model_version_t servableVersion); Status addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); - Status getOutput(const char* name, InferenceTensor** tensor); + Status getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor); // TODO consider in the future if we need getOutput by name + Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); - const InferenceParameter* getParameter(const char* name) const; + const InferenceParameter* getParameter(uint32_t id) const; const std::string& getServableName() const; model_version_t getServableVersion() const; + uint32_t getOutputCount() const; + uint32_t getParameterCount() const; Status setId(); Status getId(); diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp index eb86903487..7b6e79edbc 100644 --- a/src/inferencetensor.cpp +++ b/src/inferencetensor.cpp @@ -33,7 +33,7 @@ InferenceTensor::InferenceTensor(OVMS_DataType datatype, const size_t* shape, si Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy) { if (nullptr != buffer) { return StatusCode::DOUBLE_BUFFER_SET; - } + } // TODO validate against byteSize == 0 buffer = std::make_unique(addr, byteSize, bufferType, deviceId, createCopy); return StatusCode::OK; } diff --git a/src/pocapi.cpp b/src/pocapi.cpp index 3f6f0ce507..f53ecdd717 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -18,7 +18,21 @@ #include #include +#include "buffer.hpp" +#include "inferenceparameter.hpp" +#include "inferencerequest.hpp" +#include "inferenceresponse.hpp" +#include "inferencetensor.hpp" #include "poc_api_impl.hpp" +#include "status.hpp" + +using ovms::Buffer; +using ovms::InferenceParameter; +using ovms::InferenceRequest; +using ovms::InferenceResponse; +using ovms::InferenceTensor; +using ovms::Status; +using ovms::StatusCode; OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { *options = (OVMS_ServerGeneralOptions*)new ovms::GeneralOptionsImpl; @@ -80,3 +94,147 @@ OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOpti mmo->configPath = std::string(config_path); return 0; } +// inference API +OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion) { + // TODO should we allow to create requests to not yet loaded models? + *request = reinterpret_cast(new InferenceRequest(servableName, servableVersion)); + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* request) { + delete reinterpret_cast(request); + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->addInput(inputName, datatype, shape, dimCount); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->setInputBuffer(inputName, data, bufferSize, bufferType, deviceId); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* req, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->addParameter(parameterName, datatype, data); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* req, const char* parameterName) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->removeParameter(parameterName); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* req, const char* inputName) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->removeInput(inputName); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* req, const char* inputName) { + // TODO error handling if null + // if (nullptr == req) + InferenceRequest* request = reinterpret_cast(req); + auto status = request->removeInputBuffer(inputName); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + return nullptr; +} + +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId) { + // TODO error handling if null + // if (nullptr == req) + InferenceResponse* response = reinterpret_cast(res); + InferenceTensor* tensor = nullptr; + const std::string* cppName; + auto status = response->getOutput(id, &cppName, &tensor); + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + if ((tensor == nullptr) || + (cppName == nullptr)) { + return reinterpret_cast(new Status(StatusCode::INTERNAL_ERROR, "InferenceResponse returned nullptr tensor or name")); + } + const Buffer* buffer = tensor->getBuffer(); + if (nullptr == buffer) { + return reinterpret_cast(new Status(ovms::StatusCode::INTERNAL_ERROR, "InferenceResponse has tensor without buffer")); + } + *name = cppName->c_str(); + *datatype = tensor->getDataType(); + *shape = tensor->getShape().data(); + *dimCount = tensor->getShape().size(); + *bufferType = buffer->getBufferType(); + *deviceId = buffer->getDeviceId().value_or(0); // TODO how discriminate betwen undefined & actual device 0 + // possibly it is not neccessary to discriminate + *data = const_cast(buffer->data()); // should data return const ptr? + *bytesize = buffer->getByteSize(); + return nullptr; +} + +OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* res, uint32_t* count) { + // TODO error handling if null + // if (nullptr == req) + InferenceResponse* response = reinterpret_cast(res); + *count = response->getOutputCount(); + return nullptr; +} + +OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* res, uint32_t* count) { + // TODO error handling if null + // if (nullptr == req) + InferenceResponse* response = reinterpret_cast(res); + *count = response->getParameterCount(); + return nullptr; +} + +OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* res, uint32_t id, OVMS_DataType* datatype, const void** data) { + // TODO error handling if null + // if (nullptr == req) + InferenceResponse* response = reinterpret_cast(res); + const InferenceParameter* parameter = response->getParameter(id); + if (nullptr == parameter) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL)); + } + *datatype = parameter->getDataType(); + *data = parameter->getData(); + return nullptr; +} + +OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* res) { + // TODO error handling if null + // if (nullptr == req) + InferenceResponse* response = reinterpret_cast(res); + delete response; + return nullptr; +} diff --git a/src/pocapi.hpp b/src/pocapi.hpp index a3777f19de..1de64c10d1 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -17,6 +17,8 @@ #include #include // For precise data types +// TODO extern C + struct OVMS_Server; struct OVMS_Status; @@ -146,27 +148,27 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest -OVMS_Status* OVMS_InferenceRequestNew(char* modelName, uint32_t servableVersion); +OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion); // TODO add passing server here OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); -OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, char* inputName, OVMS_DataType datatype, uint64_t* shape, uint32_t dimCount); -OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters +OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); +OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED // ownership of data needs to be maintained during inference -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId); -OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, char* inputName); -OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, char* inputName); // this will allow for reuse of request but with different input data +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId); +OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, const char* inputName); +OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); // this will allow for reuse of request but with different input data OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs - -OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, char* paramaterName, OVMS_DataType datatype, void* data, size_t byteSize); +OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize); +OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* request, const char* parameterName); // OVMS_Inference Response OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseOutput(OVMS_InferenceResponse* response, uint32_t id, char* name, OVMS_DataType* datatype, uint64_t* shape, uint32_t dimCount, BufferType* bufferType, uint32_t* deviceId, void** data); -OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId); OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, void** data); +OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, const void** data); +OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); OVMS_Status* OVMS_Inference(OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); diff --git a/src/serialization.hpp b/src/serialization.hpp index c5248fde1b..93eb2fa261 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -146,7 +146,7 @@ Status serializePredictResponse( outputNameChooser_t outputNameChooser) { OVMS_PROFILE_FUNCTION(); Status status; - ProtoGetter protoGetter(response); + uint32_t outputId = 0; for (const auto& [outputName, outputInfo] : outputMap) { ov::Tensor tensor; status = outputGetter.get(outputNameChooser(outputName, *outputInfo), tensor); @@ -203,7 +203,9 @@ Status serializePredictResponse( outputName, response->getServableName(), response->getServableVersion()); return StatusCode::INTERNAL_ERROR; } - status = response->getOutput(outputInfo->getMappedName().c_str(), &outputTensor); + const std::string* outputNameFromCapiTensor = nullptr; + status = response->getOutput(outputId, &outputNameFromCapiTensor, &outputTensor); + ++outputId; // TODO C-API test serialization 2 outputs if (!status.ok()) { SPDLOG_ERROR("Cannot serialize output with name:{} for servable name:{}; version:{}; error: cannot find inserted input", outputName, response->getServableName(), response->getServableVersion()); diff --git a/src/status.hpp b/src/status.hpp index 9abfdefe2b..4045c6d32d 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -282,7 +282,7 @@ enum class StatusCode { NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, NONEXISTENT_TENSOR_FOR_REMOVAL, NONEXISTENT_BUFFER_FOR_REMOVAL, - NONEXISTENT_PARAMETER_FOR_REMOVAL, + NONEXISTENT_PARAMETER_FOR_REMOVAL, // rename to non existen parameter STATUS_CODE_END }; diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index b1669785e6..5dcd297230 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -16,10 +16,17 @@ #include +// TODO we should not include classes from OVMS here +// consider how to workaround test_utils #include "../config.hpp" +#include "../inferenceresponse.hpp" +#include "../logging.hpp" #include "../poc_api_impl.hpp" +#include "../pocapi.hpp" +#include "test_utils.hpp" using namespace ovms; +using testing::ElementsAreArray; class CapiConfigTest : public ::testing::Test { protected: @@ -40,3 +47,192 @@ TEST_F(CapiConfigTest, Parse) { EXPECT_EQ(ovms::Config::instance().restPort(), 234); EXPECT_EQ(ovms::Config::instance().configPath(), "/path/config.json"); } + +class CapiInferencePreparationTest : public ::testing::Test {}; + +TEST_F(CapiInferencePreparationTest, Basic) { + // request creation + OVMS_InferenceRequest* request{nullptr}; + OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); + ASSERT_EQ(nullptr, status); + ASSERT_NE(nullptr, request); + + // adding input + status = OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); + ASSERT_EQ(nullptr, status); + // setting buffer + std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + uint32_t notUsedNum = 0; + status = OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); + ASSERT_EQ(nullptr, status); + // add parameters + const uint64_t sequenceId{42}; + status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); + ASSERT_EQ(nullptr, status); + // 2nd time should get error + status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + const uint32_t sequenceControl{1}; // SEQUENCE_START + status = OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl)); + ASSERT_EQ(nullptr, status); + + ////////////////// + // INFERENCE + ////////////////// + + /////////////// + // CLEANUP + /////////////// + // here we will add additional inputs to verify 2 ways of cleanup + // - direct call to remove whole request + // - separate calls to remove partial data + // + // here we will just add inputs to remove them later + // one original will be removed together with buffer during whole request removal + // one will be removed together with request but without buffer attached + // one will be removed with buffer directly + // one will be removed without buffer directly + status = OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); + ASSERT_EQ(nullptr, status); + status = OVMS_InferenceRequestAddInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); + ASSERT_EQ(nullptr, status); + status = OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); + ASSERT_EQ(nullptr, status); + status = OVMS_InferenceRequestInputSetData(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); + ASSERT_EQ(nullptr, status); + // we will add buffer and remove it to check separate buffer removal + status = OVMS_InferenceRequestInputSetData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); + ASSERT_EQ(nullptr, status); + + status = OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); + ASSERT_EQ(nullptr, status); + // second time we should get error + status = OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + status = OVMS_InferenceRequestRemoveInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); + ASSERT_EQ(nullptr, status); + status = OVMS_InferenceRequestRemoveInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY"); + ASSERT_EQ(nullptr, status); + // we will remove 1 of two parameters + status = OVMS_InferenceRequestRemoveParameter(request, "sequence_id"); + ASSERT_EQ(nullptr, status); + // 2nd time should report error + status = OVMS_InferenceRequestRemoveParameter(request, "sequence_id"); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + + status = OVMS_InferenceRequestRemoveInput(request, "NONEXISTENT_TENSOR"); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + status = OVMS_InferenceRequestDelete(request); + ASSERT_EQ(nullptr, status); +} +// TODO negative test -> validate at the infer stage +// TODO flow with removel just request no separate input/buffer +// TODO reuse request after inference +namespace { +const std::string MODEL_NAME{"SomeModelName"}; +const uint64_t MODEL_VERSION{42}; +const std::string PARAMETER_NAME{"sequence_id"}; // TODO check if in ovms there is such constant +const OVMS_DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; + +const uint32_t PARAMETER_VALUE{13}; +const uint32_t PRIORITY{7}; +const uint64_t REQUEST_ID{3}; + +const std::string INPUT_NAME{"NOT_RANDOM_NAME"}; +const ovms::shape_t INPUT_SHAPE{1, 3, 220, 230}; +const std::array INPUT_DATA{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; +constexpr size_t INPUT_DATA_BYTESIZE{INPUT_DATA.size() * sizeof(float)}; +const OVMS_DataType DATATYPE{OVMS_DATATYPE_FP32}; +} // namespace +class CapiInferenceRetrievalTest : public ::testing::Test {}; +TEST_F(CapiInferenceRetrievalTest, Basic) { + auto cppResponse = std::make_unique(MODEL_NAME, MODEL_VERSION); + // add output + std::array cppOutputShape{1, DUMMY_MODEL_INPUT_SIZE}; + auto cppStatus = cppResponse->addOutput(INPUT_NAME.c_str(), DATATYPE, cppOutputShape.data(), cppOutputShape.size()); + ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); + InferenceTensor* cpptensor = nullptr; + const std::string* cppOutputName; + cppStatus = cppResponse->getOutput(0, &cppOutputName, &cpptensor); + ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); + + // save data into output (it should have it's own copy in contrast to request) + bool createCopy = true; + cppStatus = cpptensor->setBuffer(INPUT_DATA.data(), INPUT_DATA_BYTESIZE, OVMS_BUFFERTYPE_CPU, std::nullopt, createCopy); + ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); + // add parameter to response + uint64_t seqId = 666; + cppStatus = cppResponse->addParameter("sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&seqId)); + ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); + /////////////////////////// + // now response is prepared so we can test C-API + /////////////////////////// + OVMS_InferenceResponse* response = reinterpret_cast(cppResponse.get()); + uint32_t outputCount = -1; + auto status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(outputCount, 1); + + uint32_t parameterCount = -1; + status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(1, parameterCount); + // verify get Parameter + OVMS_DataType parameterDatatype = OVMS_DATATYPE_FP32; + const void* parameterData{nullptr}; + status = OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, ¶meterData); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(parameterDatatype, OVMS_DATATYPE_U64); + EXPECT_EQ(0, std::memcmp(parameterData, (void*)&seqId, sizeof(seqId))); + // verify get Output + void* voutputData; + size_t bytesize = -1; + uint32_t outputId = 0; + OVMS_DataType datatype = (OVMS_DataType)199; + const uint64_t* shape{nullptr}; + uint32_t dimCount = -1; + BufferType bufferType = (BufferType)199; + uint32_t deviceId = -1; + const char* outputName{nullptr}; + status = OVMS_InferenceResponseGetOutput(response, outputId + 42123, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + status = OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(INPUT_NAME, outputName); + EXPECT_EQ(datatype, OVMS_DATATYPE_FP32); + EXPECT_EQ(dimCount, 2); + EXPECT_EQ(bufferType, OVMS_BUFFERTYPE_CPU); + EXPECT_EQ(deviceId, 0); + + for (size_t i = 0; i < cppOutputShape.size(); ++i) { + EXPECT_EQ(cppOutputShape[i], shape[i]) << "Different at:" << i << " place."; + } + float* outputData = reinterpret_cast(voutputData); + ASSERT_EQ(bytesize, sizeof(float) * DUMMY_MODEL_INPUT_SIZE); + for (size_t i = 0; i < INPUT_DATA.size(); ++i) { + EXPECT_EQ(INPUT_DATA[i], outputData[i]) << "Different at:" << i << " place."; + } + + // we release unique_ptr ownership here so that we can free it safely via C-API + // test negative scenario with getting output without buffer + cppStatus = cppResponse->addOutput("outputWithNoBuffer", DATATYPE, cppOutputShape.data(), cppOutputShape.size()); + ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); + status = OVMS_InferenceResponseGetOutput(response, outputId + 1, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + // negative scenario nonexistsing parameter + status = OVMS_InferenceResponseGetParameter(response, 123, ¶meterDatatype, ¶meterData); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + // final cleanup + cppResponse.release(); + status = OVMS_InferenceResponseDelete(response); + ASSERT_EQ(nullptr, status); +} +// TODO make cleaner error codes reporting +// todo decide either use remove or delete for consistency diff --git a/src/test/inferencerequest_test.cpp b/src/test/inferencerequest_test.cpp index c92ff0efd1..7ce2c7e940 100644 --- a/src/test/inferencerequest_test.cpp +++ b/src/test/inferencerequest_test.cpp @@ -138,16 +138,22 @@ TEST(InferenceResponse, CreateAndReadData) { // add output auto status = response.addOutput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); ASSERT_EQ(status, StatusCode::OK) << status.string(); + EXPECT_EQ(response.getOutputCount(), 1); // add 2nd output with the same name should fail status = response.addOutput(INPUT_NAME.c_str(), DATATYPE, INPUT_SHAPE.data(), INPUT_SHAPE.size()); ASSERT_EQ(status, StatusCode::DOUBLE_TENSOR_INSERT) << status.string(); // get nonexistent output + const std::string* wrongOutputName = nullptr; InferenceTensor* tensor = nullptr; - status = response.getOutput("SOME_NOT_RANDOM_NAME", &tensor); + status = response.getOutput(13, &wrongOutputName, &tensor); ASSERT_EQ(status, StatusCode::NONEXISTENT_TENSOR) << status.string(); + ASSERT_EQ(nullptr, tensor); + ASSERT_EQ(nullptr, wrongOutputName); // get output - status = response.getOutput(INPUT_NAME.c_str(), &tensor); + const std::string* outputName = nullptr; + status = response.getOutput(0, &outputName, &tensor); ASSERT_NE(nullptr, tensor); + ASSERT_EQ(INPUT_NAME, *outputName); ASSERT_EQ(status, StatusCode::OK) << status.string(); // compare datatype ASSERT_EQ(tensor->getDataType(), DATATYPE); @@ -177,9 +183,11 @@ TEST(InferenceResponse, CreateAndReadData) { // verify parameter handling status = response.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); ASSERT_EQ(status, StatusCode::OK) << status.string(); + EXPECT_EQ(response.getParameterCount(), 1); - // add parameter - const InferenceParameter* parameter = response.getParameter(PARAMETER_NAME.c_str()); + const InferenceParameter* parameter = response.getParameter(1); + ASSERT_EQ(parameter, nullptr); + parameter = response.getParameter(0); ASSERT_NE(parameter, nullptr); EXPECT_EQ(parameter->getName(), PARAMETER_NAME); EXPECT_EQ(parameter->getDataType(), PARAMETER_DATATYPE); diff --git a/src/test/ovinferrequestqueue_test.cpp b/src/test/ovinferrequestqueue_test.cpp index c762ceca73..a6017cd4a8 100644 --- a/src/test/ovinferrequestqueue_test.cpp +++ b/src/test/ovinferrequestqueue_test.cpp @@ -77,7 +77,7 @@ TEST(OVInferRequestQueue, FullQueue) { EXPECT_EQ(reqid, 3); } -void inferenceSimulate(ovms::OVInferRequestsQueue& ms, std::vector& tv) { +static void inferenceSimulate(ovms::OVInferRequestsQueue& ms, std::vector& tv) { for (int i = 1; i <= 10; i++) { int st = ms.getIdleStream().get(); int rd = std::rand(); diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index 8f3ce4d8b0..84da850368 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -726,7 +726,14 @@ TEST_F(CApiSerialization, ValidSerialization) { getTensorInfoName); ASSERT_EQ(status.getCode(), ovms::StatusCode::OK); InferenceTensor* responseOutput{nullptr}; - ASSERT_EQ(response.getOutput(DUMMY_MODEL_OUTPUT_NAME, &responseOutput), StatusCode::OK); + uint32_t outputCount = response.getOutputCount(); + ASSERT_EQ(1, outputCount); + ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); + const std::string* outputName{nullptr}; + status = response.getOutput(0, &outputName, &responseOutput); + ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); + ASSERT_NE(outputName, nullptr); + ASSERT_EQ(*outputName, DUMMY_MODEL_OUTPUT_NAME); ASSERT_NE(responseOutput, nullptr); EXPECT_EQ(responseOutput->getDataType(), OVMS_DATATYPE_FP32); EXPECT_THAT(responseOutput->getShape(), ElementsAre(1, NUMBER_OF_ELEMENTS, 1, 1)); From 825337673f2c8377bd7e74b1763d2c9247950ef9 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 29 Nov 2022 13:24:07 +0100 Subject: [PATCH 095/130] Non blocking server start, C-API calls impl (#1538) - signal handlers moved out of non blocking start to OVMS only start: enabling possibility to install signal handlers in C-API apps manually - increasing total allowed cppclean errors due to static sigint handlers in main3.cpp --- check_coverage.bat | 4 +- cppclean.sh | 2 +- docs/parameters.md | 2 +- src/BUILD | 3 +- src/cli_parser.cpp | 28 +-- src/cli_parser.hpp | 1 - src/config.cpp | 12 +- src/config.hpp | 16 +- src/main3.cpp | 76 +++++++- src/modelmanager.cpp | 2 +- src/poc_api_impl.cpp | 27 --- src/pocapi.cpp | 159 +++++++++++++--- src/pocapi.hpp | 68 ++++--- src/server.cpp | 33 ++-- src/{poc_api_impl.hpp => server_options.hpp} | 14 +- src/test/c_api_tests.cpp | 188 +++++++++++++++++-- src/test/ovmsconfig_test.cpp | 14 +- src/test/test_utils.hpp | 6 + 18 files changed, 487 insertions(+), 168 deletions(-) delete mode 100644 src/poc_api_impl.cpp rename src/{poc_api_impl.hpp => server_options.hpp} (88%) diff --git a/check_coverage.bat b/check_coverage.bat index 9783dcae6a..b7c2d6811d 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,8 +5,8 @@ #MIN_FUNCTION_COV=87.4 #Rhel -MIN_LINES_COV=73.8 -MIN_FUNCTION_COV=74.0 +MIN_LINES_COV=74.2 +MIN_FUNCTION_COV=74.9 diff --git a/cppclean.sh b/cppclean.sh index 0b3b11ffd5..066da99588 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,7 +42,7 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 3 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 178 ]; then +if [ ${NO_WARNINGS} -gt 182 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi diff --git a/docs/parameters.md b/docs/parameters.md index 93f24f8d45..9d37ab3ab2 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -33,7 +33,7 @@ Configuration options for the server are defined only via command-line options a | `rest_workers` | `integer` | Number of HTTP server threads. Effective when `rest_port` > 0. Default value is set based on the number of CPUs. | | `file_system_poll_wait_seconds` | `integer` | Time interval between config and model versions changes detection in seconds. Default value is 1. Zero value disables changes monitoring. | | `sequence_cleaner_poll_wait_minutes` | `integer` | Time interval (in minutes) between next sequence cleaner scans. Sequences of the models that are subjects to idle sequence cleanup that have been inactive since the last scan are removed. Zero value disables sequence cleaner. See [idle sequence cleanup](stateful_models.md). | -| `custom_node_resources_cleaner_interval` | `integer` | Time interval (in seconds) between two consecutive resources cleanup scans. Default is 1. Must be greater than 0. See [custom node development](custom_node_development.md). | +| `custom_node_resources_cleaner_interval_seconds` | `integer` | Time interval (in seconds) between two consecutive resources cleanup scans. Default is 1. Must be greater than 0. See [custom node development](custom_node_development.md). | | `cpu_extension` | `string` | Optional path to a library with [custom layers implementation](https://docs.openvino.ai/2022.2/openvino_docs_Extensibility_UG_Intro.html). | | `log_level` | `"DEBUG"/"INFO"/"ERROR"` | Serving logging level | | `log_path` | `string` | Optional path to the log file. | diff --git a/src/BUILD b/src/BUILD index c2cf9b13c5..947260c4e8 100644 --- a/src/BUILD +++ b/src/BUILD @@ -237,8 +237,6 @@ cc_library( "pipelineeventqueue.hpp", "pipeline_factory.cpp", "pipeline_factory.hpp", - "poc_api_impl.cpp", - "poc_api_impl.hpp", "pocapi.hpp", "pocapiinternal.cpp", "pocapiinternal.hpp", @@ -267,6 +265,7 @@ cc_library( "serialization.hpp", "servablemanagermodule.cpp", "servablemanagermodule.hpp", + "server_options.hpp", "server.cpp", "server.hpp", "session_id.hpp", diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index 4188785079..53d55ca00e 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -20,7 +20,7 @@ #include -#include "poc_api_impl.hpp" +#include "server_options.hpp" #include "version.hpp" namespace ovms { @@ -37,7 +37,7 @@ void CLIParser::parse(int argc, char** argv) { "Show binary version") ("port", "gRPC server port", - cxxopts::value()->default_value("9178"), + cxxopts::value()->default_value("9178"), "PORT") ("grpc_bind_address", "Network interface address to bind to for the gRPC API", @@ -45,7 +45,7 @@ void CLIParser::parse(int argc, char** argv) { "GRPC_BIND_ADDRESS") ("rest_port", "REST server port, the REST server will not be started if rest_port is blank or set to 0", - cxxopts::value()->default_value("0"), + cxxopts::value()->default_value("0"), "REST_PORT") ("rest_bind_address", "Network interface address to bind to for the REST API", @@ -53,11 +53,11 @@ void CLIParser::parse(int argc, char** argv) { "REST_BIND_ADDRESS") ("grpc_workers", "Number of gRPC servers. Default 1. Increase for multi client, high throughput scenarios", - cxxopts::value()->default_value("1"), + cxxopts::value()->default_value("1"), "GRPC_WORKERS") ("rest_workers", "Number of worker threads in REST server - has no effect if rest_port is not set. Default value depends on number of CPUs. ", - cxxopts::value(), + cxxopts::value(), "REST_WORKERS") ("log_level", "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", @@ -75,16 +75,16 @@ void CLIParser::parse(int argc, char** argv) { cxxopts::value(), "GRPC_CHANNEL_ARGUMENTS") ("file_system_poll_wait_seconds", "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", - cxxopts::value()->default_value("1"), + cxxopts::value()->default_value("1"), "FILE_SYSTEM_POLL_WAIT_SECONDS") ("sequence_cleaner_poll_wait_minutes", "Time interval between two consecutive sequence cleanup scans. Default is 5. Zero value disables sequence cleaner.", cxxopts::value()->default_value("5"), "SEQUENCE_CLEANER_POLL_WAIT_MINUTES") - ("custom_node_resources_cleaner_interval", + ("custom_node_resources_cleaner_interval_seconds", "Time interval between two consecutive resources cleanup scans. Default is 1. Must be greater than 0.", cxxopts::value()->default_value("1"), - "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL") + "CUSTOM_NODE_RESOURCES_CLEANER_INTERVAL_SECONDS") ("cache_dir", "Overrides model cache directory. By default cache files are saved into /opt/cache if the directory is present. When enabled, first model load will produce cache files.", cxxopts::value(), @@ -182,8 +182,8 @@ void CLIParser::parse(int argc, char** argv) { } void CLIParser::prepare(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - go->grpcPort = result->operator[]("port").as(); - go->restPort = result->operator[]("rest_port").as(); + go->grpcPort = result->operator[]("port").as(); + go->restPort = result->operator[]("rest_port").as(); if (result->count("model_name")) mmo->modelName = result->operator[]("model_name").as(); @@ -203,10 +203,10 @@ void CLIParser::prepare(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { if (result->count("rest_bind_address")) go->restBindAddress = result->operator[]("rest_bind_address").as(); - go->grpcWorkers = result->operator[]("grpc_workers").as(); + go->grpcWorkers = result->operator[]("grpc_workers").as(); if (result->count("rest_workers")) - go->restWorkers = result->operator[]("rest_workers").as(); + go->restWorkers = result->operator[]("rest_workers").as(); if (result->count("batch_size")) mmo->batchSize = result->operator[]("batch_size").as(); @@ -254,9 +254,9 @@ void CLIParser::prepare(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { if (result->count("grpc_channel_arguments")) go->grpcChannelArguments = result->operator[]("grpc_channel_arguments").as(); - go->filesystemPollWaitSeconds = result->operator[]("file_system_poll_wait_seconds").as(); + go->filesystemPollWaitSeconds = result->operator[]("file_system_poll_wait_seconds").as(); go->sequenceCleanerPollWaitMinutes = result->operator[]("sequence_cleaner_poll_wait_minutes").as(); - go->resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval").as(); + go->resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as(); if (result != nullptr && result->count("cache_dir")) { go->cacheDir = result->operator[]("cache_dir").as(); diff --git a/src/cli_parser.hpp b/src/cli_parser.hpp index fb54d76eff..5d50fae184 100644 --- a/src/cli_parser.hpp +++ b/src/cli_parser.hpp @@ -22,7 +22,6 @@ namespace ovms { struct GeneralOptionsImpl; -struct SingleModelOptionsImpl; struct MultiModelOptionsImpl; class CLIParser { diff --git a/src/config.cpp b/src/config.cpp index 695c62b021..f5136547f9 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -24,7 +24,7 @@ #include "cli_parser.hpp" #include "modelconfig.hpp" -#include "poc_api_impl.hpp" +#include "server_options.hpp" namespace ovms { @@ -173,13 +173,13 @@ bool Config::validate() { } const std::string& Config::configPath() const { return this->mmo.configPath; } -uint64_t Config::port() const { return this->go.grpcPort; } +uint32_t Config::port() const { return this->go.grpcPort; } const std::string Config::cpuExtensionLibraryPath() const { return this->go.cpuExtensionLibraryPath; } const std::string Config::grpcBindAddress() const { return this->go.grpcBindAddress; } -uint64_t Config::restPort() const { return this->go.restPort; } +uint32_t Config::restPort() const { return this->go.restPort; } const std::string Config::restBindAddress() const { return this->go.restBindAddress; } -uint Config::grpcWorkers() const { return this->go.grpcWorkers; } -uint Config::restWorkers() const { return this->go.restWorkers.value_or(DEFAULT_REST_WORKERS); } +uint32_t Config::grpcWorkers() const { return this->go.grpcWorkers; } +uint32_t Config::restWorkers() const { return this->go.restWorkers.value_or(DEFAULT_REST_WORKERS); } const std::string& Config::modelName() const { return this->mmo.modelName; } const std::string& Config::modelPath() const { return this->mmo.modelPath; } const std::string& Config::batchSize() const { @@ -207,7 +207,7 @@ const std::string& Config::logPath() const { return this->go.logPath; } const std::string& Config::tracePath() const { return this->go.tracePath; } #endif const std::string& Config::grpcChannelArguments() const { return this->go.grpcChannelArguments; } -uint Config::filesystemPollWaitSeconds() const { return this->go.filesystemPollWaitSeconds; } +uint32_t Config::filesystemPollWaitSeconds() const { return this->go.filesystemPollWaitSeconds; } uint32_t Config::sequenceCleanerPollWaitMinutes() const { return this->go.sequenceCleanerPollWaitMinutes; } uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->go.resourcesCleanerPollWaitSeconds; } const std::string Config::cacheDir() const { return this->go.cacheDir; } diff --git a/src/config.hpp b/src/config.hpp index 69fb74c5e1..53247535c1 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -18,7 +18,7 @@ #include #include -#include "poc_api_impl.hpp" +#include "server_options.hpp" namespace ovms { @@ -90,9 +90,9 @@ class Config { /** * @brief Gets the grpc port * - * @return uint64_t + * @return uint32_t */ - uint64_t port() const; + uint32_t port() const; /** * @brief Get the gRPC network interface address to bind to @@ -111,9 +111,9 @@ class Config { /** * @brief Gets the REST port * - * @return uint64_t + * @return uint32_t */ - uint64_t restPort() const; + uint32_t restPort() const; /** * @brief Get the rest network interface address to bind to @@ -127,14 +127,14 @@ class Config { * * @return uint */ - uint grpcWorkers() const; + uint32_t grpcWorkers() const; /** * @brief Gets the rest workers count * * @return uint */ - uint restWorkers() const; + uint32_t restWorkers() const; /** * @brief Get the model name @@ -276,7 +276,7 @@ class Config { * * @return uint */ - uint filesystemPollWaitSeconds() const; + uint32_t filesystemPollWaitSeconds() const; /** * @brief Get the sequence cleanup poll wait time in minutes diff --git a/src/main3.cpp b/src/main3.cpp index 98c5a3f412..28e0cb1850 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -13,11 +13,54 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include +// #include +// #include +// #include + +#include +#include #include "pocapi.hpp" +namespace { +volatile sig_atomic_t shutdown_request = 0; +} + +static void onInterrupt(int status) { + shutdown_request = 1; +} + +static void onTerminate(int status) { + shutdown_request = 1; +} + +static void onIllegal(int status) { + shutdown_request = 2; +} + +static void installSignalHandlers() { + static struct sigaction sigIntHandler; + sigIntHandler.sa_handler = onInterrupt; + sigemptyset(&sigIntHandler.sa_mask); + sigIntHandler.sa_flags = 0; + sigaction(SIGINT, &sigIntHandler, NULL); + + static struct sigaction sigTermHandler; + sigTermHandler.sa_handler = onTerminate; + sigemptyset(&sigTermHandler.sa_mask); + sigTermHandler.sa_flags = 0; + sigaction(SIGTERM, &sigTermHandler, NULL); + + static struct sigaction sigIllHandler; + sigIllHandler.sa_handler = onIllegal; + sigemptyset(&sigIllHandler.sa_mask); + sigIllHandler.sa_flags = 0; + sigaction(SIGILL, &sigIllHandler, NULL); +} + int main(int argc, char** argv) { + installSignalHandlers(); + OVMS_ServerGeneralOptions* go = 0; OVMS_ServerMultiModelOptions* mmo = 0; OVMS_Server* srv; @@ -28,19 +71,38 @@ int main(int argc, char** argv) { OVMS_ServerGeneralOptionsSetGrpcPort(go, 11337); OVMS_ServerGeneralOptionsSetRestPort(go, 11338); + + OVMS_ServerGeneralOptionsSetLogLevel(go, OVMS_LOG_DEBUG); OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + if (res) { + // TODO: Better error handling? + fprintf(stderr, "Error starting the server\n"); + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + return 1; + } + + fprintf(stdout, "Server ready for inference\n"); + + // infer 1 + // infer 2 + // infer 3 + + // Application loop if required (C++): + // while (shutdown_request == 0) { + // std::this_thread::sleep_for(std::chrono::milliseconds(200)); + // } + + fprintf(stdout, "No more job to be done, will shut down\n"); + OVMS_ServerDelete(srv); OVMS_ServerMultiModelOptionsDelete(mmo); OVMS_ServerGeneralOptionsDelete(go); - if (res == 0) { - std::cout << "Finish with success" << std::endl; - } else { - std::cout << "Finish with fail" << std::endl; - } - + fprintf(stdout, "main() exit\n"); return 0; } diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index 099a8084dd..0a9335d8cd 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -152,7 +152,7 @@ Status ModelManager::start(const Config& config) { sequenceCleaupIntervalMinutes = config.sequenceCleanerPollWaitMinutes(); resourcesCleanupIntervalSec = config.resourcesCleanerPollWaitSeconds(); if (resourcesCleanupIntervalSec < 1) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Parameter: custom_node_resources_cleaner_interval has to be greater than 0. Applying default value(1 second)"); + SPDLOG_LOGGER_WARN(modelmanager_logger, "Parameter: custom_node_resources_cleaner_interval_seconds has to be greater than 0. Applying default value(1 second)"); resourcesCleanupIntervalSec = 1; } Status status; diff --git a/src/poc_api_impl.cpp b/src/poc_api_impl.cpp deleted file mode 100644 index 7411448cc4..0000000000 --- a/src/poc_api_impl.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "poc_api_impl.hpp" - -#include "server.hpp" - -namespace ovms { - -int ServerImpl::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - Server& server = Server::instance(); - return server.start(go, mmo); -} - -} // namespace ovms diff --git a/src/pocapi.cpp b/src/pocapi.cpp index f53ecdd717..f5b3dc5ee6 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -23,7 +23,8 @@ #include "inferencerequest.hpp" #include "inferenceresponse.hpp" #include "inferencetensor.hpp" -#include "poc_api_impl.hpp" +#include "server.hpp" +#include "server_options.hpp" #include "status.hpp" using ovms::Buffer; @@ -35,64 +36,170 @@ using ovms::Status; using ovms::StatusCode; OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { - *options = (OVMS_ServerGeneralOptions*)new ovms::GeneralOptionsImpl; - return 0; + *options = reinterpret_cast(new ovms::GeneralOptionsImpl); + return nullptr; } OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { - delete (ovms::GeneralOptionsImpl*)options; - return 0; + delete reinterpret_cast(options); + return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options) { - *options = (OVMS_ServerMultiModelOptions*)new ovms::MultiModelOptionsImpl; - return 0; + *options = reinterpret_cast(new ovms::MultiModelOptionsImpl); + return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { - delete (ovms::MultiModelOptionsImpl*)options; - return 0; + delete reinterpret_cast(options); + return nullptr; } OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { - *server = (OVMS_Server*)new ovms::ServerImpl; - return 0; + // Create new server once multi server configuration becomes possible. + *server = reinterpret_cast(&ovms::Server::instance()); + return nullptr; } OVMS_Status* OVMS_ServerDelete(OVMS_Server* server) { - delete (ovms::ServerImpl*)server; - return 0; + // Make use of the server pointer instead of singleton once multi server configuration becomes possible. + ovms::Server* srv = reinterpret_cast(server); + srv->shutdownModules(); + return nullptr; } OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_ServerGeneralOptions* general_options, OVMS_ServerMultiModelOptions* multi_model_specific_options) { - ovms::ServerImpl* srv = (ovms::ServerImpl*)server; - ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)general_options; - ovms::MultiModelOptionsImpl* mmo = (ovms::MultiModelOptionsImpl*)multi_model_specific_options; + ovms::Server* srv = reinterpret_cast(server); + ovms::GeneralOptionsImpl* go = reinterpret_cast(general_options); + ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(multi_model_specific_options); std::int64_t res = srv->start(go, mmo); - return (OVMS_Status*)res; // TODO: Return proper OVMS_Status instead of a raw status code + return (OVMS_Status*)res; // TODO: Return proper OVMS_Status instead of a raw status code in error handling PR } OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, - uint64_t grpcPort) { - ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)options; + uint32_t grpcPort) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->grpcPort = grpcPort; - return 0; + return nullptr; } OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, - uint64_t restPort) { - ovms::GeneralOptionsImpl* go = (ovms::GeneralOptionsImpl*)options; + uint32_t restPort) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->restPort = restPort; - return 0; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* options, + uint32_t grpc_workers) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->grpcWorkers = grpc_workers; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptions* options, + const char* grpc_bind_address) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->grpcBindAddress.assign(grpc_bind_address); + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* options, + uint32_t rest_workers) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->restWorkers = rest_workers; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptions* options, + const char* rest_bind_address) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->restBindAddress.assign(rest_bind_address); + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneralOptions* options, + const char* grpc_channel_arguments) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->grpcChannelArguments.assign(grpc_channel_arguments); + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, + uint32_t seconds) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->filesystemPollWaitSeconds = seconds; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, + uint32_t minutes) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->sequenceCleanerPollWaitMinutes = minutes; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerGeneralOptions* options, + uint32_t seconds) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->resourcesCleanerPollWaitSeconds = seconds; + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, + const char* cpu_extension_path) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->cpuExtensionLibraryPath.assign(cpu_extension_path); + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, + const char* cache_dir) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->cacheDir.assign(cache_dir); + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, + OVMS_LogLevel log_level) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + switch (log_level) { + case OVMS_LOG_INFO: + go->logLevel = "INFO"; + break; + case OVMS_LOG_ERROR: + go->logLevel = "ERROR"; + break; + case OVMS_LOG_DEBUG: + go->logLevel = "DEBUG"; + break; + case OVMS_LOG_TRACE: + go->logLevel = "TRACE"; + break; + case OVMS_LOG_WARNING: + go->logLevel = "WARNING"; + break; + default: + // TODO: Return error in error handling PR + break; + } + return nullptr; +} + +OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, + const char* log_path) { + ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + go->logPath.assign(log_path); + return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, const char* config_path) { - ovms::MultiModelOptionsImpl* mmo = (ovms::MultiModelOptionsImpl*)options; - mmo->configPath = std::string(config_path); - return 0; + ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(options); + mmo->configPath.assign(config_path); + return nullptr; } // inference API OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion) { diff --git a/src/pocapi.hpp b/src/pocapi.hpp index 1de64c10d1..08a0846839 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -63,13 +63,13 @@ struct OVMS_Status; struct OVMS_InferenceRequest; struct OVMS_InferenceResponse; -typedef enum OVMSSERVER_loglevel_enum { - OVMSSERVER_LOG_TRACE, - OVMSSERVER_LOG_DEBUG, - OVMSSERVER_LOG_INFO, - OVMSSERVER_LOG_WARNING, - OVMSSERVER_LOG_ERROR -} OVMSSERVER_LogLevel; +typedef enum OVMS_LogLevel_enum { + OVMS_LOG_TRACE, + OVMS_LOG_DEBUG, + OVMS_LOG_INFO, + OVMS_LOG_WARNING, + OVMS_LOG_ERROR +} OVMS_LogLevel; //// //// OVMS_ServerGeneralOptions @@ -82,40 +82,60 @@ OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) // --port OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, - uint64_t grpcPort); + uint32_t grpc_port); // --rest_port OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, - uint64_t restPort); + uint32_t rest_port); -// --log_level -OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, - OVMSSERVER_LogLevel log_level); +// --grpc_workers +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* options, + uint32_t grpc_workers); -// --log_path -OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, - const char* log_path); +// --grpc_bind_address +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptions* options, + const char* grpc_bind_address); + +// --rest_workers +OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* options, + uint32_t rest_workers); + +// --rest_bind_address +OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptions* options, + const char* rest_bind_address); + +// --grpc_channel_arguments +OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneralOptions* options, + const char* grpc_channel_arguments); // --file_system_poll_wait_seconds OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, - uint64_t file_system_poll_wait_seconds); + uint32_t seconds); // --sequence_cleaner_poll_wait_minutes OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, - uint64_t sequence_cleaner_poll_wait_minutes); - -// --custom_node_resources_cleaner_interval -OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerInterval(OVMS_ServerGeneralOptions* options, - uint64_t custom_node_resources_cleaner_interval); // TODO: Should include seconds or minutes in the name + uint32_t minutes); -// --cache_dir -void OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, - const char* cache_dir); +// --custom_node_resources_cleaner_interval_seconds +OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerGeneralOptions* options, + uint32_t seconds); // --cpu_extension OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, const char* cpu_extension_path); +// --cache_dir +OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, + const char* cache_dir); + +// --log_level +OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, + OVMS_LogLevel log_level); + +// --log_path +OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, + const char* log_path); + //// //// OVMS_ServerMultiModelOptions //// Options for starting multi model server controlled by config.json file diff --git a/src/server.cpp b/src/server.cpp index 0a97dec2b5..31c25e6c07 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -45,10 +45,10 @@ #include "metric_module.hpp" #include "model_service.hpp" #include "modelmanager.hpp" -#include "poc_api_impl.hpp" #include "prediction_service.hpp" #include "profiler.hpp" #include "servablemanagermodule.hpp" +#include "server_options.hpp" #include "stringutils.hpp" #include "version.hpp" @@ -119,7 +119,7 @@ static void onIllegal(int status) { shutdown_request = 2; } -static void installSignalHandlers(ovms::Server& server) { +static void installSignalHandlers() { static struct sigaction sigIntHandler; sigIntHandler.sa_handler = onInterrupt; sigemptyset(&sigIntHandler.sa_mask); @@ -325,36 +325,41 @@ void Server::shutdownModules() { // OVMS Start int Server::start(int argc, char** argv) { + installSignalHandlers(); + ovms::CLIParser parser; ovms::GeneralOptionsImpl go; ovms::MultiModelOptionsImpl mmo; parser.parse(argc, argv); parser.prepare(&go, &mmo); - return start(&go, &mmo); + + int ret = start(&go, &mmo); + ModulesShutdownGuard shutdownGuard(*this); + if (ret != 0) { + return ret; + } + while (!shutdown_request) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + if (shutdown_request == 2) { + SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); + } + SPDLOG_INFO("Shutting down"); + + return EXIT_SUCCESS; } // C-API Start int Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - ovms::Server& server = ovms::Server::instance(); - installSignalHandlers(server); try { auto& config = ovms::Config::instance(); if (!config.parse(go, mmo)) return EX_USAGE; configure_logger(config.logLevel(), config.logPath()); logConfig(config); - ModulesShutdownGuard shutdownGuard(*this); auto retCode = this->startModules(config); if (retCode) return retCode; - - while (!shutdown_request) { - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - if (shutdown_request == 2) { - SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); - } - SPDLOG_INFO("Shutting down"); } catch (std::exception& e) { SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); return EXIT_FAILURE; diff --git a/src/poc_api_impl.hpp b/src/server_options.hpp similarity index 88% rename from src/poc_api_impl.hpp rename to src/server_options.hpp index a42559306e..b8418d1481 100644 --- a/src/poc_api_impl.hpp +++ b/src/server_options.hpp @@ -21,12 +21,12 @@ namespace ovms { struct GeneralOptionsImpl { - uint64_t grpcPort = 9178; - uint64_t restPort = 0; + uint32_t grpcPort = 9178; + uint32_t restPort = 0; + uint32_t grpcWorkers = 1; std::string grpcBindAddress = "0.0.0.0"; + std::optional restWorkers; std::string restBindAddress = "0.0.0.0"; - uint grpcWorkers = 1; - std::optional restWorkers; bool metricsEnabled = false; std::string metricsList; std::string cpuExtensionLibraryPath; @@ -36,7 +36,7 @@ struct GeneralOptionsImpl { std::string tracePath; #endif std::string grpcChannelArguments; - uint filesystemPollWaitSeconds = 1; + uint32_t filesystemPollWaitSeconds = 1; uint32_t sequenceCleanerPollWaitMinutes = 5; uint32_t resourcesCleanerPollWaitSeconds = 1; std::string cacheDir; @@ -60,8 +60,4 @@ struct MultiModelOptionsImpl { std::string configPath; }; -struct ServerImpl { - int start(GeneralOptionsImpl*, MultiModelOptionsImpl*); -}; - } // namespace ovms diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 5dcd297230..77b881b367 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -21,31 +21,187 @@ #include "../config.hpp" #include "../inferenceresponse.hpp" #include "../logging.hpp" -#include "../poc_api_impl.hpp" +#include "../modelconfig.hpp" #include "../pocapi.hpp" +#include "../server_options.hpp" #include "test_utils.hpp" using namespace ovms; using testing::ElementsAreArray; -class CapiConfigTest : public ::testing::Test { -protected: - void SetUp() override { - } -}; +static void testDefaultSingleModelOptions(MultiModelOptionsImpl* mmo) { + EXPECT_EQ(mmo->modelName, ""); + EXPECT_EQ(mmo->modelPath, ""); + EXPECT_EQ(mmo->batchSize, ""); + EXPECT_EQ(mmo->shape, ""); + EXPECT_EQ(mmo->layout, ""); + EXPECT_EQ(mmo->modelVersionPolicy, ""); + EXPECT_EQ(mmo->nireq, 0); + EXPECT_EQ(mmo->targetDevice, ""); + EXPECT_EQ(mmo->pluginConfig, ""); + EXPECT_EQ(mmo->stateful, std::nullopt); + EXPECT_EQ(mmo->lowLatencyTransformation, std::nullopt); + EXPECT_EQ(mmo->maxSequenceNumber, std::nullopt); + EXPECT_EQ(mmo->idleSequenceCleanup, std::nullopt); +} + +TEST(CApiConfigTest, MultiModelConfiguration) { + OVMS_ServerGeneralOptions* _go = 0; + OVMS_ServerMultiModelOptions* _mmo = 0; + + ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&_go), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&_mmo), nullptr); + ASSERT_NE(_go, nullptr); + ASSERT_NE(_mmo, nullptr); + + GeneralOptionsImpl* go = reinterpret_cast(_go); + MultiModelOptionsImpl* mmo = reinterpret_cast(_mmo); + + // Test default values + EXPECT_EQ(go->grpcPort, 9178); + EXPECT_EQ(go->restPort, 0); + EXPECT_EQ(go->grpcWorkers, 1); + EXPECT_EQ(go->grpcBindAddress, "0.0.0.0"); + EXPECT_EQ(go->restWorkers, std::nullopt); + EXPECT_EQ(go->restBindAddress, "0.0.0.0"); + EXPECT_EQ(go->metricsEnabled, false); + EXPECT_EQ(go->metricsList, ""); + EXPECT_EQ(go->cpuExtensionLibraryPath, ""); + EXPECT_EQ(go->logLevel, "INFO"); + EXPECT_EQ(go->logPath, ""); + // trace path // not tested since it is not supported in C-API + EXPECT_EQ(go->grpcChannelArguments, ""); + EXPECT_EQ(go->filesystemPollWaitSeconds, 1); + EXPECT_EQ(go->sequenceCleanerPollWaitMinutes, 5); + EXPECT_EQ(go->resourcesCleanerPollWaitSeconds, 1); + EXPECT_EQ(go->cacheDir, ""); + + testDefaultSingleModelOptions(mmo); + EXPECT_EQ(mmo->configPath, ""); + + // Set non default values + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(_go, 5555), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(_go, 6666), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcWorkers(_go, 30), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, "2.2.2.2"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestWorkers(_go, 31), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, "3.3.3.3"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, "grpcargs"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(_go, 2), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(_go, 3), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(_go, 4), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, "/ovms/src/test"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetCacheDir(_go, "/tmp/cache"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_TRACE), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogPath(_go, "/logs"), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, "/config"), nullptr); + + // Test non default values + EXPECT_EQ(go->grpcPort, 5555); + EXPECT_EQ(go->restPort, 6666); + EXPECT_EQ(go->grpcWorkers, 30); + EXPECT_EQ(go->grpcBindAddress, "2.2.2.2"); + EXPECT_EQ(go->restWorkers, 31); + EXPECT_EQ(go->restBindAddress, "3.3.3.3"); + // EXPECT_EQ(go->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(go->metricsList, ""); + EXPECT_EQ(go->cpuExtensionLibraryPath, "/ovms/src/test"); + EXPECT_EQ(go->logLevel, "TRACE"); + EXPECT_EQ(go->logPath, "/logs"); + // trace path // not tested since it is not supported in C-API + EXPECT_EQ(go->grpcChannelArguments, "grpcargs"); + EXPECT_EQ(go->filesystemPollWaitSeconds, 2); + EXPECT_EQ(go->sequenceCleanerPollWaitMinutes, 3); + EXPECT_EQ(go->resourcesCleanerPollWaitSeconds, 4); + EXPECT_EQ(go->cacheDir, "/tmp/cache"); + + testDefaultSingleModelOptions(mmo); + EXPECT_EQ(mmo->configPath, "/config"); + + // Test config parser + ConstructorEnabledConfig cfg; + ASSERT_TRUE(cfg.parse(go, mmo)); + EXPECT_EQ(cfg.port(), 5555); + EXPECT_EQ(cfg.restPort(), 6666); + EXPECT_EQ(cfg.grpcWorkers(), 30); + EXPECT_EQ(cfg.grpcBindAddress(), "2.2.2.2"); + EXPECT_EQ(cfg.restWorkers(), 31); + EXPECT_EQ(cfg.restBindAddress(), "3.3.3.3"); + // EXPECT_EQ(go->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(go->metricsList, ""); + EXPECT_EQ(cfg.cpuExtensionLibraryPath(), "/ovms/src/test"); + EXPECT_EQ(cfg.logLevel(), "TRACE"); + EXPECT_EQ(cfg.logPath(), "/logs"); + // trace path // not tested since it is not supported in C-API + EXPECT_EQ(cfg.grpcChannelArguments(), "grpcargs"); + EXPECT_EQ(cfg.filesystemPollWaitSeconds(), 2); + EXPECT_EQ(cfg.sequenceCleanerPollWaitMinutes(), 3); + EXPECT_EQ(cfg.resourcesCleanerPollWaitSeconds(), 4); + EXPECT_EQ(cfg.cacheDir(), "/tmp/cache"); + + EXPECT_EQ(cfg.modelName(), ""); + EXPECT_EQ(cfg.modelPath(), ""); + EXPECT_EQ(cfg.batchSize(), "0"); + EXPECT_EQ(cfg.shape(), ""); + EXPECT_EQ(cfg.layout(), ""); + EXPECT_EQ(cfg.modelVersionPolicy(), ""); + EXPECT_EQ(cfg.nireq(), 0); + EXPECT_EQ(cfg.targetDevice(), "CPU"); + EXPECT_EQ(cfg.pluginConfig(), ""); + EXPECT_FALSE(cfg.stateful()); + EXPECT_FALSE(cfg.lowLatencyTransformation()); + EXPECT_EQ(cfg.maxSequenceNumber(), ovms::DEFAULT_MAX_SEQUENCE_NUMBER); + EXPECT_TRUE(cfg.idleSequenceCleanup()); + + EXPECT_EQ(cfg.configPath(), "/config"); + + ASSERT_EQ(OVMS_ServerMultiModelOptionsDelete(_mmo), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsDelete(_go), nullptr); +} + +TEST(CApiConfigTest, SingleModelConfiguration) { + GTEST_SKIP() << "Use C-API to initialize in next stages, currently not supported"; +} + +TEST(CApiStartTest, InitializingMultipleServers) { + OVMS_Server* srv1 = 0; + OVMS_Server* srv2 = 0; + + ASSERT_EQ(OVMS_ServerNew(&srv1), nullptr); + ASSERT_EQ(OVMS_ServerNew(&srv2), nullptr); + ASSERT_EQ(srv1, srv2); + ASSERT_EQ(OVMS_ServerDelete(srv1), nullptr); +} + +TEST(CApiStartTest, StartFlow) { + OVMS_Server* srv = 0; + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + + ASSERT_EQ(OVMS_ServerNew(&srv), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + + ASSERT_NE(srv, nullptr); + ASSERT_NE(go, nullptr); + ASSERT_NE(mmo, nullptr); + + // Cannot start due to configuration error + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, 5555), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(go, 5555), nullptr); // The same port + ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"), nullptr); -TEST_F(CapiConfigTest, Parse) { - GeneralOptionsImpl go; - MultiModelOptionsImpl mmo; + // Expect fail + // TODO: Check exact error code and details once error reporting becomes ready + ASSERT_NE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), nullptr); - go.grpcPort = 123; - go.restPort = 234; - mmo.configPath = "/path/config.json"; + // Fix and expect ok + ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(go, 6666), nullptr); // Different port + ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), nullptr); - ovms::Config::instance().parse(&go, &mmo); - EXPECT_EQ(ovms::Config::instance().port(), 123); - EXPECT_EQ(ovms::Config::instance().restPort(), 234); - EXPECT_EQ(ovms::Config::instance().configPath(), "/path/config.json"); + ASSERT_EQ(OVMS_ServerMultiModelOptionsDelete(mmo), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsDelete(go), nullptr); + ASSERT_EQ(OVMS_ServerDelete(srv), nullptr); } class CapiInferencePreparationTest : public ::testing::Test {}; diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index 3667920766..2e624a33eb 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -25,6 +25,7 @@ #include "spdlog/spdlog.h" #include "../config.hpp" +#include "test_utils.hpp" using testing::_; using testing::ContainerEq; @@ -322,11 +323,6 @@ TEST_F(OvmsParamsTest, hostname_ip_regex) { EXPECT_EQ(ovms::Config::check_hostname_or_ip(too_long), false); } -class MockedConfig : public ovms::Config { -public: - MockedConfig() {} -}; - TEST(OvmsConfigTest, positiveMulti) { char* n_argv[] = {"ovms", "--port", "44", @@ -338,7 +334,7 @@ TEST(OvmsConfigTest, positiveMulti) { "--grpc_channel_arguments", "grpc_channel_args", "--file_system_poll_wait_seconds", "2", "--sequence_cleaner_poll_wait_minutes", "7", - "--custom_node_resources_cleaner_interval", "8", + "--custom_node_resources_cleaner_interval_seconds", "8", "--cpu_extension", "/ovms", "--cache_dir", "/tmp/model_cache", "--log_path", "/tmp/log_path", @@ -346,7 +342,7 @@ TEST(OvmsConfigTest, positiveMulti) { "--config_path", "/config.json"}; int arg_count = 31; - MockedConfig config; + ConstructorEnabledConfig config; config.parse(arg_count, n_argv); EXPECT_EQ(config.port(), 44); @@ -388,7 +384,7 @@ TEST(OvmsConfigTest, positiveSingle) { "2", "--sequence_cleaner_poll_wait_minutes", "7", - "--custom_node_resources_cleaner_interval", + "--custom_node_resources_cleaner_interval_seconds", "8", "--cpu_extension", "/ovms", @@ -427,7 +423,7 @@ TEST(OvmsConfigTest, positiveSingle) { "52", }; int arg_count = 55; - MockedConfig config; + ConstructorEnabledConfig config; config.parse(arg_count, n_argv); EXPECT_EQ(config.port(), 44); diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 81ec3d1cb7..60daa33f7f 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -34,6 +34,7 @@ #pragma GCC diagnostic ignored "-Wall" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "../config.hpp" #include "../execution_context.hpp" #include "../inferencerequest.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" @@ -571,3 +572,8 @@ static const std::vector UNSUPPORTED_CAPI_INPUT_PRECISIONS_TENS }; void randomizePort(std::string& port); + +class ConstructorEnabledConfig : public ovms::Config { +public: + ConstructorEnabledConfig() {} +}; From 30da741ebd5daa09eba8163a27a3a7af3fe83f5f Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 2 Dec 2022 15:13:07 +0100 Subject: [PATCH 096/130] C-API ModelInstance::infer integration (#1524) * One ModelInstance::infer to rule them all * Add C-API ModelInstance::infer() support Raise cppclean threashold due to FP's: ./src/modelinstance.hpp:40: 'ovms::InferenceRequest' not used ./src/modelinstance.hpp:41: 'ovms::InferenceResponse' not used * fixed segfault in validation * Change coverage threshold * Add handling nullptrs in C-API Inference calls * Remove main2 & poc OVMS_Infer * Add getting const data ptr from C-API JIRA:CVS-95308 --- Dockerfile.redhat | 2 +- check_coverage.bat | 6 +- cppclean.sh | 2 +- src/BUILD | 19 -- src/exit_node.cpp | 5 +- src/inferenceparameter.cpp | 4 - src/inferenceparameter.hpp | 1 - src/inferencerequest.cpp | 24 +- src/inferencerequest.hpp | 6 + src/inferenceresponse.cpp | 14 +- src/inferenceresponse.hpp | 7 +- src/main2.cpp | 38 --- src/modelinstance.cpp | 180 +++++------- src/modelinstance.hpp | 34 ++- src/pocapi.cpp | 325 ++++++++++++++++++++-- src/pocapi.hpp | 8 +- src/predict_request_validation_utils.cpp | 14 +- src/prediction_service_utils.cpp | 14 + src/prediction_service_utils.hpp | 5 +- src/serialization.hpp | 8 + src/statefulmodelinstance.cpp | 188 ++++++------- src/statefulmodelinstance.hpp | 36 ++- src/status.cpp | 12 +- src/status.hpp | 12 +- src/test/c_api/config_standard_dummy.json | 8 + src/test/c_api_tests.cpp | 164 +++++++++-- src/test/capi_predict_validation_test.cpp | 14 + src/test/prediction_service_test.cpp | 209 +++++++++++--- src/test/serialization_tests.cpp | 23 +- src/test/stateful_modelinstance_test.cpp | 68 +++-- src/test/test_utils.cpp | 4 + src/test/test_utils.hpp | 7 +- src/test/tfs_rest_parser_column_test.cpp | 9 +- src/test/tfs_rest_parser_row_test.cpp | 9 +- 34 files changed, 1024 insertions(+), 455 deletions(-) delete mode 100644 src/main2.cpp create mode 100644 src/test/c_api/config_standard_dummy.json diff --git a/Dockerfile.redhat b/Dockerfile.redhat index f55d409207..76e4b30eed 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -202,7 +202,7 @@ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so -RUN cd /ovms/src/example/SampleCpuExtension/ && make +RUN cd /ovms/src/example/SampleCpuExtension/ && make # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared diff --git a/check_coverage.bat b/check_coverage.bat index b7c2d6811d..327530e9b4 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,10 +5,8 @@ #MIN_FUNCTION_COV=87.4 #Rhel -MIN_LINES_COV=74.2 -MIN_FUNCTION_COV=74.9 - - +MIN_LINES_COV=74.1 +MIN_FUNCTION_COV=75.6 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` FUNC_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | tail -n 1` diff --git a/cppclean.sh b/cppclean.sh index 066da99588..d7b656967b 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,7 +42,7 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 3 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 182 ]; then +if [ ${NO_WARNINGS} -gt 183 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi diff --git a/src/BUILD b/src/BUILD index 947260c4e8..fd119348ab 100644 --- a/src/BUILD +++ b/src/BUILD @@ -538,25 +538,6 @@ cc_binary( ] ) -cc_binary( - name = "poc", - srcs = [ - "main2.cpp", - ], - linkopts = [ - "-lxml2", - "-luuid", - "-lstdc++fs", - "-lcrypto", - ], - copts = [ - ], - deps = [ - "//src:ovms_lib", - ], - linkstatic = True, -) - # POC which can start OVMS with C-API built by bazel # Standalone example for building outside of bazel (with bare g++ is inside MakefileCapi) cc_binary( diff --git a/src/exit_node.cpp b/src/exit_node.cpp index e8db9b744a..7de469b7a0 100644 --- a/src/exit_node.cpp +++ b/src/exit_node.cpp @@ -62,7 +62,10 @@ Status OutputGetter::get(const std::string& name, ov::Tensor& template Status ExitNode::fetchResults(const TensorMap& inputTensors) { OutputGetter outputGetter(inputTensors); - return serializePredictResponse(outputGetter, this->outputsInfo, this->response, getOutputMapKeyName, useSharedOutputContent); + // TODO fix name version (it does not work properly anyway right now) + static const std::string name{""}; + static const model_version_t version{1}; + return serializePredictResponse(outputGetter, name, version, this->outputsInfo, this->response, getOutputMapKeyName); } template diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index ebb4f5351f..bb5a851322 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -62,10 +62,6 @@ OVMS_DataType InferenceParameter::getDataType() const { return this->datatype; } -size_t InferenceParameter::getByteSize() const { - return this->data.size(); -} - const void* InferenceParameter::getData() const { return reinterpret_cast(this->data.c_str()); } diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp index a5695ebbc7..c899009cd2 100644 --- a/src/inferenceparameter.hpp +++ b/src/inferenceparameter.hpp @@ -33,7 +33,6 @@ class InferenceParameter { InferenceParameter(const char* name, OVMS_DataType datatype, const void* data, size_t byteSize); const std::string& getName() const; OVMS_DataType getDataType() const; - size_t getByteSize() const; const void* getData() const; }; } // namespace ovms diff --git a/src/inferencerequest.cpp b/src/inferencerequest.cpp index a11a2df5ce..b1129c2028 100644 --- a/src/inferencerequest.cpp +++ b/src/inferencerequest.cpp @@ -17,7 +17,9 @@ #include "status.hpp" namespace ovms { - +// this constructor can be removed with prediction tests overhaul +InferenceRequest::InferenceRequest() : + InferenceRequest("CONSTRUCTOR_USED_ONLY_IN_PREDICTION_TESTS", 42) {} InferenceRequest::InferenceRequest(const char* servableName, model_version_t servableVersion) : servableName(servableName), servableVersion(servableVersion) { @@ -87,4 +89,24 @@ const InferenceParameter* InferenceRequest::getParameter(const char* name) const return &it->second; return nullptr; } +Status InferenceRequest::getBatchSize(size_t& batchSize, size_t batchSizeIndex) const { + if (inputs.size() == 0) { + return StatusCode::INTERNAL_ERROR; // TODO test + } + // we make here the same assumption as with bs=auto in TFS/KFS API + const InferenceTensor& tensor = inputs.begin()->second; + const auto& shape = tensor.getShape(); + if (batchSizeIndex >= shape.size()) { + return StatusCode::INTERNAL_ERROR; // TODO test + } + batchSize = shape[batchSizeIndex]; + return StatusCode::OK; +} +std::map InferenceRequest::getRequestShapes() const { + std::map result; + for (auto& [name, tensor] : inputs) { + result.emplace(name, tensor.getShape()); + } + return result; +} } // namespace ovms diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp index c00839e822..b5e8afe4df 100644 --- a/src/inferencerequest.hpp +++ b/src/inferencerequest.hpp @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include #include #include @@ -33,6 +34,8 @@ class InferenceRequest { std::unordered_map inputs; public: + // this constructor can be removed with prediction tests overhaul + InferenceRequest(); InferenceRequest(const char* modelName, model_version_t modelVersion); Status addInput(const char* name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); Status getInput(const char* name, const InferenceTensor** tensor) const; @@ -58,5 +61,8 @@ class InferenceRequest { Status setTimeoutMicorseconds(uint64_t microseconds); InferenceParameter* getInferenceParameter(const char* name); InferenceTensor* getTensor(const char* name); + // TODO add tests & seek if those can be removed by potentialy exposing inputs map? + Status getBatchSize(size_t& batchSize, size_t batchSizeIndex) const; + std::map getRequestShapes() const; }; } // namespace ovms diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp index f7d88353a3..268281ea64 100644 --- a/src/inferenceresponse.cpp +++ b/src/inferenceresponse.cpp @@ -26,6 +26,9 @@ #include "status.hpp" namespace ovms { +// this constructor can be removed with prediction tests overhaul +InferenceResponse::InferenceResponse() : + InferenceResponse("CONSTRUCTOR_USED_ONLY_IN_PREDICTION_TESTS", 42) {} InferenceResponse::InferenceResponse(const std::string& servableName, model_version_t servableVersion) : servableName(servableName), servableVersion(servableVersion) {} @@ -53,7 +56,7 @@ Status InferenceResponse::addOutput(const std::string& name, OVMS_DataType datat return StatusCode::OK; } -Status InferenceResponse::getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor) { +Status InferenceResponse::getOutput(uint32_t id, const std::string** name, const InferenceTensor** tensor) const { if (outputs.size() <= id) { *tensor = nullptr; return StatusCode::NONEXISTENT_TENSOR; @@ -63,6 +66,10 @@ Status InferenceResponse::getOutput(uint32_t id, const std::string** name, Infer return StatusCode::OK; } +Status InferenceResponse::getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor) { + return const_cast(this)->getOutput(id, name, const_cast(tensor)); +} + Status InferenceResponse::addParameter(const char* parameterName, OVMS_DataType datatype, const void* data) { auto it = std::find_if(parameters.begin(), parameters.end(), @@ -90,4 +97,9 @@ uint32_t InferenceResponse::getOutputCount() const { uint32_t InferenceResponse::getParameterCount() const { return this->parameters.size(); } + +void InferenceResponse::Clear() { + outputs.clear(); + parameters.clear(); +} // TODO remove } // namespace ovms diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index 42d21e1814..60ce028f23 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -34,9 +34,12 @@ class InferenceResponse { std::vector> outputs; // TODO after benchmark app verify if additional map wouldn't be better public: + // this constructor can be removed with prediction tests overhaul + InferenceResponse(); InferenceResponse(const std::string& servableName, model_version_t servableVersion); Status addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); - Status getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor); // TODO consider in the future if we need getOutput by name + Status getOutput(uint32_t id, const std::string** name, const InferenceTensor** tensor) const; // TODO consider in the future if we need getOutput by name + Status getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor); Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); const InferenceParameter* getParameter(uint32_t id) const; @@ -49,5 +52,7 @@ class InferenceResponse { Status setId(); Status getId(); InferenceParameter* getInferenceParameter(const char* name); + // TODO this can be removed with prediction tests overhaul + void Clear(); }; } // namespace ovms diff --git a/src/main2.cpp b/src/main2.cpp deleted file mode 100644 index cda8af6930..0000000000 --- a/src/main2.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include -#include - -#include "pocapi.hpp" - -int main(int argc, char** argv) { - std::thread t([&argv, &argc]() { - OVMS_Start(argc, argv); - }); - std::this_thread::sleep_for(std::chrono::milliseconds(1000)); - // get model instance and have a lock on reload - float a[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 11}; - float b[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - OVMS_Infer((char*)"dummy", a, b); - for (int i = 0; i < 10; ++i) { - std::cout << b[i] << " "; - } - std::cout << std::endl; - std::cout << __LINE__ << "FINISHED, press ctrl+c to stop " << std::endl; - t.join(); - std::cout << __LINE__ << "FINISHED" << std::endl; - return 0; -} diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 0db306e237..85acd99b8d 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -54,9 +54,11 @@ namespace { enum : unsigned int { GET_INFER_REQUEST, + PREPROCESS, DESERIALIZE, PREDICTION, SERIALIZE, + POSTPROCESS, TIMER_END }; } // namespace @@ -1102,16 +1104,20 @@ void ModelInstance::unloadModelComponents() { } } +const std::set& ModelInstance::getOptionalInputNames() { + static const std::set optionalInputNames = {}; + return optionalInputNames; +} + template const Status ModelInstance::validate(const RequestType* request) { OVMS_PROFILE_FUNCTION(); - static const std::set optionalInputNames = {}; return request_validation_utils::validate( *request, getInputsInfo(), getName(), getVersion(), - optionalInputNames, + this->getOptionalInputNames(), getModelConfig().getBatchingMode(), getModelConfig().getShapes()); } @@ -1146,57 +1152,28 @@ Status ModelInstance::performInference(ov::InferRequest& inferRequest) { return StatusCode::OK; } -#include -Status ModelInstance::infer(float* data, float* output) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - using std::chrono::microseconds; - timer.start(GET_INFER_REQUEST); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); - int executingInferId = executingStreamIdGuard.getId(); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop(GET_INFER_REQUEST); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed(GET_INFER_REQUEST) / 1000); - timer.start(DESERIALIZE); - static ov::Shape shape{1, 10}; - ov::element::Type precision = ov::element::Type_t::f32; - ov::Tensor tensor(precision, - shape, - (void*)data); - inferRequest.set_tensor("b", tensor); - timer.stop(DESERIALIZE); - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); - timer.start(PREDICTION); - auto status = performInference(inferRequest); - timer.stop(PREDICTION); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); - timer.start(SERIALIZE); - auto otensor = inferRequest.get_tensor("a"); - std::memcpy((void*)output, otensor.data(), otensor.get_byte_size()); - timer.stop(SERIALIZE); - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - getName(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); - return StatusCode::OK; -} - -Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, +template +Status ModelInstance::infer(const RequestType* requestProto, + ResponseType* responseProto, std::unique_ptr& modelUnloadGuardPtr) { OVMS_PROFILE_FUNCTION(); Timer timer; using std::chrono::microseconds; - auto status = validate(requestProto); + auto requestProcessor = createRequestProcessor(requestProto, responseProto); // request, response passed only to deduce type + auto status = requestProcessor->extractRequestParameters(requestProto); + if (!status.ok()) + return status; + status = validate(requestProto); auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); auto requestShapes = getRequestShapes(requestProto); status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); if (!status.ok()) return status; + status = requestProcessor->prepare(); + if (!status.ok()) + return status; + timer.start(GET_INFER_REQUEST); OVMS_PROFILE_SYNC_BEGIN("getInferRequest"); ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); @@ -1207,61 +1184,15 @@ Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestPr double getInferRequestTime = timer.elapsed(GET_INFER_REQUEST); OBSERVE_IF_ENABLED(this->getMetricReporter().waitForInferReqTime, getInferRequestTime); SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, getInferRequestTime / 1000); - - timer.start(DESERIALIZE); - InputSink inputSink(inferRequest); - bool isPipeline = false; - status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); - timer.stop(DESERIALIZE); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); + getName(), getVersion(), executingInferId, getInferRequestTime / 1000); - timer.start(PREDICTION); - status = performInference(inferRequest); - timer.stop(PREDICTION); + timer.start(PREPROCESS); + status = requestProcessor->preInferenceProcessing(inferRequest); + timer.stop(PREPROCESS); if (!status.ok()) return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); - - timer.start(SERIALIZE); - OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); - timer.stop(SERIALIZE); - if (!status.ok()) - return status; - - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); - - return StatusCode::OK; -} - -Status ModelInstance::infer(const ::KFSRequest* requestProto, - ::KFSResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) { - OVMS_PROFILE_FUNCTION(); - Timer timer; - using std::chrono::microseconds; - - auto status = validate(requestProto); - auto requestBatchSize = getRequestBatchSize(requestProto, this->getBatchSizeIndex()); - auto requestShapes = getRequestShapes(requestProto); - status = reloadModelIfRequired(status, requestBatchSize, requestShapes, modelUnloadGuardPtr); - if (!status.ok()) - return status; - timer.start(GET_INFER_REQUEST); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); - int executingInferId = executingStreamIdGuard.getId(); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop(GET_INFER_REQUEST); - double getInferRequestTime = timer.elapsed(GET_INFER_REQUEST); - OBSERVE_IF_ENABLED(this->getMetricReporter().waitForInferReqTime, getInferRequestTime); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, getInferRequestTime / 1000); + SPDLOG_DEBUG("Preprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed(PREPROCESS) / 1000); timer.start(DESERIALIZE); InputSink inputSink(inferRequest); @@ -1271,7 +1202,7 @@ Status ModelInstance::infer(const ::KFSRequest* requestProto, if (!status.ok()) return status; SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); + getName(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); timer.start(PREDICTION); status = performInference(inferRequest); @@ -1279,24 +1210,34 @@ Status ModelInstance::infer(const ::KFSRequest* requestProto, if (!status.ok()) return status; SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); + getName(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); timer.start(SERIALIZE); OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName, useSharedOutputContent(requestProto)); + status = serializePredictResponse(outputGetter, getName(), getVersion(), getOutputsInfo(), responseProto, getTensorInfoName); timer.stop(SERIALIZE); if (!status.ok()) return status; - - responseProto->set_model_name(getName()); - responseProto->set_model_version(std::to_string(getVersion())); - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_name(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); + getName(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); - return StatusCode::OK; -} + timer.start(POSTPROCESS); + status = requestProcessor->postInferenceProcessing(responseProto, inferRequest); + timer.stop(POSTPROCESS); + if (!status.ok()) + return status; + SPDLOG_DEBUG("Postprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", + getName(), getVersion(), executingInferId, timer.elapsed(POSTPROCESS) / 1000); + status = requestProcessor->release(); + return status; +} +template Status ModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, + tensorflow::serving::PredictResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr); +template Status ModelInstance::infer(const ::KFSRequest* requestProto, + ::KFSResponse* responseProto, + std::unique_ptr& modelUnloadGuardPtr); const size_t ModelInstance::getBatchSizeIndex() const { const auto& inputItr = this->inputsInfo.cbegin(); if (inputItr == this->inputsInfo.cend()) { @@ -1314,4 +1255,33 @@ uint32_t ModelInstance::getNumOfStreams() const { return compiledModel->get_property(ov::num_streams); } +std::unique_ptr> ModelInstance::createRequestProcessor(const tensorflow::serving::PredictRequest*, tensorflow::serving::PredictResponse*) { + return std::make_unique>(); +} +std::unique_ptr> ModelInstance::createRequestProcessor(const KFSRequest*, KFSResponse*) { + return std::make_unique>(); +} +std::unique_ptr> ModelInstance::createRequestProcessor(const InferenceRequest*, InferenceResponse*) { + return std::make_unique>(); +} + +template Status ModelInstance::infer(InferenceRequest const*, InferenceResponse*, std::unique_ptr&); + +template +RequestProcessor::RequestProcessor() = default; +template +RequestProcessor::~RequestProcessor() = default; +template +Status RequestProcessor::extractRequestParameters(const RequestType* request) { return StatusCode::OK; } +template +Status RequestProcessor::prepare() { return StatusCode::OK; } +template +Status RequestProcessor::preInferenceProcessing(ov::InferRequest& inferRequest) { return StatusCode::OK; } +template +Status RequestProcessor::postInferenceProcessing(ResponseType* response, ov::InferRequest& inferRequest) { return StatusCode::OK; } +template +Status RequestProcessor::release() { return StatusCode::OK; } + +template class RequestProcessor; +template class RequestProcessor; } // namespace ovms diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index 17c150971d..8d29465ce1 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -19,18 +19,15 @@ #include #include #include +#include #include #include #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop - +#include "inferencerequest.hpp" +#include "inferenceresponse.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "model_metric_reporter.hpp" #include "modelchangesubscription.hpp" @@ -39,12 +36,15 @@ #include "modelversionstatus.hpp" #include "ovinferrequestsqueue.hpp" #include "tensorinfo.hpp" +#include "tfs_frontend/tfs_utils.hpp" namespace ovms { class MetricRegistry; class ModelInstanceUnloadGuard; class PipelineDefinition; class Status; +template +struct RequestProcessor; class DynamicModelParameter { public: @@ -548,11 +548,9 @@ class ModelInstance { Status performInference(ov::InferRequest& inferRequest); - virtual Status infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr); - virtual Status infer(const ::KFSRequest* requestProto, - ::KFSResponse* responseProto, + template + Status infer(const RequestType* requestProto, + ResponseType* responseProto, std::unique_ptr& modelUnloadGuardPtr); ModelMetricReporter& getMetricReporter() const { return *this->reporter; } @@ -562,5 +560,19 @@ class ModelInstance { template void fetchModelFiles(bool& found, ArrayType ext); Status infer(float* data, float* output); + virtual std::unique_ptr> createRequestProcessor(const tensorflow::serving::PredictRequest*, tensorflow::serving::PredictResponse*); + virtual std::unique_ptr> createRequestProcessor(const KFSRequest*, KFSResponse*); + virtual std::unique_ptr> createRequestProcessor(const InferenceRequest*, InferenceResponse*); + virtual const std::set& getOptionalInputNames(); +}; +template +struct RequestProcessor { + RequestProcessor(); + virtual ~RequestProcessor(); + virtual Status extractRequestParameters(const RequestType* request); + virtual Status prepare(); + virtual Status preInferenceProcessing(ov::InferRequest& inferRequest); + virtual Status postInferenceProcessing(ResponseType* response, ov::InferRequest& inferRequest); + virtual Status release(); }; } // namespace ovms diff --git a/src/pocapi.cpp b/src/pocapi.cpp index f5b3dc5ee6..c5b1edd0ae 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -16,6 +16,7 @@ #include "pocapi.hpp" #include +#include #include #include "buffer.hpp" @@ -23,54 +24,94 @@ #include "inferencerequest.hpp" #include "inferenceresponse.hpp" #include "inferencetensor.hpp" +#include "model_service.hpp" +#include "modelinstance.hpp" +#include "modelinstanceunloadguard.hpp" +#include "modelmanager.hpp" +#include "profiler.hpp" +#include "servablemanagermodule.hpp" #include "server.hpp" #include "server_options.hpp" #include "status.hpp" +#include "timer.hpp" using ovms::Buffer; using ovms::InferenceParameter; using ovms::InferenceRequest; using ovms::InferenceResponse; using ovms::InferenceTensor; +using ovms::ModelInstanceUnloadGuard; +using ovms::ModelManager; +using ovms::ServableManagerModule; +using ovms::Server; using ovms::Status; using ovms::StatusCode; +using ovms::Timer; +using std::chrono::microseconds; OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } *options = reinterpret_cast(new ovms::GeneralOptionsImpl); return nullptr; } OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } delete reinterpret_cast(options); return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } *options = reinterpret_cast(new ovms::MultiModelOptionsImpl); return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } delete reinterpret_cast(options); return nullptr; } OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { // Create new server once multi server configuration becomes possible. + if (server == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); + } *server = reinterpret_cast(&ovms::Server::instance()); return nullptr; } OVMS_Status* OVMS_ServerDelete(OVMS_Server* server) { - // Make use of the server pointer instead of singleton once multi server configuration becomes possible. + if (server == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); + } ovms::Server* srv = reinterpret_cast(server); srv->shutdownModules(); + // delete passed in ptr once multi server configuration is done return nullptr; } OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_ServerGeneralOptions* general_options, OVMS_ServerMultiModelOptions* multi_model_specific_options) { + if (server == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); + } + if (general_options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (multi_model_specific_options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::Server* srv = reinterpret_cast(server); ovms::GeneralOptionsImpl* go = reinterpret_cast(general_options); ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(multi_model_specific_options); @@ -80,6 +121,9 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, uint32_t grpcPort) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->grpcPort = grpcPort; return nullptr; @@ -87,6 +131,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* opt OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, uint32_t restPort) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->restPort = restPort; return nullptr; @@ -94,6 +141,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* opt OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* options, uint32_t grpc_workers) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->grpcWorkers = grpc_workers; return nullptr; @@ -101,6 +151,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptions* options, const char* grpc_bind_address) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (grpc_bind_address == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->grpcBindAddress.assign(grpc_bind_address); return nullptr; @@ -108,6 +164,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptio OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* options, uint32_t rest_workers) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->restWorkers = rest_workers; return nullptr; @@ -115,6 +174,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptions* options, const char* rest_bind_address) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (rest_bind_address == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->restBindAddress.assign(rest_bind_address); return nullptr; @@ -122,6 +187,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptio OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneralOptions* options, const char* grpc_channel_arguments) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (grpc_channel_arguments == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->grpcChannelArguments.assign(grpc_channel_arguments); return nullptr; @@ -129,6 +200,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneral OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, uint32_t seconds) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->filesystemPollWaitSeconds = seconds; return nullptr; @@ -136,6 +210,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGe OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, uint32_t minutes) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->sequenceCleanerPollWaitMinutes = minutes; return nullptr; @@ -143,6 +220,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_Ser OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerGeneralOptions* options, uint32_t seconds) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->resourcesCleanerPollWaitSeconds = seconds; return nullptr; @@ -150,6 +230,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSecon OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, const char* cpu_extension_path) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (cpu_extension_path == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->cpuExtensionLibraryPath.assign(cpu_extension_path); return nullptr; @@ -157,6 +243,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOpti OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, const char* cache_dir) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (cache_dir == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->cacheDir.assign(cache_dir); return nullptr; @@ -164,6 +256,9 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* opt OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, OVMS_LogLevel log_level) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); switch (log_level) { case OVMS_LOG_INFO: @@ -190,6 +285,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* opt OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, const char* log_path) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (log_path == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::GeneralOptionsImpl* go = reinterpret_cast(options); go->logPath.assign(log_path); return nullptr; @@ -197,6 +298,12 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* opti OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, const char* config_path) { + if (options == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + } + if (config_path == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + } ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(options); mmo->configPath.assign(config_path); return nullptr; @@ -204,18 +311,34 @@ OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOpti // inference API OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion) { // TODO should we allow to create requests to not yet loaded models? + if (request == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (servableName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } *request = reinterpret_cast(new InferenceRequest(servableName, servableVersion)); return nullptr; } OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* request) { + if (request == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } delete reinterpret_cast(request); return nullptr; } OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (inputName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } + if (shape == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_TABLE)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->addInput(inputName, datatype, shape, dimCount); if (!status.ok()) { @@ -225,8 +348,15 @@ OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const cha } OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (inputName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } + if (data == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_DATA)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->setInputBuffer(inputName, data, bufferSize, bufferType, deviceId); if (!status.ok()) { @@ -236,8 +366,15 @@ OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const } OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* req, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (parameterName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } + if (data == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_DATA)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->addParameter(parameterName, datatype, data); if (!status.ok()) { @@ -247,8 +384,12 @@ OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* req, const } OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* req, const char* parameterName) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (parameterName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->removeParameter(parameterName); if (!status.ok()) { @@ -258,8 +399,12 @@ OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* req, co } OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* req, const char* inputName) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (inputName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->removeInput(inputName); if (!status.ok()) { @@ -269,8 +414,12 @@ OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* req, const } OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* req, const char* inputName) { - // TODO error handling if null - // if (nullptr == req) + if (req == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (inputName == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } InferenceRequest* request = reinterpret_cast(req); auto status = request->removeInputBuffer(inputName); if (!status.ok()) { @@ -279,11 +428,36 @@ OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* req, co return nullptr; } -OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId) { - // TODO error handling if null - // if (nullptr == req) +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId) { + if (res == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } + if (name == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + } + if (datatype == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } + if (shape == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_TABLE)); + } + if (dimCount == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } + if (data == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_DATA)); + } + if (bytesize == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } + if (bufferType == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } + if (deviceId == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } InferenceResponse* response = reinterpret_cast(res); - InferenceTensor* tensor = nullptr; + const InferenceTensor* tensor = nullptr; const std::string* cppName; auto status = response->getOutput(id, &cppName, &tensor); if (!status.ok()) { @@ -304,30 +478,45 @@ OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32 *bufferType = buffer->getBufferType(); *deviceId = buffer->getDeviceId().value_or(0); // TODO how discriminate betwen undefined & actual device 0 // possibly it is not neccessary to discriminate - *data = const_cast(buffer->data()); // should data return const ptr? + *data = buffer->data(); *bytesize = buffer->getByteSize(); return nullptr; } OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* res, uint32_t* count) { - // TODO error handling if null - // if (nullptr == req) + if (res == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } + if (count == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } InferenceResponse* response = reinterpret_cast(res); *count = response->getOutputCount(); return nullptr; } OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* res, uint32_t* count) { - // TODO error handling if null - // if (nullptr == req) + if (res == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } + if (count == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } InferenceResponse* response = reinterpret_cast(res); *count = response->getParameterCount(); return nullptr; } OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* res, uint32_t id, OVMS_DataType* datatype, const void** data) { - // TODO error handling if null - // if (nullptr == req) + if (res == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } + if (datatype == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + } + if (data == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_DATA)); + } InferenceResponse* response = reinterpret_cast(res); const InferenceParameter* parameter = response->getParameter(id); if (nullptr == parameter) { @@ -339,9 +528,93 @@ OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* res, uin } OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* res) { - // TODO error handling if null - // if (nullptr == req) + if (res == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } InferenceResponse* response = reinterpret_cast(res); delete response; return nullptr; } + +namespace { +enum : unsigned int { + TOTAL, + TIMER_END +}; + +static Status getModelInstance(ovms::Server& server, const InferenceRequest* request, std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr) { + OVMS_PROFILE_FUNCTION(); + auto& modelManager = dynamic_cast(server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME))->getServableManager(); + return modelManager.getModelInstance(request->getServableName(), request->getServableVersion(), modelInstance, modelInstanceUnloadGuardPtr); +} +} // namespace +OVMS_Status* OVMS_Inference(OVMS_Server* serverPtr, OVMS_InferenceRequest* request, OVMS_InferenceResponse** response) { + OVMS_PROFILE_FUNCTION(); + using std::chrono::microseconds; + Timer timer; + timer.start(TOTAL); + if (serverPtr == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); + } + if (request == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); + } + if (response == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); + } + auto req = reinterpret_cast(request); + ovms::Server& server = *reinterpret_cast(serverPtr); + std::unique_ptr res(new ovms::InferenceResponse(req->getServableName(), req->getServableVersion())); + + SPDLOG_DEBUG("Processing C-API request for model: {}; version: {}", + req->getServableName(), + req->getServableVersion()); + + std::shared_ptr modelInstance; + // std::unique_ptr pipelinePtr; + + std::unique_ptr modelInstanceUnloadGuard; + auto status = getModelInstance(server, req, modelInstance, modelInstanceUnloadGuard); + + if (status == StatusCode::MODEL_NAME_MISSING) { + SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", req->getServableName()); + // status = getPipeline(req, response, pipelinePtr); + status = Status(StatusCode::NOT_IMPLEMENTED, "Inference with DAG not supported with C-API in preview"); + } + if (!status.ok()) { + if (modelInstance) { + // INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().reqFailGrpcPredict); + } + SPDLOG_INFO("Getting modelInstance or pipeline failed. {}", status.string()); + return reinterpret_cast(new Status(status)); + } + + // ExecutionContext executionContext{ + // ExecutionContext::Interface::CAPI, + // ExecutionContext::Method::Inference}; + + // if (pipelinePtr) { + // status = pipelinePtr->execute(executionContext); + // INCREMENT_IF_ENABLED(pipelinePtr->getMetricReporter().getInferRequestMetric(executionContext, status.ok())); + // } else { + status = modelInstance->infer(req, res.get(), modelInstanceUnloadGuard); + // INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().getInferRequestMetric(executionContext, status.ok())); + //} + + if (!status.ok()) { + return reinterpret_cast(new Status(status)); + } + + timer.stop(TOTAL); + double reqTotal = timer.elapsed(TOTAL); + // if (pipelinePtr) { + // OBSERVE_IF_ENABLED(pipelinePtr->getMetricReporter().reqTimeGrpc, reqTotal); + // } else { + // OBSERVE_IF_ENABLED(modelInstance->getMetricReporter().reqTimeGrpc, reqTotal); + // } + SPDLOG_DEBUG("Total C-API req processing time: {} ms", reqTotal / 1000); + *response = reinterpret_cast(res.release()); + return nullptr; + // return grpc::Status::OK; +} diff --git a/src/pocapi.hpp b/src/pocapi.hpp index 08a0846839..7fd15eede8 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -185,13 +185,9 @@ OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* request // OVMS_Inference Response OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId); +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId); OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, const void** data); OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); -OVMS_Status* OVMS_Inference(OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); - -// POCAPI to be removed -int OVMS_Start(int argc, char** argv); -void OVMS_Infer(char* name, float* data, float* output); +OVMS_Status* OVMS_Inference(OVMS_Server* server, OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index 6ea3d8e206..0b43cd220f 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -650,14 +650,24 @@ Status RequestValidator Status RequestValidator::validateTensorContent(const InferenceTensor& tensor, ovms::Precision expectedPrecision, size_t bufferId) const { + const Buffer* buffer = tensor.getBuffer(); + if (nullptr == buffer) { + std::stringstream ss; + ss << "Servable: " << servableName + << "; version: " << servableVersion + << "; is missing buffer for tensor: " << bufferId; + const std::string details = ss.str(); + SPDLOG_DEBUG(details); + return Status(StatusCode::INVALID_CONTENT_SIZE, details); // TODO separate code? + } size_t expectedValueCount = 1; for (size_t i = 0; i < tensor.getShape().size(); i++) { expectedValueCount *= tensor.getShape()[i]; } size_t expectedContentSize = expectedValueCount * ov::element::Type(ovmsPrecisionToIE2Precision(expectedPrecision)).size(); - if (expectedContentSize != tensor.getBuffer()->getByteSize()) { + if (expectedContentSize != buffer->getByteSize()) { std::stringstream ss; - ss << "Expected: " << expectedContentSize << " bytes; Actual: " << tensor.getBuffer()->getByteSize() << " bytes; input name: " << getCurrentlyValidatedInputName(); + ss << "Expected: " << expectedContentSize << " bytes; Actual: " << buffer->getByteSize() << " bytes; input name: " << getCurrentlyValidatedInputName(); const std::string details = ss.str(); SPDLOG_DEBUG("[servable name: {} version: {}] Invalid content size of tensor - {}", servableName, servableVersion, details); return Status(StatusCode::INVALID_CONTENT_SIZE, details); diff --git a/src/prediction_service_utils.cpp b/src/prediction_service_utils.cpp index 255215adae..506e334ac8 100644 --- a/src/prediction_service_utils.cpp +++ b/src/prediction_service_utils.cpp @@ -19,6 +19,8 @@ #include "deserialization.hpp" #include "executingstreamidguard.hpp" +#include "inferencerequest.hpp" +#include "inferencetensor.hpp" #include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" @@ -96,6 +98,18 @@ std::map getRequestShapes(const tensorflow::serving::Predi } return requestShapes; } +std::optional getRequestBatchSize(const InferenceRequest* request, const size_t batchSizeIndex) { + size_t bs = 0; + auto status = request->getBatchSize(bs, batchSizeIndex); + if (!status.ok()) { + return std::nullopt; // TODO sth different? + } + return bs; +} + +std::map getRequestShapes(const InferenceRequest* request) { + return request->getRequestShapes(); +} bool useSharedOutputContent(const tensorflow::serving::PredictRequest* request) { return true; diff --git a/src/prediction_service_utils.hpp b/src/prediction_service_utils.hpp index 41badb69d0..3c91645ef5 100644 --- a/src/prediction_service_utils.hpp +++ b/src/prediction_service_utils.hpp @@ -27,14 +27,17 @@ #include "shape.hpp" namespace ovms { +class InferenceRequest; std::optional getRequestBatchSize(const ::KFSRequest* request, const size_t batchSizeIndex); - std::map getRequestShapes(const ::KFSRequest* request); std::optional getRequestBatchSize(const tensorflow::serving::PredictRequest* request, const size_t batchSizeIndex); std::map getRequestShapes(const tensorflow::serving::PredictRequest* request); +std::optional getRequestBatchSize(const InferenceRequest* request, const size_t batchSizeIndex); +std::map getRequestShapes(const InferenceRequest* request); + bool useSharedOutputContent(const tensorflow::serving::PredictRequest* request); bool useSharedOutputContent(const ::inference::ModelInferRequest* request); } // namespace ovms diff --git a/src/serialization.hpp b/src/serialization.hpp index 93eb2fa261..d663701134 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -87,6 +87,8 @@ const std::string& getOutputMapKeyName(const std::string& first, const TensorInf template Status serializePredictResponse( OutputGetter& outputGetter, + const std::string& servableName, + model_version_t servableVersion, const tensor_map_t& outputMap, tensorflow::serving::PredictResponse* response, outputNameChooser_t outputNameChooser, @@ -112,12 +114,16 @@ Status serializePredictResponse( template Status serializePredictResponse( OutputGetter& outputGetter, + const std::string& servableName, + model_version_t servableVersion, const tensor_map_t& outputMap, ::KFSResponse* response, outputNameChooser_t outputNameChooser, bool useSharedOutputContent = true) { OVMS_PROFILE_FUNCTION(); Status status; + response->set_model_name(servableName); + response->set_model_version(std::to_string(servableVersion)); ProtoGetter<::KFSResponse*, ::KFSResponse::InferOutputTensor&> protoGetter(response); for (const auto& [outputName, outputInfo] : outputMap) { ov::Tensor tensor; @@ -141,6 +147,8 @@ Status serializePredictResponse( template Status serializePredictResponse( OutputGetter& outputGetter, + const std::string& servableName, + model_version_t servableVersion, const tensor_map_t& outputMap, InferenceResponse* response, outputNameChooser_t outputNameChooser) { diff --git a/src/statefulmodelinstance.cpp b/src/statefulmodelinstance.cpp index 8a237bff13..fcceb4fb16 100644 --- a/src/statefulmodelinstance.cpp +++ b/src/statefulmodelinstance.cpp @@ -35,15 +35,15 @@ const std::set StatefulModelInstance::SPECIAL_INPUT_NAMES{"sequence const Status StatefulModelInstance::extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId) { if (!proto.tensor_shape().dim_size()) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto does not contain tensor shape information", getName(), getVersion()); + SPDLOG_DEBUG("Sequence id tensor proto does not contain tensor shape information"); return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; } else if (proto.tensor_shape().dim_size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid number of dimensions. Expecting shape with one dimension", getName(), getVersion()); + SPDLOG_DEBUG("Sequence id tensor proto shape has invalid number of dimensions. Expecting shape with one dimension"); return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_id is: (1)"); } if (proto.tensor_shape().dim(0).size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence id tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); + SPDLOG_DEBUG("Sequence id tensor proto shape has invalid shape. Expecting shape: (1)"); return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_id is: (1)"); } @@ -56,15 +56,15 @@ const Status StatefulModelInstance::extractSequenceId(const tensorflow::TensorPr const Status StatefulModelInstance::extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput) { if (proto.tensor_shape().dim_size() == 0) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto does not contain tensor shape information", getName(), getVersion()); + SPDLOG_DEBUG("Sequence control tensor proto does not contain tensor shape information"); return StatusCode::SPECIAL_INPUT_NO_TENSOR_SHAPE; } else if (proto.tensor_shape().dim_size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid number of dimensions. Expecting shape with one dimension.", getName(), getVersion()); + SPDLOG_DEBUG("Sequence control tensor proto shape has invalid number of dimensions. Expecting shape with one dimension."); return Status(StatusCode::INVALID_NO_OF_SHAPE_DIMENSIONS, "Required shape for sequence_control_input is: (1)"); } if (proto.tensor_shape().dim(0).size() != 1) { - SPDLOG_DEBUG("[Model: {} version: {}] Sequence control tensor proto shape has invalid shape. Expecting shape: (1)", getName(), getVersion()); + SPDLOG_DEBUG("Sequence control tensor proto shape has invalid shape. Expecting shape: (1)"); return Status(StatusCode::INVALID_SHAPE, "Required shape for sequence_control_input is: (1)"); } @@ -150,7 +150,7 @@ Status StatefulModelInstance::loadOVCompiledModel(const ModelConfig& config) { } template <> -const Status StatefulModelInstance::validateSpecialKeys(const tensorflow::serving::PredictRequest* request, SequenceProcessingSpec& sequenceProcessingSpec) { +const Status StatefulModelInstance::extractSpecialKeys(const tensorflow::serving::PredictRequest* request, SequenceProcessingSpec& sequenceProcessingSpec) { uint64_t sequenceId = 0; uint32_t sequenceControlInput = 0; Status status; @@ -180,118 +180,84 @@ const Status StatefulModelInstance::validateSpecialKeys(const tensorflow::servin return StatusCode::OK; } -template -const Status StatefulModelInstance::validate(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec) { - OVMS_PROFILE_FUNCTION(); - auto status = validateSpecialKeys(request, sequenceProcessingSpec); - if (!status.ok()) - return status; - - return request_validation_utils::validate( - *request, - getInputsInfo(), - getName(), - getVersion(), - SPECIAL_INPUT_NAMES, - getModelConfig().getBatchingMode(), - getModelConfig().getShapes()); +const std::set& StatefulModelInstance::getOptionalInputNames() { + return SPECIAL_INPUT_NAMES; } - -Status StatefulModelInstance::infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) { +template <> +StatefulRequestProcessor::StatefulRequestProcessor(SequenceManager& sequenceManager) : + sequenceManager(sequenceManager) { +} +template <> +Status StatefulRequestProcessor::extractRequestParameters(const tensorflow::serving::PredictRequest* request) { OVMS_PROFILE_FUNCTION(); - enum : unsigned int { - GET_INFER_REQUEST, - PREPROCESS, - DESERIALIZE, - PREDICTION, - SERIALIZE, - POSTPROCESS, - TIMER_END - }; - Timer timer; - using std::chrono::microseconds; - SequenceProcessingSpec sequenceProcessingSpec; - auto status = validate(requestProto, sequenceProcessingSpec); - if (!status.ok()) - return status; - - std::unique_lock sequenceManagerLock(sequenceManager->getMutex()); - status = sequenceManager->processRequestedSpec(sequenceProcessingSpec); + auto status = StatefulModelInstance::extractSpecialKeys(request, sequenceProcessingSpec); + return status; +} +template <> +Status StatefulRequestProcessor::prepare() { + sequenceManagerLock = std::make_unique>(sequenceManager.getMutex()); + auto status = sequenceManager.processRequestedSpec(sequenceProcessingSpec); if (!status.ok()) return status; - const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); - if (!sequenceManager->sequenceExists(sequenceId)) + this->sequenceId = sequenceProcessingSpec.getSequenceId(); + if (!sequenceManager.sequenceExists(this->sequenceId)) return StatusCode::INTERNAL_ERROR; - Sequence& sequence = sequenceManager->getSequence(sequenceId); - - std::unique_lock sequenceLock(sequence.getMutex()); - sequenceManagerLock.unlock(); - - timer.start(GET_INFER_REQUEST); - ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); - int executingInferId = executingStreamIdGuard.getId(); - ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); - timer.stop(GET_INFER_REQUEST); - double getInferRequestTime = timer.elapsed(GET_INFER_REQUEST); - OBSERVE_IF_ENABLED(this->getMetricReporter().waitForInferReqTime, getInferRequestTime); - SPDLOG_DEBUG("Getting infer req duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, getInferRequestTime / 1000); - - timer.start(PREPROCESS); - status = preInferenceProcessing(inferRequest, sequence, sequenceProcessingSpec); - timer.stop(PREPROCESS); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Preprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(PREPROCESS) / 1000); + // TODO should be able to search & get in one go + sequence = &sequenceManager.getSequence(this->sequenceId); - timer.start(DESERIALIZE); - InputSink inputSink(inferRequest); - bool isPipeline = false; - status = deserializePredictRequest(*requestProto, getInputsInfo(), inputSink, isPipeline); - timer.stop(DESERIALIZE); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Deserialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(DESERIALIZE) / 1000); - - timer.start(PREDICTION); - status = performInference(inferRequest); - timer.stop(PREDICTION); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Prediction duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(PREDICTION) / 1000); - - timer.start(SERIALIZE); - OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, getTensorInfoName); - timer.stop(SERIALIZE); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Serialization duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(SERIALIZE) / 1000); - - timer.start(POSTPROCESS); - status = postInferenceProcessing(responseProto, inferRequest, sequence, sequenceProcessingSpec); - timer.stop(POSTPROCESS); - if (!status.ok()) - return status; - SPDLOG_DEBUG("Postprocessing duration in model {}, version {}, nireq {}: {:.3f} ms", - requestProto->model_spec().name(), getVersion(), executingInferId, timer.elapsed(POSTPROCESS) / 1000); - - sequenceLock.unlock(); + sequenceLock = std::make_unique>(sequence->getMutex()); + sequenceManagerLock->unlock(); + return StatusCode::OK; +} +template <> +Status StatefulRequestProcessor::preInferenceProcessing(ov::InferRequest& inferRequest) { + if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_START) { + // On SEQUENCE_START reset memory state of infer request to default + for (auto&& state : inferRequest.query_state()) { + state.reset(); + } + } else { + // For next requests in the sequence set infer request memory state to the last state saved by the sequence + const sequence_memory_state_t& sequenceMemoryState = sequence->getMemoryState(); + for (auto&& state : inferRequest.query_state()) { + auto stateName = state.get_name(); + if (!sequenceMemoryState.count(stateName)) + return StatusCode::INTERNAL_ERROR; + state.set_state(sequenceMemoryState.at(stateName)); + } + } + return StatusCode::OK; +} +template <> +Status StatefulRequestProcessor::postInferenceProcessing(tensorflow::serving::PredictResponse* response, ov::InferRequest& inferRequest) { + // Reset inferRequest states on SEQUENCE_END if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { - sequenceManagerLock.lock(); - status = sequenceManager->removeSequence(sequenceId); - if (!status.ok()) - return status; + SPDLOG_DEBUG("Received SEQUENCE_END signal. Reseting model state"); + for (auto&& state : inferRequest.query_state()) { + state.reset(); + } + } else { + auto modelState = inferRequest.query_state(); + sequence->updateMemoryState(modelState); } - + // Include sequence_id in server response + auto& tensorProto = (*response->mutable_outputs())["sequence_id"]; + tensorProto.mutable_tensor_shape()->add_dim()->set_size(1); + tensorProto.set_dtype(tensorflow::DataType::DT_UINT64); + tensorProto.add_uint64_val(sequenceProcessingSpec.getSequenceId()); return StatusCode::OK; } +template <> +Status StatefulRequestProcessor::release() { + SPDLOG_DEBUG("Received SEQUENCE_END signal. Removing sequence"); + sequenceLock->unlock(); + Status status; + if (sequenceProcessingSpec.getSequenceControlInput() == SEQUENCE_END) { + sequenceManagerLock->lock(); + status = sequenceManager.removeSequence(this->sequenceId); + } + return status; +} const Status StatefulModelInstance::preInferenceProcessing(ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec) { @@ -334,4 +300,8 @@ const Status StatefulModelInstance::postInferenceProcessing(tensorflow::serving: return StatusCode::OK; } + +std::unique_ptr> StatefulModelInstance::createRequestProcessor(const tensorflow::serving::PredictRequest*, tensorflow::serving::PredictResponse*) { + return std::make_unique>(*this->getSequenceManager()); +} } // namespace ovms diff --git a/src/statefulmodelinstance.hpp b/src/statefulmodelinstance.hpp index f6d4c30a13..69b5afd3d0 100644 --- a/src/statefulmodelinstance.hpp +++ b/src/statefulmodelinstance.hpp @@ -22,6 +22,7 @@ #include "global_sequences_viewer.hpp" #include "modelinstance.hpp" #include "sequence_manager.hpp" +#include "sequence_processing_spec.hpp" namespace ovms { class ModelConfig; @@ -42,9 +43,9 @@ class StatefulModelInstance : public ModelInstance { return this->sequenceManager; } - const Status extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId); + static const Status extractSequenceId(const tensorflow::TensorProto& proto, uint64_t& sequenceId); - const Status extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput); + static const Status extractSequenceControlInput(const tensorflow::TensorProto& proto, uint32_t& sequenceControlInput); /* Performs pre inference operations: - for SEQUENCE_START control input - reset InferRequest memory state @@ -65,10 +66,6 @@ class StatefulModelInstance : public ModelInstance { const Status postInferenceProcessing(tensorflow::serving::PredictResponse* response, ov::InferRequest& inferRequest, Sequence& sequence, SequenceProcessingSpec& sequenceProcessingSpec); - Status infer(const tensorflow::serving::PredictRequest* requestProto, - tensorflow::serving::PredictResponse* responseProto, - std::unique_ptr& modelUnloadGuardPtr) override; - Status loadModel(const ModelConfig& config) override; Status reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; @@ -86,15 +83,32 @@ class StatefulModelInstance : public ModelInstance { GlobalSequencesViewer* globalSequencesViewer; - template - const Status validate(const RequestType* request, SequenceProcessingSpec& processingSpec); - Status loadModelImpl(const ModelConfig& config, const DynamicModelParameter& parameter = DynamicModelParameter()) override; Status loadOVCompiledModel(const ModelConfig& config) override; -private: +public: template - const Status validateSpecialKeys(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec); + static const Status extractSpecialKeys(const RequestType* request, SequenceProcessingSpec& sequenceProcessingSpec); + + std::unique_ptr> createRequestProcessor(const tensorflow::serving::PredictRequest*, tensorflow::serving::PredictResponse*) override; + const std::set& getOptionalInputNames() override; +}; + +template +struct StatefulRequestProcessor : public RequestProcessor { + SequenceManager& sequenceManager; + std::unique_ptr> sequenceManagerLock; + std::unique_ptr> sequenceLock; + SequenceProcessingSpec sequenceProcessingSpec; + Sequence* sequence; + uint64_t sequenceId; + + StatefulRequestProcessor(SequenceManager& sequenceManager); + Status extractRequestParameters(const RequestType* request) override; + Status prepare() override; + Status preInferenceProcessing(ov::InferRequest& inferRequest) override; + Status postInferenceProcessing(ResponseType* response, ov::InferRequest& inferRequest) override; + Status release() override; }; } // namespace ovms diff --git a/src/status.cpp b/src/status.cpp index 504894d792..c283a57a43 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -266,11 +266,19 @@ const std::unordered_map Status::statusMess {StatusCode::DOUBLE_BUFFER_SET, "Cannot set buffer more than once to the same tensor"}, {StatusCode::DOUBLE_TENSOR_INSERT, "Cannot insert more than one tensor with the same name"}, {StatusCode::DOUBLE_PARAMETER_INSERT, "Cannot insert more than one parameter with the same name"}, + {StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL, "Tried to remove nonexisting buffer"}, + {StatusCode::NONEXISTENT_DATA, "Tried to use nonexisting data"}, + {StatusCode::NONEXISTENT_STRING, "Tried to use nonexisting string"}, + {StatusCode::NONEXISTENT_NUMBER, "Tried to use nonexisting number"}, + {StatusCode::NONEXISTENT_OPTIONS, "Tried to use nonexisting options"}, + {StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL, "Tried to remove nonexisting parameter"}, + {StatusCode::NONEXISTENT_RESPONSE, "Tried to use nonexisting response"}, + {StatusCode::NONEXISTENT_REQUEST, "Tried to use nonexisting request"}, + {StatusCode::NONEXISTENT_SERVER, "Tried to use nonexisting server"}, + {StatusCode::NONEXISTENT_TABLE, "Tried to use nonexisting table"}, {StatusCode::NONEXISTENT_TENSOR, "Tried to get nonexisting tensor"}, {StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER, "Tried to set buffer for nonexisting tensor"}, {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, "Tried to remove buffer for nonexisting tensor"}, {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL, "Tried to remove nonexisting tensor"}, - {StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL, "Tried to remove nonexisting buffer"}, - {StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL, "Tried to remove nonexisting parameter"}, }; } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 4045c6d32d..947df37f37 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -277,12 +277,20 @@ enum class StatusCode { DOUBLE_BUFFER_SET, DOUBLE_TENSOR_INSERT, DOUBLE_PARAMETER_INSERT, + NONEXISTENT_BUFFER_FOR_REMOVAL, + NONEXISTENT_DATA, + NONEXISTENT_STRING, + NONEXISTENT_NUMBER, + NONEXISTENT_OPTIONS, + NONEXISTENT_PARAMETER_FOR_REMOVAL, // rename to non existen parameter + NONEXISTENT_SERVER, + NONEXISTENT_RESPONSE, + NONEXISTENT_REQUEST, + NONEXISTENT_TABLE, NONEXISTENT_TENSOR, NONEXISTENT_TENSOR_FOR_SET_BUFFER, NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, NONEXISTENT_TENSOR_FOR_REMOVAL, - NONEXISTENT_BUFFER_FOR_REMOVAL, - NONEXISTENT_PARAMETER_FOR_REMOVAL, // rename to non existen parameter STATUS_CODE_END }; diff --git a/src/test/c_api/config_standard_dummy.json b/src/test/c_api/config_standard_dummy.json new file mode 100644 index 0000000000..cb1c346348 --- /dev/null +++ b/src/test/c_api/config_standard_dummy.json @@ -0,0 +1,8 @@ +{ + "model_config_list": [ + {"config": { + "name": "dummy", + "base_path": "/ovms/src/test/dummy", + "shape": "(1, 10)"}} + ] +} diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 77b881b367..7f9266f36b 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -18,12 +18,8 @@ // TODO we should not include classes from OVMS here // consider how to workaround test_utils -#include "../config.hpp" #include "../inferenceresponse.hpp" -#include "../logging.hpp" -#include "../modelconfig.hpp" #include "../pocapi.hpp" -#include "../server_options.hpp" #include "test_utils.hpp" using namespace ovms; @@ -204,9 +200,9 @@ TEST(CApiStartTest, StartFlow) { ASSERT_EQ(OVMS_ServerDelete(srv), nullptr); } -class CapiInferencePreparationTest : public ::testing::Test {}; +class CapiInference : public ::testing::Test {}; -TEST_F(CapiInferencePreparationTest, Basic) { +TEST_F(CapiInference, Basic) { // request creation OVMS_InferenceRequest* request{nullptr}; OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); @@ -236,10 +232,70 @@ TEST_F(CapiInferencePreparationTest, Basic) { ////////////////// // INFERENCE ////////////////// + // remove when C-API start implemented + std::string port = "9000"; + randomizePort(port); + // prepare options + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + ASSERT_NE(go, nullptr); + ASSERT_NE(mmo, nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port)), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"), nullptr); + + OVMS_Server* cserver = nullptr; + ASSERT_EQ(OVMS_ServerNew(&cserver), nullptr); + ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo), nullptr); + + OVMS_InferenceResponse* response = nullptr; + status = OVMS_Inference(cserver, request, &response); + ASSERT_EQ(nullptr, status); + // verify GetOutputCount + uint32_t outputCount = 42; + status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(outputCount, 1); + // verify GetParameterCount + uint32_t parameterCount = 42; + status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(0, parameterCount); + // verify GetOutput + const void* voutputData; + size_t bytesize = 42; + uint32_t outputId = 0; + OVMS_DataType datatype = (OVMS_DataType)199; + const uint64_t* shape{nullptr}; + uint32_t dimCount = 42; + BufferType bufferType = (BufferType)199; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + status = OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); + ASSERT_EQ(nullptr, status); + ASSERT_EQ(std::string(DUMMY_MODEL_OUTPUT_NAME), outputName); + EXPECT_EQ(datatype, OVMS_DATATYPE_FP32); + EXPECT_EQ(dimCount, 2); + EXPECT_EQ(bufferType, OVMS_BUFFERTYPE_CPU); + EXPECT_EQ(deviceId, 0); + + for (size_t i = 0; i < DUMMY_MODEL_SHAPE.size(); ++i) { + EXPECT_EQ(DUMMY_MODEL_SHAPE[i], shape[i]) << "Different at:" << i << " place."; + } + const float* outputData = reinterpret_cast(voutputData); + ASSERT_EQ(bytesize, sizeof(float) * DUMMY_MODEL_INPUT_SIZE); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(data[i] + 1, outputData[i]) << "Different at:" << i << " place."; + } /////////////// // CLEANUP /////////////// + // cleanup response + status = OVMS_InferenceResponseDelete(response); + ASSERT_EQ(nullptr, status); + // cleanup request // here we will add additional inputs to verify 2 ways of cleanup // - direct call to remove whole request // - separate calls to remove partial data @@ -284,9 +340,83 @@ TEST_F(CapiInferencePreparationTest, Basic) { // OVMS_StatusDelete(status); // FIXME(dkalinow) status = OVMS_InferenceRequestDelete(request); ASSERT_EQ(nullptr, status); + + ASSERT_EQ(OVMS_ServerDelete(cserver), nullptr); +} + +TEST_F(CapiInference, NegativeInference) { + // first start OVMS + std::string port = "9000"; + randomizePort(port); + // prepare options + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + ASSERT_NE(go, nullptr); + ASSERT_NE(mmo, nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port)), nullptr); + ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"), nullptr); + + OVMS_Server* cserver = nullptr; + ASSERT_EQ(OVMS_ServerNew(&cserver), nullptr); + ASSERT_NE(OVMS_ServerStartFromConfigurationFile(nullptr, go, mmo), nullptr); + ASSERT_NE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, mmo), nullptr); + ASSERT_NE(OVMS_ServerStartFromConfigurationFile(cserver, go, nullptr), nullptr); + ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo), nullptr); + // TODO ensure cleanup + // TODO add check for server ready strict in 2022.3 not available in C-API + + OVMS_InferenceRequest* request{nullptr}; + OVMS_InferenceResponse* response = nullptr; + OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); + ASSERT_EQ(nullptr, status); + ASSERT_NE(nullptr, request); + // negative no inputs + status = OVMS_Inference(cserver, request, &response); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + + // negative no input buffer + status = OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); + ASSERT_EQ(nullptr, status); + status = OVMS_Inference(cserver, request, &response); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + + // setting buffer + std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + uint32_t notUsedNum = 0; + status = OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); + ASSERT_EQ(nullptr, status); + // add parameters + const uint64_t sequenceId{42}; + status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); + ASSERT_EQ(nullptr, status); + // 2nd time should get error + status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + const uint32_t sequenceControl{1}; // SEQUENCE_START + status = OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl)); + ASSERT_EQ(nullptr, status); + + // verify passing nullptrs + status = OVMS_Inference(nullptr, request, &response); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); FIXME + status = OVMS_Inference(cserver, nullptr, &response); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); FIXME + status = OVMS_Inference(cserver, request, nullptr); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); FIXME + + ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); + ASSERT_EQ(OVMS_ServerDelete(cserver), nullptr); + ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); } // TODO negative test -> validate at the infer stage -// TODO flow with removel just request no separate input/buffer // TODO reuse request after inference namespace { const std::string MODEL_NAME{"SomeModelName"}; @@ -304,8 +434,8 @@ const std::array INPUT_DATA{1, 2, 3, 4, 5, 6, 7, constexpr size_t INPUT_DATA_BYTESIZE{INPUT_DATA.size() * sizeof(float)}; const OVMS_DataType DATATYPE{OVMS_DATATYPE_FP32}; } // namespace -class CapiInferenceRetrievalTest : public ::testing::Test {}; -TEST_F(CapiInferenceRetrievalTest, Basic) { + +TEST_F(CapiInference, ResponseRetrieval) { auto cppResponse = std::make_unique(MODEL_NAME, MODEL_VERSION); // add output std::array cppOutputShape{1, DUMMY_MODEL_INPUT_SIZE}; @@ -328,12 +458,12 @@ TEST_F(CapiInferenceRetrievalTest, Basic) { // now response is prepared so we can test C-API /////////////////////////// OVMS_InferenceResponse* response = reinterpret_cast(cppResponse.get()); - uint32_t outputCount = -1; + uint32_t outputCount = 42; auto status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); ASSERT_EQ(nullptr, status); ASSERT_EQ(outputCount, 1); - uint32_t parameterCount = -1; + uint32_t parameterCount = 42; status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); ASSERT_EQ(nullptr, status); ASSERT_EQ(1, parameterCount); @@ -345,14 +475,14 @@ TEST_F(CapiInferenceRetrievalTest, Basic) { ASSERT_EQ(parameterDatatype, OVMS_DATATYPE_U64); EXPECT_EQ(0, std::memcmp(parameterData, (void*)&seqId, sizeof(seqId))); // verify get Output - void* voutputData; - size_t bytesize = -1; + const void* voutputData; + size_t bytesize = 42; uint32_t outputId = 0; OVMS_DataType datatype = (OVMS_DataType)199; const uint64_t* shape{nullptr}; - uint32_t dimCount = -1; + uint32_t dimCount = 42; BufferType bufferType = (BufferType)199; - uint32_t deviceId = -1; + uint32_t deviceId = 42; const char* outputName{nullptr}; status = OVMS_InferenceResponseGetOutput(response, outputId + 42123, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); ASSERT_NE(nullptr, status); @@ -368,13 +498,12 @@ TEST_F(CapiInferenceRetrievalTest, Basic) { for (size_t i = 0; i < cppOutputShape.size(); ++i) { EXPECT_EQ(cppOutputShape[i], shape[i]) << "Different at:" << i << " place."; } - float* outputData = reinterpret_cast(voutputData); + const float* outputData = reinterpret_cast(voutputData); ASSERT_EQ(bytesize, sizeof(float) * DUMMY_MODEL_INPUT_SIZE); for (size_t i = 0; i < INPUT_DATA.size(); ++i) { EXPECT_EQ(INPUT_DATA[i], outputData[i]) << "Different at:" << i << " place."; } - // we release unique_ptr ownership here so that we can free it safely via C-API // test negative scenario with getting output without buffer cppStatus = cppResponse->addOutput("outputWithNoBuffer", DATATYPE, cppOutputShape.data(), cppOutputShape.size()); ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); @@ -386,6 +515,7 @@ TEST_F(CapiInferenceRetrievalTest, Basic) { ASSERT_NE(nullptr, status); // OVMS_StatusDelete(status); // FIXME(dkalinow) // final cleanup + // we release unique_ptr ownership here so that we can free it safely via C-API cppResponse.release(); status = OVMS_InferenceResponseDelete(response); ASSERT_EQ(nullptr, status); diff --git a/src/test/capi_predict_validation_test.cpp b/src/test/capi_predict_validation_test.cpp index f78ae6c286..6c4209c261 100644 --- a/src/test/capi_predict_validation_test.cpp +++ b/src/test/capi_predict_validation_test.cpp @@ -29,6 +29,8 @@ using ::testing::NiceMock; using ::testing::Return; using ::testing::ReturnRef; +using ovms::InferenceRequest; + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-variable" @@ -401,6 +403,18 @@ TEST_F(CAPIPredictValidation, RequestIncorrectContentSize) { EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); } +TEST_F(CAPIPredictValidation, RequestIncorrectInputWithNoBuffer) { + servableInputs = ovms::tensor_map_t({{"Input_FP32_1_1_1_1_NHWC", + std::make_shared("Input_FP32_1_3_224_224_NHWC", ovms::Precision::FP32, ovms::shape_t{1, 1, 1, 1}, ovms::Layout{"NHWC"})}}); + ON_CALL(*instance, getInputsInfo()).WillByDefault(ReturnRef(servableInputs)); + + InferenceRequest request("NOT_USED", 42); + std::array shape{1, 1, 1, 1}; + request.addInput("Input_FP32_1_1_1_1_NHWC", OVMS_DATATYPE_FP32, shape.data(), shape.size()); + auto status = instance->mockValidate(&request); + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); // TODO change retcode? +} + TEST_F(CAPIPredictValidation, RequestIncorrectContentSizeZero) { decrementBufferSize = 602112; diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index 03e41d5e21..f18aa98981 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -26,11 +26,17 @@ #include #include +#include "../buffer.hpp" #include "../deserialization.hpp" #include "../executingstreamidguard.hpp" +#include "../inferenceparameter.hpp" // TODO move bytesize util +#include "../inferencerequest.hpp" +#include "../inferenceresponse.hpp" +#include "../inferencetensor.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" +#include "../modelversion.hpp" #include "../prediction_service_utils.hpp" #include "../sequence_processing_spec.hpp" #include "../serialization.hpp" @@ -38,8 +44,12 @@ #include "test_utils.hpp" using testing::Each; +using testing::ElementsAre; using testing::Eq; +using ovms::Buffer; +using ovms::InferenceResponse; +using ovms::InferenceTensor; using ovms::StatusCode; // TODO: These tests test both TFS and KFS for prediction, @@ -51,14 +61,14 @@ void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest, const std std::vector output(10); tensorflow::serving::PredictResponse response; ovms::OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, outputsInfo, &response, ovms::getTensorInfoName); + auto status = serializePredictResponse(outputGetter, UNUSED_SERVABLE_NAME, UNUSED_MODEL_VERSION, outputsInfo, &response, ovms::getTensorInfoName); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); ASSERT_EQ(response.outputs().count(outputName), 1) << "Did not find:" << outputName; std::memcpy(output.data(), (float*)response.outputs().at(outputName).tensor_content().data(), DUMMY_MODEL_OUTPUT_SIZE * sizeof(float)); EXPECT_THAT(output, Each(Eq(1.))); } -ovms::Status getOutput(const KFSResponse& response, const std::string& name, KFSOutputTensorIteratorType& it, size_t& bufferId) { +static ovms::Status getOutput(const KFSResponse& response, const std::string& name, KFSOutputTensorIteratorType& it, size_t& bufferId) { it = response.outputs().begin(); bufferId = 0; while (it != response.outputs().end()) { @@ -74,7 +84,7 @@ ovms::Status getOutput(const KFSResponse& response, const std::string& name, KFS return StatusCode::INVALID_MISSING_INPUT; } -ovms::Status getOutput(const TFSResponseType& response, const std::string& name, TFSOutputTensorIteratorType& it, size_t& bufferId) { +static ovms::Status getOutput(const TFSResponseType& response, const std::string& name, TFSOutputTensorIteratorType& it, size_t& bufferId) { it = response.outputs().find(name); if (it != response.outputs().end()) { return StatusCode::OK; @@ -82,6 +92,33 @@ ovms::Status getOutput(const TFSResponseType& response, const std::string& name, return StatusCode::INVALID_MISSING_INPUT; } +using inputs_info_elem_t = std::pair>; +size_t calculateByteSize(const inputs_info_elem_t& e) { + auto& [inputName, shapeDatatypeTuple] = e; + auto& [shape, precision] = shapeDatatypeTuple; + size_t shapeProduct = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + return shapeProduct * ovms::DataTypeToByteSize(ovms::getPrecisionAsOVMSDataType(precision)); +} +template +class Preparer { + std::vector>> dataKeeper; + +public: + void preparePredictRequest(RequestType& request, inputs_info_t requestInputs) { + ::preparePredictRequest(request, requestInputs); + } +}; +template <> +void Preparer::preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs) { + auto inputWithGreatestRequirements = std::max_element(requestInputs.begin(), requestInputs.end(), [](const inputs_info_elem_t& a, const inputs_info_elem_t& b) { + return calculateByteSize(a) < calculateByteSize(b); + }); + size_t byteSizeToPreserve = calculateByteSize(*inputWithGreatestRequirements); + auto& currentData = dataKeeper.emplace_back(std::make_unique>(byteSizeToPreserve)); + memset(reinterpret_cast(const_cast(currentData->data())), '1', byteSizeToPreserve); + ::preparePredictRequest(request, requestInputs, *currentData); +} + template @@ -117,7 +154,8 @@ class TestPredict : public ::testing::Test { std::thread( [this, initialBatchSize, &releaseWaitBeforeGettingModelInstance, i]() { RequestType request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{(initialBatchSize + (i % 3)), 10}, ovms::Precision::FP32}}}); @@ -130,7 +168,8 @@ class TestPredict : public ::testing::Test { std::thread( [this, initialBatchSize, &releaseWaitBeforePerformInference, i]() { RequestType request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{initialBatchSize, 10}, ovms::Precision::FP32}}}); @@ -166,7 +205,8 @@ class TestPredict : public ::testing::Test { std::thread( [this, initialBatchSize, &releaseWaitBeforeGettingModelInstance, i]() { RequestType request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{(initialBatchSize + i), 10}, ovms::Precision::FP32}}}); performPredict(config.getName(), config.getVersion(), request, @@ -194,6 +234,31 @@ class TestPredict : public ::testing::Test { ASSERT_EQ(0, std::memcmp(actualValues.data(), expectedValues.data(), expectedValues.size() * sizeof(float))) << readableError(expectedValues.data(), actualValues.data(), expectedValues.size() * sizeof(float)); } + static void checkOutputValues(const ovms::InferenceResponse& res, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { + InferenceResponse& response = const_cast(res); // TODO decide if output should be const + size_t outputCount = response.getOutputCount(); + ASSERT_GE(1, outputCount); + size_t outputId = 0; + while (outputId < outputCount) { + const std::string* cppName; + InferenceTensor* tensor; + auto status = response.getOutput(outputId, &cppName, &tensor); + ASSERT_EQ(status, StatusCode::OK) << status.string(); + ASSERT_NE(nullptr, tensor); + ASSERT_NE(nullptr, cppName); + if (outputName == *cppName) { + const Buffer* buffer = tensor->getBuffer(); + ASSERT_NE(nullptr, buffer); + ASSERT_EQ(expectedValues.size() * sizeof(float), buffer->getByteSize()); + float* bufferRaw = reinterpret_cast(const_cast(buffer->data())); + ASSERT_EQ(0, std::memcmp(bufferRaw, expectedValues.data(), expectedValues.size() * sizeof(float))) + << readableError(expectedValues.data(), bufferRaw, expectedValues.size() * sizeof(float)); + return; + } + ++outputId; + } + ASSERT_TRUE(false) << "did not found output with name: " << outputName; + } static void checkOutputValues(const KFSResponse& response, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { KFSOutputTensorIteratorType it; size_t bufferId; @@ -222,14 +287,14 @@ class TestPredict : public ::testing::Test { if (!status.ok()) { return status; } - response.Clear(); return model->infer(&request, &response, unload_guard); } ovms::Status performInferenceWithShape(ResponseType& response, const ovms::shape_t& shape = {1, 10}, const ovms::Precision precision = ovms::Precision::FP32) { RequestType request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); return performInferenceWithRequest(request, response); } @@ -238,15 +303,22 @@ class TestPredict : public ::testing::Test { ovms::shape_t shape = {1, 10}; shape[batchSizePosition] = batchSize; RequestType request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); return performInferenceWithRequest(request, response); } ovms::Status performInferenceWithImageInput(ResponseType& response, const std::vector& shape, const std::vector& data = {}, const std::string& servableName = "increment_1x3x4x5", int batchSize = 1, const ovms::Precision precision = ovms::Precision::FP32) { RequestType request; - preparePredictRequest(request, - {{INCREMENT_1x3x4x5_MODEL_INPUT_NAME, std::tuple{shape, precision}}}, data); + Preparer preparer; + if (data.size()) { + preparePredictRequest(request, + {{INCREMENT_1x3x4x5_MODEL_INPUT_NAME, std::tuple{shape, precision}}}, data); + } else { + preparer.preparePredictRequest(request, + {{INCREMENT_1x3x4x5_MODEL_INPUT_NAME, std::tuple{shape, precision}}}); + } return performInferenceWithRequest(request, response, servableName); } @@ -274,6 +346,31 @@ void TestPredict::checkOutputShape(const TFSResponseType& response } } +template <> +void TestPredict::checkOutputShape(const ovms::InferenceResponse& cresponse, const ovms::shape_t& shape, const std::string& outputName) { + size_t outputCount = cresponse.getOutputCount(); + EXPECT_GE(1, outputCount); + size_t outputId = 0; + while (outputId < outputCount) { + const std::string* cppName; + InferenceTensor* tensor; + InferenceResponse& response = const_cast(cresponse); // TODO decide if output should be const + auto status = response.getOutput(outputId, &cppName, &tensor); + EXPECT_EQ(status, StatusCode::OK) << status.string(); + EXPECT_NE(nullptr, tensor); + EXPECT_NE(nullptr, cppName); + if (outputName == *cppName) { + auto resultShape = tensor->getShape(); + EXPECT_EQ(shape.size(), resultShape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + EXPECT_EQ(resultShape[i], shape[i]); + } + } + ++outputId; + } + return; +} + template <> void TestPredict::checkOutputShape(const KFSResponse& response, const ovms::shape_t& shape, const std::string& outputName) { auto it = response.outputs().begin(); @@ -289,7 +386,7 @@ void TestPredict::checkOutputShape(const KFSResponse& response, co class MockModelInstance : public ovms::ModelInstance { public: MockModelInstance(ov::Core& ieCore) : - ModelInstance("UNUSED_NAME", 42, ieCore) {} + ModelInstance(UNUSED_SERVABLE_NAME, UNUSED_MODEL_VERSION, ieCore) {} template const ovms::Status mockValidate(const RequestType* request) { return validate(request); @@ -361,12 +458,13 @@ void TestPredict::performPredict(const std::str DUMMY_MODEL_OUTPUT_NAME); } -using MyTypes = ::testing::Types; +using MyTypes = ::testing::Types; TYPED_TEST_SUITE(TestPredict, MyTypes); TYPED_TEST(TestPredict, SuccesfullOnDummyModel) { typename TypeParam::first_type request; - preparePredictRequest(request, + Preparer preparer; + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{1, 10}, ovms::Precision::FP32}}}); ovms::ModelConfig config = DUMMY_MODEL_CONFIG; @@ -429,8 +527,9 @@ class TestPredictWithMapping : public TestWithTempDir { TYPED_TEST_SUITE(TestPredictWithMapping, MyTypes); TYPED_TEST(TestPredictWithMapping, SuccesfullOnDummyModelWithMapping) { + Preparer preparer; typename TypeParam::first_type request; - preparePredictRequest(request, + preparer.preparePredictRequest(request, {{this->dummyModelInputMapping, std::tuple{{1, 10}, ovms::Precision::FP32}}}); ovms::ModelConfig config = DUMMY_MODEL_CONFIG; @@ -441,8 +540,9 @@ TYPED_TEST(TestPredictWithMapping, SuccesfullOnDummyModelWithMapping) { } TYPED_TEST(TestPredict, SuccesfullReloadFromAlreadyLoadedWithNewBatchSize) { + Preparer preparer; typename TypeParam::first_type request; - preparePredictRequest(request, + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{1, 10}, ovms::Precision::FP32}}}); ovms::ModelConfig config = DUMMY_MODEL_CONFIG; @@ -454,12 +554,13 @@ TYPED_TEST(TestPredict, SuccesfullReloadFromAlreadyLoadedWithNewBatchSize) { TYPED_TEST(TestPredict, SuccesfullReloadWhen1InferenceInProgress) { // FIRST LOAD MODEL WITH BS=1 + Preparer preparer; typename TypeParam::first_type requestBs1; - preparePredictRequest(requestBs1, + preparer.preparePredictRequest(requestBs1, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{1, 10}, ovms::Precision::FP32}}}); typename TypeParam::first_type requestBs2; - preparePredictRequest(requestBs2, + preparer.preparePredictRequest(requestBs2, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{2, 10}, ovms::Precision::FP32}}}); @@ -488,14 +589,15 @@ TYPED_TEST(TestPredict, SuccesfullReloadWhen1InferenceInProgress) { TYPED_TEST(TestPredict, SuccesfullReloadWhen1InferenceAboutToStart) { // FIRST LOAD MODEL WITH BS=1 - typename TypeParam::first_type requestBs1; - preparePredictRequest(requestBs1, - {{DUMMY_MODEL_INPUT_NAME, - std::tuple{{1, 10}, ovms::Precision::FP32}}}); + Preparer preparer; typename TypeParam::first_type requestBs2; - preparePredictRequest(requestBs2, + preparer.preparePredictRequest(requestBs2, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{2, 10}, ovms::Precision::FP32}}}); + typename TypeParam::first_type requestBs1; + preparer.preparePredictRequest(requestBs1, + {{DUMMY_MODEL_INPUT_NAME, + std::tuple{{1, 10}, ovms::Precision::FP32}}}); this->config.setBatchingParams("auto"); this->config.setNireq(2); @@ -561,22 +663,18 @@ TYPED_TEST(TestPredict, SuccesfullReshapeViaRequestOnDummyModel) { config.parseShapeParameter("auto"); ASSERT_EQ(this->manager.reloadModelWithVersions(config), ovms::StatusCode::OK_RELOADED); - // Get dummy model instance - std::shared_ptr model; - std::unique_ptr unload_guard; - auto status = this->manager.getModelInstance("dummy", 0, model, unload_guard); - // Prepare request with 1x5 shape, expect reshape + Preparer preparer; typename TypeParam::first_type request; - preparePredictRequest(request, + preparer.preparePredictRequest(request, {{DUMMY_MODEL_INPUT_NAME, std::tuple{{1, 5}, ovms::Precision::FP32}}}); typename TypeParam::second_type response; // Do the inference - ASSERT_EQ(model->infer(&request, &response, unload_guard), ovms::StatusCode::OK); - + auto status = this->performInferenceWithRequest(request, response, "dummy"); + ASSERT_EQ(status, StatusCode::OK) << status.string(); // Expect reshape to 1x5 this->checkOutputShape(response, {1, 5}, DUMMY_MODEL_OUTPUT_NAME); } @@ -642,7 +740,6 @@ TYPED_TEST(TestPredict, ReshapeViaRequestAndConfigChange) { */ TYPED_TEST(TestPredict, ChangeBatchSizeViaRequestAndConfigChange) { using namespace ovms; - // Prepare model with shape=auto (initially (1,10) shape) ModelConfig config = DUMMY_MODEL_CONFIG; this->config.setBatchingParams("auto"); @@ -699,9 +796,9 @@ TYPED_TEST(TestPredict, PerformInferenceChangeModelInputLayout) { typename TypeParam::second_type response; // Perform inference with NHWC layout, ensure status OK and correct results - ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 4, 5, 3}), ovms::StatusCode::OK); + auto status = this->performInferenceWithImageInput(response, {1, 4, 5, 3}); + ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); this->checkOutputShape(response, {1, 3, 4, 5}, INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME); - // Perform inference with NCHW layout, ensure error ASSERT_EQ(this->performInferenceWithImageInput(response, {1, 3, 4, 5}), ovms::StatusCode::INVALID_SHAPE); @@ -936,6 +1033,9 @@ TYPED_TEST(TestPredict, NetworkNotLoadedWhenLayoutAndDimsInconsistent) { * 6. Do the inference with single binary image tensor - expect status OK and result in NCHW layout * */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputChangeModelInputLayout) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; + using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -977,6 +1077,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputChangeModelInputLayout) { * 2. Do the inference with single binary image tensor with witdth exceeding shape range - expect status OK and reshaped output tensor */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputAndShapeDynamic) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1001,6 +1103,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputAndShapeDynamic) { * 2. Do the inference with batch=5 binary image tensor - expect status OK and result in NCHW layout */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAuto) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1025,6 +1129,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAuto) { * 2. Do the inference with binary image tensor with no shape set - expect status INVALID_NO_OF_SHAPE_DIMENSIONS */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputNoInputShape) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1147,14 +1253,34 @@ TYPED_TEST(TestPredict, PerformInferenceDummyBatchSizeAny) { * 2. Do the inferences with (3, 10) shape, expect correct output shapes and precision */ -ovms::Precision getPrecisionFromResponse(KFSResponse& response, const std::string& name) { +static ovms::Precision getPrecisionFromResponse(ovms::InferenceResponse& response, const std::string& name) { + size_t outputCount = response.getOutputCount(); + EXPECT_GE(1, outputCount); + size_t outputId = 0; + while (outputId < outputCount) { + const std::string* cppName; + InferenceTensor* tensor; + auto status = response.getOutput(outputId, &cppName, &tensor); + EXPECT_EQ(status, StatusCode::OK) << status.string(); + EXPECT_NE(nullptr, tensor); + EXPECT_NE(nullptr, cppName); + if (name == *cppName) { + return ovms::getOVMSDataTypeAsPrecision(tensor->getDataType()); + } + ++outputId; + } + return ovms::getOVMSDataTypeAsPrecision(OVMS_DATATYPE_UNDEFINED); +} + +static ovms::Precision getPrecisionFromResponse(KFSResponse& response, const std::string& name) { KFSOutputTensorIteratorType it; size_t bufferId; auto status = getOutput(response, name, it, bufferId); EXPECT_TRUE(status.ok()); return ovms::KFSPrecisionToOvmsPrecision(it->datatype()); } -ovms::Precision getPrecisionFromResponse(TFSResponseType& response, const std::string& name) { + +static ovms::Precision getPrecisionFromResponse(TFSResponseType& response, const std::string& name) { TFSOutputTensorIteratorType it; size_t bufferId; auto status = getOutput(response, name, it, bufferId); @@ -1173,7 +1299,8 @@ TYPED_TEST(TestPredict, PerformInferenceDummyFp64) { typename TypeParam::first_type request; typename TypeParam::second_type response; - preparePredictRequest(request, {{"input:0", std::tuple{{3, 10}, ovms::Precision::FP64}}}); + Preparer preparer; + preparer.preparePredictRequest(request, {{"input:0", std::tuple{{3, 10}, ovms::Precision::FP64}}}); ASSERT_EQ(this->performInferenceWithRequest(request, response, "dummy_fp64"), ovms::StatusCode::OK); this->checkOutputShape(response, {3, 10}, "output:0"); ASSERT_EQ(getPrecisionFromResponse(response, "output:0"), ovms::Precision::FP64); @@ -1221,6 +1348,8 @@ TYPED_TEST(TestPredict, PerformInferenceDummyAllDimensionsHaveRange) { * 2. Do the inference with batch=5 binary image tensor 1x1 - expect status INVALID_SHAPE, because if any dimension is dynamic, we perform no resize operation. */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionNotMatching) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1243,6 +1372,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionNot * 2. Do the inference with batch=5 binary image tensor 1x1 - expect status OK, and correct results. */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionMatching) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1267,6 +1398,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputBatchSizeAnyResolutionMat * 2. Do the inference with resolution 1x1 binary image tensor - expect status OK and result in NCHW layout */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputResolutionAny) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) @@ -1291,6 +1424,8 @@ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputResolutionAny) { * 3. Do the inference with resolution 1x1 binary image tensor - expect status OK and result in NCHW layout */ TYPED_TEST(TestPredict, PerformInferenceWithBinaryInputResolutionRange) { + if (typeid(typename TypeParam::first_type) == typeid(ovms::InferenceRequest)) + GTEST_SKIP() << "Binary inputs not implemented for C-API yet"; using namespace ovms; // Prepare model with changed layout to nhwc (internal layout=nchw) diff --git a/src/test/serialization_tests.cpp b/src/test/serialization_tests.cpp index 84da850368..f5c339fc88 100644 --- a/src/test/serialization_tests.cpp +++ b/src/test/serialization_tests.cpp @@ -166,6 +166,11 @@ const std::vector UNSUPPORTED_CAPI_OUTPUT_PRECISIONS{ // ovms::Precision::UNDEFINED, // Cannot create ov tensor with such precision }; +namespace { +const std::string UNUSED_NAME{"UNUSED_NAME"}; +const model_version_t UNUSED_VERSION{0}; +} // namespace + class TensorflowGRPCPredict : public ::testing::TestWithParam { protected: void SetUp() override { @@ -286,7 +291,7 @@ TEST(SerializeTFGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName); EXPECT_TRUE(status.ok()); } @@ -510,7 +515,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecision) { ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName); ASSERT_TRUE(status.ok()); EXPECT_EQ(DUMMY_MODEL_OUTPUT_NAME, response.outputs(0).name()); EXPECT_EQ("FP32", response.outputs(0).datatype()); @@ -535,7 +540,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithuseS ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName, true); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName, true); ASSERT_TRUE(status.ok()); EXPECT_EQ(DUMMY_MODEL_INPUT_NAME, response.outputs(0).name()); EXPECT_EQ("FP32", response.outputs(0).datatype()); @@ -561,7 +566,7 @@ TEST(SerializeKFSGRPCPredictResponse, ShouldSuccessForSupportedPrecisionWithshar ov::Tensor tensor(tensorInfo->getOvPrecision(), ov::Shape{1, 10}); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName, false); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName, false); ASSERT_TRUE(status.ok()); EXPECT_EQ(DUMMY_MODEL_INPUT_NAME, response.outputs(0).name()); EXPECT_EQ("FP32", response.outputs(0).datatype()); @@ -619,7 +624,7 @@ TEST(SerializeCApiTensorSingle, NegativeMismatchBetweenTensorInfoAndTensorPrecis std::memcpy(tensor.data(), data, tensor.get_byte_size()); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName); EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); } @@ -641,7 +646,7 @@ TEST(SerializeCApiTensorSingle, NegativeMismatchBetweenTensorInfoAndTensorShape) std::memcpy(tensor.data(), data, tensor.get_byte_size()); inferRequest.set_tensor(DUMMY_MODEL_OUTPUT_NAME, tensor); OutputGetter outputGetter(inferRequest); - auto status = serializePredictResponse(outputGetter, tenMap, &response, getTensorInfoName); + auto status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, tenMap, &response, getTensorInfoName); EXPECT_EQ(status.getCode(), ovms::StatusCode::INTERNAL_ERROR); } @@ -666,6 +671,8 @@ TEST_P(SerializeCApiTensorPositive, SerializeTensorShouldSucceedForPrecision) { auto inputs = prepareInputs(testedPrecision); auto status = serializePredictResponse(outputGetter, + UNUSED_NAME, + UNUSED_VERSION, inputs, &response, getTensorInfoName); @@ -693,6 +700,8 @@ TEST_P(SerializeCApiTensorNegative, SerializeTensorShouldFailForPrecision) { auto inputs = prepareInputs(testedPrecision); auto status = serializePredictResponse(outputGetter, + UNUSED_NAME, + UNUSED_VERSION, inputs, &response, getTensorInfoName); @@ -721,6 +730,8 @@ TEST_F(CApiSerialization, ValidSerialization) { auto inputs = prepareInputs(ovms::Precision::FP32, shape); auto status = serializePredictResponse(outputGetter, + UNUSED_NAME, + UNUSED_VERSION, inputs, &response, getTensorInfoName); diff --git a/src/test/stateful_modelinstance_test.cpp b/src/test/stateful_modelinstance_test.cpp index 229fe3f3ff..121d56f76c 100644 --- a/src/test/stateful_modelinstance_test.cpp +++ b/src/test/stateful_modelinstance_test.cpp @@ -30,6 +30,7 @@ #include "../get_model_metadata_impl.hpp" #include "../global_sequences_viewer.hpp" #include "../modelinstanceunloadguard.hpp" +#include "../modelversion.hpp" #include "../ov_utils.hpp" #include "../sequence_processing_spec.hpp" #include "../serialization.hpp" @@ -41,6 +42,8 @@ using testing::Return; namespace { +const std::string UNUSED_NAME{"UNUSED_NAME"}; +const ovms::model_version_t UNUSED_VERSION{0}; static bool testWarningPrinted = false; enum SequenceTimeoutScenarios { @@ -147,8 +150,12 @@ class MockedValidateStatefulModelInstance : public ovms::StatefulModelInstance { MockedValidateStatefulModelInstance(const std::string& name, ovms::model_version_t version, ov::Core& ieCore) : StatefulModelInstance(name, version, ieCore, nullptr, nullptr, &sequencesViewer) {} - const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request, ovms::SequenceProcessingSpec& processingSpec) { - return validate(request, processingSpec); + const ovms::Status mockValidate(const tensorflow::serving::PredictRequest* request) { + ovms::StatefulRequestProcessor sp(*this->getSequenceManager()); + auto status = sp.extractRequestParameters(request); + if (!status.ok()) + return status; + return validate(request); } }; @@ -203,8 +210,11 @@ class MockedStatefulModelInstance : public ovms::StatefulModelInstance { }; ovms::Timer timer; using std::chrono::microseconds; - ovms::SequenceProcessingSpec sequenceProcessingSpec; - auto status = validate(requestProto, sequenceProcessingSpec); + ovms::StatefulRequestProcessor requestProcessor(*this->getSequenceManager()); + auto status = requestProcessor.extractRequestParameters(requestProto); + if (!status.ok()) + return status; + status = validate(requestProto); if (!status.ok()) return status; @@ -212,30 +222,22 @@ class MockedStatefulModelInstance : public ovms::StatefulModelInstance { std::cout << "Waiting before sequenceManagerLock" << std::endl; waitBeforeManagerLock->get(); } - std::unique_lock sequenceManagerLock(sequenceManager->getMutex()); - status = sequenceManager->processRequestedSpec(sequenceProcessingSpec); + status = requestProcessor.prepare(); if (!status.ok()) return status; - const uint64_t sequenceId = sequenceProcessingSpec.getSequenceId(); - if (!sequenceManager->sequenceExists(sequenceId)) - return ovms::StatusCode::INTERNAL_ERROR; - ovms::Sequence& sequence = sequenceManager->getSequence(sequenceId); - if (waitBeforeSequenceLock) { + if (waitBeforeSequenceLock) { // TODO remove since right now it is in singel step std::cout << "Waiting before waitBeforeSequenceLock" << std::endl; waitBeforeSequenceLock->get(); } - std::unique_lock sequenceLock(sequence.getMutex()); - sequenceManagerLock.unlock(); - timer.start(GET_INFER_REQUEST); ovms::ExecutingStreamIdGuard executingStreamIdGuard(getInferRequestsQueue(), this->getMetricReporter()); ov::InferRequest& inferRequest = executingStreamIdGuard.getInferRequest(); timer.stop(GET_INFER_REQUEST); timer.start(PREPROCESS); - status = preInferenceProcessing(inferRequest, sequence, sequenceProcessingSpec); + status = requestProcessor.preInferenceProcessing(inferRequest); if (!status.ok()) return status; timer.stop(PREPROCESS); @@ -257,13 +259,13 @@ class MockedStatefulModelInstance : public ovms::StatefulModelInstance { timer.start(SERIALIZE); ovms::OutputGetter outputGetter(inferRequest); - status = serializePredictResponse(outputGetter, getOutputsInfo(), responseProto, ovms::getTensorInfoName); + status = serializePredictResponse(outputGetter, UNUSED_NAME, UNUSED_VERSION, getOutputsInfo(), responseProto, ovms::getTensorInfoName); timer.stop(SERIALIZE); if (!status.ok()) return status; timer.start(POSTPROCESS); - status = postInferenceProcessing(responseProto, inferRequest, sequence, sequenceProcessingSpec); + status = requestProcessor.postInferenceProcessing(responseProto, inferRequest); timer.stop(POSTPROCESS); if (!status.ok()) return status; @@ -272,16 +274,8 @@ class MockedStatefulModelInstance : public ovms::StatefulModelInstance { std::cout << "Waiting before waitBeforeSequenceUnlocked" << std::endl; waitBeforeSequenceUnlocked->get(); } - - sequenceLock.unlock(); - if (sequenceProcessingSpec.getSequenceControlInput() == ovms::SEQUENCE_END) { - sequenceManagerLock.lock(); - status = sequenceManager->removeSequence(sequenceId); - if (!status.ok()) - return status; - } - - return ovms::StatusCode::OK; + status = requestProcessor.release(); + return status; } }; @@ -1119,21 +1113,21 @@ TEST_F(StatefulModelInstanceInputValidation, positiveValidate) { setRequestSequenceId(&request, seqId); setRequestSequenceControl(&request, ovms::SEQUENCE_START); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_TRUE(status.ok()); preparePredictRequest(request, modelInput); setRequestSequenceId(&request, seqId); setRequestSequenceControl(&request, ovms::SEQUENCE_END); - status = modelInstance->mockValidate(&request, spec); + status = modelInstance->mockValidate(&request); ASSERT_TRUE(status.ok()); preparePredictRequest(request, modelInput); setRequestSequenceId(&request, seqId); setRequestSequenceControl(&request, ovms::NO_CONTROL_INPUT); - status = modelInstance->mockValidate(&request, spec); + status = modelInstance->mockValidate(&request); ASSERT_TRUE(status.ok()); } @@ -1144,7 +1138,7 @@ TEST_F(StatefulModelInstanceInputValidation, missingSeqId) { preparePredictRequest(request, modelInput); setRequestSequenceControl(&request, ovms::SEQUENCE_END); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_ID_NOT_PROVIDED); } @@ -1157,7 +1151,7 @@ TEST_F(StatefulModelInstanceInputValidation, wrongSeqIdEnd) { uint64_t seqId = 0; setRequestSequenceId(&request, seqId); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_ID_NOT_PROVIDED); } @@ -1170,7 +1164,7 @@ TEST_F(StatefulModelInstanceInputValidation, wrongSeqIdNoControl) { uint64_t seqId = 0; setRequestSequenceId(&request, seqId); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_ID_NOT_PROVIDED); } @@ -1183,7 +1177,7 @@ TEST_F(StatefulModelInstanceInputValidation, wrongProtoKeywords) { input.set_dtype(tensorflow::DataType::DT_UINT64); input.mutable_tensor_shape()->add_dim()->set_size(1); input.add_uint64_val(12); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_ID_NOT_PROVIDED); } @@ -1196,7 +1190,7 @@ TEST_F(StatefulModelInstanceInputValidation, badControlInput) { input.set_dtype(tensorflow::DataType::DT_UINT32); input.mutable_tensor_shape()->add_dim()->set_size(1); input.add_uint32_val(999); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::INVALID_SEQUENCE_CONTROL_INPUT); } @@ -1210,7 +1204,7 @@ TEST_F(StatefulModelInstanceInputValidation, invalidProtoTypes) { input.set_dtype(tensorflow::DataType::DT_UINT32); input.mutable_tensor_shape()->add_dim()->set_size(1); input.add_uint32_val(12); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_ID_BAD_TYPE); } { @@ -1220,7 +1214,7 @@ TEST_F(StatefulModelInstanceInputValidation, invalidProtoTypes) { input.set_dtype(tensorflow::DataType::DT_UINT64); input.mutable_tensor_shape()->add_dim()->set_size(1); input.add_uint64_val(1); - auto status = modelInstance->mockValidate(&request, spec); + auto status = modelInstance->mockValidate(&request); ASSERT_EQ(status.getCode(), ovms::StatusCode::SEQUENCE_CONTROL_INPUT_BAD_TYPE); } } diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 801a6bcbff..006cfc705b 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -30,6 +30,10 @@ using tensorflow::serving::PredictResponse; using ovms::TensorInfo; +void prepareBinaryPredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize) { throw 42; } // CAPI binary not supported +void prepareBinaryPredictRequestNoShape(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize) { throw 42; } // CAPI binary not supported +void prepareBinary4x4PredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize) { throw 42; } // CAPI binary not supported + void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, const std::vector& data, bool putBufferInInputTensorContent) { request.mutable_inputs()->Clear(); request.mutable_raw_input_contents()->Clear(); diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 60daa33f7f..6db6bc32f7 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -37,6 +37,7 @@ #include "../config.hpp" #include "../execution_context.hpp" #include "../inferencerequest.hpp" +#include "../inferenceresponse.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../metric_registry.hpp" #include "../modelinstance.hpp" @@ -131,6 +132,7 @@ constexpr const char* INCREMENT_1x3x4x5_MODEL_INPUT_NAME = "input"; constexpr const char* INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME = "output"; constexpr const float INCREMENT_1x3x4x5_ADDITION_VALUE = 1.0; +const std::string UNUSED_SERVABLE_NAME = "UNUSED_SERVABLE_NAME"; constexpr const ovms::model_version_t UNUSED_MODEL_VERSION = 42; // Answer to the Ultimate Question of Life static const ovms::ExecutionContext DEFAULT_TEST_CONTEXT{ovms::ExecutionContext::Interface::GRPC, ovms::ExecutionContext::Method::Predict}; @@ -144,6 +146,7 @@ using TFSInputTensorIteratorType = google::protobuf::Map::const_iterator; using TFSInterface = std::pair; using KFSInterface = std::pair; +using CAPIInterface = std::pair; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" @@ -173,12 +176,14 @@ void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t reques void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize); +void prepareBinaryPredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize); // CAPI binary not supported void prepareBinaryPredictRequestNoShape(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequestNoShape(::KFSRequest& request, const std::string& inputName, const int batchSize); - +void prepareBinaryPredictRequestNoShape(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize); // CAPI binary not supported void prepareBinary4x4PredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize = 1); void prepareBinary4x4PredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize = 1); +void prepareBinary4x4PredictRequest(ovms::InferenceRequest& request, const std::string& inputName, const int batchSize = 1); // CAPI binary not supported template void prepareInvalidImageBinaryTensor(TensorType& tensor); diff --git a/src/test/tfs_rest_parser_column_test.cpp b/src/test/tfs_rest_parser_column_test.cpp index 13667569fe..2793d6b85c 100644 --- a/src/test/tfs_rest_parser_column_test.cpp +++ b/src/test/tfs_rest_parser_column_test.cpp @@ -29,7 +29,6 @@ using namespace ovms; using namespace testing; using ::testing::ElementsAre; -using tensorflow::DataType; using tensorflow::DataTypeSize; const char* predictRequestColumnNamedJson = R"({ @@ -80,12 +79,12 @@ TEST(TFSRestParserColumn, ParseValid2Inputs) { ASSERT_EQ(parser.getProto().inputs().count("inputB"), 1); const auto& inputA = parser.getProto().inputs().at("inputA"); const auto& inputB = parser.getProto().inputs().at("inputB"); - EXPECT_EQ(inputA.dtype(), DataType::DT_FLOAT); - EXPECT_EQ(inputB.dtype(), DataType::DT_FLOAT); + EXPECT_EQ(inputA.dtype(), tensorflow::DataType::DT_FLOAT); + EXPECT_EQ(inputB.dtype(), tensorflow::DataType::DT_FLOAT); EXPECT_THAT(asVector(inputA.tensor_shape()), ElementsAre(2, 2, 3, 2)); EXPECT_THAT(asVector(inputB.tensor_shape()), ElementsAre(2, 2, 3)); - ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(DataType::DT_FLOAT)); - ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(DataType::DT_FLOAT)); + ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); + ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); EXPECT_THAT(asVector(inputA.tensor_content()), ElementsAre( 1.0, 2.0, 3.0, 4.0, diff --git a/src/test/tfs_rest_parser_row_test.cpp b/src/test/tfs_rest_parser_row_test.cpp index b519b805ea..56d036f311 100644 --- a/src/test/tfs_rest_parser_row_test.cpp +++ b/src/test/tfs_rest_parser_row_test.cpp @@ -30,7 +30,6 @@ using namespace ovms; using namespace testing; using ::testing::ElementsAre; -using tensorflow::DataType; using tensorflow::DataTypeSize; const char* predictRequestRowNamedJson = R"({ @@ -81,12 +80,12 @@ TEST(TFSRestParserRow, ParseValid2Inputs) { ASSERT_EQ(parser.getProto().inputs().count("inputB"), 1); const auto& inputA = parser.getProto().inputs().at("inputA"); const auto& inputB = parser.getProto().inputs().at("inputB"); - EXPECT_EQ(inputA.dtype(), DataType::DT_FLOAT); - EXPECT_EQ(inputB.dtype(), DataType::DT_FLOAT); + EXPECT_EQ(inputA.dtype(), tensorflow::DataType::DT_FLOAT); + EXPECT_EQ(inputB.dtype(), tensorflow::DataType::DT_FLOAT); EXPECT_THAT(asVector(inputA.tensor_shape()), ElementsAre(2, 2, 3, 2)); EXPECT_THAT(asVector(inputB.tensor_shape()), ElementsAre(2, 2, 3)); - ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(DataType::DT_FLOAT)); - ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(DataType::DT_FLOAT)); + ASSERT_EQ(inputA.tensor_content().size(), 2 * 2 * 3 * 2 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); + ASSERT_EQ(inputB.tensor_content().size(), 2 * 2 * 3 * DataTypeSize(tensorflow::DataType::DT_FLOAT)); EXPECT_THAT(asVector(inputA.tensor_content()), ElementsAre( 1.0, 2.0, 3.0, 4.0, From 814254a1e69ddbf59e6b987dac4f74d7e7238fd4 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Mon, 5 Dec 2022 12:27:12 +0100 Subject: [PATCH 097/130] Add tests for nullptrs (#1562) * Change coverage threshold JIRA:CVS-95308 --- check_coverage.bat | 8 ++-- src/test/c_api_tests.cpp | 84 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/check_coverage.bat b/check_coverage.bat index 327530e9b4..f70c442075 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -1,12 +1,12 @@ #!/bin/bash #Ubuntu -#MIN_LINES_COV=75.2 -#MIN_FUNCTION_COV=87.4 +#MIN_LINES_COV=76.3 +#MIN_FUNCTION_COV=88.3 #Rhel -MIN_LINES_COV=74.1 -MIN_FUNCTION_COV=75.6 +MIN_LINES_COV=74.8 +MIN_FUNCTION_COV=75.7 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` FUNC_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | tail -n 1` diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 7f9266f36b..4bc8babc47 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -45,7 +45,9 @@ TEST(CApiConfigTest, MultiModelConfiguration) { OVMS_ServerGeneralOptions* _go = 0; OVMS_ServerMultiModelOptions* _mmo = 0; + ASSERT_NE(OVMS_ServerGeneralOptionsNew(nullptr), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&_go), nullptr); + ASSERT_NE(OVMS_ServerMultiModelOptionsNew(nullptr), nullptr); ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&_mmo), nullptr); ASSERT_NE(_go, nullptr); ASSERT_NE(_mmo, nullptr); @@ -88,9 +90,36 @@ TEST(CApiConfigTest, MultiModelConfiguration) { ASSERT_EQ(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(_go, 4), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, "/ovms/src/test"), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsSetCacheDir(_go, "/tmp/cache"), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_INFO), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_ERROR), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_DEBUG), nullptr); + ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_WARNING), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_TRACE), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogPath(_go, "/logs"), nullptr); ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, "/config"), nullptr); + // check nullptr + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcPort(nullptr, 5555), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetRestPort(nullptr, 6666), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcWorkers(nullptr, 30), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(nullptr, "2.2.2.2"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetRestWorkers(nullptr, 31), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetRestBindAddress(nullptr, "3.3.3.3"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(nullptr, "grpcargs"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(nullptr, 2), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(nullptr, 3), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(nullptr, 4), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(nullptr, "/ovms/src/test"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetCacheDir(nullptr, "/tmp/cache"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetCacheDir(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetLogLevel(nullptr, OVMS_LOG_TRACE), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetLogPath(nullptr, "/logs"), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsSetLogPath(_go, nullptr), nullptr); + ASSERT_NE(OVMS_ServerMultiModelOptionsSetConfigPath(nullptr, "/config"), nullptr); + ASSERT_NE(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, nullptr), nullptr); // Test non default values EXPECT_EQ(go->grpcPort, 5555); @@ -151,6 +180,8 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(cfg.configPath(), "/config"); + ASSERT_NE(OVMS_ServerMultiModelOptionsDelete(nullptr), nullptr); + ASSERT_NE(OVMS_ServerGeneralOptionsDelete(nullptr), nullptr); ASSERT_EQ(OVMS_ServerMultiModelOptionsDelete(_mmo), nullptr); ASSERT_EQ(OVMS_ServerGeneralOptionsDelete(_go), nullptr); } @@ -254,11 +285,15 @@ TEST_F(CapiInference, Basic) { ASSERT_EQ(nullptr, status); // verify GetOutputCount uint32_t outputCount = 42; + ASSERT_NE(OVMS_InferenceResponseGetOutputCount(nullptr, &outputCount), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutputCount(response, nullptr), nullptr); status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); ASSERT_EQ(nullptr, status); ASSERT_EQ(outputCount, 1); // verify GetParameterCount uint32_t parameterCount = 42; + ASSERT_NE(OVMS_InferenceResponseGetParameterCount(nullptr, ¶meterCount), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetParameterCount(response, nullptr), nullptr); status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); ASSERT_EQ(nullptr, status); ASSERT_EQ(0, parameterCount); @@ -272,6 +307,15 @@ TEST_F(CapiInference, Basic) { BufferType bufferType = (BufferType)199; uint32_t deviceId = 42; const char* outputName{nullptr}; + ASSERT_NE(OVMS_InferenceResponseGetOutput(nullptr, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, nullptr, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, nullptr, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, nullptr, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, nullptr, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, nullptr, &bytesize, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, nullptr, &bufferType, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, nullptr, &deviceId), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, nullptr), nullptr); status = OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); ASSERT_EQ(nullptr, status); ASSERT_EQ(std::string(DUMMY_MODEL_OUTPUT_NAME), outputName); @@ -333,11 +377,16 @@ TEST_F(CapiInference, Basic) { // 2nd time should report error status = OVMS_InferenceRequestRemoveParameter(request, "sequence_id"); ASSERT_NE(nullptr, status); + ASSERT_NE(OVMS_InferenceRequestRemoveParameter(nullptr, "sequence_id"), nullptr); + ASSERT_NE(OVMS_InferenceRequestRemoveParameter(request, nullptr), nullptr); // OVMS_StatusDelete(status); // FIXME(dkalinow) status = OVMS_InferenceRequestRemoveInput(request, "NONEXISTENT_TENSOR"); ASSERT_NE(nullptr, status); + ASSERT_NE(OVMS_InferenceRequestRemoveInput(nullptr, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST"), nullptr); + ASSERT_NE(OVMS_InferenceRequestRemoveInput(request, nullptr), nullptr); // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_NE(OVMS_InferenceRequestDelete(nullptr), nullptr); status = OVMS_InferenceRequestDelete(request); ASSERT_EQ(nullptr, status); @@ -359,6 +408,7 @@ TEST_F(CapiInference, NegativeInference) { ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"), nullptr); OVMS_Server* cserver = nullptr; + ASSERT_NE(OVMS_ServerNew(nullptr), nullptr); ASSERT_EQ(OVMS_ServerNew(&cserver), nullptr); ASSERT_NE(OVMS_ServerStartFromConfigurationFile(nullptr, go, mmo), nullptr); ASSERT_NE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, mmo), nullptr); @@ -369,6 +419,8 @@ TEST_F(CapiInference, NegativeInference) { OVMS_InferenceRequest* request{nullptr}; OVMS_InferenceResponse* response = nullptr; + ASSERT_NE(OVMS_InferenceRequestNew(nullptr, "dummy", 1), nullptr); + ASSERT_NE(OVMS_InferenceRequestNew(&request, nullptr, 1), nullptr); OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); ASSERT_EQ(nullptr, status); ASSERT_NE(nullptr, request); @@ -378,8 +430,13 @@ TEST_F(CapiInference, NegativeInference) { // OVMS_StatusDelete(status); // FIXME(dkalinow) // negative no input buffer + ASSERT_NE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); + ASSERT_NE(OVMS_InferenceRequestAddInput(request, nullptr, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); + ASSERT_NE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, nullptr, DUMMY_MODEL_SHAPE.size()), nullptr); status = OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); ASSERT_EQ(nullptr, status); + // fail with adding input second time + ASSERT_NE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); status = OVMS_Inference(cserver, request, &response); ASSERT_NE(nullptr, status); // OVMS_StatusDelete(status); // FIXME(dkalinow) @@ -387,10 +444,17 @@ TEST_F(CapiInference, NegativeInference) { // setting buffer std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; uint32_t notUsedNum = 0; + ASSERT_NE(OVMS_InferenceRequestInputSetData(nullptr, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); + ASSERT_NE(OVMS_InferenceRequestInputSetData(request, nullptr, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); + ASSERT_NE(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, nullptr, sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); status = OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); ASSERT_EQ(nullptr, status); + status = OVMS_InferenceRequestInputSetData(request, "NONEXISTENT_TENSOR", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); // add parameters const uint64_t sequenceId{42}; + ASSERT_NE(OVMS_InferenceRequestAddParameter(nullptr, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), nullptr); + ASSERT_NE(OVMS_InferenceRequestAddParameter(request, nullptr, OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), nullptr); + ASSERT_NE(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, nullptr, sizeof(sequenceId)), nullptr); status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); ASSERT_EQ(nullptr, status); // 2nd time should get error @@ -411,11 +475,27 @@ TEST_F(CapiInference, NegativeInference) { status = OVMS_Inference(cserver, request, nullptr); ASSERT_NE(nullptr, status); // OVMS_StatusDelete(status); FIXME + // + // negative inference with non existing model + OVMS_InferenceRequest* requestNoModel{nullptr}; + OVMS_InferenceResponse* reponseNoModel{nullptr}; + status = OVMS_InferenceRequestNew(&request, "NONEXISTENT_MODEL", 13); + ASSERT_EQ(nullptr, status); + // negative no model + status = OVMS_Inference(cserver, request, &response); + ASSERT_NE(nullptr, status); + // OVMS_StatusDelete(status); // FIXME(dkalinow) + + ASSERT_NE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); + status = OVMS_Inference(cserver, requestNoModel, &reponseNoModel); + ASSERT_NE(nullptr, status); + OVMS_InferenceRequestDelete(requestNoModel); ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); ASSERT_EQ(OVMS_ServerDelete(cserver), nullptr); ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); } + // TODO negative test -> validate at the infer stage // TODO reuse request after inference namespace { @@ -470,6 +550,9 @@ TEST_F(CapiInference, ResponseRetrieval) { // verify get Parameter OVMS_DataType parameterDatatype = OVMS_DATATYPE_FP32; const void* parameterData{nullptr}; + ASSERT_NE(OVMS_InferenceResponseGetParameter(nullptr, 0, ¶meterDatatype, ¶meterData), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetParameter(response, 0, nullptr, ¶meterData), nullptr); + ASSERT_NE(OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, nullptr), nullptr); status = OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, ¶meterData); ASSERT_EQ(nullptr, status); ASSERT_EQ(parameterDatatype, OVMS_DATATYPE_U64); @@ -517,6 +600,7 @@ TEST_F(CapiInference, ResponseRetrieval) { // final cleanup // we release unique_ptr ownership here so that we can free it safely via C-API cppResponse.release(); + ASSERT_NE(OVMS_InferenceResponseDelete(nullptr), nullptr); status = OVMS_InferenceResponseDelete(response); ASSERT_EQ(nullptr, status); } From 81c8a29fc44bb1fc67aeeeefa4adc26fa0edeafa Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 6 Dec 2022 13:00:48 +0100 Subject: [PATCH 098/130] CVS-97889 C-API error handling (#1555) - add C API calls to retrieve status code and status details string from OVMS_Status - modifications to server start to enable returning informative OVMS_Status - clean up positive unit test path from possible leaks with OVMS_StatusDelete - change C-API destructors to `void` return type since it cannot return an error (to behave similarly to free/delete in c/cpp) --- src/config.cpp | 1 + src/grpcservermodule.cpp | 25 +- src/grpcservermodule.hpp | 2 +- src/httpservermodule.cpp | 13 +- src/httpservermodule.hpp | 3 +- src/main3.cpp | 12 +- src/metric_module.cpp | 3 +- src/metric_module.hpp | 3 +- src/module.hpp | 4 +- src/pocapi.cpp | 77 ++-- src/pocapi.hpp | 25 +- src/servablemanagermodule.cpp | 6 +- src/servablemanagermodule.hpp | 3 +- src/server.cpp | 58 +-- src/server.hpp | 5 +- src/status.cpp | 9 + src/status.hpp | 9 + src/test/c_api_tests.cpp | 481 ++++++++++++------------ src/test/http_rest_api_handler_test.cpp | 4 +- src/test/server_test.cpp | 6 +- src/test/status_test.cpp | 18 + 21 files changed, 442 insertions(+), 325 deletions(-) diff --git a/src/config.cpp b/src/config.cpp index f5136547f9..1e447976da 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -34,6 +34,7 @@ const uint MAX_PORT_NUMBER = std::numeric_limits::max(); const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; const uint64_t MAX_REST_WORKERS = 10'000; +// TODO: Not used in OVMS - get rid of. Used only in tests. Config& Config::parse(int argc, char** argv) { ovms::CLIParser p; ovms::GeneralOptionsImpl go; diff --git a/src/grpcservermodule.cpp b/src/grpcservermodule.cpp index 17ac168c5b..84cb505416 100644 --- a/src/grpcservermodule.cpp +++ b/src/grpcservermodule.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -79,7 +80,7 @@ static Status parseGrpcChannelArgs(const std::string& channel_arguments_str, std for (const std::string& channel_argument : channel_arguments) { std::vector key_val = tokenize(channel_argument, '='); if (key_val.size() != 2) { - return StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT; + return Status(StatusCode::GRPC_CHANNEL_ARG_WRONG_FORMAT, channel_arguments_str); } erase_spaces(key_val[0]); erase_spaces(key_val[1]); @@ -110,14 +111,14 @@ GRPCServerModule::GRPCServerModule(Server& server) : tfsPredictService(this->server), tfsModelService(this->server), kfsGrpcInferenceService(this->server) {} -int GRPCServerModule::start(const ovms::Config& config) { +Status GRPCServerModule::start(const ovms::Config& config) { state = ModuleState::STARTED_INITIALIZE; SPDLOG_INFO("{} starting", GRPC_SERVER_MODULE_NAME); std::vector channel_arguments; auto status = parseGrpcChannelArgs(config.grpcChannelArguments(), channel_arguments); if (!status.ok()) { - SPDLOG_ERROR("grpc channel arguments passed in wrong format: {}", config.grpcChannelArguments()); - return EXIT_FAILURE; + SPDLOG_ERROR(status.string()); + return status; } ServerBuilder builder; @@ -146,21 +147,27 @@ int GRPCServerModule::start(const ovms::Config& config) { SPDLOG_DEBUG("Starting gRPC servers: {}", grpcServersCount); if (!isPortAvailable(config.port())) { - SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); - return EXIT_FAILURE; + std::stringstream ss; + ss << "at " << config.grpcBindAddress() << ":" << std::to_string(config.port()) << " - port is busy"; + auto status = Status(StatusCode::FAILED_TO_START_GRPC_SERVER, ss.str()); + SPDLOG_ERROR(status.string()); + return status; } for (uint i = 0; i < grpcServersCount; ++i) { std::unique_ptr server = builder.BuildAndStart(); if (server == nullptr) { - SPDLOG_ERROR("Failed to start gRPC server at " + config.grpcBindAddress() + ":" + std::to_string(config.port())); - return EXIT_FAILURE; + std::stringstream ss; + ss << "at " << config.grpcBindAddress() << ":" << std::to_string(config.port()); + auto status = Status(StatusCode::FAILED_TO_START_GRPC_SERVER, ss.str()); + SPDLOG_ERROR(status.string()); + return status; } servers.push_back(std::move(server)); } state = ModuleState::INITIALIZED; SPDLOG_INFO("{} started", GRPC_SERVER_MODULE_NAME); SPDLOG_INFO("Started gRPC server on port {}", config.port()); - return EXIT_SUCCESS; + return StatusCode::OK; } void GRPCServerModule::shutdown() { diff --git a/src/grpcservermodule.hpp b/src/grpcservermodule.hpp index 0be3eb39c2..e3589a5813 100644 --- a/src/grpcservermodule.hpp +++ b/src/grpcservermodule.hpp @@ -39,7 +39,7 @@ class GRPCServerModule : public Module { public: GRPCServerModule(Server& server); ~GRPCServerModule(); - int start(const ovms::Config& config) override; + Status start(const ovms::Config& config) override; void shutdown() override; const GetModelMetadataImpl& getTFSModelMetadataImpl() const; diff --git a/src/httpservermodule.cpp b/src/httpservermodule.cpp index 7812a59666..2d8840cecc 100644 --- a/src/httpservermodule.cpp +++ b/src/httpservermodule.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include "httpservermodule.hpp" +#include #include #include @@ -22,11 +23,12 @@ #include "http_server.hpp" #include "logging.hpp" #include "server.hpp" +#include "status.hpp" namespace ovms { HTTPServerModule::HTTPServerModule(ovms::Server& ovmsServer) : ovmsServer(ovmsServer) {} -int HTTPServerModule::start(const ovms::Config& config) { +Status HTTPServerModule::start(const ovms::Config& config) { state = ModuleState::STARTED_INITIALIZE; SPDLOG_INFO("{} starting", HTTP_SERVER_MODULE_NAME); const std::string server_address = config.restBindAddress() + ":" + std::to_string(config.restPort()); @@ -35,13 +37,16 @@ int HTTPServerModule::start(const ovms::Config& config) { SPDLOG_INFO("Will start {} REST workers", workers); server = ovms::createAndStartHttpServer(config.restBindAddress(), config.restPort(), workers, this->ovmsServer); if (server == nullptr) { - SPDLOG_ERROR("Failed to start REST server at " + server_address); - return EXIT_FAILURE; + std::stringstream ss; + ss << "at " << server_address; + auto status = Status(StatusCode::FAILED_TO_START_REST_SERVER, ss.str()); + SPDLOG_ERROR(status.string()); + return status; } state = ModuleState::INITIALIZED; SPDLOG_INFO("{} started", HTTP_SERVER_MODULE_NAME); SPDLOG_INFO("Started REST server at {}", server_address); - return EXIT_SUCCESS; + return StatusCode::OK; } void HTTPServerModule::shutdown() { if (server == nullptr) diff --git a/src/httpservermodule.hpp b/src/httpservermodule.hpp index ed3814f965..00043c2418 100644 --- a/src/httpservermodule.hpp +++ b/src/httpservermodule.hpp @@ -31,7 +31,8 @@ class HTTPServerModule : public Module { public: HTTPServerModule(Server& ovmsServer); ~HTTPServerModule(); - int start(const Config& config) override; + Status start(const ovms::Config& config) override; + void shutdown() override; }; } // namespace ovms diff --git a/src/main3.cpp b/src/main3.cpp index 28e0cb1850..83c580ad91 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -78,8 +78,16 @@ int main(int argc, char** argv) { OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); if (res) { - // TODO: Better error handling? - fprintf(stderr, "Error starting the server\n"); + uint32_t code = 0; + const char* details = nullptr; + + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + + fprintf(stderr, "error during start: code %d, details: %s\n", code, details); + + OVMS_StatusDelete(res); + OVMS_ServerDelete(srv); OVMS_ServerMultiModelOptionsDelete(mmo); OVMS_ServerGeneralOptionsDelete(go); diff --git a/src/metric_module.cpp b/src/metric_module.cpp index 77584057d1..a65771efee 100644 --- a/src/metric_module.cpp +++ b/src/metric_module.cpp @@ -16,6 +16,7 @@ #include "metric_module.hpp" #include "metric_registry.hpp" +#include "status.hpp" namespace ovms { @@ -24,7 +25,7 @@ MetricModule::~MetricModule() = default; MetricModule::MetricModule() : registry(std::make_unique()) {} -int MetricModule::start(const Config& config) { return EXIT_SUCCESS; } +Status MetricModule::start(const Config& config) { return StatusCode::OK; } void MetricModule::shutdown() {} diff --git a/src/metric_module.hpp b/src/metric_module.hpp index 8ae41999d4..4e0d213547 100644 --- a/src/metric_module.hpp +++ b/src/metric_module.hpp @@ -30,7 +30,8 @@ class MetricModule : public Module { public: MetricModule(); ~MetricModule(); - int start(const Config& config) override; + Status start(const Config& config) override; + void shutdown() override; MetricRegistry& getRegistry() const; diff --git a/src/module.hpp b/src/module.hpp index 12aa82e9ba..c9abcadd67 100644 --- a/src/module.hpp +++ b/src/module.hpp @@ -14,8 +14,10 @@ // limitations under the License. //***************************************************************************** #pragma once + namespace ovms { class Config; +class Status; enum class ModuleState { NOT_INITIALIZED, STARTED_INITIALIZE, @@ -30,7 +32,7 @@ class Module { ModuleState state = ModuleState::NOT_INITIALIZED; public: - virtual int start(const ovms::Config& config) = 0; + virtual Status start(const ovms::Config& config) = 0; virtual void shutdown() = 0; virtual ~Module() = default; ModuleState getState() const; diff --git a/src/pocapi.cpp b/src/pocapi.cpp index c5b1edd0ae..26326ac911 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -49,6 +49,34 @@ using ovms::StatusCode; using ovms::Timer; using std::chrono::microseconds; +void OVMS_StatusDelete(OVMS_Status* status) { + if (status == nullptr) + return; + delete reinterpret_cast(status); +} + +OVMS_Status* OVMS_StatusGetCode(OVMS_Status* status, + uint32_t* code) { + if (status == nullptr) + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STATUS)); + if (code == nullptr) + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_NUMBER)); + ovms::Status* sts = reinterpret_cast(status); + *code = static_cast(sts->getCode()); + return nullptr; +} + +OVMS_Status* OVMS_StatusGetDetails(OVMS_Status* status, + const char** details) { + if (status == nullptr) + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STATUS)); + if (details == nullptr) + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); + ovms::Status* sts = reinterpret_cast(status); + *details = sts->string().c_str(); + return nullptr; +} + OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { if (options == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); @@ -57,12 +85,10 @@ OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); - } +void OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { + if (options == nullptr) + return; delete reinterpret_cast(options); - return nullptr; } OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options) { @@ -73,12 +99,10 @@ OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** opti return nullptr; } -OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); - } +void OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { + if (options == nullptr) + return; delete reinterpret_cast(options); - return nullptr; } OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { @@ -90,14 +114,12 @@ OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { return nullptr; } -OVMS_Status* OVMS_ServerDelete(OVMS_Server* server) { - if (server == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); - } +void OVMS_ServerDelete(OVMS_Server* server) { + if (server == nullptr) + return; ovms::Server* srv = reinterpret_cast(server); srv->shutdownModules(); // delete passed in ptr once multi server configuration is done - return nullptr; } OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, @@ -115,8 +137,10 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, ovms::Server* srv = reinterpret_cast(server); ovms::GeneralOptionsImpl* go = reinterpret_cast(general_options); ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(multi_model_specific_options); - std::int64_t res = srv->start(go, mmo); - return (OVMS_Status*)res; // TODO: Return proper OVMS_Status instead of a raw status code in error handling PR + auto res = srv->start(go, mmo); + if (res.ok()) + return nullptr; + return reinterpret_cast(new ovms::Status(res)); } OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, @@ -277,8 +301,7 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* opt go->logLevel = "WARNING"; break; default: - // TODO: Return error in error handling PR - break; + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_LOG_LEVEL)); } return nullptr; } @@ -321,12 +344,10 @@ OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const cha return nullptr; } -OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* request) { - if (request == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); - } +void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* request) { + if (request == nullptr) + return; delete reinterpret_cast(request); - return nullptr; } OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount) { @@ -527,13 +548,11 @@ OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* res, uin return nullptr; } -OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* res) { - if (res == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); - } +void OVMS_InferenceResponseDelete(OVMS_InferenceResponse* res) { + if (res == nullptr) + return; InferenceResponse* response = reinterpret_cast(res); delete response; - return nullptr; } namespace { diff --git a/src/pocapi.hpp b/src/pocapi.hpp index 7fd15eede8..e3d0dc16ed 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.hpp @@ -71,6 +71,21 @@ typedef enum OVMS_LogLevel_enum { OVMS_LOG_ERROR } OVMS_LogLevel; +//// +//// OVMS_Status +//// Structure for status management. +//// Whenever C-API call returns non null pointer it should be treated as error with code and detailed message. +//// The status should be deallocated with OVMS_StatusDelete afterwards. +//// +// Deallocates status memory for given ptr +void OVMS_StatusDelete(OVMS_Status* status); + +OVMS_Status* OVMS_StatusGetCode(OVMS_Status* status, + uint32_t* code); + +OVMS_Status* OVMS_StatusGetDetails(OVMS_Status* status, + const char** details); + //// //// OVMS_ServerGeneralOptions //// Structure for general options for both: single and multi (with config.json) management @@ -78,7 +93,7 @@ typedef enum OVMS_LogLevel_enum { // Allocates memory for server general options and returns ptr OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options); // Deallocates server general options memory for given ptr -OVMS_Status* OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options); +void OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options); // --port OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, @@ -143,7 +158,7 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* opti // Allocates memory for multi model server options and returns ptr OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options); // Deallocates options memory for given ptr -OVMS_Status* OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options); +void OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options); // --config_path OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, @@ -156,7 +171,7 @@ OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOpti // Allocates memory for server and returns ptr OVMS_Status* OVMS_ServerNew(OVMS_Server** server); // Deallocates server memory for given ptr -OVMS_Status* OVMS_ServerDelete(OVMS_Server* server); +void OVMS_ServerDelete(OVMS_Server* server); // Start with configuration file config.json // Return error if already started @@ -169,7 +184,7 @@ OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion); // TODO add passing server here -OVMS_Status* OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); +void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED @@ -188,6 +203,6 @@ OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* respon OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId); OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, const void** data); -OVMS_Status* OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); +void OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); OVMS_Status* OVMS_Inference(OVMS_Server* server, OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); diff --git a/src/servablemanagermodule.cpp b/src/servablemanagermodule.cpp index 32787061b5..82b092567c 100644 --- a/src/servablemanagermodule.cpp +++ b/src/servablemanagermodule.cpp @@ -35,17 +35,17 @@ ServableManagerModule::ServableManagerModule(ovms::Server& ovmsServer) { } } -int ServableManagerModule::start(const ovms::Config& config) { +Status ServableManagerModule::start(const ovms::Config& config) { state = ModuleState::STARTED_INITIALIZE; SPDLOG_INFO("{} starting", SERVABLE_MANAGER_MODULE_NAME); auto status = servableManager->start(config); if (status.ok()) { state = ModuleState::INITIALIZED; SPDLOG_INFO("{} started", SERVABLE_MANAGER_MODULE_NAME); - return EXIT_SUCCESS; + return status; } SPDLOG_ERROR("ovms::ModelManager::Start() Error: {}", status.string()); - return EXIT_FAILURE; + return status; } void ServableManagerModule::shutdown() { if (state == ModuleState::SHUTDOWN) diff --git a/src/servablemanagermodule.hpp b/src/servablemanagermodule.hpp index 8f78414129..1ccee8862f 100644 --- a/src/servablemanagermodule.hpp +++ b/src/servablemanagermodule.hpp @@ -30,7 +30,8 @@ class ServableManagerModule : public Module { public: ServableManagerModule(ovms::Server& ovmsServer); ~ServableManagerModule(); - int start(const ovms::Config& config) override; + Status start(const ovms::Config& config) override; + void shutdown() override; virtual ModelManager& getServableManager() const; }; diff --git a/src/server.cpp b/src/server.cpp index 31c25e6c07..86f3dbc22e 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -187,6 +187,8 @@ class ProfilerModule : public Module { public: ProfilerModule() = default; + Status start(const ovms::Config& config) override { return StatusCode::OK; } + int start(const Config& config) override { state = ModuleState::STARTED_INITIALIZE; SPDLOG_INFO("{} starting", PROFILER_MODULE_NAME); @@ -248,14 +250,14 @@ std::unique_ptr Server::createModule(const std::string& name) { std::tie(IT_NAME, inserted) = this->modules.emplace(MODULE_NAME, std::move(module)); \ } \ if (!inserted) \ - return EXIT_FAILURE + return Status(StatusCode::MODULE_ALREADY_INSERTED, MODULE_NAME) -#define START_MODULE(IT_NAME) \ - retCode = IT_NAME->second->start(config); \ - if (retCode) \ - return retCode +#define START_MODULE(IT_NAME) \ + status = IT_NAME->second->start(config); \ + if (!status.ok()) \ + return status -int Server::startModules(ovms::Config& config) { +Status Server::startModules(ovms::Config& config) { // The order of starting modules is slightly different from inserting modules // due to dependency of modules on each other during runtime // To avoid unnecessary runtime calls in eg. prediction we have different order @@ -266,7 +268,7 @@ int Server::startModules(ovms::Config& config) { // while we want to start the server as quickly as possible to respond with liveness probe // thats why we delay starting the servable until the very end while we need to create it before // GRPC & REST - auto retCode = EXIT_SUCCESS; + Status status; bool inserted = false; auto it = modules.end(); #if MTR_ENABLED @@ -291,7 +293,7 @@ int Server::startModules(ovms::Config& config) { START_MODULE(itHttp); } START_MODULE(itServable); - return retCode; + return status; } void Server::ensureModuleShutdown(const std::string& name) { @@ -323,20 +325,28 @@ void Server::shutdownModules() { modules.clear(); } +static int statusToExitCode(const Status& status) { + if (status.ok()) { + return EX_OK; + } else if (status == StatusCode::OPTIONS_USAGE_ERROR) { + return EX_USAGE; + } + return EXIT_FAILURE; +} + // OVMS Start int Server::start(int argc, char** argv) { installSignalHandlers(); - - ovms::CLIParser parser; - ovms::GeneralOptionsImpl go; - ovms::MultiModelOptionsImpl mmo; + CLIParser parser; + GeneralOptionsImpl go; + MultiModelOptionsImpl mmo; parser.parse(argc, argv); parser.prepare(&go, &mmo); - - int ret = start(&go, &mmo); + Status ret = start(&go, &mmo); ModulesShutdownGuard shutdownGuard(*this); - if (ret != 0) { - return ret; + if (!ret.ok()) { + // Handle OVMS main() return code + return statusToExitCode(ret); } while (!shutdown_request) { std::this_thread::sleep_for(std::chrono::milliseconds(200)); @@ -345,28 +355,26 @@ int Server::start(int argc, char** argv) { SPDLOG_ERROR("Illegal operation. OVMS started on unsupported device"); } SPDLOG_INFO("Shutting down"); - return EXIT_SUCCESS; } // C-API Start -int Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { +Status Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { try { + if (this->isLive()) + return StatusCode::SERVER_ALREADY_STARTED; auto& config = ovms::Config::instance(); if (!config.parse(go, mmo)) - return EX_USAGE; + return StatusCode::OPTIONS_USAGE_ERROR; // TODO: Have separate code for each option validation failure configure_logger(config.logLevel(), config.logPath()); logConfig(config); - auto retCode = this->startModules(config); - if (retCode) - return retCode; + return this->startModules(config); } catch (std::exception& e) { SPDLOG_ERROR("Exception catch: {} - will now terminate.", e.what()); - return EXIT_FAILURE; + return Status(StatusCode::INTERNAL_ERROR, e.what()); } catch (...) { SPDLOG_ERROR("Unknown exception catch - will now terminate."); - return EXIT_FAILURE; + return StatusCode::INTERNAL_ERROR; } - return EXIT_SUCCESS; } } // namespace ovms diff --git a/src/server.hpp b/src/server.hpp index ea7fb8aaa8..19f41d0a9e 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -24,6 +24,7 @@ namespace ovms { class Config; +class Status; class GeneralOptionsImpl; class MultiModelOptionsImpl; @@ -45,7 +46,7 @@ class Server { public: static Server& instance(); int start(int argc, char** argv); - int start(GeneralOptionsImpl*, MultiModelOptionsImpl*); + Status start(GeneralOptionsImpl*, MultiModelOptionsImpl*); ModuleState getModuleState(const std::string& name) const; const Module* getModule(const std::string& name) const; bool isReady() const; @@ -54,7 +55,7 @@ class Server { void setShutdownRequest(int i); virtual ~Server(); - int startModules(ovms::Config& config); + Status startModules(ovms::Config& config); void shutdownModules(); private: diff --git a/src/status.cpp b/src/status.cpp index c283a57a43..23237c291e 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -280,5 +280,14 @@ const std::unordered_map Status::statusMess {StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER, "Tried to set buffer for nonexisting tensor"}, {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, "Tried to remove buffer for nonexisting tensor"}, {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL, "Tried to remove nonexisting tensor"}, + {StatusCode::NONEXISTENT_STATUS, "Tried to use nonexisting status"}, + {StatusCode::NONEXISTENT_LOG_LEVEL, "Tried to use nonexisting log level"}, + + // Server Start errors + {StatusCode::OPTIONS_USAGE_ERROR, "options validation error"}, + {StatusCode::FAILED_TO_START_GRPC_SERVER, "Failed to start gRPC server"}, + {StatusCode::FAILED_TO_START_REST_SERVER, "Failed to start REST server"}, + {StatusCode::SERVER_ALREADY_STARTED, "Server has already started"}, + {StatusCode::MODULE_ALREADY_INSERTED, "Module already inserted"}, }; } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 947df37f37..61ca7c6c4a 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -291,6 +291,15 @@ enum class StatusCode { NONEXISTENT_TENSOR_FOR_SET_BUFFER, NONEXISTENT_TENSOR_FOR_REMOVE_BUFFER, NONEXISTENT_TENSOR_FOR_REMOVAL, + NONEXISTENT_STATUS, + NONEXISTENT_LOG_LEVEL, + + // Server Start errors + OPTIONS_USAGE_ERROR, + FAILED_TO_START_GRPC_SERVER, + FAILED_TO_START_REST_SERVER, + SERVER_ALREADY_STARTED, + MODULE_ALREADY_INSERTED, STATUS_CODE_END }; diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 4bc8babc47..189631bea6 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -14,6 +14,9 @@ // limitations under the License. //***************************************************************************** +#include +#include + #include // TODO we should not include classes from OVMS here @@ -41,14 +44,54 @@ static void testDefaultSingleModelOptions(MultiModelOptionsImpl* mmo) { EXPECT_EQ(mmo->idleSequenceCleanup, std::nullopt); } +#define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ + { \ + auto* err = C_API_CALL; \ + if (err != nullptr) { \ + const char* msg = nullptr; \ + OVMS_StatusGetDetails(err, &msg); \ + std::string smsg(msg); \ + OVMS_StatusDelete(err); \ + ASSERT_TRUE(false) << smsg; \ + } \ + } + +#define ASSERT_CAPI_STATUS_NOT_NULL(C_API_CALL) \ + { \ + auto* err = C_API_CALL; \ + if (err != nullptr) { \ + OVMS_StatusDelete(err); \ + } else { \ + ASSERT_NE(err, nullptr); \ + } \ + } + +#define ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(C_API_CALL, EXPECTED_STATUS_CODE) \ + { \ + auto* err = C_API_CALL; \ + if (err != nullptr) { \ + uint32_t code = 0; \ + const char* details = nullptr; \ + ASSERT_EQ(OVMS_StatusGetCode(err, &code), nullptr); \ + ASSERT_EQ(OVMS_StatusGetDetails(err, &details), nullptr); \ + ASSERT_NE(details, nullptr); \ + ASSERT_EQ(code, static_cast(EXPECTED_STATUS_CODE)) \ + << std::string{"wrong code: "} + std::to_string(code) + std::string{"; details: "} << details; \ + OVMS_StatusDelete(err); \ + } else { \ + ASSERT_NE(err, nullptr); \ + } \ + } + TEST(CApiConfigTest, MultiModelConfiguration) { OVMS_ServerGeneralOptions* _go = 0; OVMS_ServerMultiModelOptions* _mmo = 0; - ASSERT_NE(OVMS_ServerGeneralOptionsNew(nullptr), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&_go), nullptr); - ASSERT_NE(OVMS_ServerMultiModelOptionsNew(nullptr), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&_mmo), nullptr); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&_go)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&_mmo)); + ASSERT_NE(_go, nullptr); ASSERT_NE(_mmo, nullptr); @@ -78,48 +121,49 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(mmo->configPath, ""); // Set non default values - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(_go, 5555), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(_go, 6666), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcWorkers(_go, 30), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, "2.2.2.2"), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestWorkers(_go, 31), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, "3.3.3.3"), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, "grpcargs"), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(_go, 2), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(_go, 3), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(_go, 4), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, "/ovms/src/test"), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetCacheDir(_go, "/tmp/cache"), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_INFO), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_ERROR), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_DEBUG), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_WARNING), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_TRACE), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetLogPath(_go, "/logs"), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, "/config"), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(_go, 5555)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(_go, 6666)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcWorkers(_go, 30)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, "2.2.2.2")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestWorkers(_go, 31)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, "3.3.3.3")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, "grpcargs")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(_go, 2)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(_go, 3)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(_go, 4)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, "/ovms/src/test")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCacheDir(_go, "/tmp/cache")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_INFO)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_ERROR)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_DEBUG)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_WARNING)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_TRACE)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogLevel(_go, static_cast(99)), StatusCode::NONEXISTENT_LOG_LEVEL); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogPath(_go, "/logs")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, "/config")); // check nullptr - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcPort(nullptr, 5555), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetRestPort(nullptr, 6666), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcWorkers(nullptr, 30), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(nullptr, "2.2.2.2"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetRestWorkers(nullptr, 31), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetRestBindAddress(nullptr, "3.3.3.3"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(nullptr, "grpcargs"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(nullptr, 2), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(nullptr, 3), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(nullptr, 4), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(nullptr, "/ovms/src/test"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetCacheDir(nullptr, "/tmp/cache"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetCacheDir(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetLogLevel(nullptr, OVMS_LOG_TRACE), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetLogPath(nullptr, "/logs"), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsSetLogPath(_go, nullptr), nullptr); - ASSERT_NE(OVMS_ServerMultiModelOptionsSetConfigPath(nullptr, "/config"), nullptr); - ASSERT_NE(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, nullptr), nullptr); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcPort(nullptr, 5555), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestPort(nullptr, 6666), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcWorkers(nullptr, 30), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(nullptr, "2.2.2.2"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestWorkers(nullptr, 31), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestBindAddress(nullptr, "3.3.3.3"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(nullptr, "grpcargs"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(nullptr, 2), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(nullptr, 3), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(nullptr, 4), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(nullptr, "/ovms/src/test"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCacheDir(nullptr, "/tmp/cache"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCacheDir(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogLevel(nullptr, OVMS_LOG_TRACE), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogPath(nullptr, "/logs"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogPath(_go, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsSetConfigPath(nullptr, "/config"), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, nullptr), StatusCode::NONEXISTENT_STRING); // Test non default values EXPECT_EQ(go->grpcPort, 5555); @@ -175,15 +219,15 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(cfg.pluginConfig(), ""); EXPECT_FALSE(cfg.stateful()); EXPECT_FALSE(cfg.lowLatencyTransformation()); - EXPECT_EQ(cfg.maxSequenceNumber(), ovms::DEFAULT_MAX_SEQUENCE_NUMBER); + EXPECT_EQ(cfg.maxSequenceNumber(), DEFAULT_MAX_SEQUENCE_NUMBER); EXPECT_TRUE(cfg.idleSequenceCleanup()); EXPECT_EQ(cfg.configPath(), "/config"); - ASSERT_NE(OVMS_ServerMultiModelOptionsDelete(nullptr), nullptr); - ASSERT_NE(OVMS_ServerGeneralOptionsDelete(nullptr), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsDelete(_mmo), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsDelete(_go), nullptr); + OVMS_ServerMultiModelOptionsDelete(nullptr); + OVMS_ServerMultiModelOptionsDelete(_mmo); + OVMS_ServerGeneralOptionsDelete(nullptr); + OVMS_ServerGeneralOptionsDelete(_go); } TEST(CApiConfigTest, SingleModelConfiguration) { @@ -191,13 +235,13 @@ TEST(CApiConfigTest, SingleModelConfiguration) { } TEST(CApiStartTest, InitializingMultipleServers) { - OVMS_Server* srv1 = 0; - OVMS_Server* srv2 = 0; + OVMS_Server* srv1 = nullptr; + OVMS_Server* srv2 = nullptr; - ASSERT_EQ(OVMS_ServerNew(&srv1), nullptr); - ASSERT_EQ(OVMS_ServerNew(&srv2), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&srv1)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&srv2)); ASSERT_EQ(srv1, srv2); - ASSERT_EQ(OVMS_ServerDelete(srv1), nullptr); + OVMS_ServerDelete(srv1); } TEST(CApiStartTest, StartFlow) { @@ -205,30 +249,58 @@ TEST(CApiStartTest, StartFlow) { OVMS_ServerGeneralOptions* go = 0; OVMS_ServerMultiModelOptions* mmo = 0; - ASSERT_EQ(OVMS_ServerNew(&srv), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerNew(nullptr), StatusCode::NONEXISTENT_SERVER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&srv)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); ASSERT_NE(srv, nullptr); ASSERT_NE(go, nullptr); ASSERT_NE(mmo, nullptr); // Cannot start due to configuration error - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, 5555), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(go, 5555), nullptr); // The same port - ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, 5555)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(go, 5555)); // The same port + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json")); // Expect fail - // TODO: Check exact error code and details once error reporting becomes ready - ASSERT_NE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), nullptr); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), + StatusCode::OPTIONS_USAGE_ERROR); // Fix and expect ok - ASSERT_EQ(OVMS_ServerGeneralOptionsSetRestPort(go, 6666), nullptr); // Different port - ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(go, 6666)); // Different port + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(srv, go, mmo)); - ASSERT_EQ(OVMS_ServerMultiModelOptionsDelete(mmo), nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsDelete(go), nullptr); - ASSERT_EQ(OVMS_ServerDelete(srv), nullptr); + // Try to start again, expect failure + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), + StatusCode::SERVER_ALREADY_STARTED); + + // TODO: Is infer ok? + + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + OVMS_ServerDelete(srv); +} + +TEST(CApiStatusTest, GetCodeAndDetails) { + std::unique_ptr s = std::make_unique( + StatusCode::INTERNAL_ERROR, "custom message"); + OVMS_Status* sts = reinterpret_cast(s.get()); + uint32_t code = 0; + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_StatusGetCode(nullptr, &code), StatusCode::NONEXISTENT_STATUS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_StatusGetCode(sts, nullptr), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NULL(OVMS_StatusGetCode(sts, &code)); + EXPECT_EQ(code, static_cast(StatusCode::INTERNAL_ERROR)); + const char* details = nullptr; + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_StatusGetDetails(nullptr, &details), StatusCode::NONEXISTENT_STATUS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_StatusGetDetails(sts, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NULL(OVMS_StatusGetDetails(sts, &details)); + std::stringstream ss; + ss << Status(StatusCode::INTERNAL_ERROR).string() << " - custom message"; + EXPECT_EQ(std::string(details), ss.str()); + OVMS_StatusDelete(reinterpret_cast(s.release())); } class CapiInference : public ::testing::Test {}; @@ -236,29 +308,22 @@ class CapiInference : public ::testing::Test {}; TEST_F(CapiInference, Basic) { // request creation OVMS_InferenceRequest* request{nullptr}; - OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); ASSERT_NE(nullptr, request); // adding input - status = OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); // setting buffer std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; uint32_t notUsedNum = 0; - status = OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum)); // add parameters const uint64_t sequenceId{42}; - status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId))); // 2nd time should get error - status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), StatusCode::DOUBLE_PARAMETER_INSERT); const uint32_t sequenceControl{1}; // SEQUENCE_START - status = OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl)); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl))); ////////////////// // INFERENCE @@ -269,33 +334,31 @@ TEST_F(CapiInference, Basic) { // prepare options OVMS_ServerGeneralOptions* go = 0; OVMS_ServerMultiModelOptions* mmo = 0; - ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); ASSERT_NE(go, nullptr); ASSERT_NE(mmo, nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port)), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port))); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json")); OVMS_Server* cserver = nullptr; - ASSERT_EQ(OVMS_ServerNew(&cserver), nullptr); - ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo)); + ASSERT_NE(cserver, nullptr); OVMS_InferenceResponse* response = nullptr; - status = OVMS_Inference(cserver, request, &response); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); // verify GetOutputCount uint32_t outputCount = 42; - ASSERT_NE(OVMS_InferenceResponseGetOutputCount(nullptr, &outputCount), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutputCount(response, nullptr), nullptr); - status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutputCount(nullptr, &outputCount), StatusCode::NONEXISTENT_RESPONSE); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutputCount(response, nullptr), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetOutputCount(response, &outputCount)); ASSERT_EQ(outputCount, 1); // verify GetParameterCount uint32_t parameterCount = 42; - ASSERT_NE(OVMS_InferenceResponseGetParameterCount(nullptr, ¶meterCount), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetParameterCount(response, nullptr), nullptr); - status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameterCount(nullptr, ¶meterCount), StatusCode::NONEXISTENT_RESPONSE); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameterCount(response, nullptr), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetParameterCount(response, ¶meterCount)); ASSERT_EQ(0, parameterCount); // verify GetOutput const void* voutputData; @@ -307,17 +370,16 @@ TEST_F(CapiInference, Basic) { BufferType bufferType = (BufferType)199; uint32_t deviceId = 42; const char* outputName{nullptr}; - ASSERT_NE(OVMS_InferenceResponseGetOutput(nullptr, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, nullptr, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, nullptr, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, nullptr, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, nullptr, &voutputData, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, nullptr, &bytesize, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, nullptr, &bufferType, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, nullptr, &deviceId), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, nullptr), nullptr); - status = OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(nullptr, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_RESPONSE); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, nullptr, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, nullptr, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, nullptr, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_TABLE); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, nullptr, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, nullptr, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_DATA); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, nullptr, &bufferType, &deviceId), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, nullptr, &deviceId), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, nullptr), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); ASSERT_EQ(std::string(DUMMY_MODEL_OUTPUT_NAME), outputName); EXPECT_EQ(datatype, OVMS_DATATYPE_FP32); EXPECT_EQ(dimCount, 2); @@ -337,8 +399,7 @@ TEST_F(CapiInference, Basic) { // CLEANUP /////////////// // cleanup response - status = OVMS_InferenceResponseDelete(response); - ASSERT_EQ(nullptr, status); + OVMS_InferenceResponseDelete(response); // cleanup request // here we will add additional inputs to verify 2 ways of cleanup // - direct call to remove whole request @@ -349,48 +410,32 @@ TEST_F(CapiInference, Basic) { // one will be removed together with request but without buffer attached // one will be removed with buffer directly // one will be removed without buffer directly - status = OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); - ASSERT_EQ(nullptr, status); - status = OVMS_InferenceRequestAddInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); - ASSERT_EQ(nullptr, status); - status = OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); - ASSERT_EQ(nullptr, status); - status = OVMS_InferenceRequestInputSetData(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum)); // we will add buffer and remove it to check separate buffer removal - status = OVMS_InferenceRequestInputSetData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum)); - status = OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY")); // second time we should get error - status = OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) - status = OVMS_InferenceRequestRemoveInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"); - ASSERT_EQ(nullptr, status); - status = OVMS_InferenceRequestRemoveInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY"); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestInputRemoveData(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY"), StatusCode::NONEXISTENT_BUFFER_FOR_REMOVAL); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestRemoveInput(request, "INPUT_WITHOUT_BUFFER_REMOVED_DIRECTLY")); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestRemoveInput(request, "INPUT_WITH_BUFFER_REMOVED_DIRECTLY")); // we will remove 1 of two parameters - status = OVMS_InferenceRequestRemoveParameter(request, "sequence_id"); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestRemoveParameter(request, "sequence_id")); // 2nd time should report error - status = OVMS_InferenceRequestRemoveParameter(request, "sequence_id"); - ASSERT_NE(nullptr, status); - ASSERT_NE(OVMS_InferenceRequestRemoveParameter(nullptr, "sequence_id"), nullptr); - ASSERT_NE(OVMS_InferenceRequestRemoveParameter(request, nullptr), nullptr); - // OVMS_StatusDelete(status); // FIXME(dkalinow) - - status = OVMS_InferenceRequestRemoveInput(request, "NONEXISTENT_TENSOR"); - ASSERT_NE(nullptr, status); - ASSERT_NE(OVMS_InferenceRequestRemoveInput(nullptr, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST"), nullptr); - ASSERT_NE(OVMS_InferenceRequestRemoveInput(request, nullptr), nullptr); - // OVMS_StatusDelete(status); // FIXME(dkalinow) - ASSERT_NE(OVMS_InferenceRequestDelete(nullptr), nullptr); - status = OVMS_InferenceRequestDelete(request); - ASSERT_EQ(nullptr, status); - - ASSERT_EQ(OVMS_ServerDelete(cserver), nullptr); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveParameter(request, "sequence_id"), StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveParameter(nullptr, "sequence_id"), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveParameter(request, nullptr), StatusCode::NONEXISTENT_STRING); + + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveInput(request, "NONEXISTENT_TENSOR"), StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveInput(nullptr, "INPUT_WITHOUT_BUFFER_REMOVED_WITH_REQUEST"), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestRemoveInput(request, nullptr), StatusCode::NONEXISTENT_STRING); + OVMS_InferenceRequestDelete(nullptr); + OVMS_InferenceRequestDelete(request); + + OVMS_ServerDelete(cserver); } TEST_F(CapiInference, NegativeInference) { @@ -400,100 +445,77 @@ TEST_F(CapiInference, NegativeInference) { // prepare options OVMS_ServerGeneralOptions* go = 0; OVMS_ServerMultiModelOptions* mmo = 0; - ASSERT_EQ(OVMS_ServerGeneralOptionsNew(&go), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsNew(&mmo), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); ASSERT_NE(go, nullptr); ASSERT_NE(mmo, nullptr); - ASSERT_EQ(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port)), nullptr); - ASSERT_EQ(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"), nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port))); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json")); OVMS_Server* cserver = nullptr; - ASSERT_NE(OVMS_ServerNew(nullptr), nullptr); - ASSERT_EQ(OVMS_ServerNew(&cserver), nullptr); - ASSERT_NE(OVMS_ServerStartFromConfigurationFile(nullptr, go, mmo), nullptr); - ASSERT_NE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, mmo), nullptr); - ASSERT_NE(OVMS_ServerStartFromConfigurationFile(cserver, go, nullptr), nullptr); - ASSERT_EQ(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo), nullptr); - // TODO ensure cleanup + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(nullptr, go, mmo), StatusCode::NONEXISTENT_SERVER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, mmo), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, go, nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo)); // TODO add check for server ready strict in 2022.3 not available in C-API OVMS_InferenceRequest* request{nullptr}; OVMS_InferenceResponse* response = nullptr; - ASSERT_NE(OVMS_InferenceRequestNew(nullptr, "dummy", 1), nullptr); - ASSERT_NE(OVMS_InferenceRequestNew(&request, nullptr, 1), nullptr); - OVMS_Status* status = OVMS_InferenceRequestNew(&request, "dummy", 1); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(nullptr, "dummy", 1), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(&request, nullptr, 1), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); ASSERT_NE(nullptr, request); // negative no inputs - status = OVMS_Inference(cserver, request, &response); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::INVALID_NO_OF_INPUTS); // negative no input buffer - ASSERT_NE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); - ASSERT_NE(OVMS_InferenceRequestAddInput(request, nullptr, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); - ASSERT_NE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, nullptr, DUMMY_MODEL_SHAPE.size()), nullptr); - status = OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddInput(request, nullptr, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, nullptr, DUMMY_MODEL_SHAPE.size()), StatusCode::NONEXISTENT_TABLE); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); // fail with adding input second time - ASSERT_NE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); - status = OVMS_Inference(cserver, request, &response); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), StatusCode::DOUBLE_TENSOR_INSERT); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::INVALID_CONTENT_SIZE); // setting buffer std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; uint32_t notUsedNum = 0; - ASSERT_NE(OVMS_InferenceRequestInputSetData(nullptr, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); - ASSERT_NE(OVMS_InferenceRequestInputSetData(request, nullptr, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); - ASSERT_NE(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, nullptr, sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), nullptr); - status = OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); - ASSERT_EQ(nullptr, status); - status = OVMS_InferenceRequestInputSetData(request, "NONEXISTENT_TENSOR", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestInputSetData(nullptr, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestInputSetData(request, nullptr, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, nullptr, sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), StatusCode::NONEXISTENT_DATA); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestInputSetData(request, "NONEXISTENT_TENSOR", reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum), StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER); // add parameters const uint64_t sequenceId{42}; - ASSERT_NE(OVMS_InferenceRequestAddParameter(nullptr, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), nullptr); - ASSERT_NE(OVMS_InferenceRequestAddParameter(request, nullptr, OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), nullptr); - ASSERT_NE(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, nullptr, sizeof(sequenceId)), nullptr); - status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddParameter(nullptr, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddParameter(request, nullptr, OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, nullptr, sizeof(sequenceId)), StatusCode::NONEXISTENT_DATA); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId))); // 2nd time should get error - status = OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddParameter(request, "sequence_id", OVMS_DATATYPE_U64, reinterpret_cast(&sequenceId), sizeof(sequenceId)), StatusCode::DOUBLE_PARAMETER_INSERT); const uint32_t sequenceControl{1}; // SEQUENCE_START - status = OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl)); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddParameter(request, "sequence_control_input", OVMS_DATATYPE_U32, reinterpret_cast(&sequenceControl), sizeof(sequenceControl))); // verify passing nullptrs - status = OVMS_Inference(nullptr, request, &response); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); FIXME - status = OVMS_Inference(cserver, nullptr, &response); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); FIXME - status = OVMS_Inference(cserver, request, nullptr); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); FIXME - // + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(nullptr, request, &response), StatusCode::NONEXISTENT_SERVER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, nullptr, &response), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, nullptr), StatusCode::NONEXISTENT_RESPONSE); + // negative inference with non existing model OVMS_InferenceRequest* requestNoModel{nullptr}; OVMS_InferenceResponse* reponseNoModel{nullptr}; - status = OVMS_InferenceRequestNew(&request, "NONEXISTENT_MODEL", 13); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "NONEXISTENT_MODEL", 13)); // negative no model - status = OVMS_Inference(cserver, request, &response); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::NOT_IMPLEMENTED); - ASSERT_NE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), nullptr); - status = OVMS_Inference(cserver, requestNoModel, &reponseNoModel); - ASSERT_NE(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestAddInput(nullptr, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size()), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, requestNoModel, &reponseNoModel), StatusCode::NONEXISTENT_REQUEST); OVMS_InferenceRequestDelete(requestNoModel); - ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); - ASSERT_EQ(OVMS_ServerDelete(cserver), nullptr); - ASSERT_NE(OVMS_ServerDelete(nullptr), nullptr); + OVMS_ServerDelete(nullptr); + OVMS_ServerDelete(cserver); + OVMS_ServerDelete(nullptr); } // TODO negative test -> validate at the infer stage @@ -509,7 +531,7 @@ const uint32_t PRIORITY{7}; const uint64_t REQUEST_ID{3}; const std::string INPUT_NAME{"NOT_RANDOM_NAME"}; -const ovms::shape_t INPUT_SHAPE{1, 3, 220, 230}; +const shape_t INPUT_SHAPE{1, 3, 220, 230}; const std::array INPUT_DATA{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; constexpr size_t INPUT_DATA_BYTESIZE{INPUT_DATA.size() * sizeof(float)}; const OVMS_DataType DATATYPE{OVMS_DATATYPE_FP32}; @@ -539,22 +561,19 @@ TEST_F(CapiInference, ResponseRetrieval) { /////////////////////////// OVMS_InferenceResponse* response = reinterpret_cast(cppResponse.get()); uint32_t outputCount = 42; - auto status = OVMS_InferenceResponseGetOutputCount(response, &outputCount); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetOutputCount(response, &outputCount)); ASSERT_EQ(outputCount, 1); uint32_t parameterCount = 42; - status = OVMS_InferenceResponseGetParameterCount(response, ¶meterCount); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetParameterCount(response, ¶meterCount)); ASSERT_EQ(1, parameterCount); // verify get Parameter OVMS_DataType parameterDatatype = OVMS_DATATYPE_FP32; const void* parameterData{nullptr}; - ASSERT_NE(OVMS_InferenceResponseGetParameter(nullptr, 0, ¶meterDatatype, ¶meterData), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetParameter(response, 0, nullptr, ¶meterData), nullptr); - ASSERT_NE(OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, nullptr), nullptr); - status = OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, ¶meterData); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameter(nullptr, 0, ¶meterDatatype, ¶meterData), StatusCode::NONEXISTENT_RESPONSE); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameter(response, 0, nullptr, ¶meterData), StatusCode::NONEXISTENT_NUMBER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, nullptr), StatusCode::NONEXISTENT_DATA); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetParameter(response, 0, ¶meterDatatype, ¶meterData)); ASSERT_EQ(parameterDatatype, OVMS_DATATYPE_U64); EXPECT_EQ(0, std::memcmp(parameterData, (void*)&seqId, sizeof(seqId))); // verify get Output @@ -567,11 +586,8 @@ TEST_F(CapiInference, ResponseRetrieval) { BufferType bufferType = (BufferType)199; uint32_t deviceId = 42; const char* outputName{nullptr}; - status = OVMS_InferenceResponseGetOutput(response, outputId + 42123, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) - status = OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); - ASSERT_EQ(nullptr, status); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId + 42123, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_TENSOR); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); ASSERT_EQ(INPUT_NAME, outputName); EXPECT_EQ(datatype, OVMS_DATATYPE_FP32); EXPECT_EQ(dimCount, 2); @@ -590,19 +606,14 @@ TEST_F(CapiInference, ResponseRetrieval) { // test negative scenario with getting output without buffer cppStatus = cppResponse->addOutput("outputWithNoBuffer", DATATYPE, cppOutputShape.data(), cppOutputShape.size()); ASSERT_EQ(cppStatus, StatusCode::OK) << cppStatus.string(); - status = OVMS_InferenceResponseGetOutput(response, outputId + 1, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId + 1, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::INTERNAL_ERROR); // negative scenario nonexistsing parameter - status = OVMS_InferenceResponseGetParameter(response, 123, ¶meterDatatype, ¶meterData); - ASSERT_NE(nullptr, status); - // OVMS_StatusDelete(status); // FIXME(dkalinow) + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetParameter(response, 123, ¶meterDatatype, ¶meterData), StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL); // final cleanup // we release unique_ptr ownership here so that we can free it safely via C-API cppResponse.release(); - ASSERT_NE(OVMS_InferenceResponseDelete(nullptr), nullptr); - status = OVMS_InferenceResponseDelete(response); - ASSERT_EQ(nullptr, status); + OVMS_InferenceResponseDelete(nullptr); + OVMS_InferenceResponseDelete(response); } // TODO make cleaner error codes reporting // todo decide either use remove or delete for consistency diff --git a/src/test/http_rest_api_handler_test.cpp b/src/test/http_rest_api_handler_test.cpp index 8e5bec42de..ebd3458b45 100644 --- a/src/test/http_rest_api_handler_test.cpp +++ b/src/test/http_rest_api_handler_test.cpp @@ -101,8 +101,8 @@ class ConfigApi : public TestWithTempDir { auto& config = ovms::Config::instance(); auto retCode = ovmsServer.startModules(config); - EXPECT_EQ(EXIT_SUCCESS, retCode); - if (retCode) + EXPECT_TRUE(retCode.ok()) << retCode.string(); + if (!retCode.ok()) throw std::runtime_error("Failed to start modules"); serverGuard = std::make_unique(ovmsServer); auto modulePtr = ovmsServer.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); diff --git a/src/test/server_test.cpp b/src/test/server_test.cpp index 8712606bb1..e9ee90ad57 100644 --- a/src/test/server_test.cpp +++ b/src/test/server_test.cpp @@ -157,7 +157,7 @@ class MockedServableManagerModule : public ovms::ServableManagerModule { bool waitWithChangingState = true; MockedServableManagerModule(ovms::Server& ovmsServer) : ovms::ServableManagerModule(ovmsServer) {} - int start(const Config& config) override { + ovms::Status start(const Config& config) override { state = ModuleState::STARTED_INITIALIZE; SPDLOG_INFO("Mocked {} starting", SERVABLE_MANAGER_MODULE_NAME); auto start = std::chrono::high_resolution_clock::now(); @@ -173,10 +173,10 @@ class MockedServableManagerModule : public ovms::ServableManagerModule { state = ModuleState::INITIALIZED; SPDLOG_INFO("Mocked {} started", SERVABLE_MANAGER_MODULE_NAME); - return EXIT_SUCCESS; + return status; } SPDLOG_ERROR("ovms::ModelManager::Start() Error: {}", status.string()); - return EXIT_FAILURE; + return status; } }; diff --git a/src/test/status_test.cpp b/src/test/status_test.cpp index 6055f2fbef..ad84ec02f3 100644 --- a/src/test/status_test.cpp +++ b/src/test/status_test.cpp @@ -13,11 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include +#include #include #include #include +#include "../pocapi.hpp" #include "../status.hpp" #include "test_utils.hpp" @@ -36,4 +39,19 @@ TEST(StatusCodeTest, AllStatusCodesMapped) { ASSERT_NE(status.string(), "Undefined error"); } } + +TEST(StatusCodeTest, CApi) { + for (auto statusCode = StatusCode::OK; statusCode != StatusCode::STATUS_CODE_END; ++statusCode) { + Status status = Status(statusCode); + OVMS_Status* sts = reinterpret_cast(&status); + uint32_t code = 0; + ASSERT_EQ(OVMS_StatusGetCode(sts, &code), nullptr); + EXPECT_EQ(code, static_cast(statusCode)); + const char* details = nullptr; + ASSERT_EQ(OVMS_StatusGetDetails(sts, &details), nullptr); + std::stringstream ss; + ss << Status(statusCode).string(); + EXPECT_EQ(std::string(details), ss.str()); + } +} } // namespace ovms From 85b2b24f665f0503e8c837171e969d7c2b036fe3 Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 7 Dec 2022 16:03:59 +0100 Subject: [PATCH 099/130] ovms with shared lib (#1570) * ovms with shared lib --- Dockerfile.redhat | 13 +++++++++---- Dockerfile.ubuntu | 14 ++++++++------ Makefile | 17 ++++++++++++----- src/BUILD | 16 +++++++++++++++- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 76e4b30eed..7853b4f14b 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -199,18 +199,23 @@ COPY check_coverage.bat /ovms/ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} -RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms -RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so - RUN cd /ovms/src/example/SampleCpuExtension/ && make # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 +# Copy binary for //src:ovms +RUN cp $(bazel info bazel-bin)/src/libovms_shared.so src +RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms +RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so + +RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make + COPY ${ovms_metadata_file} metadata.json -RUN ./bazel-bin/src/./ovms +RUN cd bazel-bin/src/ && ./ovms --version && ./ovms && cd /ovms + COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE COPY client /client diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 80087e2428..f49c56335b 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -293,20 +293,22 @@ COPY check_coverage.bat /ovms/ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} +# C api shared library +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 + +# Copy binary for //src:ovms +RUN cp $(bazel info bazel-bin)/src/libovms_shared.so src RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make -# C api shared library -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 - ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json -RUN ./bazel-bin/src/./ovms --version -RUN ./bazel-bin/src/./ovms +RUN cd bazel-bin/src/ && ./ovms --version && ./ovms && cd /ovms + COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE COPY client /client diff --git a/Makefile b/Makefile index d459a3f17f..80503f2832 100644 --- a/Makefile +++ b/Makefile @@ -255,7 +255,7 @@ get_coverage: @echo "Copying coverage report from build image to genhtml if exist..." @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) bash @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms/genhtml/ . || true - @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) + @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) || true @if [ -d genhtml/src ]; then $(MAKE) check_coverage; \ else echo "ERROR: genhtml/src was not generated during build"; \ fi @@ -264,17 +264,24 @@ check_coverage: @docker run $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG) ./check_coverage.bat | grep success test_checksec: - @echo "Running checksec on ovms binary..." + @echo "Running checksec on libovms_shared library..." @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) || true @docker create -ti --name $(OVMS_CPP_CONTAINTER_NAME) $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) bash + @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms_release/lib/libovms_shared.so /tmp @docker cp $(OVMS_CPP_CONTAINTER_NAME):/ovms_release/bin/ovms /tmp - @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) + @docker rm -f $(OVMS_CPP_CONTAINTER_NAME) || true + @checksec --file=/tmp/libovms_shared.so --format=csv > checksec.txt + @if ! grep -FRq "Full RELRO,Canary found,NX enabled,DSO,No RPATH,RUNPATH,Symbols,Yes" checksec.txt; then\ + echo "ERROR: OVMS shared library security settings changed. Run checksec on ovms shared library and fix issues." && exit 1;\ + fi @checksec --file=/tmp/ovms --format=csv > checksec.txt - @if ! grep -FRq "Full RELRO,Canary found,NX enabled,PIE enabled,No RPATH,RUNPATH,Symbols,Yes" checksec.txt; then\ - error Run checksec on ovms binary and fix issues.;\ +# Stack protector and canary not required if linked with libovms_shared + @if ! grep -FRq "Full RELRO,No Canary found,NX enabled,PIE enabled,No RPATH,RUNPATH,Symbols,No" checksec.txt; then\ + echo "ERROR: OVMS binary security settings changed. Run checksec on ovms binary and fix issues." && exit 1;\ fi @rm -f checksec.txt @rm -f /tmp/ovms + @rm -f /tmp/libovms_shared.so @echo "Checksec check success." test_perf: venv diff --git a/src/BUILD b/src/BUILD index fd119348ab..972197a7fd 100644 --- a/src/BUILD +++ b/src/BUILD @@ -559,6 +559,19 @@ cc_binary( linkstatic = True, ) +cc_import( + name = "shared_lib", + hdrs = ["pocapi.hpp"], + shared_library = "libovms_shared.so", +) + +cc_library( + name = "cpp_headers", + srcs = ["server.hpp", + "module.hpp",], + deps = ["//src:shared_lib"] +) + cc_binary( name = "ovms", srcs = [ @@ -569,13 +582,14 @@ cc_binary( "-luuid", "-lstdc++fs", "-lcrypto", + "-lovms_shared", ], copts = [ "-Wconversion", "-Werror", ], deps = [ - "//src:ovms_lib", + "//src:cpp_headers", ], linkstatic = False, ) From 29bf309ac4e68e82e0d178db27e1d8fdc1524f6b Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Thu, 8 Dec 2022 09:04:06 +0100 Subject: [PATCH 100/130] CVS-95633 Minimal C app using C-API (#1566) --- Dockerfile.redhat | 6 + Dockerfile.ubuntu | 6 + MakefileCapi | 8 +- src/BUILD | 6 +- src/buffer.cpp | 4 +- src/buffer.hpp | 8 +- src/inferenceparameter.cpp | 2 +- src/inferenceparameter.hpp | 2 +- src/inferencerequest.cpp | 2 +- src/inferencerequest.hpp | 4 +- src/inferenceresponse.hpp | 2 +- src/inferencetensor.cpp | 4 +- src/inferencetensor.hpp | 4 +- src/main3.c | 136 ++++++++++++++++++++++ src/main3.cpp | 2 +- src/pocapi.cpp | 14 ++- src/{pocapi.hpp => pocapi.h} | 41 ++++--- src/pocapiinternal.hpp | 2 +- src/predict_request_validation_utils.cpp | 8 +- src/test/c_api_tests.cpp | 6 +- src/test/capi_predict_validation_test.cpp | 12 +- src/test/status_test.cpp | 2 +- src/test/test_utils.cpp | 6 +- src/test/test_utils.hpp | 6 +- 24 files changed, 230 insertions(+), 63 deletions(-) create mode 100644 src/main3.c rename src/{pocapi.hpp => pocapi.h} (86%) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 7853b4f14b..c61752da1f 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -157,6 +157,7 @@ COPY .bazelrc WORKSPACE /ovms/ RUN ln -s /usr/lib64 /usr/lib/x86_64-linux-gnu COPY external /ovms/external/ COPY third_party /ovms/third_party/ +COPY MakefileCapi /ovms/ RUN alternatives --set python /usr/bin/python3 @@ -203,6 +204,8 @@ RUN cd /ovms/src/example/SampleCpuExtension/ && make # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared + +# C api app with bazel RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 # Copy binary for //src:ovms @@ -212,6 +215,9 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make +# C-api C/C++ app with gcc +RUN cd /ovms && make -f MakefileCapi cpp && make -f MakefileCapi c + COPY ${ovms_metadata_file} metadata.json RUN cd bazel-bin/src/ && ./ovms --version && ./ovms && cd /ovms diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index f49c56335b..9a2c7cef90 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -257,6 +257,7 @@ RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ WORKDIR /ovms COPY .bazelrc WORKSPACE /ovms/ COPY external /ovms/external/ +COPY MakefileCapi /ovms/ RUN apt-get update && apt-get install -y python-is-python3 && rm -rf /var/lib/apt/lists/* @@ -295,6 +296,8 @@ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared + +# C api app with bazel RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 # Copy binary for //src:ovms @@ -304,6 +307,9 @@ RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make +# C-api C/C++ app with gcc +RUN cd /ovms && make -f MakefileCapi cpp && make -f MakefileCapi c + ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json diff --git a/MakefileCapi b/MakefileCapi index 09b1dd489a..c15f5c279a 100644 --- a/MakefileCapi +++ b/MakefileCapi @@ -13,7 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -all: +cpp: bazel build //src:ovms_shared g++ src/main3.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/poc3 LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/poc3 + +c: + bazel build //src:ovms_shared + gcc -c src/main3.c -o /ovms/bazel-bin/src/main3.o -std=c99 + gcc -o /ovms/bazel-bin/src/poc3_c /ovms/bazel-bin/src/main3.o -lovms_shared -L/ovms/bazel-bin/src/ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/poc3_c diff --git a/src/BUILD b/src/BUILD index 972197a7fd..f072a2591a 100644 --- a/src/BUILD +++ b/src/BUILD @@ -87,7 +87,7 @@ cc_shared_library( cc_library( name = "ovms_lib", - hdrs = ["pocapi.hpp"], + hdrs = ["pocapi.h"], srcs = [ "aliases.hpp", "azurestorage.hpp", @@ -237,7 +237,7 @@ cc_library( "pipelineeventqueue.hpp", "pipeline_factory.cpp", "pipeline_factory.hpp", - "pocapi.hpp", + "pocapi.h", "pocapiinternal.cpp", "pocapiinternal.hpp", "pocapi.cpp", @@ -561,7 +561,7 @@ cc_binary( cc_import( name = "shared_lib", - hdrs = ["pocapi.hpp"], + hdrs = ["pocapi.h"], shared_library = "libovms_shared.so", ) diff --git a/src/buffer.cpp b/src/buffer.cpp index fce2841f8f..c05dbbf070 100644 --- a/src/buffer.cpp +++ b/src/buffer.cpp @@ -18,7 +18,7 @@ #include namespace ovms { -Buffer::Buffer(const void* pptr, size_t byteSize, BufferType bufferType, std::optional bufferDeviceId, bool createCopy) : +Buffer::Buffer(const void* pptr, size_t byteSize, OVMS_BufferType bufferType, std::optional bufferDeviceId, bool createCopy) : ptr(createCopy ? nullptr : pptr), byteSize(byteSize), bufferType(bufferType), @@ -39,7 +39,7 @@ size_t Buffer::getByteSize() const { return byteSize; } -BufferType Buffer::getBufferType() const { +OVMS_BufferType Buffer::getBufferType() const { return this->bufferType; } diff --git a/src/buffer.hpp b/src/buffer.hpp index 007765230a..1c4691a4af 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -17,21 +17,21 @@ #include #include -#include "pocapi.hpp" +#include "./pocapi.h" namespace ovms { class Buffer { const void* ptr; size_t byteSize; - BufferType bufferType; + OVMS_BufferType bufferType; std::optional bufferDeviceId; std::unique_ptr ownedCopy = nullptr; public: - Buffer(const void* ptr, size_t byteSize, BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional bufferDeviceId = std::nullopt, bool createCopy = false); + Buffer(const void* ptr, size_t byteSize, OVMS_BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional bufferDeviceId = std::nullopt, bool createCopy = false); ~Buffer(); const void* data() const; - BufferType getBufferType() const; + OVMS_BufferType getBufferType() const; const std::optional& getDeviceId() const; size_t getByteSize() const; }; diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index bb5a851322..bcfea79a2e 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -18,7 +18,7 @@ #include -#include "pocapi.hpp" +#include "./pocapi.h" namespace ovms { // TODO should we own our own copy of value? // diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp index c899009cd2..eb5b213351 100644 --- a/src/inferenceparameter.hpp +++ b/src/inferenceparameter.hpp @@ -17,7 +17,7 @@ #include #include -#include "pocapi.hpp" +#include "./pocapi.h" namespace ovms { size_t DataTypeToByteSize(OVMS_DataType datatype); diff --git a/src/inferencerequest.cpp b/src/inferencerequest.cpp index b1129c2028..e7a116a138 100644 --- a/src/inferencerequest.cpp +++ b/src/inferencerequest.cpp @@ -35,7 +35,7 @@ Status InferenceRequest::addInput(const char* name, OVMS_DataType datatype, cons auto [it, emplaced] = inputs.emplace(name, InferenceTensor{datatype, shape, dimCount}); return emplaced ? StatusCode::OK : StatusCode::DOUBLE_TENSOR_INSERT; } -Status InferenceRequest::setInputBuffer(const char* name, const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId) { +Status InferenceRequest::setInputBuffer(const char* name, const void* addr, size_t byteSize, OVMS_BufferType bufferType, std::optional deviceId) { auto it = inputs.find(name); if (it == inputs.end()) { return StatusCode::NONEXISTENT_TENSOR_FOR_SET_BUFFER; diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp index b5e8afe4df..464d492b35 100644 --- a/src/inferencerequest.hpp +++ b/src/inferencerequest.hpp @@ -18,10 +18,10 @@ #include #include +#include "./pocapi.h" #include "inferenceparameter.hpp" #include "inferencetensor.hpp" #include "modelversion.hpp" -#include "pocapi.hpp" namespace ovms { @@ -43,7 +43,7 @@ class InferenceRequest { Status removeInput(const char* name); Status removeAllInputs(); - Status setInputBuffer(const char* name, const void* addr, size_t byteSize, BufferType, std::optional deviceId); + Status setInputBuffer(const char* name, const void* addr, size_t byteSize, OVMS_BufferType, std::optional deviceId); Status removeInputBuffer(const char* name); Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); Status removeParameter(const char* parameterName); diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index 60ce028f23..983e85c54d 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -18,10 +18,10 @@ #include #include +#include "./pocapi.h" #include "inferenceparameter.hpp" #include "inferencetensor.hpp" #include "modelversion.hpp" -#include "pocapi.hpp" namespace ovms { diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp index 7b6e79edbc..3167df7fdd 100644 --- a/src/inferencetensor.cpp +++ b/src/inferencetensor.cpp @@ -17,8 +17,8 @@ #include +#include "./pocapi.h" #include "buffer.hpp" -#include "pocapi.hpp" #include "status.hpp" namespace ovms { @@ -30,7 +30,7 @@ InferenceTensor::InferenceTensor(InferenceTensor&& rhs) : InferenceTensor::InferenceTensor(OVMS_DataType datatype, const size_t* shape, size_t dimCount) : datatype(datatype), shape(shape, shape + dimCount) {} -Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy) { +Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, OVMS_BufferType bufferType, std::optional deviceId, bool createCopy) { if (nullptr != buffer) { return StatusCode::DOUBLE_BUFFER_SET; } // TODO validate against byteSize == 0 diff --git a/src/inferencetensor.hpp b/src/inferencetensor.hpp index 113ff63d5c..d1c8dd3d75 100644 --- a/src/inferencetensor.hpp +++ b/src/inferencetensor.hpp @@ -18,7 +18,7 @@ #include #include -#include "pocapi.hpp" +#include "./pocapi.h" #include "shape.hpp" namespace ovms { @@ -37,7 +37,7 @@ class InferenceTensor { InferenceTensor(const InferenceTensor&) = delete; InferenceTensor& operator=(const InferenceTensor&) = delete; InferenceTensor& operator=(const InferenceTensor&&); - Status setBuffer(const void* addr, size_t byteSize, BufferType bufferType, std::optional deviceId, bool createCopy = false); + Status setBuffer(const void* addr, size_t byteSize, OVMS_BufferType bufferType, std::optional deviceId, bool createCopy = false); Status removeBuffer(); OVMS_DataType getDataType() const; const shape_t& getShape() const; diff --git a/src/main3.c b/src/main3.c new file mode 100644 index 0000000000..8169fdd28d --- /dev/null +++ b/src/main3.c @@ -0,0 +1,136 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include "pocapi.h" + +int main() { + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_Server* srv; + + OVMS_ServerGeneralOptionsNew(&go); + OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerNew(&srv); + + OVMS_ServerGeneralOptionsSetGrpcPort(go, 11337); + OVMS_ServerGeneralOptionsSetRestPort(go, 11338); + + OVMS_ServerGeneralOptionsSetLogLevel(go, OVMS_LOG_DEBUG); + OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); + + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + + if (res) { + uint32_t code = 0; + const char* details = 0; + + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + + fprintf(stderr, "error during start: code %d, details: %s\n", code, details); + + OVMS_StatusDelete(res); + + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + return 1; + } + + printf("Server ready for inference\n"); + + const uint64_t SHAPE_N = 30; + const uint64_t SHAPE_C = 20; + const uint64_t SHAPE[2] = {SHAPE_N, SHAPE_C}; + const size_t NUM_ELEMENTS = SHAPE_N * SHAPE_C; + const size_t DATA_SIZE = NUM_ELEMENTS * sizeof(float); + const float INPUT_ELEMENT_VALUE = 3.2f; + + float inputData[DATA_SIZE]; + for (int i = 0; i < NUM_ELEMENTS; i++) { + inputData[i] = INPUT_ELEMENT_VALUE; + } + + OVMS_InferenceRequest* request = NULL; + OVMS_InferenceRequestNew(&request, "dummy", 1); + OVMS_InferenceRequestAddInput(request, "b", OVMS_DATATYPE_FP32, SHAPE, 2); + OVMS_InferenceRequestInputSetData(request, "b", inputData, DATA_SIZE, OVMS_BUFFERTYPE_CPU, 0); + + OVMS_InferenceResponse* response = NULL; + res = OVMS_Inference(srv, request, &response); + if (res) { + uint32_t code = 0; + const char* details = 0; + + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + + fprintf(stderr, "error during inference: code %d, details: %s\n", code, details); + + OVMS_StatusDelete(res); + + OVMS_InferenceRequestDelete(request); + + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + return 1; + } + + const char* oName = NULL; // not needed + OVMS_DataType oType; // not needed + const uint64_t* oShape; // not needed + uint32_t oDims; // not needed + const void* oData = NULL; + size_t oNumBytes = 0; + OVMS_BufferType oBuffType; // not needed + uint32_t oDeviceId; // not needed + OVMS_InferenceResponseGetOutput(response, 0, &oName, &oType, &oShape, &oDims, &oData, &oNumBytes, &oBuffType, &oDeviceId); + + float expectedOutput[DATA_SIZE]; + for (int i = 0; i < NUM_ELEMENTS; i++) { + expectedOutput[i] = INPUT_ELEMENT_VALUE + 1.0f; + } + if (memcmp(oData, expectedOutput, DATA_SIZE) != 0) { + fprintf(stderr, "output is not correct\n"); + + OVMS_InferenceResponseDelete(response); + OVMS_InferenceRequestDelete(request); + + OVMS_ServerDelete(srv); + + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + return 1; + } else { + printf("output is correct\n"); + } + + OVMS_InferenceResponseDelete(response); + OVMS_InferenceRequestDelete(request); + + printf("No more job to be done, will shut down\n"); + + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + + printf("main() exit\n"); + return 0; +} diff --git a/src/main3.cpp b/src/main3.cpp index 83c580ad91..5a310e66e9 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -20,7 +20,7 @@ #include #include -#include "pocapi.hpp" +#include "./pocapi.h" namespace { volatile sig_atomic_t shutdown_request = 0; diff --git a/src/pocapi.cpp b/src/pocapi.cpp index 26326ac911..ac22e9a2d9 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "pocapi.hpp" +#include "pocapi.h" // NOLINT #include #include @@ -49,6 +49,10 @@ using ovms::StatusCode; using ovms::Timer; using std::chrono::microseconds; +#ifdef __cplusplus +extern "C" { +#endif + void OVMS_StatusDelete(OVMS_Status* status) { if (status == nullptr) return; @@ -368,7 +372,7 @@ OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const cha return nullptr; } -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId) { +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId) { if (req == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); } @@ -449,7 +453,7 @@ OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* req, co return nullptr; } -OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId) { +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, OVMS_BufferType* bufferType, uint32_t* deviceId) { if (res == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_RESPONSE)); } @@ -637,3 +641,7 @@ OVMS_Status* OVMS_Inference(OVMS_Server* serverPtr, OVMS_InferenceRequest* reque return nullptr; // return grpc::Status::OK; } + +#ifdef __cplusplus +} +#endif diff --git a/src/pocapi.hpp b/src/pocapi.h similarity index 86% rename from src/pocapi.hpp rename to src/pocapi.h index e3d0dc16ed..1f8438eaf5 100644 --- a/src/pocapi.hpp +++ b/src/pocapi.h @@ -17,16 +17,18 @@ #include #include // For precise data types -// TODO extern C +#ifdef __cplusplus +extern "C" { +#endif -struct OVMS_Server; -struct OVMS_Status; +typedef struct OVMS_Server_ OVMS_Server; +typedef struct OVMS_Status_ OVMS_Status; -struct OVMS_ServerGeneralOptions; -struct OVMS_ServerMultiModelOptions; +typedef struct OVMS_ServerGeneralaOptions_ OVMS_ServerGeneralOptions; +typedef struct OVMS_ServerMultiModelOptions_ OVMS_ServerMultiModelOptions; // TODO reuse this in precision.hpp -enum OVMS_DataType { +typedef enum OVMS_DataType_enum { OVMS_DATATYPE_BF16, OVMS_DATATYPE_FP64, OVMS_DATATYPE_FP32, @@ -50,18 +52,17 @@ enum OVMS_DataType { OVMS_DATATYPE_Q78, OVMS_DATATYPE_BIN, OVMS_DATATYPE_END -}; +} OVMS_DataType; -enum BufferType { +typedef enum OVMS_BufferType_enum { OVMS_BUFFERTYPE_CPU, OVMS_BUFFERTYPE_CPU_PINNED, OVMS_BUFFERTYPE_GPU, OVMS_BUFFERTYPE_HDDL, -}; +} OVMS_BufferType; -struct OVMS_Status; -struct OVMS_InferenceRequest; -struct OVMS_InferenceResponse; +typedef struct OVMS_InferenceRequest_ OVMS_InferenceRequest; +typedef struct OVMS_InferenceResponse_ OVMS_InferenceResponse; typedef enum OVMS_LogLevel_enum { OVMS_LOG_TRACE, @@ -180,29 +181,33 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_ServerMultiModelOptions* multi_model_specific_options); // in fact only --config_path // Unload all and cleanup // TODO: Should not be possible to re-start? -OVMS_Status* OVMS_ServerStop(OVMS_Server* server); +// OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion); // TODO add passing server here void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); -OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED +// OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED // ownership of data needs to be maintained during inference -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, void* data, size_t bufferSize, BufferType bufferType, uint32_t deviceId); +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId); OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, const char* inputName); OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); // this will allow for reuse of request but with different input data -OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); -OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs +// OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); +// OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize); OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* request, const char* parameterName); // OVMS_Inference Response OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, BufferType* bufferType, uint32_t* deviceId); +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, OVMS_BufferType* bufferType, uint32_t* deviceId); OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, const void** data); void OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); OVMS_Status* OVMS_Inference(OVMS_Server* server, OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); + +#ifdef __cplusplus +} +#endif diff --git a/src/pocapiinternal.hpp b/src/pocapiinternal.hpp index e0303221ac..ea79b190a4 100644 --- a/src/pocapiinternal.hpp +++ b/src/pocapiinternal.hpp @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "pocapi.hpp" +#include "./pocapi.h" #include "precision.hpp" namespace ovms { diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index 0b43cd220f..e77d560fd6 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -487,8 +487,8 @@ Status RequestValidator:: template Status RequestValidator::validateInferenceTensorBufferType(const InferenceTensor& it) const { const Buffer* buffer = it.getBuffer(); - const BufferType bufType = buffer->getBufferType(); - if (bufType < BufferType::OVMS_BUFFERTYPE_CPU || bufType > BufferType::OVMS_BUFFERTYPE_HDDL) { + const OVMS_BufferType bufType = buffer->getBufferType(); + if (bufType < OVMS_BUFFERTYPE_CPU || bufType > OVMS_BUFFERTYPE_HDDL) { std::stringstream ss; ss << "Required input "; const std::string details = ss.str(); @@ -497,7 +497,7 @@ Status RequestValidator:: } else { // Remove this when other buffer types are supported - if (bufType != BufferType::OVMS_BUFFERTYPE_CPU) { + if (bufType != OVMS_BUFFERTYPE_CPU) { std::stringstream ss; ss << "Required input "; const std::string details = ss.str(); @@ -506,7 +506,7 @@ Status RequestValidator:: } } - if (buffer->getBufferType() == BufferType::OVMS_BUFFERTYPE_CPU && buffer->getDeviceId() != std::nullopt && buffer->getDeviceId() != 0) { + if (buffer->getBufferType() == OVMS_BUFFERTYPE_CPU && buffer->getDeviceId() != std::nullopt && buffer->getDeviceId() != 0) { std::stringstream ss; ss << "Required input "; const std::string details = ss.str(); diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 189631bea6..cdbfba38f7 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -22,7 +22,7 @@ // TODO we should not include classes from OVMS here // consider how to workaround test_utils #include "../inferenceresponse.hpp" -#include "../pocapi.hpp" +#include "../pocapi.h" #include "test_utils.hpp" using namespace ovms; @@ -367,7 +367,7 @@ TEST_F(CapiInference, Basic) { OVMS_DataType datatype = (OVMS_DataType)199; const uint64_t* shape{nullptr}; uint32_t dimCount = 42; - BufferType bufferType = (BufferType)199; + OVMS_BufferType bufferType = (OVMS_BufferType)199; uint32_t deviceId = 42; const char* outputName{nullptr}; ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(nullptr, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_RESPONSE); @@ -583,7 +583,7 @@ TEST_F(CapiInference, ResponseRetrieval) { OVMS_DataType datatype = (OVMS_DataType)199; const uint64_t* shape{nullptr}; uint32_t dimCount = 42; - BufferType bufferType = (BufferType)199; + OVMS_BufferType bufferType = (OVMS_BufferType)199; uint32_t deviceId = 42; const char* outputName{nullptr}; ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceResponseGetOutput(response, outputId + 42123, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId), StatusCode::NONEXISTENT_TENSOR); diff --git a/src/test/capi_predict_validation_test.cpp b/src/test/capi_predict_validation_test.cpp index 6c4209c261..c15ac55959 100644 --- a/src/test/capi_predict_validation_test.cpp +++ b/src/test/capi_predict_validation_test.cpp @@ -440,7 +440,7 @@ TEST_F(CAPIPredictValidation, RequestIncorrectBufferType) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, static_cast(999)); + requestData, decrementBufferSize, static_cast(999)); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); } @@ -455,7 +455,7 @@ TEST_F(CAPIPredictValidation, RequestNegativeBufferType) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, static_cast(-22)); + requestData, decrementBufferSize, static_cast(-22)); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); } @@ -470,7 +470,7 @@ TEST_F(CAPIPredictValidation, RequestIncorectDeviceId) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_CPU, 1); + requestData, decrementBufferSize, OVMS_BUFFERTYPE_CPU, 1); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::INVALID_DEVICE_ID) << status.string(); } @@ -485,7 +485,7 @@ TEST_F(CAPIPredictValidation, RequestIncorectBufferType) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_GPU); + requestData, decrementBufferSize, OVMS_BUFFERTYPE_GPU); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::INVALID_BUFFER_TYPE) << status.string(); } @@ -501,7 +501,7 @@ TEST_F(CAPIPredictValidation, RequestCorectDeviceId) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_GPU, 1); + requestData, decrementBufferSize, OVMS_BUFFERTYPE_GPU, 1); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::OK) << status.string(); } @@ -516,7 +516,7 @@ TEST_F(CAPIPredictValidation, RequestNotNullDeviceId) { std::tuple{{1, 6, 128, 128, 16}, ovms::Precision::I64}}, {"Input_U16_1_2_8_4_NCHW", std::tuple{{1, 2, 8, 4}, ovms::Precision::U16}}}, - requestData, decrementBufferSize, BufferType::OVMS_BUFFERTYPE_CPU, 1); + requestData, decrementBufferSize, OVMS_BUFFERTYPE_CPU, 1); auto status = instance->mockValidate(&request); EXPECT_EQ(status, ovms::StatusCode::INVALID_DEVICE_ID) << status.string(); } diff --git a/src/test/status_test.cpp b/src/test/status_test.cpp index ad84ec02f3..6bb6ef3d24 100644 --- a/src/test/status_test.cpp +++ b/src/test/status_test.cpp @@ -20,7 +20,7 @@ #include #include -#include "../pocapi.hpp" +#include "../pocapi.h" #include "../status.hpp" #include "test_utils.hpp" diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 006cfc705b..23cb7c9bfa 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -42,7 +42,7 @@ void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, c } } -void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs, const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { +void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs, const std::vector& data, uint32_t decrementBufferSize, OVMS_BufferType bufferType, std::optional deviceId) { request.removeAllInputs(); for (auto const& it : requestInputs) { prepareCAPIInferInputTensor(request, it.first, it.second, data, decrementBufferSize, bufferType, deviceId); @@ -389,7 +389,7 @@ void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, } void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, - const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { + const std::vector& data, uint32_t decrementBufferSize, OVMS_BufferType bufferType, std::optional deviceId) { auto [shape, type] = inputInfo; prepareCAPIInferInputTensor(request, name, {shape, getPrecisionAsOVMSDataType(type)}, @@ -397,7 +397,7 @@ void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::str } void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, - const std::vector& data, uint32_t decrementBufferSize, BufferType bufferType, std::optional deviceId) { + const std::vector& data, uint32_t decrementBufferSize, OVMS_BufferType bufferType, std::optional deviceId) { auto [shape, datatype] = inputInfo; size_t elementsCount = 1; for (auto const& dim : shape) { diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 6db6bc32f7..8c56a5c6d3 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -165,14 +165,14 @@ void prepareKFSInferInputTensor(::KFSRequest& request, const std::string& name, const std::vector& data = {}, bool putBufferInInputTensorContent = false); void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, - const std::vector& data, uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); + const std::vector& data, uint32_t decrementBufferSize = 0, OVMS_BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); void prepareCAPIInferInputTensor(ovms::InferenceRequest& request, const std::string& name, const std::tuple& inputInfo, - const std::vector& data, uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); + const std::vector& data, uint32_t decrementBufferSize = 0, OVMS_BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); void preparePredictRequest(::KFSRequest& request, inputs_info_t requestInputs, const std::vector& data = {}, bool putBufferInInputTensorContent = false); void preparePredictRequest(ovms::InferenceRequest& request, inputs_info_t requestInputs, const std::vector& data, - uint32_t decrementBufferSize = 0, BufferType bufferType = BufferType::OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); + uint32_t decrementBufferSize = 0, OVMS_BufferType bufferType = OVMS_BUFFERTYPE_CPU, std::optional deviceId = std::nullopt); void prepareBinaryPredictRequest(tensorflow::serving::PredictRequest& request, const std::string& inputName, const int batchSize); void prepareBinaryPredictRequest(::KFSRequest& request, const std::string& inputName, const int batchSize); From 2fcdebfa16ea4bcce475107df2a9cbd52edbfaa2 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Mon, 12 Dec 2022 14:35:55 +0100 Subject: [PATCH 101/130] Klocwork fixes (#1578) + do not copy libovms_shared.so to src folder to build //src:ovms --- Dockerfile.redhat | 1 - Dockerfile.ubuntu | 1 - src/BUILD | 2 +- src/pocapi.cpp | 13 ++++++++++++- src/statefulmodelinstance.cpp | 14 +++++++++++--- src/statefulmodelinstance.hpp | 5 +++-- src/status.cpp | 1 + src/status.hpp | 1 + src/test/c_api_tests.cpp | 18 ++++++++++++++++++ 9 files changed, 47 insertions(+), 9 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index c61752da1f..8397c72a4c 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -209,7 +209,6 @@ RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 # Copy binary for //src:ovms -RUN cp $(bazel info bazel-bin)/src/libovms_shared.so src RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 9a2c7cef90..ba0b327577 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -301,7 +301,6 @@ RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 # Copy binary for //src:ovms -RUN cp $(bazel info bazel-bin)/src/libovms_shared.so src RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so diff --git a/src/BUILD b/src/BUILD index f072a2591a..6962604e15 100644 --- a/src/BUILD +++ b/src/BUILD @@ -562,7 +562,7 @@ cc_binary( cc_import( name = "shared_lib", hdrs = ["pocapi.h"], - shared_library = "libovms_shared.so", + shared_library = "ovms_shared", ) cc_library( diff --git a/src/pocapi.cpp b/src/pocapi.cpp index ac22e9a2d9..18c7e48af9 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -568,7 +568,18 @@ enum : unsigned int { static Status getModelInstance(ovms::Server& server, const InferenceRequest* request, std::shared_ptr& modelInstance, std::unique_ptr& modelInstanceUnloadGuardPtr) { OVMS_PROFILE_FUNCTION(); - auto& modelManager = dynamic_cast(server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME))->getServableManager(); + if (!server.isLive()) { + return ovms::Status(ovms::StatusCode::SERVER_NOT_READY_FOR_INFERENCE, "not live"); + } + if (!server.isReady()) { + return ovms::Status(ovms::StatusCode::SERVER_NOT_READY_FOR_INFERENCE, "not ready"); + } + const ovms::Module* servableModule = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); + if (!servableModule) { + // TODO: Should we wait for servable manager to appear? And for model to get loaded? + return ovms::Status(ovms::StatusCode::INTERNAL_ERROR, "missing servable manager"); + } + auto& modelManager = dynamic_cast(servableModule)->getServableManager(); return modelManager.getModelInstance(request->getServableName(), request->getServableVersion(), modelInstance, modelInstanceUnloadGuardPtr); } } // namespace diff --git a/src/statefulmodelinstance.cpp b/src/statefulmodelinstance.cpp index fcceb4fb16..368e17db5d 100644 --- a/src/statefulmodelinstance.cpp +++ b/src/statefulmodelinstance.cpp @@ -200,10 +200,10 @@ Status StatefulRequestProcessorsequenceId = sequenceProcessingSpec.getSequenceId(); - if (!sequenceManager.sequenceExists(this->sequenceId)) + if (!sequenceManager.sequenceExists(this->sequenceId.value())) return StatusCode::INTERNAL_ERROR; // TODO should be able to search & get in one go - sequence = &sequenceManager.getSequence(this->sequenceId); + sequence = &sequenceManager.getSequence(this->sequenceId.value()); sequenceLock = std::make_unique>(sequence->getMutex()); sequenceManagerLock->unlock(); @@ -238,6 +238,10 @@ Status StatefulRequestProcessorupdateMemoryState(modelState); } // Include sequence_id in server response @@ -254,7 +258,11 @@ Status StatefulRequestProcessorlock(); - status = sequenceManager.removeSequence(this->sequenceId); + if (!this->sequenceId.has_value()) { + SPDLOG_DEBUG("sequenceId is not set"); + return StatusCode::INTERNAL_ERROR; + } + status = sequenceManager.removeSequence(this->sequenceId.value()); } return status; } diff --git a/src/statefulmodelinstance.hpp b/src/statefulmodelinstance.hpp index 69b5afd3d0..c4cb7bf99b 100644 --- a/src/statefulmodelinstance.hpp +++ b/src/statefulmodelinstance.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -101,8 +102,8 @@ struct StatefulRequestProcessor : public RequestProcessor> sequenceManagerLock; std::unique_ptr> sequenceLock; SequenceProcessingSpec sequenceProcessingSpec; - Sequence* sequence; - uint64_t sequenceId; + Sequence* sequence{nullptr}; + std::optional sequenceId; StatefulRequestProcessor(SequenceManager& sequenceManager); Status extractRequestParameters(const RequestType* request) override; diff --git a/src/status.cpp b/src/status.cpp index 23237c291e..346af48258 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -282,6 +282,7 @@ const std::unordered_map Status::statusMess {StatusCode::NONEXISTENT_TENSOR_FOR_REMOVAL, "Tried to remove nonexisting tensor"}, {StatusCode::NONEXISTENT_STATUS, "Tried to use nonexisting status"}, {StatusCode::NONEXISTENT_LOG_LEVEL, "Tried to use nonexisting log level"}, + {StatusCode::SERVER_NOT_READY_FOR_INFERENCE, "Server not in a state to perform inference"}, // Server Start errors {StatusCode::OPTIONS_USAGE_ERROR, "options validation error"}, diff --git a/src/status.hpp b/src/status.hpp index 61ca7c6c4a..586fd4d19e 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -293,6 +293,7 @@ enum class StatusCode { NONEXISTENT_TENSOR_FOR_REMOVAL, NONEXISTENT_STATUS, NONEXISTENT_LOG_LEVEL, + SERVER_NOT_READY_FOR_INFERENCE, // Server Start errors OPTIONS_USAGE_ERROR, diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index cdbfba38f7..4f57db26a3 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -615,5 +615,23 @@ TEST_F(CapiInference, ResponseRetrieval) { OVMS_InferenceResponseDelete(nullptr); OVMS_InferenceResponseDelete(response); } + +TEST_F(CapiInference, CallInferenceServerNotStarted) { + OVMS_Server* cserver = nullptr; + OVMS_InferenceRequest* request{nullptr}; + OVMS_InferenceResponse* response = nullptr; + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); + ASSERT_NE(nullptr, cserver); + ASSERT_NE(nullptr, request); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); + std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + uint32_t notUsedNum = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestInputSetData(request, DUMMY_MODEL_INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, notUsedNum)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::SERVER_NOT_READY_FOR_INFERENCE); + OVMS_InferenceResponseDelete(response); + OVMS_InferenceRequestDelete(request); + OVMS_ServerDelete(cserver); +} // TODO make cleaner error codes reporting // todo decide either use remove or delete for consistency From 8a25b3b659538d0a43ccb605533eb4f3b8954542 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 13 Dec 2022 10:49:06 +0100 Subject: [PATCH 102/130] Perf issue fix - link ovms_lib statically into ovms (#1587) - revert checksec settings for ovms binary - remove unused targets --- Makefile | 3 +-- src/BUILD | 9 +++++---- src/kfserving_api/BUILD | 8 -------- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 80503f2832..69853ff0be 100644 --- a/Makefile +++ b/Makefile @@ -275,8 +275,7 @@ test_checksec: echo "ERROR: OVMS shared library security settings changed. Run checksec on ovms shared library and fix issues." && exit 1;\ fi @checksec --file=/tmp/ovms --format=csv > checksec.txt -# Stack protector and canary not required if linked with libovms_shared - @if ! grep -FRq "Full RELRO,No Canary found,NX enabled,PIE enabled,No RPATH,RUNPATH,Symbols,No" checksec.txt; then\ + @if ! grep -FRq "Full RELRO,Canary found,NX enabled,PIE enabled,No RPATH,RUNPATH,Symbols,Yes" checksec.txt; then\ echo "ERROR: OVMS binary security settings changed. Run checksec on ovms binary and fix issues." && exit 1;\ fi @rm -f checksec.txt diff --git a/src/BUILD b/src/BUILD index 6962604e15..19eea17790 100644 --- a/src/BUILD +++ b/src/BUILD @@ -320,7 +320,6 @@ cc_library( "@openvino//:openvino", "@opencv//:opencv", "@com_github_jupp0r_prometheus_cpp//core", - #":kfserving_api_cpp", "//src/kfserving_api:kfserving_api_cpp", ], local_defines = [ @@ -559,6 +558,7 @@ cc_binary( linkstatic = True, ) +# Use for dynamic linking when neccessary cc_import( name = "shared_lib", hdrs = ["pocapi.h"], @@ -582,16 +582,17 @@ cc_binary( "-luuid", "-lstdc++fs", "-lcrypto", - "-lovms_shared", + # "-lovms_shared", # Use for dynamic linking when neccessary ], copts = [ "-Wconversion", "-Werror", ], deps = [ - "//src:cpp_headers", + "//src:ovms_lib", + # "//src:cpp_headers", # Use for dynamic linking when neccessary ], - linkstatic = False, + # linkstatic = False, # Use for dynamic linking when neccessary ) cc_test( diff --git a/src/kfserving_api/BUILD b/src/kfserving_api/BUILD index ae34fdc8f7..7991db132f 100644 --- a/src/kfserving_api/BUILD +++ b/src/kfserving_api/BUILD @@ -28,11 +28,3 @@ cc_proto_library( "//visibility:public", ], ) - -cc_library( - name = "kfserving_api_lib", - hdrs = ["kfserving_api/grpc_predict_v2.pb.h", - "kfserving_api/grpc_predict_v2.grpc.pb.h", - ], - deps = [":kfserving_api_cpp"], -) From f37bcb2f212eee73d699c4c84597e933ae3e0267 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Wed, 14 Dec 2022 11:44:44 +0100 Subject: [PATCH 103/130] C-API Benchmark app (#1582) * C-API Benchmark app * C++ C-API example app inference Example run command: bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 8 --threads_per_ireq 2 --nireq 2 --servable_name "resnet" --inputs_names "map/TensorArrayStack/TensorArrayGatherV3" --shape "map/TensorArrayStack/TensorArrayGatherV3[1,3,224,224]" JIRA:CVS-95632 --- Dockerfile.redhat | 2 + Dockerfile.ubuntu | 2 + cppclean.sh | 2 +- src/BUILD | 22 ++ src/main3.cpp | 95 +++-- src/main_benchmark.cpp | 435 ++++++++++++++++++++++ src/pocapi.cpp | 2 +- src/pocapi.h | 2 +- src/test/c_api/config_benchmark.json | 27 ++ src/test/c_api/config_standard_dummy.json | 4 +- 10 files changed, 568 insertions(+), 25 deletions(-) create mode 100644 src/main_benchmark.cpp create mode 100644 src/test/c_api/config_benchmark.json diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 8397c72a4c..7619744fd9 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -207,6 +207,8 @@ RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 +# C-API benchmark app +RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" # Copy binary for //src:ovms RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index ba0b327577..0c51b3c8c1 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -299,6 +299,8 @@ RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 +# C-API benchmark app +RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" # Copy binary for //src:ovms RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms diff --git a/cppclean.sh b/cppclean.sh index d7b656967b..990771fa56 100755 --- a/cppclean.sh +++ b/cppclean.sh @@ -42,7 +42,7 @@ if [ ${NO_WARNINGS_NOTUSED} -gt 3 ]; then echo "Failed probably due to unnecessary forward includes: ${NO_WARNINGS_NOTUSED}"; exit 1; fi -if [ ${NO_WARNINGS} -gt 183 ]; then +if [ ${NO_WARNINGS} -gt 187 ]; then echo "Failed due to higher than allowed number of issues in code: ${NO_WARNINGS}" exit 1 fi diff --git a/src/BUILD b/src/BUILD index 19eea17790..f235dad492 100644 --- a/src/BUILD +++ b/src/BUILD @@ -572,6 +572,28 @@ cc_library( deps = ["//src:shared_lib"] ) +cc_binary( + name = "capi_benchmark", + srcs = [ + "main_benchmark.cpp", + "stringutils.cpp", + "stringutils.hpp", + ], + linkopts = [ + "-lpthread", + "-lxml2", + "-luuid", + "-lstdc++fs", + "-lcrypto", + ], + copts = [ + ], + deps = [ + "//src:ovms_lib", + ], + linkstatic = True, +) + cc_binary( name = "ovms", srcs = [ diff --git a/src/main3.cpp b/src/main3.cpp index 5a310e66e9..a630ee8366 100644 --- a/src/main3.cpp +++ b/src/main3.cpp @@ -13,15 +13,27 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -// #include -// #include -// #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include "./pocapi.h" +const char* MODEL_NAME = "dummy"; +const uint64_t MODEL_VERSION = 1; +const char* INPUT_NAME = "b"; +constexpr size_t DIM_COUNT = 2; +constexpr size_t SHAPE[DIM_COUNT] = {1, 10}; + namespace { volatile sig_atomic_t shutdown_request = 0; } @@ -60,7 +72,6 @@ static void installSignalHandlers() { int main(int argc, char** argv) { installSignalHandlers(); - OVMS_ServerGeneralOptions* go = 0; OVMS_ServerMultiModelOptions* mmo = 0; OVMS_Server* srv; @@ -69,22 +80,20 @@ int main(int argc, char** argv) { OVMS_ServerMultiModelOptionsNew(&mmo); OVMS_ServerNew(&srv); - OVMS_ServerGeneralOptionsSetGrpcPort(go, 11337); + OVMS_ServerGeneralOptionsSetGrpcPort(go, 9178); OVMS_ServerGeneralOptionsSetRestPort(go, 11338); OVMS_ServerGeneralOptionsSetLogLevel(go, OVMS_LOG_DEBUG); - OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); + OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"); OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); - if (res) { uint32_t code = 0; const char* details = nullptr; OVMS_StatusGetCode(res, &code); OVMS_StatusGetDetails(res, &details); - - fprintf(stderr, "error during start: code %d, details: %s\n", code, details); + std::cerr << "error during start: code:" << code << "; details:" << details << std::endl; OVMS_StatusDelete(res); @@ -94,23 +103,67 @@ int main(int argc, char** argv) { return 1; } - fprintf(stdout, "Server ready for inference\n"); + std::cout << "Server ready for inference" << std::endl; - // infer 1 - // infer 2 - // infer 3 + // prepare request + OVMS_InferenceRequest* request{nullptr}; + OVMS_InferenceRequestNew(&request, MODEL_NAME, MODEL_VERSION); + OVMS_InferenceRequestAddInput(request, INPUT_NAME, OVMS_DATATYPE_FP32, SHAPE, DIM_COUNT); + std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + OVMS_InferenceRequestInputSetData(request, INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, 0); - // Application loop if required (C++): - // while (shutdown_request == 0) { - // std::this_thread::sleep_for(std::chrono::milliseconds(200)); - // } - - fprintf(stdout, "No more job to be done, will shut down\n"); + // run sync request + OVMS_InferenceResponse* response = nullptr; + res = OVMS_Inference(srv, request, &response); + if (res != nullptr) { + uint32_t code = 0; + const char* details = 0; + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + std::cout << "Error occured during inference. Code:" << code + << ", details:" << details << std::endl; + } + // read output + uint32_t outputCount = 0; + OVMS_InferenceResponseGetOutputCount(response, &outputCount); + const void* voutputData; + size_t bytesize = 0; + uint32_t outputId = outputCount - 1; + OVMS_DataType datatype = (OVMS_DataType)42; + const uint64_t* shape{nullptr}; + uint32_t dimCount = 0; + OVMS_BufferType bufferType = (OVMS_BufferType)42; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + OVMS_InferenceResponseGetOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId); + + std::stringstream ss; + ss << "Got response from OVMS via C-API. " + << "Request for model: " << MODEL_NAME + << "; version: " << MODEL_VERSION + << "ms; output name: " << outputName + << "; response with values:\n"; + for (size_t i = 0; i < shape[1]; ++i) { + ss << *(reinterpret_cast(voutputData) + i) << " "; + } + std::vector expectedOutput; + std::transform(data.begin(), data.end(), std::back_inserter(expectedOutput), + [](const float& s) -> float { + return s + 1; + }); + + if (std::memcmp(voutputData, expectedOutput.data(), expectedOutput.size() * sizeof(float)) != 0) { + std::cout << "Incorrect result of inference" << std::endl; + } + // comment line below to have app running similarly to OVMS + shutdown_request = 1; + while (shutdown_request == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + std::cout << "No more job to be done, will shut down" << std::endl; OVMS_ServerDelete(srv); OVMS_ServerMultiModelOptionsDelete(mmo); OVMS_ServerGeneralOptionsDelete(go); - - fprintf(stdout, "main() exit\n"); return 0; } diff --git a/src/main_benchmark.cpp b/src/main_benchmark.cpp new file mode 100644 index 0000000000..af10987fbb --- /dev/null +++ b/src/main_benchmark.cpp @@ -0,0 +1,435 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "pocapi.h" // NOLINT +#include "stringutils.hpp" + +namespace { + +class BenchmarkCLIParser { + std::unique_ptr options; + +public: + std::unique_ptr result; + + BenchmarkCLIParser() = default; + void parse(int argc, char** argv); + void prepare(OVMS_ServerGeneralOptions*, OVMS_ServerMultiModelOptions*); +}; + +void BenchmarkCLIParser::parse(int argc, char** argv) { + try { + options = std::make_unique(argv[0], "OpenVINO Model Server"); + + // clang-format off + options->add_options() + ("h, help", + "Show this help message and exit") + // server options + ("port", + "gRPC server port", + cxxopts::value()->default_value("9178"), + "PORT") + ("rest_port", + "REST server port, the REST server will not be started if rest_port is blank or set to 0", + cxxopts::value()->default_value("0"), + "REST_PORT") + ("log_level", + "serving log level - one of TRACE, DEBUG, INFO, WARNING, ERROR", + cxxopts::value()->default_value("ERROR"), + "LOG_LEVEL") + ("config_path", + "Config file path for OVMS to read", + cxxopts::value()->default_value("/ovms/src/test/c_api/config_benchmark.json"), + "CONFIG_PATH") +// benchmark options + ("niter", + "number of inferences to conduct", + cxxopts::value()->default_value("1000"), + "NITER") + ("nireq", + "nireq from OVMS configuration", + cxxopts::value()->default_value("1"), + "NIREQ") + ("threads_per_ireq", + "workload threads per ireq", + cxxopts::value()->default_value("2"), + "THREADS_PER_IREQ") + // inference data + ("servable_name", + "Model name to sent request to", + cxxopts::value(), + "MODEL_NAME") + ("servable_version", + "workload threads per ireq", + cxxopts::value()->default_value("0"), + "MODEL_VERSION") + ("inputs_names", + "Comma separated list of inputs names", + cxxopts::value(), + "INPUTS_NAMES") + ("shape", + "Semicolon separated list of inputs names followed by their shapes in brackers. For example: \"inputA[1,3,224,224],inputB[1,10]\"", + cxxopts::value(), + "INPUTS_NAMES"); + + result = std::make_unique(options->parse(argc, argv)); + + if (result->count("help") || result->arguments().size() == 0) { + std::cout << options->help({"", "multi model", "single model"}) << std::endl; + exit(EX_OK); + } + } catch (const cxxopts::OptionException& e) { + std::cerr << "error parsing options: " << e.what() << std::endl; + exit(EX_USAGE); + } +} + +std::vector parseShapes(const std::string& cliInputShapes) { + auto inputShapes = ovms::tokenize(cliInputShapes, ';'); + if (inputShapes.size() != 1) { + std::cout << __LINE__ << std::endl; + throw std::invalid_argument("Invalid shape argument"); + } + std::string firstShape = inputShapes[0]; + size_t leftBracket = firstShape.find("["); + size_t rightBracket = firstShape.find("]"); + if ((leftBracket == std::string::npos) || + (rightBracket == std::string::npos) || + (leftBracket > rightBracket)) { + std::cout << __LINE__ << std::endl; + throw std::invalid_argument("Invalid shape argument"); + } + std::string shapeString = firstShape.substr(leftBracket + 1, rightBracket - leftBracket - 1); + auto dimsString = ovms::tokenize(shapeString, ','); + std::vector shape; + std::transform(dimsString.begin(), dimsString.end(), std::back_inserter(shape), + [](const std::string& s) -> std::size_t { + auto dimOpt = ovms::stoi64(s); + if (!dimOpt.has_value()) { + std::cout << __LINE__ << " " << s << std::endl; + throw std::invalid_argument("Invalid shape argument"); + } + return dimOpt.value(); }); + return shape; +} + +volatile sig_atomic_t shutdown_request = 0; + +static void onInterrupt(int status) { + shutdown_request = 1; +} + +static void onTerminate(int status) { + shutdown_request = 1; +} + +static void onIllegal(int status) { + shutdown_request = 2; +} + +static void installSignalHandlers() { + static struct sigaction sigIntHandler; + sigIntHandler.sa_handler = onInterrupt; + sigemptyset(&sigIntHandler.sa_mask); + sigIntHandler.sa_flags = 0; + sigaction(SIGINT, &sigIntHandler, NULL); + + static struct sigaction sigTermHandler; + sigTermHandler.sa_handler = onTerminate; + sigemptyset(&sigTermHandler.sa_mask); + sigTermHandler.sa_flags = 0; + sigaction(SIGTERM, &sigTermHandler, NULL); + + static struct sigaction sigIllHandler; + sigIllHandler.sa_handler = onIllegal; + sigemptyset(&sigIllHandler.sa_mask); + sigIllHandler.sa_flags = 0; + sigaction(SIGILL, &sigIllHandler, NULL); +} + +using shape_t = std::vector; + +OVMS_InferenceRequest* prepareRequest(const std::string& servableName, uint32_t servableVersion, OVMS_DataType datatype, const shape_t& shape, const std::string& inputName, const void* data) { + OVMS_InferenceRequest* request{nullptr}; + OVMS_InferenceRequestNew(&request, servableName.c_str(), servableVersion); + OVMS_InferenceRequestAddInput(request, inputName.c_str(), datatype, shape.data(), shape.size()); + size_t elementsCount = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + OVMS_InferenceRequestInputSetData(request, inputName.c_str(), data, sizeof(float) * elementsCount, OVMS_BUFFERTYPE_CPU, 0); // TODO sizeof + return request; +} + +void triggerInferenceInALoop( + std::future& startSignal, + std::future& stopSignal, + const size_t niterPerThread, + size_t& wholeThreadTimeUs, + double& averageWholeLatency, + double& averagePureLatency, + OVMS_Server* server, + OVMS_InferenceRequest* request) { + OVMS_InferenceResponse* response{nullptr}; + std::vector latenciesWhole(niterPerThread); + std::vector latenciesPure(niterPerThread); + startSignal.get(); + auto workloadStart = std::chrono::high_resolution_clock::now(); + size_t iter = niterPerThread; + while (iter-- > 0) { + // stopSignal will be used with ctrl-c app stopping or with total requestCount + // stopSignal.wait_for(std::chrono::milliseconds(0)); + auto iterationWholeStart = std::chrono::high_resolution_clock::now(); + auto iterationPureStart = std::chrono::high_resolution_clock::now(); + // aternatively we are changing request + OVMS_Inference(server, request, &response); + auto iterationPureEnd = std::chrono::high_resolution_clock::now(); + OVMS_InferenceResponseDelete(response); + auto iterationWholeEnd = std::chrono::high_resolution_clock::now(); + latenciesWhole[iter] = std::chrono::duration_cast(iterationWholeEnd - iterationWholeStart).count(); + latenciesPure[iter] = std::chrono::duration_cast(iterationPureEnd - iterationPureStart).count(); + } + auto workloadEnd = std::chrono::high_resolution_clock::now(); + wholeThreadTimeUs = std::chrono::duration_cast(workloadEnd - workloadStart).count(); + averageWholeLatency = std::accumulate(latenciesWhole.begin(), latenciesWhole.end(), 0) / (double(niterPerThread) * 1'000); + averagePureLatency = std::accumulate(latenciesPure.begin(), latenciesPure.end(), 0) / (double(niterPerThread) * 1'000); +} +} // namespace +// TODO support more than 1 input +// TODO support 3 modes: +// * fair play -> don't recreate request, but set buffer inputs +// * full request prepare -> recreate request +// * inference only -> dont reset buffers +// TODO change plugin_config +// TODO change nireq in model + +int main(int argc, char** argv) { + installSignalHandlers(); + BenchmarkCLIParser cliparser; + cliparser.parse(argc, argv); + + OVMS_ServerGeneralOptions* go = 0; + OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_Server* srv; + + OVMS_ServerGeneralOptionsNew(&go); + OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerNew(&srv); + + uint32_t grpcPort = cliparser.result->operator[]("port").as(); + uint32_t restPort = cliparser.result->operator[]("rest_port").as(); + OVMS_ServerGeneralOptionsSetGrpcPort(go, grpcPort); + OVMS_ServerGeneralOptionsSetRestPort(go, restPort); + + std::string cliLogLevel(cliparser.result->operator[]("log_level").as()); + OVMS_LogLevel_enum logLevel; + if (cliLogLevel == "TRACE") { + logLevel = OVMS_LOG_TRACE; + } else if (cliLogLevel == "DEBUG") { + logLevel = OVMS_LOG_DEBUG; + } else if (cliLogLevel == "INFO") { + logLevel = OVMS_LOG_INFO; + } else if (cliLogLevel == "WARN") { + logLevel = OVMS_LOG_WARNING; + } else if (cliLogLevel == "ERROR") { + logLevel = OVMS_LOG_ERROR; + } else { + std::cout << __LINE__ << std::endl; + return EX_USAGE; + } + OVMS_ServerGeneralOptionsSetLogLevel(go, logLevel); + OVMS_ServerMultiModelOptionsSetConfigPath(mmo, cliparser.result->operator[]("config_path").as().c_str()); + + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + + if (res) { + uint32_t code = 0; + const char* details = nullptr; + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + std::cout << "Error starting the server. Code:" << code + << "; details:" << details << std::endl; + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + return 1; + } + + std::cout << "Server ready for inference" << std::endl; + + /////////////////////// + // model parameters + /////////////////////// + std::string servableName(cliparser.result->operator[]("servable_name").as()); + uint64_t servableVersion(cliparser.result->operator[]("servable_version").as()); + // input names handling + std::string cliInputsNames(cliparser.result->operator[]("inputs_names").as()); + auto inputsNames = ovms::tokenize(cliInputsNames, ','); + if (inputsNames.size() != 1) { + std::cout << __LINE__ << std::endl; + return EX_USAGE; + } + std::string inputName = inputsNames[0]; + // TODO add support for many shapes/inputs + // datatype handling + OVMS_DataType datatype = OVMS_DATATYPE_FP32; + // shape handling + std::vector shape = parseShapes(cliparser.result->operator[]("shape").as()); + /////////////////////// + // benchmark parameters + /////////////////////// + size_t nireq = cliparser.result->operator[]("nireq").as(); + size_t niter = cliparser.result->operator[]("niter").as(); + size_t threadsPerIreq = cliparser.result->operator[]("threads_per_ireq").as(); + size_t threadCount = nireq * threadsPerIreq; + size_t niterPerThread = niter / threadCount; + + size_t elementsCount = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + std::vector data(elementsCount, 0.1); + + /////////////////////// + // prepare time measuring + /////////////////////// + std::vector wholeTimes; + wholeTimes.resize(threadCount); + std::vector pureTimes; + pureTimes.resize(threadCount); + + /////////////////////// + // prepare requests + /////////////////////// + OVMS_InferenceRequest* request = prepareRequest(servableName, servableVersion, datatype, shape, inputName, (const void*)data.data()); + /////////////////////// + // check request + /////////////////////// + OVMS_InferenceResponse* response; + res = OVMS_Inference(srv, request, &response); + if (res != nullptr) { + uint32_t code = 0; + const char* details = 0; + OVMS_StatusGetCode(res, &code); + OVMS_StatusGetDetails(res, &details); + std::cerr << "Error occured during inference. Code:" << code + << ", details:" << details << std::endl; + OVMS_StatusDelete(res); + OVMS_InferenceRequestDelete(request); + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + exit(EX_CONFIG); + } + OVMS_InferenceResponseDelete(response); + + /////////////////////// + // prepare response data + /////////////////////// + uint32_t outputCount = 0; + const void* voutputData; + size_t bytesize = 0; + uint32_t outputId = outputCount - 1; + OVMS_DataType responseDatatype = (OVMS_DataType) 42; + const uint64_t* outputShape{nullptr}; + uint32_t dimCount = 0; + OVMS_BufferType bufferType = (OVMS_BufferType)42; + uint32_t deviceId = 42; + const char* outputName{nullptr}; + /////////////////////// + // setup workload machinery + /////////////////////// + std::vector> workerThreads; + std::vector> startSignals(threadCount); + std::vector> stopSignals(threadCount); + std::vector> futureStartSignals; + std::vector> futureStopSignals; + std::vector wholeThreadsTimesUs(threadCount, 0); + std::transform(startSignals.begin(), + startSignals.end(), + std::back_inserter(futureStartSignals), + [](auto& p) { return p.get_future(); }); + std::transform(stopSignals.begin(), + stopSignals.end(), + std::back_inserter(futureStopSignals), + [](auto& p) { return p.get_future(); }); + + /////////////////////// + // prepare threads + /////////////////////// + for (size_t i = 0; i < threadCount; ++i) { + workerThreads.emplace_back(std::make_unique( + [&futureStartSignals, + &futureStopSignals, + &niterPerThread, + &wholeThreadsTimesUs, + &wholeTimes, + &pureTimes, + &srv, + &request, + i]() { + triggerInferenceInALoop( + futureStartSignals[i], + futureStopSignals[i], + niterPerThread, + wholeThreadsTimesUs[i], + wholeTimes[i], + pureTimes[i], + srv, + request); + })); + } + // sleep to allow all threads to initialize + // TODO rework to use signals + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + /////////////////////// + // start workload + /////////////////////// + std::cout << "Benchmark starting workload" << std::endl; + auto workloadStart = std::chrono::high_resolution_clock::now(); + std::for_each(startSignals.begin(), startSignals.end(), [](auto& startSignal) { startSignal.set_value(); }); + /////////////////////// + // end workload + /////////////////////// + std::for_each(workerThreads.begin(), workerThreads.end(), [](auto& t) { t->join(); }); + auto workloadEnd = std::chrono::high_resolution_clock::now(); + auto wholeTimeUs = std::chrono::duration_cast(workloadEnd - workloadStart).count(); + std::cout << "FPS: " << double(niter) / wholeTimeUs * 1'000'000 << std::endl; + auto totalUs = std::accumulate(wholeThreadsTimesUs.begin(), wholeThreadsTimesUs.end(), 0); + std::cout << "Average per thread FPS: " << double(niter) * threadCount/totalUs * 1'000'000 << std::endl; + OVMS_InferenceRequestDelete(request); + double totalWhole = std::accumulate(wholeTimes.begin(), wholeTimes.end(), double(0)) / threadCount; + double totalPure = std::accumulate(pureTimes.begin(), pureTimes.end(), double(0)) / threadCount; + std::cout << std::fixed << std::setprecision(3); + std::cout << "Average latency whole prediction path:" << totalWhole << "ms" << std::endl; + std::cout << "Average latency pure C-API inference:" << totalPure << "ms" << std::endl; + // OVMS cleanup + OVMS_ServerDelete(srv); + OVMS_ServerMultiModelOptionsDelete(mmo); + OVMS_ServerGeneralOptionsDelete(go); + std::cout << "main() exit" << std::endl; + return 0; +} +// adjustable nireq, adjustable shape, model name diff --git a/src/pocapi.cpp b/src/pocapi.cpp index 18c7e48af9..3d3c258d6f 100644 --- a/src/pocapi.cpp +++ b/src/pocapi.cpp @@ -372,7 +372,7 @@ OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* req, const cha return nullptr; } -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId) { +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* req, const char* inputName, const void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId) { if (req == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); } diff --git a/src/pocapi.h b/src/pocapi.h index 1f8438eaf5..d9eacebc47 100644 --- a/src/pocapi.h +++ b/src/pocapi.h @@ -191,7 +191,7 @@ OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const // OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED // ownership of data needs to be maintained during inference -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId); +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, const void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId); OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, const char* inputName); OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); // this will allow for reuse of request but with different input data // OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); diff --git a/src/test/c_api/config_benchmark.json b/src/test/c_api/config_benchmark.json new file mode 100644 index 0000000000..0b6f2e3f3d --- /dev/null +++ b/src/test/c_api/config_benchmark.json @@ -0,0 +1,27 @@ +{ + "model_config_list": [ + {"config": { + "name": "dummy", + "base_path": "/ovms/src/test/dummy", + "plugin_config": { + "NUM_STREAMS":1, + "AFFINITY": "CORE", + "PERFORMANCE_HINT": "THROUGHPUT", + "PERF_COUNT":"NO" + }, + "shape": "(1, 10)", + "nireq": 1} + }, + {"config": { + "name": "resnet", + "base_path": "/models/resnet-50-tf", + "plugin_config": { + "NUM_STREAMS":12, + "AFFINITY": "CORE", + "PERFORMANCE_HINT": "THROUGHPUT", + "PERF_COUNT":"NO" + }, + "nireq": 12} + } + ] +} diff --git a/src/test/c_api/config_standard_dummy.json b/src/test/c_api/config_standard_dummy.json index cb1c346348..6fe2321258 100644 --- a/src/test/c_api/config_standard_dummy.json +++ b/src/test/c_api/config_standard_dummy.json @@ -3,6 +3,8 @@ {"config": { "name": "dummy", "base_path": "/ovms/src/test/dummy", - "shape": "(1, 10)"}} + "shape": "(1, 10)" + } + } ] } From b810eb97a3a6af1673a1d272a547f7d688b98436 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Wed, 14 Dec 2022 14:20:29 +0100 Subject: [PATCH 104/130] Calculate KFS REST binary inputs size from shape (#1568) * Calculate KFS REST binary inputs size from shape --- src/http_rest_api_handler.cpp | 174 +++++++++++++++++---------- src/rest_parser.cpp | 77 ++++++------ src/rest_parser.hpp | 4 +- src/test/kfs_rest_parser_test.cpp | 120 ++++++++++++++++--- src/test/kfs_rest_test.cpp | 191 +++++++++++++++++++++--------- 5 files changed, 401 insertions(+), 165 deletions(-) diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index c6850d3fac..336102bfd9 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -34,6 +34,7 @@ #include "get_model_metadata_impl.hpp" #include "grpcservermodule.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" +#include "kfs_frontend/kfs_utils.hpp" #include "metric_module.hpp" #include "metric_registry.hpp" #include "model_metric_reporter.hpp" @@ -257,59 +258,59 @@ static Status convertStringToVectorOfSizes(const std::string& comma_separated_nu return StatusCode::OK; } -static Status parseBinaryInput(KFSTensorInputProto* input, size_t binary_input_size, const char* buffer) { - if (input->datatype() == "FP32") { +static Status parseBinaryInput(KFSTensorInputProto& input, size_t binary_input_size, const char* buffer) { + if (input.datatype() == "FP32") { for (size_t i = 0; i < binary_input_size; i += sizeof(float)) { - auto value = input->mutable_contents()->mutable_fp32_contents()->Add(); + auto value = input.mutable_contents()->mutable_fp32_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "INT64") { + } else if (input.datatype() == "INT64") { for (size_t i = 0; i < binary_input_size; i += sizeof(int64_t)) { - auto value = input->mutable_contents()->mutable_int64_contents()->Add(); + auto value = input.mutable_contents()->mutable_int64_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "INT32") { + } else if (input.datatype() == "INT32") { for (size_t i = 0; i < binary_input_size; i += sizeof(int32_t)) { - auto value = input->mutable_contents()->mutable_int_contents()->Add(); + auto value = input.mutable_contents()->mutable_int_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "INT16") { + } else if (input.datatype() == "INT16") { for (size_t i = 0; i < binary_input_size; i += sizeof(int16_t)) { - auto value = input->mutable_contents()->mutable_int_contents()->Add(); + auto value = input.mutable_contents()->mutable_int_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "INT8") { + } else if (input.datatype() == "INT8") { for (size_t i = 0; i < binary_input_size; i += sizeof(int8_t)) { - auto value = input->mutable_contents()->mutable_int_contents()->Add(); + auto value = input.mutable_contents()->mutable_int_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "UINT64") { + } else if (input.datatype() == "UINT64") { for (size_t i = 0; i < binary_input_size; i += sizeof(uint64_t)) { - auto value = input->mutable_contents()->mutable_uint64_contents()->Add(); + auto value = input.mutable_contents()->mutable_uint64_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "UINT32") { + } else if (input.datatype() == "UINT32") { for (size_t i = 0; i < binary_input_size; i += sizeof(uint32_t)) { - auto value = input->mutable_contents()->mutable_uint_contents()->Add(); + auto value = input.mutable_contents()->mutable_uint_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "UINT16") { + } else if (input.datatype() == "UINT16") { for (size_t i = 0; i < binary_input_size; i += sizeof(uint16_t)) { - auto value = input->mutable_contents()->mutable_uint_contents()->Add(); + auto value = input.mutable_contents()->mutable_uint_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "UINT8") { + } else if (input.datatype() == "UINT8") { for (size_t i = 0; i < binary_input_size; i += sizeof(uint8_t)) { - auto value = input->mutable_contents()->mutable_uint_contents()->Add(); + auto value = input.mutable_contents()->mutable_uint_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "FP64") { + } else if (input.datatype() == "FP64") { for (size_t i = 0; i < binary_input_size; i += sizeof(double)) { - auto value = input->mutable_contents()->mutable_fp64_contents()->Add(); + auto value = input.mutable_contents()->mutable_fp64_contents()->Add(); *value = (*(reinterpret_cast(buffer + i))); } - } else if (input->datatype() == "BYTES") { - input->mutable_contents()->add_bytes_contents(buffer, binary_input_size); + } else if (input.datatype() == "BYTES") { + input.mutable_contents()->add_bytes_contents(buffer, binary_input_size); } else { return StatusCode::REST_UNSUPPORTED_PRECISION; } @@ -319,59 +320,59 @@ static Status parseBinaryInput(KFSTensorInputProto* input, size_t binary_input_s #define CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE " contents is not empty. Content field should be empty when using binary inputs extension." -static Status validateContentFieldsEmptiness(KFSTensorInputProto* input) { - if (input->datatype() == "FP32") { - if (input->contents().fp32_contents_size() > 0) { +static Status validateContentFieldsEmptiness(KFSTensorInputProto& input) { + if (input.datatype() == "FP32") { + if (input.contents().fp32_contents_size() > 0) { SPDLOG_DEBUG("FP32" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "INT64") { - if (input->contents().int64_contents_size() > 0) { + } else if (input.datatype() == "INT64") { + if (input.contents().int64_contents_size() > 0) { SPDLOG_DEBUG("INT64" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "INT32") { - if (input->contents().int_contents_size() > 0) { + } else if (input.datatype() == "INT32") { + if (input.contents().int_contents_size() > 0) { SPDLOG_DEBUG("INT32" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "INT16") { - if (input->contents().int_contents_size() > 0) { + } else if (input.datatype() == "INT16") { + if (input.contents().int_contents_size() > 0) { SPDLOG_DEBUG("INT16" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "INT8") { - if (input->contents().int_contents_size() > 0) { + } else if (input.datatype() == "INT8") { + if (input.contents().int_contents_size() > 0) { SPDLOG_DEBUG("INT8" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "UINT64") { - if (input->contents().uint64_contents_size() > 0) { + } else if (input.datatype() == "UINT64") { + if (input.contents().uint64_contents_size() > 0) { SPDLOG_DEBUG("UINT64" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "UINT32") { - if (input->contents().uint_contents_size() > 0) { + } else if (input.datatype() == "UINT32") { + if (input.contents().uint_contents_size() > 0) { SPDLOG_DEBUG("UINT32" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "UINT16") { - if (input->contents().uint_contents_size() > 0) { + } else if (input.datatype() == "UINT16") { + if (input.contents().uint_contents_size() > 0) { SPDLOG_DEBUG("UINT16" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "UINT8") { - if (input->contents().uint_contents_size() > 0) { + } else if (input.datatype() == "UINT8") { + if (input.contents().uint_contents_size() > 0) { SPDLOG_DEBUG("UINT8" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "FP64") { - if (input->contents().fp64_contents_size() > 0) { + } else if (input.datatype() == "FP64") { + if (input.contents().fp64_contents_size() > 0) { SPDLOG_DEBUG("FP64" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } - } else if (input->datatype() == "BYTES") { - if (input->contents().bytes_contents_size() > 0) { + } else if (input.datatype() == "BYTES") { + if (input.contents().bytes_contents_size() > 0) { SPDLOG_DEBUG("BYTES" CONTENT_FIELD_NOT_EMPTY_ERROR_MESSAGE); return StatusCode::REST_CONTENTS_FIELD_NOT_EMPTY; } @@ -382,32 +383,72 @@ static Status validateContentFieldsEmptiness(KFSTensorInputProto* input) { return StatusCode::OK; } +static bool isInputEmpty(const ::KFSRequest::InferInputTensor& input) { + if (input.datatype() == "FP32") + return input.contents().fp32_contents_size() == 0; + if (input.datatype() == "INT64") + return input.contents().int64_contents_size() == 0; + if (input.datatype() == "INT32") + return input.contents().int_contents_size() == 0; + if (input.datatype() == "INT16") + return input.contents().int_contents_size() == 0; + if (input.datatype() == "INT8") + return input.contents().int_contents_size() == 0; + if (input.datatype() == "UINT64") + return input.contents().uint64_contents_size() == 0; + if (input.datatype() == "UINT32") + return input.contents().uint_contents_size() == 0; + if (input.datatype() == "UINT16") + return input.contents().uint_contents_size() == 0; + if (input.datatype() == "UINT8") + return input.contents().uint_contents_size() == 0; + if (input.datatype() == "FP64") + return input.contents().fp64_contents_size() == 0; + if (input.datatype() == "BYTES") + return input.contents().bytes_contents_size() == 0; + return true; +} + +static Status handleBinaryInput(const int binary_input_size, size_t& binary_input_offset, const size_t binary_buffer_size, const char* binary_inputs, ::KFSRequest::InferInputTensor& input) { + if (binary_input_offset + binary_input_size > binary_buffer_size) { + SPDLOG_DEBUG("Binary inputs size exceeds provided buffer size {}", binary_buffer_size); + return StatusCode::REST_BINARY_BUFFER_EXCEEDED; + } + auto status = parseBinaryInput(input, binary_input_size, binary_inputs + binary_input_offset); + if (!status.ok()) { + SPDLOG_DEBUG("Parsing binary inputs failed"); + return status; + } + binary_input_offset += binary_input_size; + return StatusCode::OK; +} + +static size_t calculateBinaryDataSize(::KFSRequest::InferInputTensor& input, size_t& binary_data_size) { + auto element_size = KFSDataTypeSize(input.datatype()); + size_t elements_number = std::accumulate(std::begin(input.shape()), std::end(input.shape()), 1, std::multiplies()); + binary_data_size = elements_number * element_size; + return binary_data_size; +} + static Status handleBinaryInputs(::KFSRequest& grpc_request, const std::string& request_body, size_t endOfJson) { const char* binary_inputs = &(request_body[endOfJson]); - size_t binary_inputs_size = request_body.length() - endOfJson; + size_t binary_buffer_size = request_body.length() - endOfJson; size_t binary_input_offset = 0; for (int i = 0; i < grpc_request.mutable_inputs()->size(); i++) { auto input = grpc_request.mutable_inputs()->Mutable(i); auto binary_data_size_parameter = input->parameters().find("binary_data_size"); if (binary_data_size_parameter != input->parameters().end()) { - auto status = validateContentFieldsEmptiness(input); + auto status = validateContentFieldsEmptiness(*input); if (!status.ok()) { SPDLOG_DEBUG("Request contains both data in json and binary inputs"); return status; } if (binary_data_size_parameter->second.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kInt64Param) { auto binary_input_size = binary_data_size_parameter->second.int64_param(); - if (binary_input_offset + binary_input_size > binary_inputs_size) { - SPDLOG_DEBUG("Binary inputs size exceeds provided buffer size {}", binary_inputs_size); - return StatusCode::REST_BINARY_BUFFER_EXCEEDED; - } - status = parseBinaryInput(input, binary_input_size, binary_inputs + binary_input_offset); - if (!status.ok()) { - SPDLOG_DEBUG("Parsing binary inputs failed"); + auto status = handleBinaryInput(binary_input_size, binary_input_offset, binary_buffer_size, binary_inputs, *input); + if (!status.ok()) return status; - } - binary_input_offset += binary_input_size; } else if (binary_data_size_parameter->second.parameter_choice_case() == inference::InferParameter::ParameterChoiceCase::kStringParam) { std::vector binary_inputs_sizes; status = convertStringToVectorOfSizes(binary_data_size_parameter->second.string_param(), binary_inputs_sizes); @@ -415,11 +456,11 @@ static Status handleBinaryInputs(::KFSRequest& grpc_request, const std::string& return status; } for (auto size : binary_inputs_sizes) { - if (binary_input_offset + size > binary_inputs_size) { - SPDLOG_DEBUG("Binary inputs size exceeds provided buffer size {}", binary_inputs_size); + if (binary_input_offset + size > binary_buffer_size) { + SPDLOG_DEBUG("Binary inputs size exceeds provided buffer size {}", binary_buffer_size); return StatusCode::REST_BINARY_BUFFER_EXCEEDED; } - status = parseBinaryInput(input, size, binary_inputs + binary_input_offset); + status = parseBinaryInput(*input, size, binary_inputs + binary_input_offset); if (!status.ok()) { SPDLOG_DEBUG("Parsing binary inputs failed"); return status; @@ -430,6 +471,19 @@ static Status handleBinaryInputs(::KFSRequest& grpc_request, const std::string& SPDLOG_DEBUG("binary_data_size parameter type should be int64 or string"); return StatusCode::REST_BINARY_DATA_SIZE_PARAMETER_INVALID; } + } else { + if (!isInputEmpty(*input)) + continue; + if (grpc_request.mutable_inputs()->size() == 1 && input->datatype() == "BYTES") { + auto status = handleBinaryInput(binary_buffer_size, binary_input_offset, binary_buffer_size, binary_inputs, *input); + if (!status.ok()) + return status; + continue; + } + size_t binary_data_size = calculateBinaryDataSize(*input, binary_data_size); + auto status = handleBinaryInput(binary_data_size, binary_input_offset, binary_buffer_size, binary_inputs, *input); + if (!status.ok()) + return status; } } return StatusCode::OK; diff --git a/src/rest_parser.cpp b/src/rest_parser.cpp index 6e83b22ec4..af002bc110 100644 --- a/src/rest_parser.cpp +++ b/src/rest_parser.cpp @@ -543,47 +543,47 @@ Status KFSRestParser::parseOutputs(rapidjson::Value& node) { return StatusCode::OK; } -#define HANDLE_VALUE(CONTENTS, TYPE_GETTER, TYPE_CHECK) \ - for (auto& value : node.GetArray()) { \ - if (value.IsArray()) { \ - for (auto& v : node.GetArray()) { \ - auto status = parseData(v, input); \ - if (!status.ok()) { \ - return status; \ - } \ - } \ - return StatusCode::OK; \ - } \ - if (!value.TYPE_CHECK()) { \ - return StatusCode::REST_COULD_NOT_PARSE_INPUT; \ - } \ - input->mutable_contents()->CONTENTS()->Add(value.TYPE_GETTER()); \ - } - -Status KFSRestParser::parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor* input) { - if (input->datatype() == "FP32") { +#define HANDLE_VALUE(CONTENTS, TYPE_GETTER, TYPE_CHECK) \ + for (auto& value : node.GetArray()) { \ + if (value.IsArray()) { \ + for (auto& v : node.GetArray()) { \ + auto status = parseData(v, input); \ + if (!status.ok()) { \ + return status; \ + } \ + } \ + return StatusCode::OK; \ + } \ + if (!value.TYPE_CHECK()) { \ + return StatusCode::REST_COULD_NOT_PARSE_INPUT; \ + } \ + input.mutable_contents()->CONTENTS()->Add(value.TYPE_GETTER()); \ + } + +Status KFSRestParser::parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor& input) { + if (input.datatype() == "FP32") { HANDLE_VALUE(mutable_fp32_contents, GetFloat, IsNumber) - } else if (input->datatype() == "INT64") { + } else if (input.datatype() == "INT64") { HANDLE_VALUE(mutable_int64_contents, GetInt64, IsInt) - } else if (input->datatype() == "INT32") { + } else if (input.datatype() == "INT32") { HANDLE_VALUE(mutable_int_contents, GetInt, IsInt) - } else if (input->datatype() == "INT16") { + } else if (input.datatype() == "INT16") { HANDLE_VALUE(mutable_int_contents, GetInt, IsInt) - } else if (input->datatype() == "INT8") { + } else if (input.datatype() == "INT8") { HANDLE_VALUE(mutable_int_contents, GetInt, IsInt) - } else if (input->datatype() == "UINT64") { + } else if (input.datatype() == "UINT64") { HANDLE_VALUE(mutable_uint64_contents, GetUint64, IsUint) - } else if (input->datatype() == "UINT32") { + } else if (input.datatype() == "UINT32") { HANDLE_VALUE(mutable_uint_contents, GetUint, IsUint) - } else if (input->datatype() == "UINT16") { + } else if (input.datatype() == "UINT16") { HANDLE_VALUE(mutable_uint_contents, GetUint, IsUint) - } else if (input->datatype() == "UINT8") { + } else if (input.datatype() == "UINT8") { HANDLE_VALUE(mutable_uint_contents, GetUint, IsUint) - } else if (input->datatype() == "FP64") { + } else if (input.datatype() == "FP64") { HANDLE_VALUE(mutable_fp64_contents, GetFloat, IsNumber) - } else if (input->datatype() == "BOOL") { + } else if (input.datatype() == "BOOL") { HANDLE_VALUE(mutable_bool_contents, GetBool, IsBool) - } else if (input->datatype() == "BYTES") { + } else if (input.datatype() == "BYTES") { SPDLOG_DEBUG("For REST datatype BYTES is supported only with binary data extension"); return StatusCode::REST_COULD_NOT_PARSE_INPUT; } else { @@ -592,7 +592,15 @@ Status KFSRestParser::parseData(rapidjson::Value& node, ::KFSRequest::InferInput return StatusCode::OK; } -Status KFSRestParser::parseInput(rapidjson::Value& node) { +static Status binaryDataSizeCanBeCalculated(::KFSRequest::InferInputTensor& input, bool onlyOneInput) { + if (input.datatype() == "BYTES" && (!onlyOneInput || input.shape_size() != 1 || input.shape()[0] != 1)) { + SPDLOG_DEBUG("Tensor: {} with datatype BYTES has no binary_data_size parameter and the size of the data cannot be calculated from shape.", input.name()); + return StatusCode::REST_COULD_NOT_PARSE_INPUT; + } + return StatusCode::OK; +} + +Status KFSRestParser::parseInput(rapidjson::Value& node, bool onlyOneInput) { if (!node.IsObject()) { return StatusCode::REST_COULD_NOT_PARSE_INPUT; } @@ -634,14 +642,13 @@ Status KFSRestParser::parseInput(rapidjson::Value& node) { if (!(dataItr->value.IsArray())) { return StatusCode::REST_COULD_NOT_PARSE_INPUT; } - return parseData(dataItr->value, input); + return parseData(dataItr->value, *input); } else { auto binary_data_size_parameter = input->parameters().find("binary_data_size"); if (binary_data_size_parameter != input->parameters().end()) { return StatusCode::OK; - } else { - return StatusCode::REST_COULD_NOT_PARSE_INPUT; } + return binaryDataSizeCanBeCalculated(*input, onlyOneInput); } } @@ -654,7 +661,7 @@ Status KFSRestParser::parseInputs(rapidjson::Value& node) { } requestProto.mutable_inputs()->Clear(); for (auto& input : node.GetArray()) { - auto status = parseInput(input); + auto status = parseInput(input, (node.GetArray().Size() == 1)); if (!status.ok()) { return status; } diff --git a/src/rest_parser.hpp b/src/rest_parser.hpp index e575876b70..43d6bcea9d 100644 --- a/src/rest_parser.hpp +++ b/src/rest_parser.hpp @@ -227,8 +227,8 @@ class KFSRestParser : RestParser { Status parseOutputParameters(rapidjson::Value& node, ::KFSRequest::InferRequestedOutputTensor& input); Status parseOutput(rapidjson::Value& node); Status parseOutputs(rapidjson::Value& node); - Status parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor* input); - Status parseInput(rapidjson::Value& node); + Status parseData(rapidjson::Value& node, ::KFSRequest::InferInputTensor& input); + Status parseInput(rapidjson::Value& node, bool onlyOneInput); Status parseInputs(rapidjson::Value& node); public: diff --git a/src/test/kfs_rest_parser_test.cpp b/src/test/kfs_rest_parser_test.cpp index c6777f8aed..fc8995fd71 100644 --- a/src/test/kfs_rest_parser_test.cpp +++ b/src/test/kfs_rest_parser_test.cpp @@ -319,6 +319,112 @@ TEST_F(KFSRestParserTest, parseValidRequestBYTES) { ASSERT_EQ(binary_data_size_parameter->second.int64_param(), 4); } +TEST_F(KFSRestParserTest, parseValidRequestBYTES_noBinaryDataSizeParameter) { + std::string request = R"({ + "inputs" : [ + { + "name" : "input0", + "shape" : [1], + "datatype" : "BYTES" + } + ] + })"; + auto status = parser.parse(request.c_str()); + ASSERT_EQ(status, StatusCode::OK); + + auto proto = parser.getProto(); + ASSERT_EQ(proto.inputs_size(), 1); + ASSERT_EQ(proto.inputs()[0].name(), "input0"); + ASSERT_THAT(proto.inputs()[0].shape(), ElementsAre(1)); + ASSERT_EQ(proto.inputs()[0].datatype(), "BYTES"); + ASSERT_EQ(proto.inputs()[0].contents().bytes_contents_size(), 0); +} + +TEST_F(KFSRestParserTest, parseInvalidRequestBYTES_noBinaryDataSizeParameter) { + std::string request = R"({ + "inputs" : [ + { + "name" : "input0", + "shape" : [2], + "datatype" : "BYTES" + } + ] + })"; + auto status = parser.parse(request.c_str()); + ASSERT_EQ(status, StatusCode::REST_COULD_NOT_PARSE_INPUT); +} + +TEST_F(KFSRestParserTest, parseInvalidRequestBYTES_noBinaryDataSizeParameter_twoInputs) { + std::string request = R"({ + "inputs" : [ + { + "name" : "input0", + "shape" : [ 2, 2 ], + "datatype" : "BYTES" + }, + { + "name" : "input1", + "shape" : [ 1, 2 ], + "datatype" : "INT32" + } + ] + })"; + auto status = parser.parse(request.c_str()); + ASSERT_EQ(status, StatusCode::REST_COULD_NOT_PARSE_INPUT); +} + +TEST_F(KFSRestParserTest, parseValidRequestFP32_noData_noBinaryDataSizeParameter) { + std::string request = R"({ + "inputs" : [ + { + "name" : "input0", + "shape" : [ 2, 2 ], + "datatype" : "FP32" + } + ] + })"; + auto status = parser.parse(request.c_str()); + ASSERT_EQ(status, StatusCode::OK); + + auto proto = parser.getProto(); + ASSERT_EQ(proto.inputs_size(), 1); + ASSERT_EQ(proto.inputs()[0].name(), "input0"); + ASSERT_THAT(proto.inputs()[0].shape(), ElementsAre(2, 2)); + ASSERT_EQ(proto.inputs()[0].datatype(), "FP32"); + ASSERT_EQ(proto.inputs()[0].contents().fp32_contents_size(), 0); +} + +TEST_F(KFSRestParserTest, parseValidRequestFP32_noData_noBinaryDataSizeParameter_twoInputs) { + std::string request = R"({ + "inputs" : [ + { + "name" : "input0", + "shape" : [ 2, 2 ], + "datatype" : "FP32" + }, + { + "name" : "input1", + "shape" : [ 1, 1 ], + "datatype" : "INT32" + } + ] + })"; + auto status = parser.parse(request.c_str()); + ASSERT_EQ(status, StatusCode::OK); + + auto proto = parser.getProto(); + ASSERT_EQ(proto.inputs_size(), 2); + ASSERT_EQ(proto.inputs()[0].name(), "input0"); + ASSERT_THAT(proto.inputs()[0].shape(), ElementsAre(2, 2)); + ASSERT_EQ(proto.inputs()[0].datatype(), "FP32"); + ASSERT_EQ(proto.inputs()[0].contents().fp32_contents_size(), 0); + + ASSERT_EQ(proto.inputs()[1].name(), "input1"); + ASSERT_THAT(proto.inputs()[1].shape(), ElementsAre(1, 1)); + ASSERT_EQ(proto.inputs()[1].datatype(), "INT32"); + ASSERT_EQ(proto.inputs()[1].contents().int_contents_size(), 0); +} + TEST_F(KFSRestParserTest, parseValidRequestWithStringRequestParameter) { std::string request = R"({ "parameters" : {"param" : "value"}, @@ -662,20 +768,6 @@ TEST_F(KFSRestParserTest, parseInvalidRequestBoolWithIntData) { ASSERT_EQ(status, StatusCode::REST_COULD_NOT_PARSE_INPUT); } -TEST_F(KFSRestParserTest, parseInvalidRequestWithNoDataAndNoBinaryInputsParameter) { - std::string request = R"({ - "inputs" : [ - { - "name" : "input0", - "shape" : [ 2, 2 ], - "datatype" : "UINT32" - } - ] - })"; - auto status = parser.parse(request.c_str()); - ASSERT_EQ(status, StatusCode::REST_COULD_NOT_PARSE_INPUT); -} - TEST_F(KFSRestParserTest, parseInvalidRequestWithDataAndBYTESdatatype) { std::string request = R"({ "inputs" : [ diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index f6e132a2c4..907574948f 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -363,26 +363,57 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT8) { ASSERT_EQ(i, 4); } +static void assertSingleBinaryInput(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request) { + ASSERT_EQ(grpc_request.inputs_size(), 1); + ASSERT_EQ(grpc_request.model_name(), modelName); + ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); + + ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); +} + +static void assertBinaryInputsBYTES(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request, std::string binaryData) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "BYTES"); + ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); + ASSERT_EQ(grpc_request.inputs()[0].contents().bytes_contents()[0], binaryData); + ASSERT_EQ(grpc_request.inputs()[0].contents().bytes_contents_size(), 1); +} + TEST_F(HttpRestApiHandlerTest, binaryInputsBYTES) { std::string binaryData{0x00, 0x01, 0x02, 0x03}; - std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"BYTES\",\"parameters\":{\"binary_data_size\":4}}]}"; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1],\"datatype\":\"BYTES\",\"parameters\":{\"binary_data_size\":4}}]}"; request_body += binaryData; ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + ASSERT_EQ(grpc_request.inputs()[0].parameters().count("binary_data_size"), 1); + assertBinaryInputsBYTES(modelName, modelVersion, grpc_request, binaryData); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "BYTES"); +TEST_F(HttpRestApiHandlerTest, binaryInputsBYTES_noBinaryDataSizeParameter) { + std::string binaryData{0x00, 0x01, 0x02, 0x03}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1],\"datatype\":\"BYTES\"}]}"; + request_body += binaryData; + + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsBYTES(modelName, modelVersion, grpc_request, binaryData); +} +static void assertBinaryInputsINT16(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request, std::string binaryData) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT16"); ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); - ASSERT_EQ(grpc_request.inputs()[0].contents().bytes_contents()[0], binaryData); - ASSERT_EQ(grpc_request.inputs()[0].contents().bytes_contents_size(), 1); + int i = 0; + for (auto content : grpc_request.inputs()[0].contents().int_contents()) { + ASSERT_EQ(content, i++); + } + ASSERT_EQ(i, 4); } TEST_F(HttpRestApiHandlerTest, binaryInputsINT16) { @@ -393,17 +424,26 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT16) { ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT16(modelName, modelVersion, grpc_request, binaryData); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT16"); +TEST_F(HttpRestApiHandlerTest, binaryInputsINT16_noBinaryDataSizeParameter) { + std::string binaryData{0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT16\"}]}"; + request_body += binaryData; + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT16(modelName, modelVersion, grpc_request, binaryData); +} + +static void assertBinaryInputsINT32(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request, std::string binaryData) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT32"); ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); - int i = 0; for (auto content : grpc_request.inputs()[0].contents().int_contents()) { ASSERT_EQ(content, i++); @@ -419,19 +459,28 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT32) { ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT32(modelName, modelVersion, grpc_request, binaryData); +} + +TEST_F(HttpRestApiHandlerTest, binaryInputsINT32_noBinaryDataSizeParameter) { + std::string binaryData{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\"}]}"; + request_body += binaryData; + + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT32(modelName, modelVersion, grpc_request, binaryData); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT32"); +static void assertBinaryInputsINT64(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request, std::string binaryData) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT64"); ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); - int i = 0; - for (auto content : grpc_request.inputs()[0].contents().int_contents()) { + for (auto content : grpc_request.inputs()[0].contents().int64_contents()) { ASSERT_EQ(content, i++); } ASSERT_EQ(i, 4); @@ -445,19 +494,28 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsINT64) { ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT64(modelName, modelVersion, grpc_request, binaryData); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "INT64"); +TEST_F(HttpRestApiHandlerTest, binaryInputsINT64_noBinaryDataSizeParameter) { + std::string binaryData{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT64\"}]}"; + request_body += binaryData; + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsINT64(modelName, modelVersion, grpc_request, binaryData); +} + +static void assertBinaryInputsFP32(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "FP32"); ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); - int i = 0; - for (auto content : grpc_request.inputs()[0].contents().int64_contents()) { + for (auto content : grpc_request.inputs()[0].contents().fp32_contents()) { ASSERT_EQ(content, i++); } ASSERT_EQ(i, 4); @@ -471,19 +529,28 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsFP32) { ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - 16); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsFP32(modelName, modelVersion, grpc_request); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "FP32"); +TEST_F(HttpRestApiHandlerTest, binaryInputsFP32_noBinaryDataSizeParameter) { + float values[] = {0.0, 1.0, 2.0, 3.0}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"FP32\"}]}"; + request_body.append((char*)values, 16); + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - 16); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsFP32(modelName, modelVersion, grpc_request); +} + +static void assertBinaryInputsFP64(const std::string& modelName, const std::optional& modelVersion, ::KFSRequest& grpc_request) { + assertSingleBinaryInput(modelName, modelVersion, grpc_request); + + ASSERT_EQ(grpc_request.inputs()[0].datatype(), "FP64"); ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); - int i = 0; - for (auto content : grpc_request.inputs()[0].contents().fp32_contents()) { + for (auto content : grpc_request.inputs()[0].contents().fp64_contents()) { ASSERT_EQ(content, i++); } ASSERT_EQ(i, 4); @@ -497,22 +564,18 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsFP64) { ::KFSRequest grpc_request; int inferenceHeaderContentLength = (request_body.size() - 32); ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsFP64(modelName, modelVersion, grpc_request); +} - ASSERT_EQ(grpc_request.inputs_size(), 1); - ASSERT_EQ(grpc_request.model_name(), modelName); - ASSERT_EQ(grpc_request.model_version(), std::to_string(modelVersion.value())); - auto params = grpc_request.parameters(); - ASSERT_EQ(grpc_request.inputs()[0].name(), "b"); - ASSERT_EQ(grpc_request.inputs()[0].datatype(), "FP64"); - - ASSERT_EQ(grpc_request.inputs()[0].shape()[0], 1); - ASSERT_EQ(grpc_request.inputs()[0].shape()[1], 4); +TEST_F(HttpRestApiHandlerTest, binaryInputsFP64_noBinaryDataSizeParameter) { + double values[] = {0.0, 1.0, 2.0, 3.0}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"FP64\"}]}"; + request_body.append((char*)values, 32); - int i = 0; - for (auto content : grpc_request.inputs()[0].contents().fp64_contents()) { - ASSERT_EQ(content, i++); - } - ASSERT_EQ(i, 4); + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - 32); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::OK); + assertBinaryInputsFP64(modelName, modelVersion, grpc_request); } TEST_F(HttpRestApiHandlerTest, binaryInputsBinaryDataAndContentField) { @@ -534,6 +597,26 @@ TEST_F(HttpRestApiHandlerTest, binaryInputsBufferSmallerThanExpected) { ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_BINARY_BUFFER_EXCEEDED); } +TEST_F(HttpRestApiHandlerTest, binaryInputsBufferSmallerThanExpected_noBinaryDataSizeParameter) { + std::string binaryData{0x00, 0x00, 0x00, 0x00}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\"}]}"; + request_body += binaryData; + + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_BINARY_BUFFER_EXCEEDED); +} + +TEST_F(HttpRestApiHandlerTest, binaryInputsBytes_noBinaryDataSizeParameter_twoInputs) { + std::string binaryData{0x00, 0x00, 0x00, 0x00}; + std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1],\"datatype\":\"BYTES\"}, {\"name\":\"a\",\"shape\":[1],\"datatype\":\"BYTES\"}]}"; + request_body += binaryData; + + ::KFSRequest grpc_request; + int inferenceHeaderContentLength = (request_body.size() - binaryData.size()); + ASSERT_EQ(HttpRestApiHandler::prepareGrpcRequest(modelName, modelVersion, request_body, grpc_request, inferenceHeaderContentLength), ovms::StatusCode::REST_COULD_NOT_PARSE_INPUT); +} + TEST_F(HttpRestApiHandlerTest, binaryInputsInvalidBinaryDataSizeParameter) { std::string binaryData{0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00}; std::string request_body = "{\"inputs\":[{\"name\":\"b\",\"shape\":[1,4],\"datatype\":\"INT32\",\"parameters\":{\"binary_data_size\":true}}]}"; From 14c107ff96b1c9c9000de74a0fa35f13fa6b477b Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Wed, 14 Dec 2022 16:52:13 +0100 Subject: [PATCH 105/130] Change prediction fail due to not getting model to DEBUG (#1507) --- src/prediction_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prediction_service.cpp b/src/prediction_service.cpp index 0000925945..a28702499a 100644 --- a/src/prediction_service.cpp +++ b/src/prediction_service.cpp @@ -110,7 +110,7 @@ grpc::Status ovms::PredictionServiceImpl::Predict( if (modelInstance) { INCREMENT_IF_ENABLED(modelInstance->getMetricReporter().requestFailGrpcPredict); } - SPDLOG_INFO("Getting modelInstance or pipeline failed. {}", status.string()); + SPDLOG_DEBUG("Getting modelInstance or pipeline failed. {}", status.string()); return grpc(status); } From 51411662cc8f0c67a0aa5aa18522f0661c999a5b Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Wed, 14 Dec 2022 17:42:14 +0100 Subject: [PATCH 106/130] Update OV package link (#1598) add tbb package and ignore missing 3rdparty folder --- Dockerfile.redhat | 3 ++- Dockerfile.ubuntu | 1 + DockerfileMakePackage | 2 +- Makefile | 4 ++-- release_files/Dockerfile.redhat | 4 ++-- release_files/Dockerfile.ubuntu | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 7619744fd9..2af2d68875 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -72,7 +72,8 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n yum-utils \ unzip \ vim \ - xz && \ + xz \ + https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm && \ yum clean all # Build and install pugixml diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 0c51b3c8c1..f57ebd3fc3 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -66,6 +66,7 @@ RUN apt-get update && apt-get install -y \ libtool \ libxml2-dev \ libnuma-dev \ + libtbb2 \ libssl-dev \ nlohmann-json3-dev \ patch \ diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 6879fac178..64a7c7ce87 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -61,7 +61,7 @@ libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so lib RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; -RUN find /opt/intel/openvino/runtime/3rdparty/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; +RUN if [ -d /opt/intel/openvino/runtime/3rdparty ] ; then find /opt/intel/openvino/runtime/3rdparty/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \;; fi RUN find /opt/opencv/lib/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN cp /opt/opencv/share/licenses/opencv4/* /ovms/release_files/thirdparty-licenses/ RUN if [ "$ov_use_binary" == "1" ] ; then true ; else exit 0 ; fi ; if [ -f /opt/intel/openvino/docs/licensing/EULA.txt ] ; then true ; else exit 0; fi ; cp /opt/intel/openvino/docs/licensing/EULA.txt /ovms/release_files/thirdparty-licenses/openvino.LICENSE.txt; diff --git a/Makefile b/Makefile index 69853ff0be..36600d2764 100644 --- a/Makefile +++ b/Makefile @@ -80,14 +80,14 @@ ifeq ($(BASE_OS),ubuntu) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) endif INSTALL_DRIVER_VERSION ?= "22.35.24055" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0357a97c40a41be0f362a3d0907e378b236b3c71/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.8963.0357a97c40a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/releases/2022/3/commit/b84161848eaf02d8004f92cb530c1f092eac5782/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.9038.b84161848ea_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/0357a97c40a41be0f362a3d0907e378b236b3c71/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.8963.0357a97c40a_x86_64.tgz + DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/releases/2022/3/commit/b84161848eaf02d8004f92cb530c1f092eac5782/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.9038.b84161848ea_x86_64.tgz endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index ca59e05084..f0aaceba25 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -30,7 +30,7 @@ RUN mkdir /licenses && ln -s /ovms/LICENSE /licenses && ln -s /ovms/thirdparty-l FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 as release LABEL "name"="OVMS" LABEL "vendor"="Intel Corporation" -LABEL "version"="2022.1" +LABEL "version"="2022.3" LABEL "release"="2022" LABEL "summary"="OpenVINO(TM) Model Server" LABEL "description"="OpenVINO(TM) Model Server is a solution for serving AI models" @@ -46,7 +46,7 @@ WORKDIR / RUN set -e ; \ set -x ; \ microdnf upgrade -y ; \ - microdnf install -y pkg-config ; \ + microdnf install -y pkg-config && rpm -ivh https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm && \ if [ "$GPU" == "1" ] ; then \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 9ad40a2b61..baf49328b1 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -84,7 +84,7 @@ COPY drivers /drivers RUN set -e ; \ set -x ; \ apt update -y ; \ - apt install -y curl libpugixml1v5 ; \ + apt install -y curl libpugixml1v5 libtbb2 && \ if [ "$GPU" == "1" ] ; then \ apt-get update && apt-get install -y libnuma1 ocl-icd-libopencl1 && rm -rf /var/lib/apt/lists/* && \ case $INSTALL_DRIVER_VERSION in \ From f059d13fa5f810fc51c1f9a85dffffa85b6a2363 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Wed, 14 Dec 2022 20:47:58 +0100 Subject: [PATCH 107/130] fix building nvidia plugin (#1599) --- Dockerfile.ubuntu | 6 +++--- Makefile | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index f57ebd3fc3..85a930b966 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -14,7 +14,7 @@ # limitations under the License. # -ARG BASE_IMAGE=ubuntu:20.04 +ARG BASE_IMAGE FROM $BASE_IMAGE as base_build LABEL version="1.0.0" @@ -105,7 +105,7 @@ RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"; \ apt-get update && apt install -y \ libzstd-dev \ - cuda \ + cuda-11-8 \ libcudnn8 \ libcudnn8-dev \ libcutensor1 \ @@ -240,7 +240,7 @@ ENV OPENVINO_HOME=/openvino ENV OPENVINO_CONTRIB=/openvino_contrib RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ - git clone --recurse-submodules --branch=$ov_contrib_branch https://github.com/openvinotoolkit/openvino_contrib.git /openvino_contrib && cd /openvino_contrib && \ + git clone https://github.com/openvinotoolkit/openvino_contrib.git /openvino_contrib && cd /openvino_contrib && git checkout $ov_contrib_branch && git submodule update --init --recursive && \ mkdir ${OPENVINO_BUILD_PATH} && \ cd "${OPENVINO_BUILD_PATH}" && \ cmake "${OPENVINO_HOME}" \ diff --git a/Makefile b/Makefile index 36600d2764..3631e434c9 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ NVIDIA ?=0 # - uncomment source build section, comment binary section # - adjust binary version path - version variable is not passed to WORKSPACE file! OV_SOURCE_BRANCH ?= master +OV_CONTRIB_BRANCH ?= master OV_USE_BINARY ?= 1 APT_OV_PACKAGE ?= openvino-2022.1.0 @@ -212,7 +213,7 @@ endif --build-arg PROJECT_NAME=${PROJECT_NAME} \ --build-arg PROJECT_VERSION=${PROJECT_VERSION} \ --build-arg BASE_IMAGE=$(BASE_IMAGE) \ - --build-arg NVIDIA=$(NVIDIA) \ + --build-arg NVIDIA=$(NVIDIA) --build-arg ov_contrib_branch="$(OV_CONTRIB_BRANCH)" \ -t $(OVMS_CPP_DOCKER_IMAGE)-build:$(OVMS_CPP_IMAGE_TAG)$(IMAGE_TAG_SUFFIX) \ --build-arg JOBS=$(JOBS) docker build $(NO_CACHE_OPTION) -f DockerfileMakePackage . \ From 4b868eb7a3356a94d2b0bf3d790e9c24b5229978 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 15 Dec 2022 09:28:00 +0100 Subject: [PATCH 108/130] C-API renaming (#1586) -> pocapi.h -> ovms.h -> pocapi.cpp -> capi.cpp Moved pocapiinternal.[ch]pp to capi_utils Add server parameter to OVMS_InferenceRequestNew This is to potentially enable reserving ov::InferRequest during request creation, and to avoid getting model instance during OVMS_Inference() call in the future Rename GeneralOptions & MultiModelOptions JIRA:CVS-98960 Co-authored-by: rasapala --- Dockerfile.redhat | 2 +- Dockerfile.ubuntu | 2 +- DockerfileMakePackage | 15 +- MakefileCapi | 10 +- src/BUILD | 16 +- src/buffer.hpp | 2 +- src/{pocapi.cpp => capi_frontend/capi.cpp} | 242 ++++++------- src/capi_frontend/capi_utils.cpp | 100 ++++++ src/capi_frontend/capi_utils.hpp | 5 + src/cli_parser.cpp | 66 ++-- src/cli_parser.hpp | 6 +- src/config.cpp | 86 ++--- src/config.hpp | 8 +- src/inferenceparameter.cpp | 2 +- src/inferenceparameter.hpp | 2 +- src/inferencerequest.hpp | 2 +- src/inferenceresponse.hpp | 2 +- src/inferencetensor.cpp | 2 +- src/inferencetensor.hpp | 2 +- src/main_benchmark.cpp | 40 +-- src/{main3.c => main_capi.c} | 38 +- src/{main3.cpp => main_capi.cpp} | 34 +- src/{pocapi.h => ovms.h} | 64 ++-- src/pocapiinternal.cpp | 119 ------- src/pocapiinternal.hpp | 23 -- src/predict_request_validation_utils.cpp | 1 - src/serialization.hpp | 2 +- src/server.cpp | 14 +- src/server.hpp | 6 +- ...server_options.hpp => server_settings.hpp} | 4 +- src/status.cpp | 2 +- src/status.hpp | 2 +- src/test/c_api_tests.cpp | 330 +++++++++--------- src/test/deserialization_tests.cpp | 2 +- src/test/status_test.cpp | 2 +- src/test/test_utils.cpp | 1 - 36 files changed, 617 insertions(+), 639 deletions(-) rename src/{pocapi.cpp => capi_frontend/capi.cpp} (75%) rename src/{main3.c => main_capi.c} (78%) rename src/{main3.cpp => main_capi.cpp} (85%) rename src/{pocapi.h => ovms.h} (72%) delete mode 100644 src/pocapiinternal.cpp delete mode 100644 src/pocapiinternal.hpp rename src/{server_options.hpp => server_settings.hpp} (97%) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 2af2d68875..66e8ac0e7c 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -207,7 +207,7 @@ RUN cd /ovms/src/example/SampleCpuExtension/ && make RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:capi_cpp_example # C-API benchmark app RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 85a930b966..6563674e47 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -299,7 +299,7 @@ RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel -RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:poc3 +RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:capi_cpp_example # C-API benchmark app RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 64a7c7ce87..347fe7a7c7 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -53,11 +53,15 @@ RUN rm -f /ovms_release/lib/libcustom_node* # Remove coverage libaries RUN if [ -f /ovms_release/lib/libjava.so ] ; then true ; else exit 0 ; fi ;cd /ovms_release/lib/ &&\ -rm -rf libatk-wrapper.so libattach.so libawt_headless.so libawt.so libawt_xawt.so libdt_socket.so \ -libextnet.so libfontmanager.so libinstrument.so libj2gss.so libj2pcsc.so libj2pkcs11.so libjaas.so \ -libjavajpeg.so libjava.so libjawt.so libjdwp.so libjimage.so libjli.so libjsig.so libjsound.so libjvm.so \ -liblcms.so libmanagement_agent.so libmanagement_ext.so libmanagement.so libmlib_image.so libnet.so libnio.so \ -libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libsystemconf.so libunpack.so libverify.so libzip.so + rm -rf libatk-wrapper.so libattach.so libawt_headless.so libawt.so libawt_xawt.so libdt_socket.so \ + libextnet.so libfontmanager.so libinstrument.so libj2gss.so libj2pcsc.so libj2pkcs11.so libjaas.so \ + libjavajpeg.so libjava.so libjawt.so libjdwp.so libjimage.so libjli.so libjsig.so libjsound.so libjvm.so \ + liblcms.so libmanagement_agent.so libmanagement_ext.so libmanagement.so libmlib_image.so libnet.so libnio.so \ + libprefs.so librmi.so libsaproc.so libsctp.so libsplashscreen.so libsunec.so libsystemconf.so libunpack.so libverify.so libzip.so + +# Remove capi temp libraries +RUN if [ -f /ovms_release/lib/libsrc_Slibovms_Ushared.so ] ; then true ; else exit 0 ; fi ;\ + rm -rf /ovms_release/lib/libsrc_Slibovms_Ushared.so RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; @@ -78,6 +82,7 @@ RUN find /ovms_release/lib/ -iname '*.so*' -exec patchelf --debug --set-rpath '$ WORKDIR /ovms RUN cp -v /ovms/release_files/LICENSE /ovms_release/ RUN cp -rv /ovms/release_files/thirdparty-licenses /ovms_release/ +RUN mkdir -vp /ovms_release/include && cp /ovms/src/ovms.h /ovms_release/include RUN ls -lahR /ovms_release/ diff --git a/MakefileCapi b/MakefileCapi index c15f5c279a..a21aefaa28 100644 --- a/MakefileCapi +++ b/MakefileCapi @@ -15,11 +15,11 @@ # cpp: bazel build //src:ovms_shared - g++ src/main3.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/poc3 - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/poc3 + g++ src/main_capi.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/capi_cpp_example + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/capi_cpp_example c: bazel build //src:ovms_shared - gcc -c src/main3.c -o /ovms/bazel-bin/src/main3.o -std=c99 - gcc -o /ovms/bazel-bin/src/poc3_c /ovms/bazel-bin/src/main3.o -lovms_shared -L/ovms/bazel-bin/src/ - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/poc3_c + gcc -c src/main_capi.c -o /ovms/bazel-bin/src/main_capi.o -std=c99 + gcc -o /ovms/bazel-bin/src/capi_c_example /ovms/bazel-bin/src/main_capi.o -lovms_shared -L/ovms/bazel-bin/src/ + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/capi_c_example diff --git a/src/BUILD b/src/BUILD index f235dad492..72f9707dc3 100644 --- a/src/BUILD +++ b/src/BUILD @@ -87,7 +87,7 @@ cc_shared_library( cc_library( name = "ovms_lib", - hdrs = ["pocapi.h"], + hdrs = ["ovms.h"], srcs = [ "aliases.hpp", "azurestorage.hpp", @@ -96,6 +96,7 @@ cc_library( "azurefilesystem.hpp", "buffer.cpp", "buffer.hpp", + "capi_frontend/capi.cpp", "capi_frontend/capi_utils.cpp", "capi_frontend/capi_utils.hpp", "cleaner_utils.cpp", @@ -226,6 +227,7 @@ cc_library( "ovinferrequestsqueue.hpp", "ov_utils.cpp", "ov_utils.hpp", + "ovms.h", "pipeline.cpp", "pipeline.hpp", "pipelinedefinition.cpp", @@ -237,10 +239,6 @@ cc_library( "pipelineeventqueue.hpp", "pipeline_factory.cpp", "pipeline_factory.hpp", - "pocapi.h", - "pocapiinternal.cpp", - "pocapiinternal.hpp", - "pocapi.cpp", "precision.cpp", "precision.hpp", "prediction_service.cpp", @@ -265,7 +263,7 @@ cc_library( "serialization.hpp", "servablemanagermodule.cpp", "servablemanagermodule.hpp", - "server_options.hpp", + "server_settings.hpp", "server.cpp", "server.hpp", "session_id.hpp", @@ -540,9 +538,9 @@ cc_binary( # POC which can start OVMS with C-API built by bazel # Standalone example for building outside of bazel (with bare g++ is inside MakefileCapi) cc_binary( - name = "poc3", + name = "capi_cpp_example", srcs = [ - "main3.cpp", + "main_capi.cpp", ], linkopts = [ "-lxml2", @@ -561,7 +559,7 @@ cc_binary( # Use for dynamic linking when neccessary cc_import( name = "shared_lib", - hdrs = ["pocapi.h"], + hdrs = ["ovms.h"], shared_library = "ovms_shared", ) diff --git a/src/buffer.hpp b/src/buffer.hpp index 1c4691a4af..c5b17f263f 100644 --- a/src/buffer.hpp +++ b/src/buffer.hpp @@ -17,7 +17,7 @@ #include #include -#include "./pocapi.h" +#include "ovms.h" // NOLINT namespace ovms { class Buffer { diff --git a/src/pocapi.cpp b/src/capi_frontend/capi.cpp similarity index 75% rename from src/pocapi.cpp rename to src/capi_frontend/capi.cpp index 3d3c258d6f..fda2790b12 100644 --- a/src/pocapi.cpp +++ b/src/capi_frontend/capi.cpp @@ -13,27 +13,26 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "pocapi.h" // NOLINT - #include #include #include -#include "buffer.hpp" -#include "inferenceparameter.hpp" -#include "inferencerequest.hpp" -#include "inferenceresponse.hpp" -#include "inferencetensor.hpp" -#include "model_service.hpp" -#include "modelinstance.hpp" -#include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" -#include "profiler.hpp" -#include "servablemanagermodule.hpp" -#include "server.hpp" -#include "server_options.hpp" -#include "status.hpp" -#include "timer.hpp" +#include "../buffer.hpp" +#include "../inferenceparameter.hpp" +#include "../inferencerequest.hpp" +#include "../inferenceresponse.hpp" +#include "../inferencetensor.hpp" +#include "../model_service.hpp" +#include "../modelinstance.hpp" +#include "../modelinstanceunloadguard.hpp" +#include "../modelmanager.hpp" +#include "../ovms.h" // NOLINT +#include "../profiler.hpp" +#include "../servablemanagermodule.hpp" +#include "../server.hpp" +#include "../server_settings.hpp" +#include "../status.hpp" +#include "../timer.hpp" using ovms::Buffer; using ovms::InferenceParameter; @@ -81,32 +80,32 @@ OVMS_Status* OVMS_StatusGetDetails(OVMS_Status* status, return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); +OVMS_Status* OVMS_ServerSettingsNew(OVMS_ServerSettings** settings) { + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - *options = reinterpret_cast(new ovms::GeneralOptionsImpl); + *settings = reinterpret_cast(new ovms::ServerSettingsImpl); return nullptr; } -void OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options) { - if (options == nullptr) +void OVMS_ServerSettingsDelete(OVMS_ServerSettings* settings) { + if (settings == nullptr) return; - delete reinterpret_cast(options); + delete reinterpret_cast(settings); } -OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); +OVMS_Status* OVMS_ModelsSettingsNew(OVMS_ModelsSettings** settings) { + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - *options = reinterpret_cast(new ovms::MultiModelOptionsImpl); + *settings = reinterpret_cast(new ovms::ModelsSettingsImpl); return nullptr; } -void OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options) { - if (options == nullptr) +void OVMS_ModelsSettingsDelete(OVMS_ModelsSettings* settings) { + if (settings == nullptr) return; - delete reinterpret_cast(options); + delete reinterpret_cast(settings); } OVMS_Status* OVMS_ServerNew(OVMS_Server** server) { @@ -127,182 +126,182 @@ void OVMS_ServerDelete(OVMS_Server* server) { } OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, - OVMS_ServerGeneralOptions* general_options, - OVMS_ServerMultiModelOptions* multi_model_specific_options) { + OVMS_ServerSettings* server_settings, + OVMS_ModelsSettings* models_settings) { if (server == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); } - if (general_options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (server_settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - if (multi_model_specific_options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (models_settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } ovms::Server* srv = reinterpret_cast(server); - ovms::GeneralOptionsImpl* go = reinterpret_cast(general_options); - ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(multi_model_specific_options); - auto res = srv->start(go, mmo); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(server_settings); + ovms::ModelsSettingsImpl* modelsSettings = reinterpret_cast(models_settings); + auto res = srv->start(serverSettings, modelsSettings); if (res.ok()) return nullptr; return reinterpret_cast(new ovms::Status(res)); } -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcPort(OVMS_ServerSettings* settings, uint32_t grpcPort) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->grpcPort = grpcPort; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->grpcPort = grpcPort; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestPort(OVMS_ServerSettings* settings, uint32_t restPort) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->restPort = restPort; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->restPort = restPort; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcWorkers(OVMS_ServerSettings* settings, uint32_t grpc_workers) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->grpcWorkers = grpc_workers; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->grpcWorkers = grpc_workers; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcBindAddress(OVMS_ServerSettings* settings, const char* grpc_bind_address) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (grpc_bind_address == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->grpcBindAddress.assign(grpc_bind_address); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->grpcBindAddress.assign(grpc_bind_address); return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestWorkers(OVMS_ServerSettings* settings, uint32_t rest_workers) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->restWorkers = rest_workers; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->restWorkers = rest_workers; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestBindAddress(OVMS_ServerSettings* settings, const char* rest_bind_address) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (rest_bind_address == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->restBindAddress.assign(rest_bind_address); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->restBindAddress.assign(rest_bind_address); return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcChannelArguments(OVMS_ServerSettings* settings, const char* grpc_channel_arguments) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (grpc_channel_arguments == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->grpcChannelArguments.assign(grpc_channel_arguments); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->grpcChannelArguments.assign(grpc_channel_arguments); return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetFileSystemPollWaitSeconds(OVMS_ServerSettings* settings, uint32_t seconds) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->filesystemPollWaitSeconds = seconds; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->filesystemPollWaitSeconds = seconds; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetSequenceCleanerPollWaitMinutes(OVMS_ServerSettings* settings, uint32_t minutes) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->sequenceCleanerPollWaitMinutes = minutes; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->sequenceCleanerPollWaitMinutes = minutes; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerSettings* settings, uint32_t seconds) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->resourcesCleanerPollWaitSeconds = seconds; + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->resourcesCleanerPollWaitSeconds = seconds; return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCpuExtensionPath(OVMS_ServerSettings* settings, const char* cpu_extension_path) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (cpu_extension_path == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->cpuExtensionLibraryPath.assign(cpu_extension_path); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->cpuExtensionLibraryPath.assign(cpu_extension_path); return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCacheDir(OVMS_ServerSettings* settings, const char* cache_dir) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (cache_dir == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->cacheDir.assign(cache_dir); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->cacheDir.assign(cache_dir); return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetLogLevel(OVMS_ServerSettings* settings, OVMS_LogLevel log_level) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); switch (log_level) { case OVMS_LOG_INFO: - go->logLevel = "INFO"; + serverSettings->logLevel = "INFO"; break; case OVMS_LOG_ERROR: - go->logLevel = "ERROR"; + serverSettings->logLevel = "ERROR"; break; case OVMS_LOG_DEBUG: - go->logLevel = "DEBUG"; + serverSettings->logLevel = "DEBUG"; break; case OVMS_LOG_TRACE: - go->logLevel = "TRACE"; + serverSettings->logLevel = "TRACE"; break; case OVMS_LOG_WARNING: - go->logLevel = "WARNING"; + serverSettings->logLevel = "WARNING"; break; default: return reinterpret_cast(new Status(StatusCode::NONEXISTENT_LOG_LEVEL)); @@ -310,37 +309,40 @@ OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* opt return nullptr; } -OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetLogPath(OVMS_ServerSettings* settings, const char* log_path) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (log_path == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::GeneralOptionsImpl* go = reinterpret_cast(options); - go->logPath.assign(log_path); + ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); + serverSettings->logPath.assign(log_path); return nullptr; } -OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, +OVMS_Status* OVMS_ModelsSettingsSetConfigPath(OVMS_ModelsSettings* settings, const char* config_path) { - if (options == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_OPTIONS)); + if (settings == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (config_path == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string } - ovms::MultiModelOptionsImpl* mmo = reinterpret_cast(options); - mmo->configPath.assign(config_path); + ovms::ModelsSettingsImpl* modelsSettings = reinterpret_cast(settings); + modelsSettings->configPath.assign(config_path); return nullptr; } // inference API -OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion) { +OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, OVMS_Server* server, const char* servableName, uint32_t servableVersion) { // TODO should we allow to create requests to not yet loaded models? if (request == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); } + if (server == nullptr) { + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SERVER)); + } if (servableName == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } diff --git a/src/capi_frontend/capi_utils.cpp b/src/capi_frontend/capi_utils.cpp index db89d15e7b..979fac325f 100644 --- a/src/capi_frontend/capi_utils.cpp +++ b/src/capi_frontend/capi_utils.cpp @@ -25,4 +25,104 @@ std::string tensorShapeToString(const Shape& shape) { return shape.toString(); } +OVMS_DataType getPrecisionAsOVMSDataType(Precision precision) { + switch (precision) { + case Precision::BF16: + return OVMS_DATATYPE_BF16; + case Precision::FP64: + return OVMS_DATATYPE_FP64; + case Precision::FP32: + return OVMS_DATATYPE_FP32; + case Precision::FP16: + return OVMS_DATATYPE_FP16; + case Precision::I64: + return OVMS_DATATYPE_I64; + case Precision::I32: + return OVMS_DATATYPE_I32; + case Precision::I16: + return OVMS_DATATYPE_I16; + case Precision::I8: + return OVMS_DATATYPE_I8; + case Precision::I4: + return OVMS_DATATYPE_I4; + case Precision::U64: + return OVMS_DATATYPE_U64; + case Precision::U32: + return OVMS_DATATYPE_U32; + case Precision::U16: + return OVMS_DATATYPE_U16; + case Precision::U8: + return OVMS_DATATYPE_U8; + case Precision::U4: + return OVMS_DATATYPE_U4; + case Precision::U1: + return OVMS_DATATYPE_U1; + case Precision::BOOL: + return OVMS_DATATYPE_BOOL; + case Precision::CUSTOM: + return OVMS_DATATYPE_CUSTOM; + case Precision::UNDEFINED: + return OVMS_DATATYPE_UNDEFINED; + case Precision::DYNAMIC: + return OVMS_DATATYPE_DYNAMIC; + case Precision::MIXED: + return OVMS_DATATYPE_MIXED; + case Precision::Q78: + return OVMS_DATATYPE_Q78; + case Precision::BIN: + return OVMS_DATATYPE_BIN; + default: + return OVMS_DATATYPE_UNDEFINED; + } +} +Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype) { + switch (datatype) { + case OVMS_DATATYPE_BF16: + return Precision::BF16; + case OVMS_DATATYPE_FP64: + return Precision::FP64; + case OVMS_DATATYPE_FP32: + return Precision::FP32; + case OVMS_DATATYPE_FP16: + return Precision::FP16; + case OVMS_DATATYPE_I64: + return Precision::I64; + case OVMS_DATATYPE_I32: + return Precision::I32; + case OVMS_DATATYPE_I16: + return Precision::I16; + case OVMS_DATATYPE_I8: + return Precision::I8; + case OVMS_DATATYPE_I4: + return Precision::I4; + case OVMS_DATATYPE_U64: + return Precision::U64; + case OVMS_DATATYPE_U32: + return Precision::U32; + case OVMS_DATATYPE_U16: + return Precision::U16; + case OVMS_DATATYPE_U8: + return Precision::U8; + case OVMS_DATATYPE_U4: + return Precision::U4; + case OVMS_DATATYPE_U1: + return Precision::U1; + case OVMS_DATATYPE_BOOL: + return Precision::BOOL; + case OVMS_DATATYPE_CUSTOM: + return Precision::CUSTOM; + case OVMS_DATATYPE_UNDEFINED: + return Precision::UNDEFINED; + case OVMS_DATATYPE_DYNAMIC: + return Precision::DYNAMIC; + case OVMS_DATATYPE_MIXED: + return Precision::MIXED; + case OVMS_DATATYPE_Q78: + return Precision::Q78; + case OVMS_DATATYPE_BIN: + return Precision::BIN; + default: + return Precision::UNDEFINED; + } +} } // namespace ovms diff --git a/src/capi_frontend/capi_utils.hpp b/src/capi_frontend/capi_utils.hpp index d30158d2aa..998564deea 100644 --- a/src/capi_frontend/capi_utils.hpp +++ b/src/capi_frontend/capi_utils.hpp @@ -16,8 +16,13 @@ #pragma once #include +#include "../ovms.h" // NOLINT +#include "../precision.hpp" + namespace ovms { class Shape; std::string tensorShapeToString(const Shape& tensorShape); +OVMS_DataType getPrecisionAsOVMSDataType(Precision precision); +Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype); } // namespace ovms diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index 53d55ca00e..723596c295 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -20,7 +20,7 @@ #include -#include "server_options.hpp" +#include "server_settings.hpp" #include "version.hpp" namespace ovms { @@ -181,89 +181,89 @@ void CLIParser::parse(int argc, char** argv) { } } -void CLIParser::prepare(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - go->grpcPort = result->operator[]("port").as(); - go->restPort = result->operator[]("rest_port").as(); +void CLIParser::prepare(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* modelsSettings) { + serverSettings->grpcPort = result->operator[]("port").as(); + serverSettings->restPort = result->operator[]("rest_port").as(); if (result->count("model_name")) - mmo->modelName = result->operator[]("model_name").as(); + modelsSettings->modelName = result->operator[]("model_name").as(); if (result->count("model_path")) - mmo->modelPath = result->operator[]("model_path").as(); + modelsSettings->modelPath = result->operator[]("model_path").as(); if (result->count("max_sequence_number")) - mmo->maxSequenceNumber = result->operator[]("max_sequence_number").as(); + modelsSettings->maxSequenceNumber = result->operator[]("max_sequence_number").as(); if (result != nullptr && result->count("cpu_extension")) { - go->cpuExtensionLibraryPath = result->operator[]("cpu_extension").as(); + serverSettings->cpuExtensionLibraryPath = result->operator[]("cpu_extension").as(); } if (result->count("grpc_bind_address")) - go->grpcBindAddress = result->operator[]("grpc_bind_address").as(); + serverSettings->grpcBindAddress = result->operator[]("grpc_bind_address").as(); if (result->count("rest_bind_address")) - go->restBindAddress = result->operator[]("rest_bind_address").as(); + serverSettings->restBindAddress = result->operator[]("rest_bind_address").as(); - go->grpcWorkers = result->operator[]("grpc_workers").as(); + serverSettings->grpcWorkers = result->operator[]("grpc_workers").as(); if (result->count("rest_workers")) - go->restWorkers = result->operator[]("rest_workers").as(); + serverSettings->restWorkers = result->operator[]("rest_workers").as(); if (result->count("batch_size")) - mmo->batchSize = result->operator[]("batch_size").as(); + modelsSettings->batchSize = result->operator[]("batch_size").as(); if (result->count("shape")) - mmo->shape = result->operator[]("shape").as(); + modelsSettings->shape = result->operator[]("shape").as(); if (result->count("layout")) - mmo->layout = result->operator[]("layout").as(); + modelsSettings->layout = result->operator[]("layout").as(); if (result->count("model_version_policy")) - mmo->modelVersionPolicy = result->operator[]("model_version_policy").as(); + modelsSettings->modelVersionPolicy = result->operator[]("model_version_policy").as(); if (result->count("nireq")) - mmo->nireq = result->operator[]("nireq").as(); + modelsSettings->nireq = result->operator[]("nireq").as(); if (result->count("target_device")) - mmo->targetDevice = result->operator[]("target_device").as(); + modelsSettings->targetDevice = result->operator[]("target_device").as(); if (result->count("plugin_config")) - mmo->pluginConfig = result->operator[]("plugin_config").as(); + modelsSettings->pluginConfig = result->operator[]("plugin_config").as(); if (result->count("stateful")) - mmo->stateful = result->operator[]("stateful").as(); + modelsSettings->stateful = result->operator[]("stateful").as(); - go->metricsEnabled = result->operator[]("metrics_enable").as(); - go->metricsList = result->operator[]("metrics_list").as(); + serverSettings->metricsEnabled = result->operator[]("metrics_enable").as(); + serverSettings->metricsList = result->operator[]("metrics_list").as(); if (result->count("idle_sequence_cleanup")) - mmo->idleSequenceCleanup = result->operator[]("idle_sequence_cleanup").as(); + modelsSettings->idleSequenceCleanup = result->operator[]("idle_sequence_cleanup").as(); if (result->count("low_latency_transformation")) - mmo->lowLatencyTransformation = result->operator[]("low_latency_transformation").as(); + modelsSettings->lowLatencyTransformation = result->operator[]("low_latency_transformation").as(); if (result->count("log_level")) - go->logLevel = result->operator[]("log_level").as(); + serverSettings->logLevel = result->operator[]("log_level").as(); if (result->count("log_path")) - go->logPath = result->operator[]("log_path").as(); + serverSettings->logPath = result->operator[]("log_path").as(); #ifdef MTR_ENABLED if (result->count("trace_path")) - go->tracePath = result->operator[]("trace_path").as(); + serverSettings->tracePath = result->operator[]("trace_path").as(); #endif if (result->count("grpc_channel_arguments")) - go->grpcChannelArguments = result->operator[]("grpc_channel_arguments").as(); + serverSettings->grpcChannelArguments = result->operator[]("grpc_channel_arguments").as(); - go->filesystemPollWaitSeconds = result->operator[]("file_system_poll_wait_seconds").as(); - go->sequenceCleanerPollWaitMinutes = result->operator[]("sequence_cleaner_poll_wait_minutes").as(); - go->resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as(); + serverSettings->filesystemPollWaitSeconds = result->operator[]("file_system_poll_wait_seconds").as(); + serverSettings->sequenceCleanerPollWaitMinutes = result->operator[]("sequence_cleaner_poll_wait_minutes").as(); + serverSettings->resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as(); if (result != nullptr && result->count("cache_dir")) { - go->cacheDir = result->operator[]("cache_dir").as(); + serverSettings->cacheDir = result->operator[]("cache_dir").as(); } if (result->count("config_path")) - mmo->configPath = result->operator[]("config_path").as(); + modelsSettings->configPath = result->operator[]("config_path").as(); } } // namespace ovms diff --git a/src/cli_parser.hpp b/src/cli_parser.hpp index 5d50fae184..d40507b3b0 100644 --- a/src/cli_parser.hpp +++ b/src/cli_parser.hpp @@ -21,8 +21,8 @@ namespace ovms { -struct GeneralOptionsImpl; -struct MultiModelOptionsImpl; +struct ServerSettingsImpl; +struct ModelsSettingsImpl; class CLIParser { std::unique_ptr options; @@ -32,7 +32,7 @@ class CLIParser { CLIParser() = default; void parse(int argc, char** argv); - void prepare(GeneralOptionsImpl*, MultiModelOptionsImpl*); + void prepare(ServerSettingsImpl*, ModelsSettingsImpl*); }; } // namespace ovms diff --git a/src/config.cpp b/src/config.cpp index 1e447976da..ddaaf3b3e5 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -24,7 +24,7 @@ #include "cli_parser.hpp" #include "modelconfig.hpp" -#include "server_options.hpp" +#include "server_settings.hpp" namespace ovms { @@ -37,18 +37,18 @@ const uint64_t MAX_REST_WORKERS = 10'000; // TODO: Not used in OVMS - get rid of. Used only in tests. Config& Config::parse(int argc, char** argv) { ovms::CLIParser p; - ovms::GeneralOptionsImpl go; - ovms::MultiModelOptionsImpl mmo; + ovms::ServerSettingsImpl serverSettings; + ovms::ModelsSettingsImpl modelsSettings; p.parse(argc, argv); - p.prepare(&go, &mmo); - if (!this->parse(&go, &mmo)) + p.prepare(&serverSettings, &modelsSettings); + if (!this->parse(&serverSettings, &modelsSettings)) exit(EX_USAGE); return *this; } -bool Config::parse(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { - this->go = *go; - this->mmo = *mmo; +bool Config::parse(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* modelsSettings) { + this->serverSettings = *serverSettings; + this->modelsSettings = *modelsSettings; return validate(); } @@ -85,8 +85,8 @@ bool Config::validate() { return false; } - if (!configPath().empty() && (!this->mmo.batchSize.empty() || !shape().empty() || - nireq() != 0 || !modelVersionPolicy().empty() || !this->mmo.targetDevice.empty() || + if (!configPath().empty() && (!this->modelsSettings.batchSize.empty() || !shape().empty() || + nireq() != 0 || !modelVersionPolicy().empty() || !this->modelsSettings.targetDevice.empty() || !pluginConfig().empty())) { std::cerr << "Model parameters in CLI are exclusive with the config file" << std::endl; return false; @@ -104,7 +104,7 @@ bool Config::validate() { return false; } - if (this->go.restWorkers.has_value() && restPort() == 0) { + if (this->serverSettings.restWorkers.has_value() && restPort() == 0) { std::cerr << "rest_workers is set but rest_port is not set. rest_port is required to start rest servers" << std::endl; return false; } @@ -166,51 +166,51 @@ bool Config::validate() { return false; } // check stateful flags: - if ((this->mmo.lowLatencyTransformation.has_value() || this->mmo.maxSequenceNumber.has_value() || this->mmo.idleSequenceCleanup.has_value()) && !stateful()) { + if ((this->modelsSettings.lowLatencyTransformation.has_value() || this->modelsSettings.maxSequenceNumber.has_value() || this->modelsSettings.idleSequenceCleanup.has_value()) && !stateful()) { std::cerr << "Setting low_latency_transformation, max_sequence_number and idle_sequence_cleanup require setting stateful flag for the model." << std::endl; return false; } return true; } -const std::string& Config::configPath() const { return this->mmo.configPath; } -uint32_t Config::port() const { return this->go.grpcPort; } -const std::string Config::cpuExtensionLibraryPath() const { return this->go.cpuExtensionLibraryPath; } -const std::string Config::grpcBindAddress() const { return this->go.grpcBindAddress; } -uint32_t Config::restPort() const { return this->go.restPort; } -const std::string Config::restBindAddress() const { return this->go.restBindAddress; } -uint32_t Config::grpcWorkers() const { return this->go.grpcWorkers; } -uint32_t Config::restWorkers() const { return this->go.restWorkers.value_or(DEFAULT_REST_WORKERS); } -const std::string& Config::modelName() const { return this->mmo.modelName; } -const std::string& Config::modelPath() const { return this->mmo.modelPath; } +const std::string& Config::configPath() const { return this->modelsSettings.configPath; } +uint32_t Config::port() const { return this->serverSettings.grpcPort; } +const std::string Config::cpuExtensionLibraryPath() const { return this->serverSettings.cpuExtensionLibraryPath; } +const std::string Config::grpcBindAddress() const { return this->serverSettings.grpcBindAddress; } +uint32_t Config::restPort() const { return this->serverSettings.restPort; } +const std::string Config::restBindAddress() const { return this->serverSettings.restBindAddress; } +uint32_t Config::grpcWorkers() const { return this->serverSettings.grpcWorkers; } +uint32_t Config::restWorkers() const { return this->serverSettings.restWorkers.value_or(DEFAULT_REST_WORKERS); } +const std::string& Config::modelName() const { return this->modelsSettings.modelName; } +const std::string& Config::modelPath() const { return this->modelsSettings.modelPath; } const std::string& Config::batchSize() const { static const std::string defaultBatch = "0"; - return this->mmo.batchSize.empty() ? defaultBatch : this->mmo.batchSize; + return this->modelsSettings.batchSize.empty() ? defaultBatch : this->modelsSettings.batchSize; } -const std::string& Config::Config::shape() const { return this->mmo.shape; } -const std::string& Config::layout() const { return this->mmo.layout; } -const std::string& Config::modelVersionPolicy() const { return this->mmo.modelVersionPolicy; } -uint32_t Config::nireq() const { return this->mmo.nireq; } +const std::string& Config::Config::shape() const { return this->modelsSettings.shape; } +const std::string& Config::layout() const { return this->modelsSettings.layout; } +const std::string& Config::modelVersionPolicy() const { return this->modelsSettings.modelVersionPolicy; } +uint32_t Config::nireq() const { return this->modelsSettings.nireq; } const std::string& Config::targetDevice() const { static const std::string defaultTargetDevice = "CPU"; - return this->mmo.targetDevice.empty() ? defaultTargetDevice : this->mmo.targetDevice; + return this->modelsSettings.targetDevice.empty() ? defaultTargetDevice : this->modelsSettings.targetDevice; } -const std::string& Config::Config::pluginConfig() const { return this->mmo.pluginConfig; } -bool Config::stateful() const { return this->mmo.stateful.value_or(false); } -bool Config::metricsEnabled() const { return this->go.metricsEnabled; } -std::string Config::metricsList() const { return this->go.metricsList; } -bool Config::idleSequenceCleanup() const { return this->mmo.idleSequenceCleanup.value_or(true); } -uint32_t Config::maxSequenceNumber() const { return this->mmo.maxSequenceNumber.value_or(DEFAULT_MAX_SEQUENCE_NUMBER); } -bool Config::lowLatencyTransformation() const { return this->mmo.lowLatencyTransformation.value_or(false); } -const std::string& Config::logLevel() const { return this->go.logLevel; } -const std::string& Config::logPath() const { return this->go.logPath; } +const std::string& Config::Config::pluginConfig() const { return this->modelsSettings.pluginConfig; } +bool Config::stateful() const { return this->modelsSettings.stateful.value_or(false); } +bool Config::metricsEnabled() const { return this->serverSettings.metricsEnabled; } +std::string Config::metricsList() const { return this->serverSettings.metricsList; } +bool Config::idleSequenceCleanup() const { return this->modelsSettings.idleSequenceCleanup.value_or(true); } +uint32_t Config::maxSequenceNumber() const { return this->modelsSettings.maxSequenceNumber.value_or(DEFAULT_MAX_SEQUENCE_NUMBER); } +bool Config::lowLatencyTransformation() const { return this->modelsSettings.lowLatencyTransformation.value_or(false); } +const std::string& Config::logLevel() const { return this->serverSettings.logLevel; } +const std::string& Config::logPath() const { return this->serverSettings.logPath; } #ifdef MTR_ENABLED -const std::string& Config::tracePath() const { return this->go.tracePath; } +const std::string& Config::tracePath() const { return this->serverSettings.tracePath; } #endif -const std::string& Config::grpcChannelArguments() const { return this->go.grpcChannelArguments; } -uint32_t Config::filesystemPollWaitSeconds() const { return this->go.filesystemPollWaitSeconds; } -uint32_t Config::sequenceCleanerPollWaitMinutes() const { return this->go.sequenceCleanerPollWaitMinutes; } -uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->go.resourcesCleanerPollWaitSeconds; } -const std::string Config::cacheDir() const { return this->go.cacheDir; } +const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; } +uint32_t Config::filesystemPollWaitSeconds() const { return this->serverSettings.filesystemPollWaitSeconds; } +uint32_t Config::sequenceCleanerPollWaitMinutes() const { return this->serverSettings.sequenceCleanerPollWaitMinutes; } +uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; } +const std::string Config::cacheDir() const { return this->serverSettings.cacheDir; } } // namespace ovms diff --git a/src/config.hpp b/src/config.hpp index 53247535c1..822c80ad88 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -18,7 +18,7 @@ #include #include -#include "server_options.hpp" +#include "server_settings.hpp" namespace ovms { @@ -43,8 +43,8 @@ class Config { */ const std::string empty; - GeneralOptionsImpl go; - MultiModelOptionsImpl mmo; + ServerSettingsImpl serverSettings; + ModelsSettingsImpl modelsSettings; public: /** @@ -64,7 +64,7 @@ class Config { * @return Config& */ Config& parse(int argc, char** argv); - bool parse(GeneralOptionsImpl*, MultiModelOptionsImpl*); + bool parse(ServerSettingsImpl*, ModelsSettingsImpl*); /** * @brief Validate passed arguments diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index bcfea79a2e..4f63384bea 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -18,7 +18,7 @@ #include -#include "./pocapi.h" +#include "ovms.h" // NOLINT namespace ovms { // TODO should we own our own copy of value? // diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp index eb5b213351..72a5821cc3 100644 --- a/src/inferenceparameter.hpp +++ b/src/inferenceparameter.hpp @@ -17,7 +17,7 @@ #include #include -#include "./pocapi.h" +#include "ovms.h" // NOLINT namespace ovms { size_t DataTypeToByteSize(OVMS_DataType datatype); diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp index 464d492b35..8439323204 100644 --- a/src/inferencerequest.hpp +++ b/src/inferencerequest.hpp @@ -18,10 +18,10 @@ #include #include -#include "./pocapi.h" #include "inferenceparameter.hpp" #include "inferencetensor.hpp" #include "modelversion.hpp" +#include "ovms.h" // NOLINT namespace ovms { diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index 983e85c54d..5fb8b6474e 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -18,10 +18,10 @@ #include #include -#include "./pocapi.h" #include "inferenceparameter.hpp" #include "inferencetensor.hpp" #include "modelversion.hpp" +#include "ovms.h" // NOLINT namespace ovms { diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp index 3167df7fdd..13efc6606d 100644 --- a/src/inferencetensor.cpp +++ b/src/inferencetensor.cpp @@ -17,8 +17,8 @@ #include -#include "./pocapi.h" #include "buffer.hpp" +#include "ovms.h" // NOLINT #include "status.hpp" namespace ovms { diff --git a/src/inferencetensor.hpp b/src/inferencetensor.hpp index d1c8dd3d75..d742b41c95 100644 --- a/src/inferencetensor.hpp +++ b/src/inferencetensor.hpp @@ -18,7 +18,7 @@ #include #include -#include "./pocapi.h" +#include "ovms.h" // NOLINT #include "shape.hpp" namespace ovms { diff --git a/src/main_benchmark.cpp b/src/main_benchmark.cpp index af10987fbb..b9d9d12c66 100644 --- a/src/main_benchmark.cpp +++ b/src/main_benchmark.cpp @@ -28,7 +28,7 @@ #include #include -#include "pocapi.h" // NOLINT +#include "ovms.h" // NOLINT #include "stringutils.hpp" namespace { @@ -41,7 +41,7 @@ class BenchmarkCLIParser { BenchmarkCLIParser() = default; void parse(int argc, char** argv); - void prepare(OVMS_ServerGeneralOptions*, OVMS_ServerMultiModelOptions*); + void prepare(OVMS_ServerSettings*, OVMS_ModelsSettings*); }; void BenchmarkCLIParser::parse(int argc, char** argv) { @@ -177,9 +177,9 @@ static void installSignalHandlers() { using shape_t = std::vector; -OVMS_InferenceRequest* prepareRequest(const std::string& servableName, uint32_t servableVersion, OVMS_DataType datatype, const shape_t& shape, const std::string& inputName, const void* data) { +OVMS_InferenceRequest* prepareRequest(OVMS_Server* server, const std::string& servableName, uint32_t servableVersion, OVMS_DataType datatype, const shape_t& shape, const std::string& inputName, const void* data) { OVMS_InferenceRequest* request{nullptr}; - OVMS_InferenceRequestNew(&request, servableName.c_str(), servableVersion); + OVMS_InferenceRequestNew(&request, server, servableName.c_str(), servableVersion); OVMS_InferenceRequestAddInput(request, inputName.c_str(), datatype, shape.data(), shape.size()); size_t elementsCount = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); OVMS_InferenceRequestInputSetData(request, inputName.c_str(), data, sizeof(float) * elementsCount, OVMS_BUFFERTYPE_CPU, 0); // TODO sizeof @@ -233,18 +233,18 @@ int main(int argc, char** argv) { BenchmarkCLIParser cliparser; cliparser.parse(argc, argv); - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; OVMS_Server* srv; - OVMS_ServerGeneralOptionsNew(&go); - OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerSettingsNew(&serverSettings); + OVMS_ModelsSettingsNew(&modelsSettings); OVMS_ServerNew(&srv); uint32_t grpcPort = cliparser.result->operator[]("port").as(); uint32_t restPort = cliparser.result->operator[]("rest_port").as(); - OVMS_ServerGeneralOptionsSetGrpcPort(go, grpcPort); - OVMS_ServerGeneralOptionsSetRestPort(go, restPort); + OVMS_ServerSettingsSetGrpcPort(serverSettings, grpcPort); + OVMS_ServerSettingsSetRestPort(serverSettings, restPort); std::string cliLogLevel(cliparser.result->operator[]("log_level").as()); OVMS_LogLevel_enum logLevel; @@ -262,10 +262,10 @@ int main(int argc, char** argv) { std::cout << __LINE__ << std::endl; return EX_USAGE; } - OVMS_ServerGeneralOptionsSetLogLevel(go, logLevel); - OVMS_ServerMultiModelOptionsSetConfigPath(mmo, cliparser.result->operator[]("config_path").as().c_str()); + OVMS_ServerSettingsSetLogLevel(serverSettings, logLevel); + OVMS_ModelsSettingsSetConfigPath(modelsSettings, cliparser.result->operator[]("config_path").as().c_str()); - OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings); if (res) { uint32_t code = 0; @@ -275,8 +275,8 @@ int main(int argc, char** argv) { std::cout << "Error starting the server. Code:" << code << "; details:" << details << std::endl; OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); return 1; } @@ -323,7 +323,7 @@ int main(int argc, char** argv) { /////////////////////// // prepare requests /////////////////////// - OVMS_InferenceRequest* request = prepareRequest(servableName, servableVersion, datatype, shape, inputName, (const void*)data.data()); + OVMS_InferenceRequest* request = prepareRequest(srv, servableName, servableVersion, datatype, shape, inputName, (const void*)data.data()); /////////////////////// // check request /////////////////////// @@ -339,8 +339,8 @@ int main(int argc, char** argv) { OVMS_StatusDelete(res); OVMS_InferenceRequestDelete(request); OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); exit(EX_CONFIG); } OVMS_InferenceResponseDelete(response); @@ -427,8 +427,8 @@ int main(int argc, char** argv) { std::cout << "Average latency pure C-API inference:" << totalPure << "ms" << std::endl; // OVMS cleanup OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); std::cout << "main() exit" << std::endl; return 0; } diff --git a/src/main3.c b/src/main_capi.c similarity index 78% rename from src/main3.c rename to src/main_capi.c index 8169fdd28d..21910e6646 100644 --- a/src/main3.c +++ b/src/main_capi.c @@ -17,24 +17,24 @@ #include #include -#include "pocapi.h" +#include "ovms.h" int main() { - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; OVMS_Server* srv; - OVMS_ServerGeneralOptionsNew(&go); - OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerSettingsNew(&serverSettings); + OVMS_ModelsSettingsNew(&modelsSettings); OVMS_ServerNew(&srv); - OVMS_ServerGeneralOptionsSetGrpcPort(go, 11337); - OVMS_ServerGeneralOptionsSetRestPort(go, 11338); + OVMS_ServerSettingsSetGrpcPort(serverSettings, 11337); + OVMS_ServerSettingsSetRestPort(serverSettings, 11338); - OVMS_ServerGeneralOptionsSetLogLevel(go, OVMS_LOG_DEBUG); - OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json"); + OVMS_ServerSettingsSetLogLevel(serverSettings, OVMS_LOG_DEBUG); + OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/ovms/src/test/c_api/config.json"); - OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings); if (res) { uint32_t code = 0; @@ -48,8 +48,8 @@ int main() { OVMS_StatusDelete(res); OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); return 1; } @@ -68,7 +68,7 @@ int main() { } OVMS_InferenceRequest* request = NULL; - OVMS_InferenceRequestNew(&request, "dummy", 1); + OVMS_InferenceRequestNew(&request, srv, "dummy", 1); OVMS_InferenceRequestAddInput(request, "b", OVMS_DATATYPE_FP32, SHAPE, 2); OVMS_InferenceRequestInputSetData(request, "b", inputData, DATA_SIZE, OVMS_BUFFERTYPE_CPU, 0); @@ -88,8 +88,8 @@ int main() { OVMS_InferenceRequestDelete(request); OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); return 1; } @@ -115,8 +115,8 @@ int main() { OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); return 1; } else { printf("output is correct\n"); @@ -128,8 +128,8 @@ int main() { printf("No more job to be done, will shut down\n"); OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); printf("main() exit\n"); return 0; diff --git a/src/main3.cpp b/src/main_capi.cpp similarity index 85% rename from src/main3.cpp rename to src/main_capi.cpp index a630ee8366..fb65880059 100644 --- a/src/main3.cpp +++ b/src/main_capi.cpp @@ -26,7 +26,7 @@ #include #include -#include "./pocapi.h" +#include "ovms.h" // NOLINT const char* MODEL_NAME = "dummy"; const uint64_t MODEL_VERSION = 1; @@ -72,21 +72,23 @@ static void installSignalHandlers() { int main(int argc, char** argv) { installSignalHandlers(); - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; + + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; OVMS_Server* srv; - OVMS_ServerGeneralOptionsNew(&go); - OVMS_ServerMultiModelOptionsNew(&mmo); + OVMS_ServerSettingsNew(&serverSettings); + OVMS_ModelsSettingsNew(&modelsSettings); OVMS_ServerNew(&srv); - OVMS_ServerGeneralOptionsSetGrpcPort(go, 9178); - OVMS_ServerGeneralOptionsSetRestPort(go, 11338); + OVMS_ServerSettingsSetGrpcPort(serverSettings, 9178); + OVMS_ServerSettingsSetRestPort(serverSettings, 11338); + + OVMS_ServerSettingsSetLogLevel(serverSettings, OVMS_LOG_DEBUG); + OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/ovms/src/test/c_api/config_standard_dummy.json"); - OVMS_ServerGeneralOptionsSetLogLevel(go, OVMS_LOG_DEBUG); - OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json"); + OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings); - OVMS_Status* res = OVMS_ServerStartFromConfigurationFile(srv, go, mmo); if (res) { uint32_t code = 0; const char* details = nullptr; @@ -98,8 +100,8 @@ int main(int argc, char** argv) { OVMS_StatusDelete(res); OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); return 1; } @@ -107,7 +109,7 @@ int main(int argc, char** argv) { // prepare request OVMS_InferenceRequest* request{nullptr}; - OVMS_InferenceRequestNew(&request, MODEL_NAME, MODEL_VERSION); + OVMS_InferenceRequestNew(&request, srv, MODEL_NAME, MODEL_VERSION); OVMS_InferenceRequestAddInput(request, INPUT_NAME, OVMS_DATATYPE_FP32, SHAPE, DIM_COUNT); std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; OVMS_InferenceRequestInputSetData(request, INPUT_NAME, reinterpret_cast(data.data()), sizeof(float) * data.size(), OVMS_BUFFERTYPE_CPU, 0); @@ -163,7 +165,9 @@ int main(int argc, char** argv) { std::cout << "No more job to be done, will shut down" << std::endl; OVMS_ServerDelete(srv); - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); + + fprintf(stdout, "main() exit\n"); return 0; } diff --git a/src/pocapi.h b/src/ovms.h similarity index 72% rename from src/pocapi.h rename to src/ovms.h index d9eacebc47..d458c16228 100644 --- a/src/pocapi.h +++ b/src/ovms.h @@ -24,8 +24,8 @@ extern "C" { typedef struct OVMS_Server_ OVMS_Server; typedef struct OVMS_Status_ OVMS_Status; -typedef struct OVMS_ServerGeneralaOptions_ OVMS_ServerGeneralOptions; -typedef struct OVMS_ServerMultiModelOptions_ OVMS_ServerMultiModelOptions; +typedef struct OVMS_ServerSettings_ OVMS_ServerSettings; +typedef struct OVMS_ModelsSettings_ OVMS_ModelsSettings; // TODO reuse this in precision.hpp typedef enum OVMS_DataType_enum { @@ -88,81 +88,83 @@ OVMS_Status* OVMS_StatusGetDetails(OVMS_Status* status, const char** details); //// -//// OVMS_ServerGeneralOptions -//// Structure for general options for both: single and multi (with config.json) management +//// OVMS_ServerSettings +//// Structure for server settings for both: single and multi (with config.json) management //// -// Allocates memory for server general options and returns ptr -OVMS_Status* OVMS_ServerGeneralOptionsNew(OVMS_ServerGeneralOptions** options); -// Deallocates server general options memory for given ptr -void OVMS_ServerGeneralOptionsDelete(OVMS_ServerGeneralOptions* options); +// Allocates memory for server settings and returns ptr +OVMS_Status* OVMS_ServerSettingsNew(OVMS_ServerSettings** settings); +// Deallocates server settings memory for given ptr +void OVMS_ServerSettingsDelete(OVMS_ServerSettings* settings); // --port -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcPort(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcPort(OVMS_ServerSettings* settings, uint32_t grpc_port); // --rest_port -OVMS_Status* OVMS_ServerGeneralOptionsSetRestPort(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestPort(OVMS_ServerSettings* settings, uint32_t rest_port); // --grpc_workers -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcWorkers(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcWorkers(OVMS_ServerSettings* settings, uint32_t grpc_workers); // --grpc_bind_address -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcBindAddress(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcBindAddress(OVMS_ServerSettings* settings, const char* grpc_bind_address); // --rest_workers -OVMS_Status* OVMS_ServerGeneralOptionsSetRestWorkers(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestWorkers(OVMS_ServerSettings* settings, uint32_t rest_workers); // --rest_bind_address -OVMS_Status* OVMS_ServerGeneralOptionsSetRestBindAddress(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetRestBindAddress(OVMS_ServerSettings* settings, const char* rest_bind_address); // --grpc_channel_arguments -OVMS_Status* OVMS_ServerGeneralOptionsSetGrpcChannelArguments(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetGrpcChannelArguments(OVMS_ServerSettings* settings, const char* grpc_channel_arguments); // --file_system_poll_wait_seconds -OVMS_Status* OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetFileSystemPollWaitSeconds(OVMS_ServerSettings* settings, uint32_t seconds); // --sequence_cleaner_poll_wait_minutes -OVMS_Status* OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetSequenceCleanerPollWaitMinutes(OVMS_ServerSettings* settings, uint32_t minutes); // --custom_node_resources_cleaner_interval_seconds -OVMS_Status* OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerSettings* settings, uint32_t seconds); // --cpu_extension -OVMS_Status* OVMS_ServerGeneralOptionsSetCpuExtensionPath(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCpuExtensionPath(OVMS_ServerSettings* settings, const char* cpu_extension_path); // --cache_dir -OVMS_Status* OVMS_ServerGeneralOptionsSetCacheDir(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetCacheDir(OVMS_ServerSettings* settings, const char* cache_dir); // --log_level -OVMS_Status* OVMS_ServerGeneralOptionsSetLogLevel(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetLogLevel(OVMS_ServerSettings* settings, OVMS_LogLevel log_level); // --log_path -OVMS_Status* OVMS_ServerGeneralOptionsSetLogPath(OVMS_ServerGeneralOptions* options, +OVMS_Status* OVMS_ServerSettingsSetLogPath(OVMS_ServerSettings* settings, const char* log_path); //// -//// OVMS_ServerMultiModelOptions +//// OVMS_ModelsSettings //// Options for starting multi model server controlled by config.json file +//// Models management settings for starting OVMS. Right now only using config.json file +//// is supported //// -// Allocates memory for multi model server options and returns ptr -OVMS_Status* OVMS_ServerMultiModelOptionsNew(OVMS_ServerMultiModelOptions** options); -// Deallocates options memory for given ptr -void OVMS_ServerMultiModelOptionsDelete(OVMS_ServerMultiModelOptions* options); +// Allocates memory for models settings and returns ptr +OVMS_Status* OVMS_ModelsSettingsNew(OVMS_ModelsSettings** settings); +// Deallocates models settings memory for given ptr +void OVMS_ModelsSettingsDelete(OVMS_ModelsSettings* settings); // --config_path -OVMS_Status* OVMS_ServerMultiModelOptionsSetConfigPath(OVMS_ServerMultiModelOptions* options, +OVMS_Status* OVMS_ModelsSettingsSetConfigPath(OVMS_ModelsSettings* settings, const char* config_path); //// @@ -177,14 +179,14 @@ void OVMS_ServerDelete(OVMS_Server* server); // Start with configuration file config.json // Return error if already started OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, - OVMS_ServerGeneralOptions* general_options, - OVMS_ServerMultiModelOptions* multi_model_specific_options); // in fact only --config_path + OVMS_ServerSettings* server_settings, + OVMS_ModelsSettings* models_settings); // in fact only --config_path // Unload all and cleanup // TODO: Should not be possible to re-start? // OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest -OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, const char* servableName, uint32_t servableVersion); // TODO add passing server here +OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, OVMS_Server* server, const char* servableName, uint32_t servableVersion); void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); diff --git a/src/pocapiinternal.cpp b/src/pocapiinternal.cpp deleted file mode 100644 index 6137983d41..0000000000 --- a/src/pocapiinternal.cpp +++ /dev/null @@ -1,119 +0,0 @@ -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "pocapiinternal.hpp" - -namespace ovms { -OVMS_DataType getPrecisionAsOVMSDataType(Precision precision) { - switch (precision) { - case Precision::BF16: - return OVMS_DATATYPE_BF16; - case Precision::FP64: - return OVMS_DATATYPE_FP64; - case Precision::FP32: - return OVMS_DATATYPE_FP32; - case Precision::FP16: - return OVMS_DATATYPE_FP16; - case Precision::I64: - return OVMS_DATATYPE_I64; - case Precision::I32: - return OVMS_DATATYPE_I32; - case Precision::I16: - return OVMS_DATATYPE_I16; - case Precision::I8: - return OVMS_DATATYPE_I8; - case Precision::I4: - return OVMS_DATATYPE_I4; - case Precision::U64: - return OVMS_DATATYPE_U64; - case Precision::U32: - return OVMS_DATATYPE_U32; - case Precision::U16: - return OVMS_DATATYPE_U16; - case Precision::U8: - return OVMS_DATATYPE_U8; - case Precision::U4: - return OVMS_DATATYPE_U4; - case Precision::U1: - return OVMS_DATATYPE_U1; - case Precision::BOOL: - return OVMS_DATATYPE_BOOL; - case Precision::CUSTOM: - return OVMS_DATATYPE_CUSTOM; - case Precision::UNDEFINED: - return OVMS_DATATYPE_UNDEFINED; - case Precision::DYNAMIC: - return OVMS_DATATYPE_DYNAMIC; - case Precision::MIXED: - return OVMS_DATATYPE_MIXED; - case Precision::Q78: - return OVMS_DATATYPE_Q78; - case Precision::BIN: - return OVMS_DATATYPE_BIN; - default: - return OVMS_DATATYPE_UNDEFINED; - } -} -Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype) { - switch (datatype) { - case OVMS_DATATYPE_BF16: - return Precision::BF16; - case OVMS_DATATYPE_FP64: - return Precision::FP64; - case OVMS_DATATYPE_FP32: - return Precision::FP32; - case OVMS_DATATYPE_FP16: - return Precision::FP16; - case OVMS_DATATYPE_I64: - return Precision::I64; - case OVMS_DATATYPE_I32: - return Precision::I32; - case OVMS_DATATYPE_I16: - return Precision::I16; - case OVMS_DATATYPE_I8: - return Precision::I8; - case OVMS_DATATYPE_I4: - return Precision::I4; - case OVMS_DATATYPE_U64: - return Precision::U64; - case OVMS_DATATYPE_U32: - return Precision::U32; - case OVMS_DATATYPE_U16: - return Precision::U16; - case OVMS_DATATYPE_U8: - return Precision::U8; - case OVMS_DATATYPE_U4: - return Precision::U4; - case OVMS_DATATYPE_U1: - return Precision::U1; - case OVMS_DATATYPE_BOOL: - return Precision::BOOL; - case OVMS_DATATYPE_CUSTOM: - return Precision::CUSTOM; - case OVMS_DATATYPE_UNDEFINED: - return Precision::UNDEFINED; - case OVMS_DATATYPE_DYNAMIC: - return Precision::DYNAMIC; - case OVMS_DATATYPE_MIXED: - return Precision::MIXED; - case OVMS_DATATYPE_Q78: - return Precision::Q78; - case OVMS_DATATYPE_BIN: - return Precision::BIN; - default: - return Precision::UNDEFINED; - } -} -} // namespace ovms diff --git a/src/pocapiinternal.hpp b/src/pocapiinternal.hpp deleted file mode 100644 index ea79b190a4..0000000000 --- a/src/pocapiinternal.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once -//***************************************************************************** -// Copyright 2022 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -//***************************************************************************** -#include "./pocapi.h" -#include "precision.hpp" - -namespace ovms { -OVMS_DataType getPrecisionAsOVMSDataType(Precision precision); -Precision getOVMSDataTypeAsPrecision(OVMS_DataType datatype); -} // namespace ovms diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index e77d560fd6..d4b4d84ed0 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -30,7 +30,6 @@ #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "kfs_frontend/kfs_utils.hpp" #include "modelconfig.hpp" -#include "pocapiinternal.hpp" #include "profiler.hpp" #include "status.hpp" #include "tfs_frontend/tfs_utils.hpp" diff --git a/src/serialization.hpp b/src/serialization.hpp index d663701134..b925aa6081 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -27,10 +27,10 @@ #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #pragma GCC diagnostic pop +#include "capi_frontend/capi_utils.hpp" #include "inferenceresponse.hpp" #include "inferencetensor.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" -#include "pocapiinternal.hpp" #include "profiler.hpp" #include "status.hpp" #include "tensorinfo.hpp" diff --git a/src/server.cpp b/src/server.cpp index 86f3dbc22e..67300e7fd6 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -48,7 +48,7 @@ #include "prediction_service.hpp" #include "profiler.hpp" #include "servablemanagermodule.hpp" -#include "server_options.hpp" +#include "server_settings.hpp" #include "stringutils.hpp" #include "version.hpp" @@ -338,11 +338,11 @@ static int statusToExitCode(const Status& status) { int Server::start(int argc, char** argv) { installSignalHandlers(); CLIParser parser; - GeneralOptionsImpl go; - MultiModelOptionsImpl mmo; + ServerSettingsImpl serverSettings; + ModelsSettingsImpl modelsSettings; parser.parse(argc, argv); - parser.prepare(&go, &mmo); - Status ret = start(&go, &mmo); + parser.prepare(&serverSettings, &modelsSettings); + Status ret = start(&serverSettings, &modelsSettings); ModulesShutdownGuard shutdownGuard(*this); if (!ret.ok()) { // Handle OVMS main() return code @@ -359,12 +359,12 @@ int Server::start(int argc, char** argv) { } // C-API Start -Status Server::start(GeneralOptionsImpl* go, MultiModelOptionsImpl* mmo) { +Status Server::start(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* modelsSettings) { try { if (this->isLive()) return StatusCode::SERVER_ALREADY_STARTED; auto& config = ovms::Config::instance(); - if (!config.parse(go, mmo)) + if (!config.parse(serverSettings, modelsSettings)) return StatusCode::OPTIONS_USAGE_ERROR; // TODO: Have separate code for each option validation failure configure_logger(config.logLevel(), config.logPath()); logConfig(config); diff --git a/src/server.hpp b/src/server.hpp index 19f41d0a9e..f103989802 100644 --- a/src/server.hpp +++ b/src/server.hpp @@ -26,8 +26,8 @@ namespace ovms { class Config; class Status; -class GeneralOptionsImpl; -class MultiModelOptionsImpl; +class ServerSettingsImpl; +class ModelsSettingsImpl; extern const std::string PROFILER_MODULE_NAME; extern const std::string GRPC_SERVER_MODULE_NAME; @@ -46,7 +46,7 @@ class Server { public: static Server& instance(); int start(int argc, char** argv); - Status start(GeneralOptionsImpl*, MultiModelOptionsImpl*); + Status start(ServerSettingsImpl*, ModelsSettingsImpl*); ModuleState getModuleState(const std::string& name) const; const Module* getModule(const std::string& name) const; bool isReady() const; diff --git a/src/server_options.hpp b/src/server_settings.hpp similarity index 97% rename from src/server_options.hpp rename to src/server_settings.hpp index b8418d1481..ccb1191d57 100644 --- a/src/server_options.hpp +++ b/src/server_settings.hpp @@ -20,7 +20,7 @@ namespace ovms { -struct GeneralOptionsImpl { +struct ServerSettingsImpl { uint32_t grpcPort = 9178; uint32_t restPort = 0; uint32_t grpcWorkers = 1; @@ -42,7 +42,7 @@ struct GeneralOptionsImpl { std::string cacheDir; }; -struct MultiModelOptionsImpl { +struct ModelsSettingsImpl { std::string modelName; std::string modelPath; std::string batchSize; diff --git a/src/status.cpp b/src/status.cpp index 346af48258..bb8a49f5cd 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -270,7 +270,7 @@ const std::unordered_map Status::statusMess {StatusCode::NONEXISTENT_DATA, "Tried to use nonexisting data"}, {StatusCode::NONEXISTENT_STRING, "Tried to use nonexisting string"}, {StatusCode::NONEXISTENT_NUMBER, "Tried to use nonexisting number"}, - {StatusCode::NONEXISTENT_OPTIONS, "Tried to use nonexisting options"}, + {StatusCode::NONEXISTENT_SETTINGS, "Tried to use nonexisting settings"}, {StatusCode::NONEXISTENT_PARAMETER_FOR_REMOVAL, "Tried to remove nonexisting parameter"}, {StatusCode::NONEXISTENT_RESPONSE, "Tried to use nonexisting response"}, {StatusCode::NONEXISTENT_REQUEST, "Tried to use nonexisting request"}, diff --git a/src/status.hpp b/src/status.hpp index 586fd4d19e..206d0dd212 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -281,7 +281,7 @@ enum class StatusCode { NONEXISTENT_DATA, NONEXISTENT_STRING, NONEXISTENT_NUMBER, - NONEXISTENT_OPTIONS, + NONEXISTENT_SETTINGS, NONEXISTENT_PARAMETER_FOR_REMOVAL, // rename to non existen parameter NONEXISTENT_SERVER, NONEXISTENT_RESPONSE, diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 4f57db26a3..23e8ec28f1 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -22,26 +22,26 @@ // TODO we should not include classes from OVMS here // consider how to workaround test_utils #include "../inferenceresponse.hpp" -#include "../pocapi.h" +#include "../ovms.h" #include "test_utils.hpp" using namespace ovms; using testing::ElementsAreArray; -static void testDefaultSingleModelOptions(MultiModelOptionsImpl* mmo) { - EXPECT_EQ(mmo->modelName, ""); - EXPECT_EQ(mmo->modelPath, ""); - EXPECT_EQ(mmo->batchSize, ""); - EXPECT_EQ(mmo->shape, ""); - EXPECT_EQ(mmo->layout, ""); - EXPECT_EQ(mmo->modelVersionPolicy, ""); - EXPECT_EQ(mmo->nireq, 0); - EXPECT_EQ(mmo->targetDevice, ""); - EXPECT_EQ(mmo->pluginConfig, ""); - EXPECT_EQ(mmo->stateful, std::nullopt); - EXPECT_EQ(mmo->lowLatencyTransformation, std::nullopt); - EXPECT_EQ(mmo->maxSequenceNumber, std::nullopt); - EXPECT_EQ(mmo->idleSequenceCleanup, std::nullopt); +static void testDefaultSingleModelOptions(ModelsSettingsImpl* modelsSettings) { + EXPECT_EQ(modelsSettings->modelName, ""); + EXPECT_EQ(modelsSettings->modelPath, ""); + EXPECT_EQ(modelsSettings->batchSize, ""); + EXPECT_EQ(modelsSettings->shape, ""); + EXPECT_EQ(modelsSettings->layout, ""); + EXPECT_EQ(modelsSettings->modelVersionPolicy, ""); + EXPECT_EQ(modelsSettings->nireq, 0); + EXPECT_EQ(modelsSettings->targetDevice, ""); + EXPECT_EQ(modelsSettings->pluginConfig, ""); + EXPECT_EQ(modelsSettings->stateful, std::nullopt); + EXPECT_EQ(modelsSettings->lowLatencyTransformation, std::nullopt); + EXPECT_EQ(modelsSettings->maxSequenceNumber, std::nullopt); + EXPECT_EQ(modelsSettings->idleSequenceCleanup, std::nullopt); } #define ASSERT_CAPI_STATUS_NULL(C_API_CALL) \ @@ -84,120 +84,120 @@ static void testDefaultSingleModelOptions(MultiModelOptionsImpl* mmo) { } TEST(CApiConfigTest, MultiModelConfiguration) { - OVMS_ServerGeneralOptions* _go = 0; - OVMS_ServerMultiModelOptions* _mmo = 0; + OVMS_ServerSettings* _serverSettings = 0; + OVMS_ModelsSettings* _modelsSettings = 0; - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&_go)); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&_mmo)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsNew(nullptr), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsNew(&_serverSettings)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ModelsSettingsNew(nullptr), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&_modelsSettings)); - ASSERT_NE(_go, nullptr); - ASSERT_NE(_mmo, nullptr); + ASSERT_NE(_serverSettings, nullptr); + ASSERT_NE(_modelsSettings, nullptr); - GeneralOptionsImpl* go = reinterpret_cast(_go); - MultiModelOptionsImpl* mmo = reinterpret_cast(_mmo); + ServerSettingsImpl* serverSettings = reinterpret_cast(_serverSettings); + ModelsSettingsImpl* modelsSettings = reinterpret_cast(_modelsSettings); // Test default values - EXPECT_EQ(go->grpcPort, 9178); - EXPECT_EQ(go->restPort, 0); - EXPECT_EQ(go->grpcWorkers, 1); - EXPECT_EQ(go->grpcBindAddress, "0.0.0.0"); - EXPECT_EQ(go->restWorkers, std::nullopt); - EXPECT_EQ(go->restBindAddress, "0.0.0.0"); - EXPECT_EQ(go->metricsEnabled, false); - EXPECT_EQ(go->metricsList, ""); - EXPECT_EQ(go->cpuExtensionLibraryPath, ""); - EXPECT_EQ(go->logLevel, "INFO"); - EXPECT_EQ(go->logPath, ""); + EXPECT_EQ(serverSettings->grpcPort, 9178); + EXPECT_EQ(serverSettings->restPort, 0); + EXPECT_EQ(serverSettings->grpcWorkers, 1); + EXPECT_EQ(serverSettings->grpcBindAddress, "0.0.0.0"); + EXPECT_EQ(serverSettings->restWorkers, std::nullopt); + EXPECT_EQ(serverSettings->restBindAddress, "0.0.0.0"); + EXPECT_EQ(serverSettings->metricsEnabled, false); + EXPECT_EQ(serverSettings->metricsList, ""); + EXPECT_EQ(serverSettings->cpuExtensionLibraryPath, ""); + EXPECT_EQ(serverSettings->logLevel, "INFO"); + EXPECT_EQ(serverSettings->logPath, ""); // trace path // not tested since it is not supported in C-API - EXPECT_EQ(go->grpcChannelArguments, ""); - EXPECT_EQ(go->filesystemPollWaitSeconds, 1); - EXPECT_EQ(go->sequenceCleanerPollWaitMinutes, 5); - EXPECT_EQ(go->resourcesCleanerPollWaitSeconds, 1); - EXPECT_EQ(go->cacheDir, ""); + EXPECT_EQ(serverSettings->grpcChannelArguments, ""); + EXPECT_EQ(serverSettings->filesystemPollWaitSeconds, 1); + EXPECT_EQ(serverSettings->sequenceCleanerPollWaitMinutes, 5); + EXPECT_EQ(serverSettings->resourcesCleanerPollWaitSeconds, 1); + EXPECT_EQ(serverSettings->cacheDir, ""); - testDefaultSingleModelOptions(mmo); - EXPECT_EQ(mmo->configPath, ""); + testDefaultSingleModelOptions(modelsSettings); + EXPECT_EQ(modelsSettings->configPath, ""); // Set non default values - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(_go, 5555)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(_go, 6666)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcWorkers(_go, 30)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, "2.2.2.2")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestWorkers(_go, 31)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, "3.3.3.3")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, "grpcargs")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(_go, 2)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(_go, 3)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(_go, 4)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, "/ovms/src/test")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetCacheDir(_go, "/tmp/cache")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_INFO)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_ERROR)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_DEBUG)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_WARNING)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogLevel(_go, OVMS_LOG_TRACE)); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogLevel(_go, static_cast(99)), StatusCode::NONEXISTENT_LOG_LEVEL); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetLogPath(_go, "/logs")); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, "/config")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(_serverSettings, 5555)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(_serverSettings, 6666)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcWorkers(_serverSettings, 30)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcBindAddress(_serverSettings, "2.2.2.2")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestWorkers(_serverSettings, 31)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestBindAddress(_serverSettings, "3.3.3.3")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcChannelArguments(_serverSettings, "grpcargs")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetFileSystemPollWaitSeconds(_serverSettings, 2)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetSequenceCleanerPollWaitMinutes(_serverSettings, 3)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetCustomNodeResourcesCleanerIntervalSeconds(_serverSettings, 4)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetCpuExtensionPath(_serverSettings, "/ovms/src/test")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetCacheDir(_serverSettings, "/tmp/cache")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_INFO)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_ERROR)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_DEBUG)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_WARNING)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogLevel(_serverSettings, OVMS_LOG_TRACE)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetLogLevel(_serverSettings, static_cast(99)), StatusCode::NONEXISTENT_LOG_LEVEL); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetLogPath(_serverSettings, "/logs")); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(_modelsSettings, "/config")); // check nullptr - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcPort(nullptr, 5555), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestPort(nullptr, 6666), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcWorkers(nullptr, 30), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(nullptr, "2.2.2.2"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcBindAddress(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestWorkers(nullptr, 31), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestBindAddress(nullptr, "3.3.3.3"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetRestBindAddress(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(nullptr, "grpcargs"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetGrpcChannelArguments(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetFileSystemPollWaitSeconds(nullptr, 2), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetSequenceCleanerPollWaitMinutes(nullptr, 3), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCustomNodeResourcesCleanerIntervalSeconds(nullptr, 4), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(nullptr, "/ovms/src/test"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCpuExtensionPath(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCacheDir(nullptr, "/tmp/cache"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetCacheDir(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogLevel(nullptr, OVMS_LOG_TRACE), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogPath(nullptr, "/logs"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsSetLogPath(_go, nullptr), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsSetConfigPath(nullptr, "/config"), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsSetConfigPath(_mmo, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcPort(nullptr, 5555), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetRestPort(nullptr, 6666), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcWorkers(nullptr, 30), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcBindAddress(nullptr, "2.2.2.2"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcBindAddress(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetRestWorkers(nullptr, 31), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetRestBindAddress(nullptr, "3.3.3.3"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetRestBindAddress(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcChannelArguments(nullptr, "grpcargs"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetGrpcChannelArguments(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetFileSystemPollWaitSeconds(nullptr, 2), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetSequenceCleanerPollWaitMinutes(nullptr, 3), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetCustomNodeResourcesCleanerIntervalSeconds(nullptr, 4), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetCpuExtensionPath(nullptr, "/ovms/src/test"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetCpuExtensionPath(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetCacheDir(nullptr, "/tmp/cache"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetCacheDir(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetLogLevel(nullptr, OVMS_LOG_TRACE), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetLogPath(nullptr, "/logs"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsSetLogPath(_serverSettings, nullptr), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ModelsSettingsSetConfigPath(nullptr, "/config"), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ModelsSettingsSetConfigPath(_modelsSettings, nullptr), StatusCode::NONEXISTENT_STRING); // Test non default values - EXPECT_EQ(go->grpcPort, 5555); - EXPECT_EQ(go->restPort, 6666); - EXPECT_EQ(go->grpcWorkers, 30); - EXPECT_EQ(go->grpcBindAddress, "2.2.2.2"); - EXPECT_EQ(go->restWorkers, 31); - EXPECT_EQ(go->restBindAddress, "3.3.3.3"); - // EXPECT_EQ(go->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api - // EXPECT_EQ(go->metricsList, ""); - EXPECT_EQ(go->cpuExtensionLibraryPath, "/ovms/src/test"); - EXPECT_EQ(go->logLevel, "TRACE"); - EXPECT_EQ(go->logPath, "/logs"); + EXPECT_EQ(serverSettings->grpcPort, 5555); + EXPECT_EQ(serverSettings->restPort, 6666); + EXPECT_EQ(serverSettings->grpcWorkers, 30); + EXPECT_EQ(serverSettings->grpcBindAddress, "2.2.2.2"); + EXPECT_EQ(serverSettings->restWorkers, 31); + EXPECT_EQ(serverSettings->restBindAddress, "3.3.3.3"); + // EXPECT_EQ(serverSettings->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(serverSettings->metricsList, ""); + EXPECT_EQ(serverSettings->cpuExtensionLibraryPath, "/ovms/src/test"); + EXPECT_EQ(serverSettings->logLevel, "TRACE"); + EXPECT_EQ(serverSettings->logPath, "/logs"); // trace path // not tested since it is not supported in C-API - EXPECT_EQ(go->grpcChannelArguments, "grpcargs"); - EXPECT_EQ(go->filesystemPollWaitSeconds, 2); - EXPECT_EQ(go->sequenceCleanerPollWaitMinutes, 3); - EXPECT_EQ(go->resourcesCleanerPollWaitSeconds, 4); - EXPECT_EQ(go->cacheDir, "/tmp/cache"); + EXPECT_EQ(serverSettings->grpcChannelArguments, "grpcargs"); + EXPECT_EQ(serverSettings->filesystemPollWaitSeconds, 2); + EXPECT_EQ(serverSettings->sequenceCleanerPollWaitMinutes, 3); + EXPECT_EQ(serverSettings->resourcesCleanerPollWaitSeconds, 4); + EXPECT_EQ(serverSettings->cacheDir, "/tmp/cache"); - testDefaultSingleModelOptions(mmo); - EXPECT_EQ(mmo->configPath, "/config"); + testDefaultSingleModelOptions(modelsSettings); + EXPECT_EQ(modelsSettings->configPath, "/config"); // Test config parser ConstructorEnabledConfig cfg; - ASSERT_TRUE(cfg.parse(go, mmo)); + ASSERT_TRUE(cfg.parse(serverSettings, modelsSettings)); EXPECT_EQ(cfg.port(), 5555); EXPECT_EQ(cfg.restPort(), 6666); EXPECT_EQ(cfg.grpcWorkers(), 30); EXPECT_EQ(cfg.grpcBindAddress(), "2.2.2.2"); EXPECT_EQ(cfg.restWorkers(), 31); EXPECT_EQ(cfg.restBindAddress(), "3.3.3.3"); - // EXPECT_EQ(go->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api - // EXPECT_EQ(go->metricsList, ""); + // EXPECT_EQ(serverSettings->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(serverSettings->metricsList, ""); EXPECT_EQ(cfg.cpuExtensionLibraryPath(), "/ovms/src/test"); EXPECT_EQ(cfg.logLevel(), "TRACE"); EXPECT_EQ(cfg.logPath(), "/logs"); @@ -224,10 +224,10 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(cfg.configPath(), "/config"); - OVMS_ServerMultiModelOptionsDelete(nullptr); - OVMS_ServerMultiModelOptionsDelete(_mmo); - OVMS_ServerGeneralOptionsDelete(nullptr); - OVMS_ServerGeneralOptionsDelete(_go); + OVMS_ModelsSettingsDelete(nullptr); + OVMS_ModelsSettingsDelete(_modelsSettings); + OVMS_ServerSettingsDelete(nullptr); + OVMS_ServerSettingsDelete(_serverSettings); } TEST(CApiConfigTest, SingleModelConfiguration) { @@ -246,41 +246,41 @@ TEST(CApiStartTest, InitializingMultipleServers) { TEST(CApiStartTest, StartFlow) { OVMS_Server* srv = 0; - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerNew(nullptr), StatusCode::NONEXISTENT_SERVER); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerGeneralOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerMultiModelOptionsNew(nullptr), StatusCode::NONEXISTENT_OPTIONS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerSettingsNew(nullptr), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ModelsSettingsNew(nullptr), StatusCode::NONEXISTENT_SETTINGS); ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&srv)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsNew(&serverSettings)); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&modelsSettings)); ASSERT_NE(srv, nullptr); - ASSERT_NE(go, nullptr); - ASSERT_NE(mmo, nullptr); + ASSERT_NE(serverSettings, nullptr); + ASSERT_NE(modelsSettings, nullptr); // Cannot start due to configuration error - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, 5555)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(go, 5555)); // The same port - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config.json")); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(serverSettings, 5555)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(serverSettings, 5555)); // The same port + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/ovms/src/test/c_api/config.json")); // Expect fail - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings), StatusCode::OPTIONS_USAGE_ERROR); // Fix and expect ok - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetRestPort(go, 6666)); // Different port - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(srv, go, mmo)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(serverSettings, 6666)); // Different port + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings)); // Try to start again, expect failure - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, go, mmo), + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings), StatusCode::SERVER_ALREADY_STARTED); // TODO: Is infer ok? - OVMS_ServerMultiModelOptionsDelete(mmo); - OVMS_ServerGeneralOptionsDelete(go); + OVMS_ModelsSettingsDelete(modelsSettings); + OVMS_ServerSettingsDelete(serverSettings); OVMS_ServerDelete(srv); } @@ -306,9 +306,31 @@ TEST(CApiStatusTest, GetCodeAndDetails) { class CapiInference : public ::testing::Test {}; TEST_F(CapiInference, Basic) { + ////////////////////// + // start server + ////////////////////// + // remove when C-API start implemented + std::string port = "9000"; + randomizePort(port); + // prepare options + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsNew(&serverSettings)); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&modelsSettings)); + ASSERT_NE(serverSettings, nullptr); + ASSERT_NE(modelsSettings, nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(serverSettings, std::stoi(port))); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/ovms/src/test/c_api/config_standard_dummy.json")); + + OVMS_Server* cserver = nullptr; + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, modelsSettings)); + ASSERT_NE(cserver, nullptr); + /////////////////////// // request creation + /////////////////////// OVMS_InferenceRequest* request{nullptr}; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, "dummy", 1)); ASSERT_NE(nullptr, request); // adding input @@ -328,23 +350,6 @@ TEST_F(CapiInference, Basic) { ////////////////// // INFERENCE ////////////////// - // remove when C-API start implemented - std::string port = "9000"; - randomizePort(port); - // prepare options - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); - ASSERT_NE(go, nullptr); - ASSERT_NE(mmo, nullptr); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port))); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json")); - - OVMS_Server* cserver = nullptr; - ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo)); - ASSERT_NE(cserver, nullptr); OVMS_InferenceResponse* response = nullptr; ASSERT_CAPI_STATUS_NULL(OVMS_Inference(cserver, request, &response)); @@ -443,28 +448,29 @@ TEST_F(CapiInference, NegativeInference) { std::string port = "9000"; randomizePort(port); // prepare options - OVMS_ServerGeneralOptions* go = 0; - OVMS_ServerMultiModelOptions* mmo = 0; - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsNew(&go)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsNew(&mmo)); - ASSERT_NE(go, nullptr); - ASSERT_NE(mmo, nullptr); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerGeneralOptionsSetGrpcPort(go, std::stoi(port))); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerMultiModelOptionsSetConfigPath(mmo, "/ovms/src/test/c_api/config_standard_dummy.json")); + OVMS_ServerSettings* serverSettings = 0; + OVMS_ModelsSettings* modelsSettings = 0; + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsNew(&serverSettings)); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsNew(&modelsSettings)); + ASSERT_NE(serverSettings, nullptr); + ASSERT_NE(modelsSettings, nullptr); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(serverSettings, std::stoi(port))); + ASSERT_CAPI_STATUS_NULL(OVMS_ModelsSettingsSetConfigPath(modelsSettings, "/ovms/src/test/c_api/config_standard_dummy.json")); OVMS_Server* cserver = nullptr; ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(nullptr, go, mmo), StatusCode::NONEXISTENT_SERVER); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, mmo), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, go, nullptr), StatusCode::NONEXISTENT_OPTIONS); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, go, mmo)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(nullptr, serverSettings, modelsSettings), StatusCode::NONEXISTENT_SERVER); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, modelsSettings), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, nullptr), StatusCode::NONEXISTENT_SETTINGS); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, modelsSettings)); // TODO add check for server ready strict in 2022.3 not available in C-API OVMS_InferenceRequest* request{nullptr}; OVMS_InferenceResponse* response = nullptr; - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(nullptr, "dummy", 1), StatusCode::NONEXISTENT_REQUEST); - ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(&request, nullptr, 1), StatusCode::NONEXISTENT_STRING); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(nullptr, cserver, "dummy", 1), StatusCode::NONEXISTENT_REQUEST); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(&request, cserver, nullptr, 1), StatusCode::NONEXISTENT_STRING); + ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_InferenceRequestNew(&request, nullptr, "dummy", 1), StatusCode::NONEXISTENT_SERVER); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, "dummy", 1)); ASSERT_NE(nullptr, request); // negative no inputs ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::INVALID_NO_OF_INPUTS); @@ -505,7 +511,7 @@ TEST_F(CapiInference, NegativeInference) { // negative inference with non existing model OVMS_InferenceRequest* requestNoModel{nullptr}; OVMS_InferenceResponse* reponseNoModel{nullptr}; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "NONEXISTENT_MODEL", 13)); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, "NONEXISTENT_MODEL", 13)); // negative no model ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_Inference(cserver, request, &response), StatusCode::NOT_IMPLEMENTED); @@ -621,7 +627,7 @@ TEST_F(CapiInference, CallInferenceServerNotStarted) { OVMS_InferenceRequest* request{nullptr}; OVMS_InferenceResponse* response = nullptr; ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, "dummy", 1)); + ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestNew(&request, cserver, "dummy", 1)); ASSERT_NE(nullptr, cserver); ASSERT_NE(nullptr, request); ASSERT_CAPI_STATUS_NULL(OVMS_InferenceRequestAddInput(request, DUMMY_MODEL_INPUT_NAME, OVMS_DATATYPE_FP32, DUMMY_MODEL_SHAPE.data(), DUMMY_MODEL_SHAPE.size())); diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index 9cea22f66e..548b904ee2 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -29,11 +29,11 @@ #pragma GCC diagnostic pop #include "../buffer.hpp" +#include "../capi_frontend/capi_utils.hpp" #include "../deserialization.hpp" #include "../inferencerequest.hpp" #include "../inferencetensor.hpp" #include "../kfs_frontend/kfs_utils.hpp" -#include "../pocapiinternal.hpp" #include "../tfs_frontend/tfs_utils.hpp" #include "test_utils.hpp" diff --git a/src/test/status_test.cpp b/src/test/status_test.cpp index 6bb6ef3d24..89396f6925 100644 --- a/src/test/status_test.cpp +++ b/src/test/status_test.cpp @@ -20,7 +20,7 @@ #include #include -#include "../pocapi.h" +#include "../ovms.h" #include "../status.hpp" #include "test_utils.hpp" diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 23cb7c9bfa..5033043912 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -20,7 +20,6 @@ #include "../capi_frontend/capi_utils.hpp" #include "../inferenceparameter.hpp" #include "../kfs_frontend/kfs_utils.hpp" -#include "../pocapiinternal.hpp" #include "../prediction_service_utils.hpp" #include "../tensorinfo.hpp" #include "../tfs_frontend/tfs_utils.hpp" From 218400c6291c5d8c92a99db795b5149094e086e3 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Thu, 15 Dec 2022 10:44:13 +0100 Subject: [PATCH 109/130] Kfs samples with ssl (#1595) * added support for ssl in samples * doc update --- client/python/kserve-api/samples/README.md | 18 ++++++++++++++++-- .../kserve-api/samples/grpc_infer_resnet.py | 8 ++++++++ .../kserve-api/samples/http_infer_resnet.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/client/python/kserve-api/samples/README.md b/client/python/kserve-api/samples/README.md index 7e7f1eff05..88c3c64075 100644 --- a/client/python/kserve-api/samples/README.md +++ b/client/python/kserve-api/samples/README.md @@ -226,7 +226,7 @@ python3 grpc_infer_resnet.py --help usage: grpc_infer_resnet.py [-h] --images_numpy_path IMAGES_NUMPY_PATH [--labels_numpy_path LABELS_NUMPY_PATH] [--grpc_address GRPC_ADDRESS] [--grpc_port GRPC_PORT] [--input_name INPUT_NAME] [--output_name OUTPUT_NAME] [--transpose_input {False,True}] [--transpose_method {nchw2nhwc,nhwc2nchw}] [--iterations ITERATIONS] [--batchsize BATCHSIZE] [--model_name MODEL_NAME] - [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] + [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] [--tls] [--server_cert SERVER_CERT] [--client_cert CLIENT_CERT] [--client_key CLIENT_KEY] Sends requests via KServe gRPC API using images in numpy format. It displays performance statistics and optionally the model accuracy @@ -258,6 +258,13 @@ optional arguments: Define pipeline name, must be same as is in service --dag-batch-size-auto Add demultiplexer dimension at front + --tls use TLS communication with gRPC endpoint + --server_cert SERVER_CERT + Path to server certificate + --client_cert CLIENT_CERT + Path to client certificate + --client_key CLIENT_KEY + Path to client key ``` - Usage Example @@ -544,7 +551,7 @@ python3 ./http_infer_resnet.py --help usage: http_infer_resnet.py [-h] --images_numpy_path IMAGES_NUMPY_PATH [--labels_numpy_path LABELS_NUMPY_PATH] [--http_address HTTP_ADDRESS] [--http_port HTTP_PORT] [--input_name INPUT_NAME] [--output_name OUTPUT_NAME] [--transpose_input {False,True}] [--transpose_method {nchw2nhwc,nhwc2nchw}] [--iterations ITERATIONS] [--batchsize BATCHSIZE] [--model_name MODEL_NAME] - [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] [--binary_data] + [--pipeline_name PIPELINE_NAME] [--dag-batch-size-auto] [--binary_data] [--tls] [--server_cert SERVER_CERT] [--client_cert CLIENT_CERT] [--client_key CLIENT_KEY] Sends requests via KServe REST API using images in numpy format. It displays performance statistics and optionally the model accuracy @@ -577,6 +584,13 @@ optional arguments: --dag-batch-size-auto Add demultiplexer dimension at front --binary_data Send input data in binary format + --tls use TLS communication with gRPC endpoint + --server_cert SERVER_CERT + Path to server certificate + --client_cert CLIENT_CERT + Path to client certificate + --client_key CLIENT_KEY + Path to client key ``` - Usage Example #1 - Input data placed in JSON object. diff --git a/client/python/kserve-api/samples/grpc_infer_resnet.py b/client/python/kserve-api/samples/grpc_infer_resnet.py index b4481a1fd4..a190eb0256 100644 --- a/client/python/kserve-api/samples/grpc_infer_resnet.py +++ b/client/python/kserve-api/samples/grpc_infer_resnet.py @@ -53,6 +53,10 @@ parser.add_argument('--pipeline_name', default='', help='Define pipeline name, must be same as is in service', dest='pipeline_name') parser.add_argument('--dag-batch-size-auto', default=False, action='store_true', help='Add demultiplexer dimension at front', dest='dag-batch-size-auto') + parser.add_argument('--tls', default=False, action='store_true', help='use TLS communication with gRPC endpoint') + parser.add_argument('--server_cert', required=False, help='Path to server certificate', default=None) + parser.add_argument('--client_cert', required=False, help='Path to client certificate', default=None) + parser.add_argument('--client_key', required=False, help='Path to client key', default=None) args = vars(parser.parse_args()) @@ -60,6 +64,10 @@ triton_client = grpcclient.InferenceServerClient( url=address, + ssl=args['tls'], + root_certificates=args['server_cert'], + private_key=args['client_key'], + certificate_chain=args['client_cert'], verbose=False) processing_times = np.zeros((0),int) diff --git a/client/python/kserve-api/samples/http_infer_resnet.py b/client/python/kserve-api/samples/http_infer_resnet.py index af5dc35f1a..0dab976e77 100644 --- a/client/python/kserve-api/samples/http_infer_resnet.py +++ b/client/python/kserve-api/samples/http_infer_resnet.py @@ -54,13 +54,29 @@ dest='pipeline_name') parser.add_argument('--dag-batch-size-auto', default=False, action='store_true', help='Add demultiplexer dimension at front', dest='dag-batch-size-auto') parser.add_argument('--binary_data', default=False, action='store_true', help='Send input data in binary format', dest='binary_data') + parser.add_argument('--tls', default=False, action='store_true', help='use TLS communication with gRPC endpoint') + parser.add_argument('--server_cert', required=False, help='Path to server certificate', default=None) + parser.add_argument('--client_cert', required=False, help='Path to client certificate', default=None) + parser.add_argument('--client_key', required=False, help='Path to client key', default=None) args = vars(parser.parse_args()) address = "{}:{}".format(args['http_address'],args['http_port']) + if args['tls']: + ssl_options = { + 'keyfile':args['client_key'], + 'cert_file':args['client_cert'], + 'ca_certs':args['server_cert'] + } + else: + ssl_options = None + + triton_client = httpclient.InferenceServerClient( url=address, + ssl=args['tls'], + ssl_options=ssl_options, verbose=False) processing_times = np.zeros((0),int) From d77eb9fa2c2b9ef5b22d5dbe9b99edf79221059a Mon Sep 17 00:00:00 2001 From: Tatiana Savina Date: Thu, 15 Dec 2022 13:56:11 +0100 Subject: [PATCH 110/130] Documentation: Start Page and Quickstart changes (#1588) Co-authored-by: ngrozae <104074686+ngrozae@users.noreply.github.com> --- docs/home.md | 103 ++++++++----------------------------- docs/ovms_diagram.png | Bin 0 -> 169474 bytes docs/ovms_high_level.png | Bin 0 -> 121586 bytes docs/ovms_quickstart.md | 87 +++++++++++++++++-------------- docs/quickstart_result.jpg | Bin 0 -> 93039 bytes 5 files changed, 70 insertions(+), 120 deletions(-) create mode 100644 docs/ovms_diagram.png create mode 100644 docs/ovms_high_level.png create mode 100644 docs/quickstart_result.jpg diff --git a/docs/home.md b/docs/home.md index f998ffbd43..d768453330 100644 --- a/docs/home.md +++ b/docs/home.md @@ -10,99 +10,38 @@ ovms_docs_models_repository ovms_docs_deploying_server ovms_docs_serving_model - ovms_docs_features ovms_docs_server_app + ovms_docs_features ovms_docs_performance_tuning ovms_docs_demos ovms_docs_troubleshooting @endsphinxdirective +Model Server hosts models and makes them accessible to software components over standard network protocols: a client sends a request to the model server, which performs model inference and sends a response back to the client. Model Server offers many advantages for efficient model deployment: +- Remote inference enables using lightweight clients with only the necessary functions to perform API calls to edge or cloud deployments. +- Applications are independent of the model framework, hardware device, and infrastructure. +- Client applications in any programming language that supports REST or gRPC calls can be used to run inference remotely on the model server. +- Clients require fewer updates since client libraries change very rarely. +- Model topology and weights are not exposed directly to client applications, making it easier to control access to the model. +- Ideal architecture for microservices-based applications and deployments in cloud environments – including Kubernetes and OpenShift clusters. +- Efficient resource utilization with horizontal and vertical inference scaling. -![OVMS picture](ovms.png) - -OpenVINO™ Model Server (OVMS) is a high-performance system for serving machine learning models. It is based on C++ for high scalability -and optimized for Intel solutions, so that you can take advantage of all the power of the Intel® Xeon® processor or Intel’s AI accelerators -and expose it over a network interface. OVMS uses the same architecture and API as [TensorFlow Serving](https://github.com/tensorflow/serving), -while applying OpenVINO for inference execution. Inference service is provided via gRPC or REST API, making it easy to deploy new algorithms and AI experiments. - -Model repositories may reside on a locally accessible file system (e.g. NFS), as well as online storage compatible with -Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage. - -Read [release notes](https://github.com/openvinotoolkit/model_server/releases) to find out what’s new. - -Key features: -- support for multiple frameworks, such as Caffe, TensorFlow, MXNet, PaddlePaddle and ONNX -- online deployment of new [model versions](model_version_policy.md) -- [configuration updates in runtime](online_config_changes.md) -- support for AI accelerators, such as -[Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), -[GPU](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html), and -[HDDL](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_HDDL.html) -- works with [Bare Metal Hosts](deploying_server.md) as well as [Docker containers](deploying_server.md) -- [model reshaping](shape_batch_size_and_layout.md) in runtime -- [directed Acyclic Graph Scheduler](dag_scheduler.md) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead -- [custom nodes in DAG pipelines](custom_node_development.md) - allowing model inference and data transformations to be implemented with a custom node C/C++ dynamic library -- [serving stateful models](stateful_models.md) - models that operate on sequences of data and maintain their state between inference requests -- [binary format of the input data](binary_input.md) - data can be sent in JPEG or PNG formats to reduce traffic and offload the client applications -- [model caching](model_cache.md) - cache the models on first load and re-use models from cache on subsequent loads -- [metrics](metrics.md) - metrics compatible with Prometheus standard - -**Note:** OVMS has been tested on RedHat, CentOS, and Ubuntu. The latest publicly released docker images are based on Ubuntu and UBI. -They are stored in: -- [Dockerhub](https://hub.docker.com/r/openvino/model_server) -- [RedHat Ecosystem Catalog](https://catalog.redhat.com/software/containers/intel/openvino-model-server/607833052937385fc98515de) - - -## Run OpenVINO Model Server - -A demonstration on how to use OpenVINO Model Server can be found in [our quick-start guide](ovms_quickstart.md). -For more information on using Model Server in various scenarios you can check the following guides: - -* [Model repository configuration](models_repository.md) - -* [Using a docker container](deploying_server.md) - -* [Landing on bare metal or virtual machine](deploying_server.md) - -* [Performance tuning](performance_tuning.md) - -* [Directed Acyclic Graph Scheduler](dag_scheduler.md) - -* [Custom nodes development](custom_node_development.md) - -* [Serving stateful models](stateful_models.md) - -* [Deploy using a Kubernetes Helm Chart](https://github.com/openvinotoolkit/operator/tree/main/helm-charts/ovms) - -* [Deployment using Kubernetes Operator](https://operatorhub.io/operator/ovms-operator) - -* [Using binary input data](binary_input.md) - - - -## References - -* [OpenVINO™](https://software.intel.com/en-us/openvino-toolkit) - -* [TensorFlow Serving](https://github.com/tensorflow/serving) - -* [gRPC](https://grpc.io/) - -* [RESTful API](https://restfulapi.net/) - -* [Benchmarking results](https://docs.openvino.ai/2022.1/openvino_docs_performance_benchmarks_ovms.html) - -* [Speed and Scale AI Inference Operations Across Multiple Architectures](https://techdecoded.intel.io/essentials/speed-and-scale-ai-inference-operations-across-multiple-architectures/?elq_cid=3646480_ts1607680426276&erpm_id=6470692_ts1607680426276) - webinar recording +![OVMS diagram](ovms_diagram.png) -* [What is new in OpenVINO Model Server C++](https://www.intel.com/content/www/us/en/artificial-intelligence/posts/whats-new-openvino-model-server.html) +## Serving with OpenVINO Model Server -* [Capital Health Improves Stroke Care with AI](https://www.intel.co.uk/content/www/uk/en/customer-spotlight/stories/capital-health-ai-customer-story.html) - use case example +OpenVINO™ Model Server (OVMS) is a high-performance system for serving models. Implemented in C++ for scalability and optimized for deployment on Intel architectures, the model server uses the same architecture and API as [TensorFlow Serving](https://github.com/tensorflow/serving) and [KServe](https://github.com/kserve/kserve) while applying OpenVINO for inference execution. Inference service is provided via gRPC or REST API, making deploying new algorithms and AI experiments easy. -## Contact +![OVMS picture](ovms_high_level.png) -If you have a question, a feature request, or a bug report, feel free to submit a Github issue. +The models used by the server need to be stored locally or hosted remotely by object storage services. For more details, refer to [Preparing Model Repository](./models_repository.md) documentation. Model server works inside [Docker containers](deploying_server.md), on [Bare Metal](deploying_server.md), and in [Kubernetes environment](deploying_server.md). +Start using OpenVINO Model Server with a fast-forward serving example from the [Quickstart guide](ovms_quickstart.md) or explore [Model Server features](features.md). +## Additional Resources ---- -\* Other names and brands may be claimed as the property of others. +* [Simplified Deployments with OpenVINO™ Model Server and TensorFlow Serving](https://community.intel.com/t5/Blogs/Tech-Innovation/Artificial-Intelligence-AI/Simplified-Deployments-with-OpenVINO-Model-Server-and-TensorFlow/post/1353218) +* [Inference Scaling with OpenVINO™ Model Server in Kubernetes and OpenShift Clusters](https://www.intel.com/content/www/us/en/developer/articles/technical/deploy-openvino-in-openshift-and-kubernetes.html) +* [Benchmarking results](https://docs.openvino.ai/2022.1/openvino_docs_performance_benchmarks_ovms.html) +* [Speed and Scale AI Inference Operations Across Multiple Architectures Demo Recording](https://techdecoded.intel.io/essentials/speed-and-scale-ai-inference-operations-across-multiple-architectures/?elq_cid=3646480_ts1607680426276&erpm_id=6470692_ts1607680426276) +* [Release Notes](https://github.com/openvinotoolkit/model_server/releases) diff --git a/docs/ovms_diagram.png b/docs/ovms_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..9f3ac71dd6de5d885d65e42c3bca28c8c7c66a1f GIT binary patch literal 169474 zcmeFY`#;m~A3u(gkaQ9yhmI;SohWBYDkMq@8OFrUA<5nA|Ad9oGUR|i3j14|8Vi;EIxqqJ zvf0hZ!bnJ{B0+N5SyV_QDf!xEquU|EQ#8r)2i;)|e&A>W(ddoRnX}>FZpcMH(9w)9 zQ|Nx>RDF~dU7vX7#@2+-#}7V#bV}u=cg7j)_3wX?Ex*4Xy{K4Up!`kfjmw+gJNy1C zlDm2IwZg;l`(|Iyeua;uHT0zKgK%_lYkDPT*Erd05vAQ>za}~fWZtd7@#&Sjh+s?J z^usl7L}}qf)@ERj|3-cEUF82h13y!gHxK!ar|`xfkx z@*h|3sL7t|>Vef1JW)QdF+GPrzFz^D{^HyFY)VqK?AFn`MF~-VMCmgrFVab_Q4_Og zOxJyosb(!i=7Es7jb_sQy$XAOI{TZIQ{ff$Ih-vX=smy47?u(5Jt`9!QHPu;rDi8cs$Oo@ zf5$C;*cNfN5jX%aA{Sfa*2BZ@`zbR&xA@i=U&OhR*~TfFrCuev1yiNYn@To9glr_YIx-wF8(sdtJ9uw2}7y5{XT0yHa)pJJLdQ~ z5GlO06UxVG6BkHZ1?y>_Y!ns}()QLbiX{bA70lg1Gd);z-J)ggNqF8OYQz__N`$XM)wyb71?vJu34Cnd zFJ(5rV@$dKPOCPypQ?}R=QmJ&stnq(0y*6@-H?-T)Mg51LF}F-nrZW#f`oaTO>>l7 z@#OFLneHVT))iDpv+fQkTrR(i()+$0*tOt4J5A2^j8j*`Y@=D{QZ~NnFsDOBeV!0 zR2YW*Dd4|YTn4ZI0nRXag5apg@v0O-(Kc3DwIL$K3Y^ zx1_U0^H4K>>RU>~p){X8zpn@{D3kM1tdfNq%rAK^2KT+J}AYUyjWC)p;qy&imI@!m%GcY2<^kxckTOO z=${38>>lhrJF<^f(OzupsKl_lPF7FYDb_6y22rEC_zo94jJYT2Mp{W2MYNg>h0U`E zG0v0|BrJv(?^7HY3R}x%Avb(8u+EV_tynOLrLaPJLVbG*qyXH2pBve zFUXVngFRMos1`M+;~AmeDp=24Td7_?%A3!lPrF?Ta zL2xl#H!h96c8vq>nMNW$pbH%1Ti`Egxc3zK4^Ju7@QIqPj}#FmG8B>?b=wJncJ0c- z`FI$R>Z@)qLaEP<*W9VtFjzXK1;hKDBO7dPQ}lhLmQqUM9SC5Hpz87ScoV)1R7+yO z3?1yDM_75&?j)p^KO!`3a}*&_F;A)maO8b(@(Y@caW&TEN7{b zq^FkD*kx3@>7gyxNi(Oq&aV{ccoM=u-kDkI!4ptw7yq~D z5QKo@dL?~9K`~X`)-}2|g;#mpx~dFH7OpJZJHH#tg4PA5y06(~)$uZ>u=Zo!{gq7Y z4IUTgst20-)r$NG!~)L^g9Chr^`(vb+(OKL0QqYp;Yg})FK)v?$6HWNmFL|P-eJKK zNc$%Gab0u_gC5>%*pZH=ev=vLj48TL3a>J-Y6Dxt!E^KDm@59mZhA7xl>~*eL4-2J zbUGGAX(#`r^F~cuL&%_hEch6ll~-@afRlDyrYb#fWUW?VMK=VX;jyuuz4uNGyhbjF z%vk5JFjf+MUQ9|Oqa%EjkEcG;sWYh}Qi48k?QolmYcl)&For>iV~%!MO){0y5kwpj z3w;%+)2`jl|6sFc4&iC^c+U-l?A0z2c8C6D;qZCg+&GO~kD%;lp<8SOq>Gx>z@ox*m@dsJAs&1Ge=yPgo`t3^)4ZW{Hxi<;eI+P{}OT-2yg6&l- zIdl$2d4L)e{mq&GkcpbL8!oI}{bHj-X?)inl&l{e2r)JYX8M;`zeJX$& zq(H01{j^qQ?5gY=bIlJWgCs_CmCyG-gXD92C5`QyY!QwiM?1~5hrCXz8%!Rq#Nr?9 zC%Zf5QjS?t-uu?FR|rxB@X7_xv!Ju*KZ^>}nF^({BoL{LLvq4*O)FEY4&uUZz#XUX z?aW(z;y*VjaLXn=yG~MbS|#(%IC;M3JoC??&o6VchLu~ps-i!Z3L|RSuUs-spnu|| z?Rca;0j8}|OTChC(#zj9t7t#&6u6!6}-&_dxTltsx0{H$(N2USJh=#NI$!8Rb)hdt85|ReC9P z#<&H^70YB)5`%{fPb`<}qHQMOz<-6fRBhLfblkR!%{zc8q|4@ ze9S^df%LZEN)L2*l1_5Yd-WU<;#le@nFnT}z!S6RzV z$4o)ACW8#@n)m*jU+KCb&tXwJcv|V;{7Dk#_&FR;ygIV|KQi^6TV_&E8Cd5mwwj`I z>DRwR+_BM)8fNlhaQ^(Dn>oDQImCVQ5g=S7^V3dQWZ2jNFiY7|Qz0SJs>_ux7 zwoDnuI!VJ)`gaS$K>d-8d|0l+F1Z=ItE;TISeK)HNX)gJTd+;6IaaD^VGP%$8qHfo zi_%JEJ2zGmGT$YDJG%qpGGJy0@11+C(IBf4{)a<^FHDBZDZUzphGu)|{0i|{A%Zui zL|)rS5H#+iiY#<~p-Z>%o|YBpj3BRD3B{?}K8v$1hf1Bf{gje~r|7pxZcNd*w&A$0 zl$>jZJ#2!3rm*jAPVC4#b3o2 ziU!{vDb1%&wiisE-LG>1kj3|zPrtN_?~0chjBz;%VBI`}HAvfy1+9&_DYsR30Uxe> z4~m$!2^d?{{M!K?$9f}ng?G8Ti2TMye&5mIA9-K2dd*JZKwE7PR zH;|8u+Hj;PXCAMqA*PY_=Mx@~WGX+Ex40nUL%UJ?Gdm*ez;pF7fgerTBT%p-leN9Y z)7<8rAWU99!UiSOOqm;L?R+VtIuK_+KgMA0JQVlQL}q03?~SFLIs}+U*q$`;v-Kb{ zz_lcwy5Q*|x=Ajg_)e)G;VLD2twUog>U1+{*w}}H6m)CFPMow8UV)#4?S-tKohScLW_%9pzPa)85e z1yyeyc)v2Z2)H3&1PBYJbQ@10um3=5oWhCb5dNGcK3?<-hU{^osF!_XVGr#P2mpLp zfY)-1ENC|Pq^MJRH+-w~!3Q?cT#FlHR+sJD$LazyDp>ulTbnj^WJnPRIz{#>b@Hcs z#*D5bqn}@+mU`@9YMG)~n;yg@WREAag-f>A#|!LmT#!NRjC*(}od zGz{*WZqIA$S5%oZIp2y@({Tw8pTZ171)e6yOJF0%B|Zgy`KVQkE?wm|Z|ULKUGf*A zB}UCygLc-W2ZY79wM5y;ghrnfM8h_djSD@z$HocXJ-Ap~CPDU#$;Z_`jCv%ZmXm&` z_Ac8v#5ZvLR zPe#bBp4_VKGFkP1+FB&`Ne}2e>=hbd>Vs_~Oj|@@a$@Qq1v9`K6$lOEQlmGD@(v7H z&FVQ3RnYRG&mwpW+i24UoF3$|-qqe=Gq1s)j|6p%fd9qTgF&^9^1~F#5~9<~a)}h( zGdHR!-8C{{tE5K_oHBn8@Nwa0hN12s}=M`=Nw{wf?_H_q`&P{0On zwHKaNvnA9&bKbl0{Qs{0g&prRWGh4hNAk2hdGQGfCC7Z_Sa^w18 zvu4vr4@uMpf_7QeLkXA3xTwt&W_?YTM}IWPz}90B;K}4RNVdIJ=hUlh-%sfE&=gEt z-8b7C;wGGQ@Pj6?R21Z3Zn@6n6@`uGbIb&?y0_;XCSFXR0u>rZcJvN%YagoCd9`dt zEDy}=RSg)IR};v zRwZRUy8fu@ufo-H}+J2;qZpjJ`H-OE^3ZxD|FRwkkJUu9n{8aG(&t{ zF8Lwa<4M_+p~n3i&$kH@fx{07pkJL;9VsL$p%I&q?Vq+q&bl?RdL+R_pH4GcF>}Ea zTq-I0gFiGq`sdAd3NT~R$x7(> zho%#RvP@e~26M|kLtHohM8dOs&tHqSi)tMZggHao4Y~;cHBNR|?|Yrv7lm{}=l+x| zLcW?5h?-|TeiEm^vnP}1<{v+PQPe^TG-C-YF8-_l%BhOp)Xteh#x=7s&=8jN8Kg$! z?1SQZdv%|0t-X?NxeFwh*JXxdPyfGX-zz%Ueoh`fkz;S*Bi^#LJaW3&<_UEY3QG}% z;h`-)QIZ}TvNB8-IM17EA#+7oXyC(ARTueZ?Wofl7yWE-vJXu4uJqE6PWfq?gBFa9 z$m@fuxE=JFRe^A()Su?7)PqCV!DfPL<>25Jjzl zbjPJ-rr`kdM&|Ipyy1-C!f)=EeB35sLCEk?5p#gxz6s%56+yY781kCNd#xlvAphr- zt`wZ>!Ie&8y@My(mkXpCRh`cC9^5tW##nX5+KQva#Pt|uF1QW55Ep7bAkdM};Seex zRo2E?DNy}m9cdCG8jvm3uu#gfI;5>Q?qX6{IY9mALTm-#_f3Ba(W*BonPXg5knSRK z%IVDI?Ah_(sms+}dC{fHxQ3ePhqQEo`DwfsNU7vp8CnYoyR8P9#8&TlDLh^~-YBPs z#&l5Cy)JS3%LOj&f5!Tmc=N=54n-0J34?J7Xtbp;>$pl!SK9|$XT_~!vZ|44FUZ;Ew(tT9aI7UXZ9!J1e7suQz z2-E#u9gAbIvNFWXU_%v?t3yr+Zv_!`puOb>U-g&PEoY_jsaS9p)*-+ut(b)1p<)%o zAB@hvj^NOv7k7hoZ!v9h1GdectTk&nEMyq0H{W)6g>fcUTIq&6?$L1b{8=iaeT@VJ zxyacKoLzpEc=KAYmv?b(-pv7huCXr~zQ_NQ8+mxe&>k_G8a0tcU>B9~2cr_^R~7YL zmbmFKyLZYC>{b(5NUvmLXT~?L_tc|Xvz4^pL9ivIQRtrQ?D3QjutGVp$Zs?DglXl? zN>7)$ZzZ>GTsQv7hCko?pVn*G_EMIv-Zbo5AvNICslun>Gw);LTEB{S^`EIXp21t= zJY`0NBELC-Mlb28D+4kd-!3qfmqpc3$~ad3_j0u>(GjLC+%rDaAL!<~s*v%x`D&7S zb$ja{k9L!D^QQ#2^naUh(e&}0j-lIoypx}vFfMLyqopRFthM}!_zaeIe}9Q(7lYID z86UnreMgT>X*yHo;2`js8q)$z{c*>+-vgedr4BRWHN!797RLkwmxfaFbYf-d!2Ah6 z-+B77mY8MNq7s$E)>b!puxrCE1-71GUX}MRyqX;7%CA(C)1$>jOO-BEG{{35ux}Gi z;}y`sFCW(hmIwe>1k^)S>qxG|P!-Fpj8O07PfuiM`x2S~@n6@PqEYHV zPdL*EHq!#p^5e0+RBagF!eh8bzL=He$y}WujFQgY=>($_FM=ah&QE-Y+U}tyN-V^} zsdWDL)fcbiwLWSsG$Q&Edct~1{Mxf3A$>t>HQpr#vOluNjZ?=tjuPd0aSy%AA$9_+ z;xzCtV}u-M1v)zv&6ma4Mc>X-kr~3De9W8XyTvr)_do9Z_{zinC~@zu$7_FVY-5LC zDqZkZroowr4xNZ;uun{NiGLFnHU-MNG^1-NGM8OD)cjljtLB`32sKfYzJzUEGSa!y zB3YPM#2OFrE#R>iS9xo7L*|*Ud<%O>`r3iWdj}gyu8tVX1qY%3_18Y^qDQ zFtDt-Dv#mjDqI_~urcUF zJ7lZ+gLLc6Ov1HfzV_zinH5>s;$nD8DXs!=4^3fzQTB|pyPvj(GiZ|iT7~Luu1fS( zt~}ql`V@{sf$1EMZ&{uzsw0sjAXhBkwpi*fwoD{PjF2Y!+_IAjS3+^E2t#DjLV!s% zy)0aJ;d{i^ubshlsMUK{%$4MYw>{(sy*2<2U?>I71I!^lrEGkU_K)ug3zeaE9wjiH z?d7y*knNIQEw6dM29ka6;?*eKgIWv+rMJ{OiM*B}&7qpj9ZJiCVh?ScnA6{aS=tJ| z0R4`h4q=q+aQJ#{9ys~2&?A>Av_i(5Bs)vy=~!6rEbNAX<2^c4UqP}-@ocX8ue0~2 z2OHepzQX7G%U|4e=FzfhnhKx@|S%rD`MAp4>}-(G_FItS2&)$wpP1Cq5N_8!kGXisRGy7^^{E~?Ka$x0Sxb70kY|%bI0V|s_zY~GzgNN>Nct!t7@-?*$ zAe+Mqw#aI4W%Nq+K#yXvfy|>GW51|3(5kfe>CA%=M>(9mv3K7ysT%2mu*5`uDU5Im zlXK|t?6}M2!r-m?dR-DT^Qi#BpdGC1eDF6z5246jqj%L& zHDUA&deul{z{&!J8zFDkVV_tVVhxx4E_>dTQM}%=@M#&!e*3L4I|t- zFbgGth}OQ8xm1X}d{mJi^%dU7l(E((W{_x?;aH&G(%8`=rM4fWJ9$U1Xe;lvCE#R< z-`HP+R|g0Wr6{)!KUr*`!+t2}(9`kWJQ8oAnDJ#XT`S+-Co~v{+-8y+_Is#UvcmKvz4{#WYwKlay}b!im}E8kJH*6e64fO7aR4U#{dBN;8A9-* zj;t`?b5yIZF-ddj*3zuQ;DXjlwV_$0n(VIuMzx2_e%Ui!}Ja5N!+W=($m%nezx^dnY6|~v(u!w#F`^QN|@oJ?o)OVBdA`W1F4@YY= zgt}-|#;`%Bj$>V_bENzeBC7&;qru9(YWx)SP3*G zstHNh1K?@yUmKqW*hyPO@WPK-NEV&tSP z`;f+_rOC@`pKlrhn{CQHv|i?k_C6ZyacbtrooGrwcKAu>QMlI1Z#DJ}gRrJoE57$u zea30Y-!nN{*_7YHMYs0cd$npsENePup7(R}U|j$c{XsfxMr+sJ8vzYwCqDNuM_>BN zM^dp<+zZcuIHNX4wQ!s{UMIt@<(Gx~D*P8(`NPH*Op7=*9GMK?yg#V!L2s56Q)`~* zk)jwg2fKSOS@8wnCdCDMQrTd$okY5+x?jmbY%s_U0#U?mnUs@Gcih)<6MvO##L3c2 zSXe80eE(MKFx)IS<>EG8^QL4chfzfLz^;ZHwYwbnRXEvqt>UpbAk2c@2O40pS3 zd3j0XyE&FqtV5@w%_)Dh*l5z3+lkA+-Z3x``MLDPB_=vG(5T>crCGvtcnt}WozZ9M z!B*r@_X!D^9}{4%lcMIR*9LPMMa=zrQ*MtpOrC_H_g}fO z?Ydz&;dYe4{MBloLVJI4oNGx772zDRy-Tc5><0)StHK@Z=*2?Gk}kP{fhM7~Uq5O- z1Bw`OAn8IzICd;2&4{2yk+oCQ5BM7&+EIM7YxjTizDI;H%cBEj&lWFsd6}XQLl0c= z9gJUaO`CTLm>K+ZmZ_x4g!2JP5V_@l08t~bpu3|qwDkK~iCXcBQL>8uYQ;HQTS%zM z8u~~A;W+rJVsFIYZQD1b`g zb2O_(G5dUc!ZzWm?%Qw2f+}ChlUZW?U5}Y@=Ui9R?A0@PWdp)OhB!e-M@s4Vug=_h z>K&KK9DCf+gDKOi#68t{h$8_$tgMd1&Y0WDtASdm*`UR($X4|S7lzP698_Xun7Z_YAchVLcU*E{BexD zO*8`F6C!5=vEq@^3i=sWEP{WkAK9FoLtV8%^~o^#Yg3U_dXL%4^;xK`gTB9GBd;lL z#z7cxpn?`VllxJs3ip|dDBo7evIqF-LtvbA4nH>z>@7P6U({u&!x9vtyn!ecuTmI| ztUDmVrt%)_#|+RygSsujv?}_OO_tfGEwaqcT~3UbE&Jw=O&SE;)x`h2)(qHQb3fS@ z&5nUsja-QI`IH^en(jj|=c!kwn!#j%N0*5ntubeLN8i21AK;yOXtS^+#U!U(Sf&x9}Tx0Pu9LegBJ-Rhuy*$~>2&911 z6=|HmKZe1CX>;B{m{liUC%D#NrEfLA6Ba(c=~6;Il)?w1UfUjb*udw%G++;_|5%a} zgFT(G6Y#sSx1;Ix9S=7>`k8*GXo;&{bZ{G>K}2YupH&@o#x3h&Z^S8=AU1yGSJldB z`b*|Dr`5qr3G?i_1!yt~FNca4*=o8hN$VvU9}sj1t%62wXnj^&f;;`Z#z#A zMf`+piu$=l&3R7Wx7eYug}cP?WXW8A*$K!94tIl8x5s9O>Gd7_;hYV6h* zar0*d-|fk9qs&mXCe>b_4)JSj2 zA)~0 zXF=|dTXtBGNSg&&k2heB!qqA_6V!gT;;dY2AO?6ju@idBw;K;Q%s$)f^dE6HHoWS> z+S%;Y+jifNtt?#h9W<=FzLaWvvhnDV@J?jh68IVDNFV*lir7PJddp6g4}h1Ej1f@5 zky<@CEwcP3-J;1Z=TOKqFXbmbpITnJ?;8q?fTYQ)jajVYUQ8%wbd^zXZHDCWq$vo1 z8m|Pz!xI)j$_%PGzm+QTPRhR`wB39QKQsZJ@?4*T4}RWJaRro*1LgyxO=iPG#XTr< zVI!ZGS!)3V8%Tl>c+-cV&6`HF9AxM2NBk>5P7C^wcaE2zOjyj!M?4AaTB*s(AEWx3 z1^@SY@zLLzp(-G~=%kbQ9jhy1O9?UYrPUst%~eq9$l-lnJCDzvtmDiYC7jR7XO7sb z|7(?$vlZ;%vt(NC^vk@{^-$gyu0Hq!C_b|;+`ga0*AJ#QM2Ybub|uCEI8GDF2*1@| z?tDpP^gDB;;SPd2Y{aTiNPCYBdwqE0NI8;%eSRiZkVw<9(CTixosdbt%#*)r_&^%D z``KU?jYm%o2LK;`=z@kV_r7dcae}D!AFg`Ji5IVTV;=o$c5N1y84}V?*j*`v|1gpw z6XOsH@i`nEoOMmBr6xU4hm4IH(gpN&;yvB*m<=7!a*O`Tk=*d$@eaMEv*ZKj2(E6A zJ@>!Vkt3TUsNmv^B$=! z1Q5(y|01ola@1s!t!$e^sY?~SWubaz*}$C`riNa+ito?j{&_#VI%OX-BZi2`WVo%5aQVXcy;vS4qe-~O5v-%+I6xY6NXcMgL_+0#-_W&9s8PIzsj!SkORj`PGl0H-TU>W8t*`Rum(!4LMhu&C)tY^M&HxkL$>P1 z+ik54X4kHhg?wCpGbC}H3|i-WGAe%AGlKeltVoBAZ}I>LsfqWoiW9M2zmfLzOdmET zbiA!FOutUHIbu9YX)AB2C>sxd*r$pn>O$)l6+Vq-Pwra%?=07lb?{j5{cY9>4U}(*em|iv-LAvE4+W4(Nh#RukBxYCnI5Z zmE_J-NkuL8lvYobLg9b}DIBW`xiCMBl!R+kf3jHkw3#6Pi$1ueyEJHZ3F-mRN&}(R zVc}jNbHHp?!`(}Y(Sfc@zcO)|J8vpY$MoBSYO3oJLPHBjxhJb#!QvgCpp%PVwb{!x z!I|yhL4);AR`nHo*8&V4w1L?uFIJ#dOUqYF$!`HR}{^r~Fe`pWl#q@GyeMVnWBhtp1o=t!7~v6!J28u9_o}C@4%Kjhg{iO zs}Yb0z2u|!bJb@?(+A=${#&8f2E%EA`7Ni&4a@29 zhJ|$)Y5DfMA3>_HA7FD$A1X3EeJ8eq4M2nhDz5G@YhZCTyq6Ahe9EPtX|EpIm21Wi zRhrlvc7bbb$hSX=hc(R<9CAR*f{y@E8W?S&t-FMX#@SO2Pp~^> zRz^lZY-*tAGT1# z$_wk{ac-@x%yeM-uUP;}10z5|W~hFbR<_DT((X^4Mz$n174N(jtLL{RGRmQxqw__)v2jAlrE$!b*%s3>iT)r6?PQy_46qSS0 zx-$v4W0*yL3PH??l`-GJ`RXBAaDa1lLb4Aj+}6XLjhiXTF;lshSJsr5s57apClTyrM9A2KjQq;< ztez+4B@&K$_fRYNg}g8NZq(O2c-0iFRriMHraalOy`|@JBz{VIaxU-5uRpT6MUgCV*T9ynBRW zobzXF>Bp_kA*|z~#IghPVf~l2#yLNWr}lDrm(;3#R%o)|+gpWF%wfy#l`K*m&P@GU zPtZb%3vc2Ob^Acqz->zR$LXE2{&awboL<8wF{*VUR0+RIGAeC!L8HM_)j0Qqr#$!bC#e^yPb+^ZkJ$$EN~sl>$iKv6g&vd z!hCYJE&Prs%N?YtH@$#4`CM-S@%zvUpe+|a_75M}i_C~80H)SCbSc)Yc@g>6-^yme zAtNlPCN5~=wXWFQO5|!oEMKKJfH$>%`m;ei2k+i;dS-agC9OI4yad@A+kNQ#baSy$ zI!p5atRwnV<>j{bx9Z$Jv9uUnYpk&GnW+hRvL0?X`Pwt}@3N&n zeSQbmnF|ikcROBf?c|yK?fOBV6Nw90n%|+WVjr#u&{9oO?gy%b`$`>)a(^7OR(s%C zbJg-?@ZZqolw8GVa?abR5 zH5+a@!oL-h6+Dl(N^d`-Ru>8&(Z6rox*-`SI*%lFUm9_m{s0H7Ykv(7Y(dc0);Vj* zSmgsLJFwQi{e=fk*~ncXO5|DZC{6z6CyKSbr@)HU|1bTL67|Z74?V0u^@}@Q->$y*2Zsn0rB8p@n?@eV1)4xKYNts&{b2mOGJw-* z+Z$22+SChME^)q!RxN5p{8|1vwy>(R@_hvx2L0w(<4f*N`bG95KiUc`*!!kQ)9N)@2!M7 zohS$knmXM?@NHJKh31SXD2d}nZ{RPGhh~8g4B&fKU2*&Dvh#S-)*aNP+-`3B_ zn?Ba@r2UQo}!X% z#)9lIGc?K7z^@QONKV(!M_m^9J8(!)&Emsstw{WM2rF#*lRKQnQBug($WW67_bLU? z1SHKjgu5=3PUv~~gWP}7p0>)xVF}j?@wK7+iTZmte$Wu?m34r%(gJv{c7w(0K<1Z~ z-ETUYFOhcW*(P3{Q8(f`6kYy+;K*mZd{@w6kGG5%A2MCyzu`Hv=_>prO~2gPRJ#8_ zr_O(AyxC0lV_P`Drv7TZJN^iqz?SmZkZey)X&xBhht!D>#qj3L+bNqRrIy>pu zmJ-4E9+NT3~K1#<-Y=ZoDcFJ&xI6gom--v1MiH0GRL7j8k+AkMSL!4DEu>3NdJ?QVGEh!J*F z5#Q?*_xHhS9%r<+B7~ED%pz93sD_#G!z=Zh(gld)LdLhh5j!t_3khZ)$&-3+@V7zw zhi9>Yrb}DkZ-DbSJ65*q=u8hSQ&Rj--k=?FoYA}MH-M7D2|u5>Wgp* z*dPPP8QUd=H!zaM0DpCipa?X=tR6EYSh%!}a5cQoSI3K~v!k@2HNFp*rfEE0?AnXQ zcAy4OXU^vt(CR>SD}Qhww$*(?@WMXPE$0Zg;D(hpeY+GHzD=*rCalr)f)^s(uCuTD zJ#qKBgawgK06ABu;rr_7rPOV%4`SDLR;txhcSEXA%3?>y{`{x-WZX6Y?7@7(SaI*K;jFB@O`v~sFRVqr{m(x0ZAeq%^shjz zDQX#!SWV2>@ovW&7aaU!a+EfmcYAwM^NyC~3gN(czKQRZ@msEls z!>;6dQB(wwhK{}cluTRFa!eC&S649x} z{@1fcdL`s21_#1W>W6JIO__bWyRol2xTNovd<0vu*v zze15CChjXCoOg^SZyu=eJd}_hGM=%GsN-`g)b2e59Ps9#%78M3j*W%J$@iy}S9iH|(;!f)AojFL(IL3m?nj z3nD^qP1leO9pzq>iMWG-Xu~;z#j0PMs;t2kqw0^KH}513QImfG7s2ZEOb?z9KU>Cj z?44*o_;+zEo3a*m;)!>y;idk!JC?EtTE7Y<{X9&cIw@@9sn&o<)k-Nql$-ydi9-B+ zmHsQU&n>m=l5fq{bHOO}2EW?p%3`A(wB^SY2SdsRW#QSb)OA97K3nl`0iWt%z+Q%- z9NiAC-I*KskPpP`V?gmlV@qyZ`5xq-nNH?pha|*I<_oIDs-7uIKm|gbyj6)_!XbMe zn?ey!tgGs<*58Zk=OIz<5u_qj%RSA>S;<*$m)4fEW{QScq7uLhQDeEu-AOLo#Uy$$ zog<9YW>+8bUci?1baj4|SAX*o>9s<7D6YBzHQOa{ ztf_3s^?J2STP6+$9f_weUdPdQ-8WO#O^&<=zuZA~#EVc|-nZ(n!C{Dm+B2KmNIbd$ z0B8N*LXx;Vh-}S;uI-{ng#D%;6*`KqefQ$MNQgM4H~YCZfC;K|d)SAVzNqJ5&C0#! zVHP_$r~S<<2a#5ZzNPj&d2$IlDI!N&EgbW6gM|ULkopJrkV5gdw`(jgG@#>=pCzva z7*2{FMk!ODRnu4D?@PiDuyD9F{3~4~^u$dxCKJXA`4hBw+@VO%GOVL!o_*oGOu`B) zW`fS?L7Nsnb*TH8)!}us_&bFOmpyWstk<8=&)i~;ALoF)W!zXP6ugws+a?{&ZM&~i zC)ovfT^R7fK|pf6ROYqwIX+29HnA=%C?m~)D36FV519XxM*zwml8>-&sJ+I;*vSE=t}5HpYc;7C}B2=X*I zI*uLw*19C`xLDlYv~Ps+Z9DD{>Bg?lh$G94hwvwf3m_VNb|Q>nFyOUKXrmqTO3A`y zSF^7}qMtRFyNx{eeD5hjKus_>FxvX~(kH$vwSwG>jo|QZ*=n3i&z5vyQc~q5^ITnWXVv3A^w3e1u zW2|9c8Pz6S{J2i|T8ozwxV3%z#Oy@w6vBVqc*O=Q8q_jg1+|@hv$+;dr;{x4t>0>U zeYHN(KC#0Zr9-@?dg&B!#4AF^RLiD1zK+{wY8vs_uDxcxC2IkLr7h0ck|YMeI9bdF z&OHtBve($tcr-aO!w*8@WVWx^O@G(b%b_ZyJ)$l0TDz?|~cBYQGd@0Mq zGFi2pa|V83;&=*%HPO;^TL!^bu;BD_M#oU?7eTml4PXkJvp3_U_2;kdZw6om**?(!&s5I$P<;jcbGWgYw?N^zF2yNB6{VFy znn43(EZAhn_8&&A*FHN_Pg;OtqA|O3LifpEK;8&~AFK$U5-p+Y$`?t%4#4EOJ;tm9 zFA-z}9Dkq(_pA{WiS5d=4Li}z0$$*x+7jZW>dpB3NV24bDNTP-kBP8dHqF!(ADNht zdh1;Z?90=shd^Us-_DNhkv=EGR49$JJ&Yu5u=rCLfFdVekeUKj)R$bm7zoD;5$sx( zIHuA~7dWgxpp5V%+ot)0zI#zk?E98sY-1KEH7wsg4ZuN2;ku#%l^ zs+8S&nU+cZVT&Gael>PXNXU0T&@AMy(S1SF<<<7HEIbB!x%&$~jbaZ<8;vPe^2QRSs5RM=Y9Reg^4@^YX^32si_I%@Rx7+A~>P{lJ3_KrG<3NaLpDrZ`5% z5#kho0zwh79Dlm=Y-nacwZZ>I*t-WZ{eSV}a;XrBYL$C8m6&eieklu@Pd;48z!r-%Ib$_xt_x_x!`3UeCQ==XoxV^Ei*^ zJdK~{-^B7CM;+b>`y&wH(d-#_CjGMJnjgI;NIzfXJi#B&eQhvh1+lEH{;4ihU`j(z zx+{Gzr6W9HZd~e5!@Xo?Wydi6`XoL?$C9&loM-$Nx*&#X@=|*aEj@=z$kfnCxSkPI zvdZDt!_KA4;TIu+6b1@ax2kLvJy^v7+Utn@+AGHFsx$_Cv9#nuN_dlQpRg%Vt!XK# z3L5(vu2#do)jn~?Xw3m_ggo8T)pL<}&RWgXvko+Er_IH!MK2@|lom597{NpHs?Wu2 z!(bx`ckM}3%>c40)9%MtnTH;XEgS*U_o>`L>WkY{-leUW=}Q$AtdH#-V|{4Y=wvgo zmQLdq2O){hgdspY!Np~JS;wd>{%#R1N_yI_FlJozzj^07#4LlHCcsq%+78f%K}0>IiaxW@ozI+8-n0P&tsSx?2?$FaH_h{o7mj!8{l} zuf++T*STe@Y|!mFJY<9*LJ(rPQJ|;FpD}rt$KAt+VqvAZ+C1ZjF{K2GDBz=ER~M=T zc;;d$qkenj*N?vuvs2bP5-GPS4*(JTIW<3PZWy>1M0YVVLe2zZWlMiDF*K60DS*ZF z3KtSh_hevHwb^sd6=cA@Q2O_0pSZZS2AEfBmu{#pqQZ-JPqH={*oqSNnGhE_|1r=!!K&qVau9Fz*;IOh)~ysCD}$PT%cCA`;!{$Wm{ zH)wown9SPZ=BEmI5x8^Tt3``U8+}&BOc=RVfcciZQg)n}*1{Dc&#xp55CQ@-PB4ba zD8M_4s5W$A$qe1vSQe?T`rc^ij{q=Fv>H?}y60G=u$V`{_z*PU78BR2gbgnde_pRdx|J8f7V1z(6|Gnqi|1%cv_5N1Rz-@jFFhtAm2Nc3N= z|6Y)N0?*y5-JX`-J>__i73x{WOLOnPZsH!OPZcopJtTil5Et)JAY}Q+@Y$F2N6jgp zwQGyXpxW)8tw0u`+K+=umwrA}P4%Qy!&4s4k|3PqztkRdc4Y|drdL^sD`mCbcLl%d zS=p%%K7N+2)_rO%sSH8i)UpUo_xx;)%WPBLpGlf;dCgW!eaN>wXV-n~weR^|PkZ9e z-3{~HXHXbfRnmBM>_9F4?s3S)YAk=1m8X{{IT#iS^E6wA)0Ix>f90=9h;~yRk4Lx{ zKPY>v+%6k9k9TDtEaA@ zIWhuq{&0JiwXRpe^`X7C9R$m)UGKLc2|p=V%_p*tLg4<6}3#W z;>Z0z4!TuUTP;ehAUT_K-14!%vo@Lq3xwG{+C2+|Jy-9=J(-r?Hj#g?bK`^Z&n0EY zrG*UNE_FdYHK38~TTvHEhv&(XNV{br1{mdO^7Qx8EXyde3+0wQIGv>$ceL z$*rISzb}FRs-v4ofM=Xqa5?d@n_z7Oa({YJ0)U$RUFLjzzNe2w26|%7g5J!e!QRqe zy0su3Qvoa}mIxZ@0{YEtY`L^xs!P2o>HW`Dhod2DBa&?56M0rod=WFk1EPH6$f^#I z$Xx+a8vJpgkP7>@!bGq@simXSY+`jj-vIn`C%TVEY+SxRcgJ-GlfWs`hHzUdF;t-!PgV{@7OSw!O|Ux zUPVQ;2&XSGi4oGF*?XTo92yvkdd!$|tJ`csgq)9gS7OI67XR9tI(YBL$%G4uWjSmK z=LgI`o!t7GH;ZV~V%P~ti;wFr0?4ow6K(bb@X>4If!Xg%0|twlK>5T?@U@nu$Dh~9AJOG1#B8bY=R zQ+b3lKiqx0y`}V6bCl~%nl{(#&zf(ae)F+R%Jjj8)&*Gw^Uj2YW-BWuyHET_yD3=8 zoWedgeu~fZmDD>3ua>s{W(z^_8XHngdF*<4_EZECXyZcbBSAB)7M~B_d zl8lrcJ}QoYg)NjG%YNJX5#UAa=#;g-QyaS7Abif`SspoE)WvTRDx^T58_ByLJh%VY z8tKcoN|>q2f_5oab5727`<27Sd}qh}FyLXpYZQZRs#|$#T^ZP+`_R09y@!gdz+Znj zURd164-0_CMY5z&Fjpf8wEO*_XOL)5%dy;eHY&N)5gakKMmPqYWY@ZUCcr1zI?#l}$mKn_VfX4N#*u65!H@1p(> z-h|RVst!P$;V~WWzx?R>*Km>LwK4&u>bGvEz{|TUp+;Dm=bb%4J#}b(94L8ihCnwY zw(c&o?6yax&HFR)hZJpA7_%xiUNdD?8g344w2bFZFh$KimYT0TEc6n|&n-1A9gAg2 zF7wq)XRt$Tb^h$oXcaMc&|irkVL7Vccra_KPK_86H_a*|=>2>_ZVK6+ZkN)p#z)Ec!RdZ$~F1=4q zVH)5sc$t6#uTwy5rXAa%zmCF#wCE6cK@M<0rY?>Cq6}40zU=IghiB@woRp~nJsfIk zJYoH7^eNH#n#cH!{_&v00v+U=MAz+=WxyXL%q{lp-hhcAupee;y&8QdN*KPqD+hz5 zmkoImMBA8L`y}mO5mF@#2}0c>&WAWzZTt1Fz^WyPm^=G`Pq#J%X;nYt_fpw)*Wu>@ zJyGePlBut)UH~!xvwIjssmQIiuQ&1|{clGA?ri!3R9K}<(daB1jWU><<9BfjD#IY| zmH6{setZeH8rC_MTfZ70$K*0Y)wwAUbI;>JOHZV50U#M3cU(H$-KcU#kpZ z_C!^K?2(8$-UFSQd9c{685wG;EPeqFLM#6#7o7415CF9DjL6)9wBD23~rsZW15%K9L>Sb=vyztyy6H+ zCvj(T5B^4*SWh+tD@w;jfF6u6c5&2KVp(Lv6aFc{Ok?er^T`NsdOyw3^E#;DiPL`r zfpbD(-e$!K=MI-;owi^ltQJ-f-sofViHP_Dc2cf|phDpG$TpEL!s3#c5k3F zc_>PaQk2ut=Q-e>zEA=m&2UX&ha4QE&dgu*54lXNR&>5Kt4vh?sJ<~Bz79x}3~Wn? zzn0ZTA#KjcL<|FzCYdPn1oR|!sPv2TN1Dqjdt*9e1;&y=TYEguVv8`_>-KxLVAJ#S z(V)OU+OS&{0QHRj#$Zv6fhCED#?$MDJ0Y!Gr-ASQtT{kT2bmBc{-x66pbB>PsH*W| zVMUdzeSc>SR6$6}(5?6hL_KM!!E0y7K&1ttaf`QsVx2q@3E z3;O<7ti};+AAw0vqn73W3taU!@hiwlWO;+PTbivoU|zwT|C-rvL)VO9X#Lwjw?NK=x$wDg zvK-CDvfD9nh-C`ZC;jx0)S31~H3l5?zYfhH0n|2id=AO?7YK79g8#Sr5s;-`v7%*F zFJ~mAn((F;c@b_=K_|s;S?1PU(bk4)*}Sh7t~^6v*$=T$yU+e(Pvd$u12A5p0iz7F zqJu9lI=B7$>HSQ)tb9ceXYcW z&&Gkl3^l>;aWdM+V%3T*y3hkt>J5qlq_wYJrif;*p%+j6xA9T;vplPh%QdK&w4%@l zUZy4LqL*vlaMo(&!60ZbZzc&ZtVReT2V~_YuGXp1eX0P;@6P%GfczStkoR@%>-?9A znoG0RHKXxS{_wo#h0M<-R+)(B%+H<$y`AG3eAy=(cVv48w&RuiPKhO+14fAtfHNi5 zFi>3gci~=w1Xt<97jnymo|S}&2KS)5FyW^c`m0&V_^9E3f&e^#phN6e)wY{g%Y?=>-{#kT=!_$HR;BY7 z3*1-g7!|YFGoikj$CjKjAy;f*l3NcDt$%BLlOVUq01iYIJ&w00Ke{SZvD$J0=JySB zo`-o#D0>;HIscTUeh#A>1j?FQ3$ow?V(sD#^o$Lla$4AK+V{U-9{xbYxm~#2J?pkU z5$hBN{Q()WnJ}CTBS%^pTM9Q{qN417IOmlggdPWZv}ND};-u7j_y3^Jo>siY)Zz3z z`gHKzs2^csTv*T^OIMoGhU>I`$zmMmgscGCa>Aw_fgix^FY=mC+ka!WT$1;>{Vu?O ztF~bOd7&eZ_$QA*N~|ZEXvtDR3WG9cNDN==H?C`Rgs% zvifEhxt)zqAHxz<0~^ah$=JK;G|ZPt=RXls{;*12y%-i|k`R!+VQK6|S|zvQZJ)J& zdhW&GbaG02u`qO%aEA%)-*h0)jzU6oOigMM*xKi{RpyL#m%?GsOh>GQc5IWNrcDsc z{D*ylkUdVf_2kd-ynkL;PT9lQvo?GXFKldouGP=F*ps+mbWjG+NVWyX7Mv}K%Pr`S zsIji)$@XtF?g!K;zJk1htSRJ{MD~)3evRjU4|W`Ti*q`^59L#b{7or(su&>*DDsw; z!m2zz=Upp5Bgew!ewcuqYY=Ssdaw-*-O8y&3Vydpy*l}Vl&pP^Vnqd1Gj%$0e-3|s z=l}SpEDL|?vMcnkX=Fgp`!p|budADBnJq+Ak2S(*(ozaVi~wPPlpqWMyE_Z#c#rq6 zWy#N8hz`nw&$ad!@^%Tmgd(mw_f+#Sk({V!zF;b_K3e6@2@RjHdpxaO?=_Cw;^^9b zr4rxK%e5$@PW$Y2Dtazt-@OR|BnU8zv2dli3APttLKw9bUyv@Q%}!FA4=UA^iN4;SZG3TqMV*D z{2U>@IK4{_n#q^nHtWb`uUTo=8a^Tiq`U+Orr_Sj!1bm_tXw16ZVyphJU(Pz$@b)bdrK~G>-z3|U>~zOs!18`(?zRxp!{_lQ+7^_!@-DH)JIo! zebAgc38>A;UhoIh; zxUF$nbw>{E6|EQm%^;dUeo!$~HT+6YX~c--Q!i4~ zs5qOMkytVHg0u=T*e(51;(@Ys59>n+JdhHJf|lj;A%b|~vvac`x?T0RObOle@L$;SjR_0r#N+`7jw~?ZGTG z4b8-JHojI8fC>R%CZ%fe|GB~YWJ43R;#%LaVii%%^&rJ>vyUHTWE~whN2a!Q^e4nG z!fbP}um{=LXYI2wwGejf63TEXA!_gwYnNmB^-tK4Vm|K!m@P=%wQRpH>nDe|{@CvC z(yCi2OhJV*l7h}NLzoFgkACdi?pKh#sq5Wn_+K%Y3kcDi|EzD0qs6<`2;IeIBM4s7 z?$*;m+K?W)rvgfuUFQk0{pxdHEJ=5H9{#SP9}6;1#OYxtKa`4N8{flz2%tHg7(kudmd`o(7D40ZMI!S7%hf(A)7?ryCYpj%MVp3-HA&DLg-utQ{>i)UD1F8TsSQayfXPj3S2qR}+&8D7 zpxLAgFHApuJb%LZsVUEbbsn!>VDBy5-N=qdJujYGws``XLtz)BRR+CM;(}!;xfk_i zKb49vs4@-a67CA+dIbECY*i-6oJBUcD*pEH zOD(DFymo$HZ(1?g+_OH*O)B7Y27M5ZT(ZrX(Y}5}6cFZ#enTltR#O54urr2KN|5D} z{}+T6_$QOpN)!gOc{CN%ygRU}cgrUO4<4R=qg}{5z;Wfg!)W;zX^8`M5Yt0bWYxWm&)O&b z5HhRc2L#CeFwri;lte249s;WykUC6Kdwy5*#(*2lo1WH?A@rlKlNLzwa7j#Dgcx}9 zo^iyd`3>IVP738{Yz}{Q=DNWeK404(`+X>05LYDb^qvkaEL#AyX=qi_9S;wYiCpz01r>z zOL(eY{>g_HtPQ1%>eiaOX;#r)s>>a0>x&`S*wXH2f^r#71)}g;fBMlwUgTNbsMu>g zEtf4Jss|A@Z}Lw)kb60&!??efp=#fXrVPV&n=lM{#vj&O%+FGk6Gs>y6QZ26cBa~Q z)(*IG&0WOfu>yIIx;`Vq0w*HrF!Ds4sRiAYp5L+FWTkInJJ;}_ePD9=C@c1WrJT&F z^4k%ir881213!eMB`3+u62ZqFr0F|$SI=K2+crEn>*uB;?eHsiC`M*~`G4~#4d0G0 zHS6sz=sw)@>LP=mpz{_aVC^rBNDi5g5Ypf$=G=eJBbDT^Jd&6$lhZML!YKBaS>JdT ztu*I^5@;cCpe6{la05yKSM`i~YJ9rRs^stOsVA`N7VAt27n&vd-&{*ofLZ`>^N-^*m|@KHK^$|a!v!ixrEB? zX43tyj6bBc7)cRvPTeYJ*#T7`pUPiT+Pr2j0C_AMf6`;d=jD_y|LLxoiJxogIVluo zKTDd)di>tX^#N&dw39{9yJ-t6g;~{7BBZ)Sr|EzY@o?!} zlclv#CE@2~bXI4PN0Dc9wTs0Tw(?_EX5q{rC5U9NcZT2P&3WsIC;!$$wDzHyMzp+CzChbF=&cA?|8t6(pC``uIMnkDRt8> z0OE8%3c4u0u*b6CVc{*xUZjt(V3#~}Gt-)a4_yVNaS*{I;+KBE3&OS*LnfZ@B@NMOObhW zLXp6EcDVgrD!pV%X;#3Jt!nFGg8cy)poO9hAgrij>vO)x>6`6QkN_F>^7daLl6xQN z26g!8EwFUJ1(ZhSxd>9$%J?G&WE0I7@2PXHVYM>BSy=l&HO-yT$5uV$5FU2L+0^)8 zgjViE8*R6CB&A09{?Wd~rG{O<)X>jvAyX?pRR_RjXCLpnQ5jv=@Ody6tNi0RNQ=r) zmwZo(YR+2ofb=-;fXw?{s-oPpRF{2n(~Xx=cccCp`Ho%Od)NdDw?@w^mlp zoFq0@i^L+2c(EWj{lp6x^`8BX@@-#F?XS5lM941G;10`7&HHR5%}QU(c6OkBjhncy z_5R!y&&9MVWL#O|*JnX=ZT?J>Nd`TEA;tlQUv)Y?QNpLG$gG@6CRHB{5lEO@e{XmmebM z#y_zfK6xlv++x6cDJTrmK2)NYv@Xjc#H#uxz_UF03SDm{}gTlau}I89t{8Y zlXgxlyvhDtH~zv+Z5*2EdVb03OnNgbIE5q!1imXizuq|~zH$D@e_$i)Q^Gk|(Uq$R z^*=Xgqi%LmY)&5sTZ%!DgcY)s`vt#0YtI3{ex;oLSQ?%h<@)d6?`VC)kK-krEGG_F zKcqF-gWXAAJ6vHc-^RV_z#l#UVF70x3-#I8r~N3fwf_u0c5Au(W9i|wbCDGZPg63T zk@!%Bvfs7+&rRI4O+&HOYv*|!gGU-PD|USwB89W0e^<$JZ?rx@ZL~B~_lGd*R#LDI zz!VCwl1?g$pN3JxvD)|kSo_YB90pv6DoAgFa2Fl+Ou>vxhdP>WOp9?^yW1&oyo@^nR=d$g;{ZcI-Jt1& zfj3XQoGmb8?hyE)rb;-zGT}lNeQlm?S-W;5uICFwQ1eD$&SgsNw_VaF2_{;hXL3RC zd9Gx-*&X-6+yYOR_6Eu^1y3e|4w|;1O7>g z(z`WjvS?_*sGdU^$;V)v6iZkwv>GhF7jIMQ@DV5-jMb>(2Kde84zJv*g(oL8WQ5D? zwJfk%uNU+Sqb_LVJ~rbZWH5WspiIjVil=yp?Dl*{o`ePWW^%|<#64!7YHEDK_C%0d zi=pn@Jev#AzpQL=FHc8Ktl5c;1I@JR$c4AB849wYS26ySDJPyf6#WJN&O2&2DZe+v z8#e4QyEBaUgcq2ADE7wG-{k_+ba&0g!}yY}UmSZbMe{kLoHgZJ;J#wz2PA`{Vi;f?~D%7y_G^fSx#*x=1y(9BQ%z3bWIgQ zfcbz!v-KC8s682I#MOP0VMd^yIZ|-_!56;IZec5tp<6g%Wi!MUX(&%Nu=#)3%S{6@ zokn|$7enR^vu-ef`!qHg#ZFl0-esnt{V=e=6e8^y2--j6^7ygS<5#xOmm8O?kLY|y z)BV2Wf{hggDAgsdN24>QLd(^px3jks^5SI6lr_0cOzN<+EQ6x353YgyBK-;_x3tiQ z{{G-Gb!&5{reiRcSNK3;PydCV(-QjY;t!hV&TFVISf6_xEKY{l1gz3J`#eo@Ps(3? znkyG~Gz2K>Kf3jq56z-*w$w*MRo0=ZcC69%xh?-E!=(#^(3CmOP*;ZXin5p|Xo@-w zmo{tNnusTa;^X;>k;kP;OnBciV>BrbwwfH?Zp$`*`G>li)yhr^_j>o5L0rI`*WPG1 zO8&9E16K7G)jK=?uk1V!Zjq#K;Xu6&puk0U`&k4NKz7jmZqJ#%lLy}|`R;Noc;lQd zbt3oTm~heR($zxIL8c*0!S=0DtH&+bW(cp%FEFF_&h>#5vNIue3tJp9c?8}n!zp_n zF>w!(mQC=pNxE)cOACQ-s}jcI&Dy5!9P7*|85wc-D{*8^<#n$&wEMQk@M%oF1R7 z6@)9=TONN5p-{iP0k z($>eeF59-&ug|)j(qDgf|2cvK{Vi>};)5EYW@@f}W+_BRKB`AfM7vpWF^z_aIl|z& zEZGQGe-bYRx=ZTD#aX#fI2$iwi60h66GA75l>l1B^OZ<`{0(b3ttAvcOt&l z`5uvxX1RmzJi*VU#{l3sbS*BVF7%(CNrRPL2q7##R@_b+OoJ_zU1EK(bc)U{VOI5e?YPWi;PvGv3TCz2dV zwO2bK`<>^emoD4(8Ntnyf?+Q`XGh$cIB5*-7T7w1rmw8n6R9avnl{Q#n+7R|e_;R}L``#$< z8rhe?T1obE`WgZ(t)ZGpqCfx{`Z*N_+mlL<%Fx;hn!~G||%bKk(O~fTm`8i^x6oTmV78=*|WJ^s;T%jz<@+c6hh-hf??A${ize!mm?E)O2;G>9({na!Ee4TRZ0 z>9JVh-Hn@SUH{qI27(7au_=?xeeI9;k+Wnf*0x* zOPZ;DaIOt|+4m18k7copnJqTeU7?qmm!SvPmtO*xQ{)mtSE$IrzRI)J_`-H0V|-{kTjL+l@ulDF zmak1^pZ&Q8=1pbDl9NRei3+pE)*%#I=vw1j>{Xh&(~F>h|Lf;v)vX-79*?stf46Si z*gx5llu+rK4rDh8gMVp+jCR{oM&0~zY5764QC0iUe<8sBsU- znnmgkKa>vB2suvQScc~jLz&)wBlT;4MW{*?&S<5VOE7`jlik`oD;j*3I~lrt7xlN# zB0mwX@viRCc=-w#u8U3gwc&C@Qxh<)?h>a-zC>*`-?2{lZy`IvTdc;r-Q1cwxMr#E zm@a)Xzjt`2J(8L}&&4cNSR78|%+{~<7|NP%{=Y_zeqdzPR$ov@kisvzGx#b|fkM}S z@uVY8@xxVH7YugM)-&dBwQkSV=TfI)oC}p9={B%s)h6#~!W3YxQC#J%{71Yo=OZPQ z8gD2~FP>{A1fAlSZ)S!q&ee{XGnKDb&=TuXX5!@V<58`#`FnI(AgVjKIDF_qU11KcS`aV#5k^s3Wu*JSFW|p_NpX!vdNWye-=rc z&Jbxb-G$}KeC;r#?8%+dY~339abYhp=h$Y?o`tK~q!o|VQ4y|JAKJ@~=oifxd-X9}xPZ;%ffa`;A1YlL?yAOq0?v;u7L*CpR26{rf1%F+DQ1cU6P)Up$ z&1;2;{n{ruI#t6Xbf&5W;V9~5qwH|gD88muQfJLKXsnlr@UPd`rf8~4G0pHPuC(J zE_8g&T!DVd4+7a3j%Nb*lDSE7EF@>;wBiBTbyX<(e-3G*t=2+eN23%_uR9sGDBAN0 zd6tKj!1^;1KZ_eB2{$lTMm_6xz_l$|6-sRI$^vqk1~+ZCt6r+@E36LyetjDMq)}YD zQ1i*@?oZfm`4@D*6NU4VmG{RWT|b$EEhLd+;pjLzt4hu6Rredk?W&b+*lp z4Ypfof6&?f{)nxu{a^=KDC0B&0+~ypTXw)t#bE4f{Q(73^WZ#Cy~u4D02QnboO5v9an`s%xvPI)o_`DcN(m zj4&un^&Y(gxT)kQ5S&@~QEYQ8hVrp`bA5lllI~J7l~m6inA9L*hFJNL+mG+zHc7Gh zRf<2ab(obvGS=|ll)1z*ZjcF_6U?Sl*v9OU0Pa+Hu2Z-08@!C}{8YrK>W|2~!5x%F zu&nX+6l+1ac1Iix0N=&M!UNh=6u!708e0pgOkb={D>6o6|6cU$Ouiy$w^u<2tJ^uS z1l|g=zbO_g`bNdU;Ibrg^fvxr!e1A>UVjm9f1JBhtI8*Jt5uE`F7Q*8c3D11TiVYk zHp3XMvw1%`>I!95nyR22bcbE9jJY%Vjaqe!wmD1TUPmo&qm{WMaP!<0GW(ia-_mSf zds*n}3UBYBwIX1~$|69#9Q(&Y>bOCvMgE3(q8iE$+CZ>MK-J$M=7aKo*Bgb#BXX zNtEF=X!6JeIA9_@{U%|!}}A)V@uEX=b+w7INRM;h|8}@)?xeb;tW{j z2UdFzM2y-TOxWLhMeXnnrDEkyFeVh;v4j}?Gp$811E)moK0%ZXxxig83MmT>9cNcj z%N3Q?uUu0$4qC;F8i&iUy={3Ms*ghlx?l*-Yl!1*u#m_Wh(6HMQjfm z^}U59#kD^Gdx`fv8Vdn-G=Q!4G*hKs++jTfm8XFi`AL;fgX@M>+XZRl2RqW%4(~}; zQmPH66LK8G%{@VNWK3_t35!-5g(#^bgZ+gy0kyhS!9l#Z>#-$8Hl3wssz4fsAhQ`a zh0fVBqyrXodtk7jv-u-_@oeF6CDRntGLUzteZ*^Ed1K=vKKLGq#)Zp_#@DTOUgL7M zCq_GZi(tO~$HF&jw<Xln_O;duQfZZK1kPToqsu&cwAN?OXk|+jO_&B!mA-W zJKq6|s^;HV1$?&5wQ~m*R!c+ly=FrT8MW30_omMreFvv99}qZoF-YT{^4+ zV6)5>=yDy{)7jebAq{40svvt`;f&QyF2AAOfL2x+boE4-C!X154|^T?^g- z_s!M^4X1$-AHf(}#1;jPUVX8Wz`%IgV=wCLlh#I^$1F4?_R4OtEOPU8xJc9n@J-Q! zN~VZFF77g@qmJb#ICorqP)0XoOavDpf?K_qo33|JJ8va(S0q_x(m2J+(NmG>UglQt zni2xs3;k@&s`~N9@(LHo>);nbej>kHp2Kg3u5=>uwxF&8zj=P7ASKf27NurKc%g7W@esoaSWB9Py0PETfq&vpELODI*N4 zZmngonwGB_#peD?Oq!6AocT-en;B!w>+D|{GRE-jQS8G`&JFuCm;>6}p7Yz+?MR0z z24TM@U3NBO!Fb0E}K%BEV^x`)=ftnLLSG+?CKv=@sZ z74p^q%Mi zk5k`7j7@eK*D*xKgX@R%{(CrN`ViOdf!=^T z>dHsT2H_qENyuZRu>!Wqxzr{l#5^~|{$t%{?sKk{zr;t`-mVBZy?E~~iyd12R{PnPJ%@J5H&*>T{_=Q~rjXr} zYoD#%58(#~A5#gV_7L=T?&_dBL7-$XSI!TdX{RsvP43J$xxIoZnPUEXM_8M)W=Y;Y zK3%`?jFfQJt#rohrJzS$s-_RSKB^NpxT5z_NT$|q%LR$Jx~DOcz=h)Z={>T#b+p~D zCl8oAIXOwlkKLUuI*wJ2Irlzgw{3e++l9j+BI3ML7JTyIVg`GGKfU*F@7?b_cD{Zf z)~8;2wwwe>6DyVIx#f1>+4*f!(q4mb#T7$HfV|z~`t@TFT_~b9*k6ogn-QYmxGpd| zQCzX-_}PN59B%R|JZqVIYSFanlnj2$Q$VOTBV2PK-R!{A>sl@@E+Wi`qWjg)rAj5* zTl~$;%p_>KU||EH$xqpm4S&LG@438Qvpe){W_jAr`h38$@R@Lqkr`*{207rW-5=uA zt4xRwZ>fy8cvcCG+x`>0)Rn?e6aTImKz#st3Ep5u4%ai@(8al+`Z_+~ba)xxsgmD6 zgHXOPFi0Q8d|#tI|NbiRgfi{mkn@4Q*S}D~rP>289Vgtl!=g8?0w}4QcR;Doa2^U$ z8ke@#;F)=l7O)Mx77?Mz|Ggyx`*%qcyQksOAwLx6!uO^beyb>zP(e!1{?$Rwyuu@2 znM>)gy<~;X6^fld^}+f~+xWvW*01>&7eM>5v&Pcs_ModvIq5I^ew~1cr9D;isner` zANy{IMh>3NpCBaf!`IL7h|cRqBLb$U#wICz@^7P$od5j`v~W6jE?o5Jz_LY*FC-xR zh=j+9m8A1Z1OCvhj5hjLQLMfcCu03X7miq=k)QRsd{n$yQL7JBOo@J#ziODj za`5~2>r*-8kKerKRgl!=XVRpAPk219OYV#vR&{^wDHs3rj@Gy)R+r51?Hknk!=I1~ zMW3}D9LM5b7CF=$voU=7?HOM*^?r%!%DjI52eJf_41heQ^nu2JwAf8$AQxe10EUA!;1Gd2@lh8y4&&Z-dx8>@I3^f>wM!N8$UY z(&806-pSJ%*_ZB{>q<&1ux-CO?U zRZ{r`^_z*>j9-W$>IyV7;h9&fadMW>#5vq9ci8XuDT^g&wM;RYBVPwE`IzM2T`{|P z-SC#u&%#2>6Hw;*uq(1^G`=$`VX6Tn`4vIjh1T(ooSQp?c1%1n_kn!jP;CIcqQyIs zkF$Ry%rEYj^5@Fyi_0w?6N!J7D4Eqc2c^wrU3Ef=Jmf1Yq&+0B-f^DeM}T`EFtvM8 zwB@8_ao`*n$fy2yAPtO74Tk2NUc%kI(;^Z}_oF+M$ z%b)Bqn8~}t$@G0!&-u$7fKO5Hfk$Yw@T*7ExZYvk^et_@Kg0ILz0$vX|6-O^pNIaK z{$s`0`Kzdkmc>vlriM-FdawB< zdivR`O!U25%Y*hoTNK1Yr`Z5>@^|R)siTQfBax2ATRjKA@7Hd<67xvYuG}=KlFzzj z@Y!p9)db`8VF6?8^|hRIxyffqC&n5JXnuY;z;%V(;`0FJECv~?-B(bqfM&=4F|UO1 z!;?-g;0GL99;7+_)f@V7HX!Wdvj1H7Gn)rF2O>TU40})PHE_|pajf_Elvdv%(1+jc z3V%jQUE=S&Tg=uhyJm17uM}u>>h!(e0UB9WRtFAcbQ|-7_C|f1oXv>wOsTJeHZPFS zVw8p97)IKqH;12`ynnzqc*rg*^?Weu)r1D^$sNhbhX-8GQ9RT7rv}I!zu&hrqvE?Z z2Dq}f5`KGhT)SrXG9US}f>d&{$n%x9hGFZ|XPBj>$c#_rSwf`h;HU$jedQ#XkF3LG z4GOYlRYPS-DnieR+RfEsWoMLHSFMf2=5IHTrqv$M{q1nQrt;Abe{aX#_vkLp zaU?7De%=oPgFfG2Qo0XbaUvi}Rpa8#&Uf@;!Ceyy140fZ;8c|jThK#xtV%>m)2L!4 z;MeTKCH^^ogS~Dy=Sa91C!c^)UrdpyLUFel^TX@SN`8fNOBXut#C*Tbq20!~>x&AB z4nYi{p?z18zxMtrv>fZu-HMa+WYNMzXsT0ZP3_7)f998eYjXeh*8=`jf4#MJ`BwFM zb9`c8VRNTQfZTC3y0tUn>)%d8>FG@S=nlToyOw6ROzmU8f9H1ynQ=vPyBm4WV^~u+ zmiFvF=r~C;rLQi0ggzNL1iC*x_bG6$J~T`dd@TGj|F&-72v+tA$v-gf4ykcp@N-cT zod2g*p8DAe{^eUlzCD9a!}k}m_Ux%A>-PsgPE}6--?7>&f6@v#;t%=Vp-$rUGhIld zX&cSXnN4%lfD6tpd~a7!<0})BNV{;8V2^zl#qoRY1#<@;;_Fu=(A@q$j&s|WO9%Ef zg$1*<@4Ug->!Q(9Ly#qG{gy=d^>bMZ2`%0VW{KuE{1QvtL3a<>_*IlJPyZg0yook? zG@uljGIppo=*3%R%`4}enIo7bl@%&Fj8KPMY8|fSB-PACfzRvt`Bf2gX8s7i)Es^B zqow5&;+B9xt`GMHoSN#t_(Rd+$1P_~J5~g7wz3PQ`wPDNpckKdUDA2N0K2(TX|`uT z8}`LZ;s3-dDN52Lxuis0{O`A}*<@AxjN800=I+umrA_+6-Su}Ey!okxNjnxK1{!q9IDjgxC0 zDZ7cE`;AGb*#Ae_Uq?mVeqX?_B8_xOhlDh!bT>##cZn!S zGjs?H-6AEWbaxCbT{?6QIW$9e^L)JT+wX6!=Y7|D*ZT)7=FhpfuCwvYuBH8#XcsSpRF{lDzMU>Idwl)cJ`E^fPq)F z<$zG>7pJ_s7V(ptvRhHPKuVekA$=V85(ky53GRi@EDL6a|Gr)Yk7~1yw9Bqv;y(%t z(YwV{Q&SbVnwrVLa5cq0gmI4J}Ssai|(S)+bB zUtO9udDMQt-asK^FDbX_YUFvQ_mtdONXRv;D`xL1InAzMB2l>uUL?`zCV8}svgdH!KbymhgoEvy-cc}XirX}7=6G&WI}qRd-O&7iTd zu@JkAvr5I?IDZvQ^ZqHYiPRffr#CxQN$dUb^tb2j z_ZFhD(LEbk_i&PT*t;+W-=lFC^~P%~aOX6rDn+*}%Z7oElVqXUy*Iqod(Pq(^0+aq z7_*app6&-0_fMPHoGr8BIu>@3bAOkecb@pekXL#GMMINKhIVLcO}z(N3Z>%+{Yzd2 z^eA!ce_l;43S@1}segDF4I_PGLaX%l^2wi5B7Pr`(!n2;3a{R+CwpxEzM#N>XEeK3b95))-bn8*x%az;-$$A* z1=ueL@#hxJ#f2X8ljhhU-1fvQZq6j(-9z}vpa?|O*JN$8sa2OML5Jp6(ZgCl$m;&U zKc}dp^WHF|L@E@I-<7_;g_@9X9526y@^x8rFbctv$zk=B4hOf@iV<<+`f0wTy$vOkkWoo7UpVu4+um=p`oUoHsC zuJZGTUivX-uI~y5;Qa9ZKo{V&){nnO%dCHzwCXy}wADEwkfDpmoKCXR2SieGqE3uI zIBXCyJ}f}N;4xK!W%pf(j=|@9ssr-(ri`-$o$qeh!ia3m2_MgnWsE+Ho+4S1Urj1@cJ>t6+p`>H;N2)Xt4&tNe!|u&08)z z14BeCxhO^zkh$)r>AyaB7a{zL8~lQ@A5w$6hx+J`cAR=$J5{bvz4MogfS}xH>d+%+ zdzDt_chSlp1ia#>X2s>@_Vij|*QU&l0)=*AmKs~%?bD2kn-i68aX)MJF!5$Vt*SVG z>GOwY12H;o0qqEbN?pU`qeCpjERM&na|{0m#>&)*;BmbzPq9l#tG=dYWX96IoTMZ^ zK;6MeC3cH7*5YljDvFN-dFW(L4%*4;=;AmjGM_vblycp6LcV+<_~m!(aM^PwKHa3x zzn+J#`ahnm@EIU7p9ff#_b&NB76|q>l$hm}_jer4s=7mpZ{B1pxe^d9ktCVzwRQD# zvC7K+uJYy+<0H=~jk)F`+4ydTaPJqZquFnK2vs?bygACbm^K@L(Xf)hshkFgNtfqO zpi;j(-PP~+Z_vS)JfyT&AtlD+=tGAE*6mRMgu`G@s&6c=)f}>|1i}Ttc;!tB$z>?A z#$2WC>*sQtI=UImQ$t_Wmzlm?DvnbZx>TW0V=M|G9E?+HObVQ$Ir~QGTR8ouxMd z`EphWgBn#SNfAvqY~r4n?4^J#D)#QJ@EAPumaH07Dee`Gf48*S;W#piBLiE9bk zB4%_9G#M(+9p@GLg*EN&-%@(i%6T=)I>sAK80#blt-2}p#uqq0k7CRE zbWPXz!j`j*35O@XsZE)8_jdsQ+zLK5_~npBofDxA7E)TXuuu-+mo{;z|20n02NH_} zfbZ>mW)w71Sq1x^&u(6xNw@dxFEmq(p=-=@=CpmcM9>2*Y_p^YqIAcV4BC!Ywo?hT zf8yT9Smi9}g#z#gcD7^;7L$gQ%4$Q`oE?tFLb_kERZ6<59=fxM;}~VP;lXlpg)<6B$~Kb-JA6{*W=ZS}LGzCWK%R@XSBs@&E$8xsHgB!~mCqur!G_JW55+4sUcAb6+_^J5xC^|;#i@=MM}VYEBv~ zCaJpO{2tD42>i~64CB9QJ=pU;OftS0uk{Yxf@Z{B*9hZh?3)(G?579R-W*72**t=~ z`N-39%J7I(g=(uZ$4&KVIi&5F)@4E)kPlz5#8SmrS~LvB?7aq^kxBl2t#}0cBfQs6 zy~Fd~OpiO?jh5Xc4t-t7-5FzMVxs5d!~xeb3@sW@(h>l0`Yehid<3&WT__dhYBcu<=Z3X zJR0u`Y)_NcqmxNj7u?0FryWw9896DvIg1y3O!To5nWugv6TkdPPxn8RcN84;Yp%SToREM;x{6(hm9)#3X&n$8|kcD5Z zQr0m5!hbWR^&RPlMx$R27oCw_h+4ZfrkJlcr(7nzQPmA4zm~PA!`0Ad9I~XHMN%-O zTDOa)zJB<(Z{Orrgd*(|B8}1NS~X6#^EbG|S%gZ~&jS^;-olR62hVLD6o;y;#$!d`r$33|N{ySrrDoohsMXm5NP6q*qeLp5G&HKVgAo)yEYgkE8E!uUqqG_)|= zBJAP4&-Byo3^6>X&Gc|csV3fQW7NlJQ;&Gp(NiEqIOmvz%4hZaX%b~Q%xZ0qEqkMS zcbT(nw30;HAwtU`XK4RswRS=0oTtv;AJOfhHOb);YOP4%xI5MmxWGZ?Op#8bg5|xV zj6xq(%G>T=-ZYffQdYFL%$S(AY(4p8Wa{wP{VI_Z{`GMy#d?vT$Fm=AYo;G=;TTTZ z&r*qJjLP^xNpbBi@pU!oR^C!l@*daR`p-(9WKWp#{)$C^y9wWu8CeV_($;zzuE!c+ z#D8W`Qa+(Lq^_?YOq%JrU_u(Co4ztg5%{m z4O~}S(WBqsTqPqam!h5K8#-s0O6TUy4Pc7?c_Zl|D!7748*&h}5(t%TqAb73bnNs# z-vtIKpYyAXHBJ(jO8?&@V;P}uNyLz$jN{Ug@X4Hl{v}9=27k>3q{vVd@wTY;YhQ+-)(a&{PmQ60taO?~hExXGzM39{u zl}F>b_@h~JFc-3-o*Q0yM%8Fqkx|6e;(97zv7XC4|RHnsEQ>_++9A}4Cc6~ zlwht^8@fs=D?f#=u1YG{zFL`!tU8a3e!;=f*FsA28m!iOX5+d$F=6$|^$n&}US&0T znsfVTZ%=Pew)X{8pvqm%Fp7qsmXr<~SMZCjc&H5#m1Ep$BwlB5q$`yWzRa@xPdCE1 zEslhSb1(Bd9GoF7d)+r@)H$$Z&KTt0`X{oNL!qXtX+bKjV4t>=M;jHe7zmueRasRo z3;Gerc^&Npr%n6?>(}fXV6D@perT`!e635w(H!*YFSD-%&#yP`wqI-s;baWyA~81q zO8uf%+JLOq{|Lj>^(;UY*}c9MRVp>#?nA{H|9|BlJSloC%$2^!{PB{Zx}gaJa3$sx zVSywc9HNeaBYII$VqFtal8P2cA>D2Z7F2Rd;%isxFN#S)}dWza}R}o z#Q^)H8tTd>j)!o9Lxm~a;YG3h*;AI>lG<-D&$B>$oOg=@y+HP7h|1Y^>%;DGC%QY6 z8GAwE9~H?(5in&j>s>-Fa|q24hVN%B`Aj&simTV72~?B4_}yTiL|!tgf=p26#y@G3 zkl+;vMib(q%)jaYku0l{^ZuGm5`9~ElX&qLm>61Ku(EdEqF}`*)KE4ydY;!y>*$b# z;E;ifG##Ygzki?5;_2y`^2*($>paSIJ`~7unxj(veQN1^er;(yd-eV`dN|PwFb%*+ zYP4+av~x%F-=&nkdgr9Jho0OJocKMZ?@Qcj&*xpSFtx4E5yxJ(lR)+AWxmQYQ6wt| z&*2iA?wxrmPgvcv$;QR(viy7!;r4~-z(9J+8Gm$?B!3^!31R3`G&v-{qq+S$Msj{A@DA3GAYOIyxv`1~@d>hMQJ z7_`^k3!m(Pfx#a0mqACD?cxtldV2CwE@JteU>Vcd zj8~3VHZAvK?R#ecjyYbmic4%@EJ)Jhf|essq9TQ6ezS$LWEnX0w>y*l4L6;*qr(>o z-upc}llM16!DgB5Y&5YV$K3R(f-a`DWlJct`uZt6t73TBQgkheqd@x7jqC}-ADjg7 zMypctha;f%j^Xys{t61Vhm)j@dUbFtZ{zt$_`S)=fKb#k!Dq7tI}16?G3}Jv6#PHx zzjMbK)BV5MtpbEL^OeTM&>tD8D9zBw^Drqb%{Zgdp1>KFCQwq}!)mfJQsf0V$f}bgL#yV8dhAvkLNplWX$`%^@c+2*-^6-{XiolqK#6Ccfy#(4EzRwR%l9 zODtK1FJh6fci*Bo@Z}P6PRJUF!;j+V`a~e_&j)27ami`S8N%&`;nb|(%fO$)t)%wC z#y~dP?5D0IE7cJDL9SzGdq-yx7Eab4wv(MrR5ew_&S5{#k_Abwl!qOHuZh|pfGvvq z;EAk-@GaK|sk%xewdMy-nLJcl`#yJMxD-)WaOCWzDlJ1g9tdz)ao|F}z}U5S-DODa%OnAt64Fp8Gz39V%(pPm^oeHVl!Tuc52QKm24Yzy?6>jf;|Ti}hP%B2gU;J&v6Gp4v zOU{Lt-1eQ%ynSIw2Zy7_%lUWQ06TOO5k)X+>FDSm_)<9nZ|cVq3v+;s*v?J6#MwSw zf%f!zle8{ZAsjN`^;gpAKcW*Cr>S`_K3mCwybr8`oFge$2@{By(+h-6Ew@8(rBQL^ zTxgQ<%_t8}78X+J>e%Ui{qPt|>{k9241TyPitT*y#yg%>i1v_kFH?@qQ02VsGOyr$WwC!jD^ z){wXbc=LPO{nLOj;Z5c4E4-m-S2ASEN^tl$+U__A5u5)>n*!dLQ5fI%;^M&%Dy2}~ z63kMj4jom8&e&iNZ`wDnVw*nx{4CTjQP+0SwAciKtD1WCeLP_MHbkB(@u?2ED| z;zC9Ux@?+5(@E39rY*$V{nyN4FO8X)t;z1WPe!jTt7UH6MeeQb$n=3|B?NMQuHGJM7vDBJErFSWG^sH>BFbyibcS1+UqJ5Z_= zFMa&|wY1S*P|NC1A~19x@%)G+1J@$QXhV{LP2+DZ2;|mU@}1XT7l(~4$`CL7tTQ`C zdCM8h_RuZIKU`N_1d1iyTf}u}?=>OS3~4zkc5mJc#_OPY7(W5>6q#YeeMl0$2SxWV zEyKdefwsK&Uf0*8zltKFO5)prA$ofNgq`%`UBog{pA@WqjFa7l!2NeF1vgL%Fu<~3 zX);Ei0Pg7Iihky9)kli_L2+m&aoEVjwoDp+E9J#NpFFKCQMgy%pZ10%`9-{Q-xrK0e z{J`;B9*DhF3)(UobPQ|ClEZJ`$h{3W=Zxy8T|1Tc8>lAuR8%Sq(ubV(6wP?Twd}WD z3xlx0J&>J`y%i&>JBn(bG9`ny!xs;X3iS6(blgnW*6q8P+1k2Mazfy_i=y~XZ!W02 z2_H_7Vu*-lA8wAOjle&;pvWJE3zsmRsmdtKaIf-2Mx0^VY$`LtK*D&M#F3ikRh0MT`ugbwEm=n^SIX>VK87O26D<6}kME zTzhfE%@2uc9*0Xj&>)dn1M0@lkA!RW@>F_^GTIIGig&zq?)FdWCQj{A&7j8oXb_fI zLo%YSyKxR)d@{u!t*_3MBc%Ch0-G+Elp8@v0A$#)?dxuGVD9*X4kaD~_O0R7!yi)K zH#-Q? zzfEGko1%g3B*j2#ZsB<^*d)sg6sQ*?w&weqeqZgqrz>Yu!x<0Xu?P)=~`oO&xOx`Xmg6N5%F3urr5>tHr2aWQ$nGm(Ktxn1|iMomnsx1M-I52t817ZN?zdt}eO zyb4|`KWu;P-G>o+>vdkBNGu@&{1O5ZD(b;bN}m)kq0t<4eH5PMj{1&?9LU-lGQ@Tn zA~r2e9DY<-F3wM9eXI8USKRDCRssmWWBgaEs=^opRDo%VFPgHE2CkZSizGH~?@T=| z+o}8BDB-!U79YgQzOnfNOkv%_t+d$myY<~8VHbTHkxaco z&TjHE2$DvNUG{&tOt-_gr@6S)fVfg}bifFONVn`$+NdT>1^+6^3LbI!=s$zMH8vWO z&+;})ge@N%!J!1fy(EZ6ij&&)Xfj~~3NiCrZES>;50?X4<+v$pWlsIeVD?esdr|k#JOG{EcW;xTQqW!4 zAfhhIp?k<>sahxjv3oUe(B#f4>o{hwprG(UPq#H%zbq>yosVEx-TrQ{V>739-{7vG6fzmr+ZS7!lRyUmU$An^1)8($BnvE$aeG%+(C!0SOS=5;5lW6r z{tJt{8I=@M4jnaWtr6e886Z5vfyGlQJs8aAK-m#xR8>G{r8uS7CDpc?_H`pX-y^&4 zH^Hkoj|?`ijO1q($6=X6XqjOF3&IxX5z+k%hGxp{HnBi(3>=S7&78m^p;>xN8b$tz zt5e7aP@%Gy9qG>34*FdDy@|1wJf=x1(6B|R&<0BmkIL`DB96p$v~%|aH$>9ioVko# zx~ypTp463VmvsQgp%P@b=IpxC^x9{z})7`X*+aRdnkwpv) zF7>^3qG_Wp?2WRVmY2NuKH|K)Jdj|Q1JiDQ|Mq#Ffco1I>LSItzsUelz%l*=JEjsd zXhS89q7K}!7|P%C@S0L8EG?}@iA)2=n4Ai)GsNk99@fyKn~Qjh@Lw_eCax1R9?tIU zr=RXLwgmnY=eF%2{rqwqkTC!={J%t$qV}tB3P0YL@+G z{zzUqcer-x^1n3q&qY;}H`Y8P=~_^@^$k-uF1`w9zEg74Ocg5OO7ND}$gD&z;Pp76 zlWE?;HvO@Is4HaS_((*W_$?;I?+Ko;hd?%*aD@5=dxcElgu#Wqq<)?TI%!nXmQhQC z2g;iWrK&^I1M|a1Yg!(bQ`;6{f13N_Ev)*w#^od$wNCxbT2r1Mb;~Y9nOhr}xX9v3 zVMy{(X9SI~rjcx0*h1;RO+xL7F)uM~hcRib-%@~xiRUhxvY!{; zJM-G`Imql*#)thDqq3t)5I&J8`jb+0b^s{_s=x0ioQTLbN4+JHPUF7`@i3*3{cqKV zVf2tngVtHrEP9WjuFV%MPBj3shSk{`(?mzAvkG;u3JUW+pDn=0?7Y5B z%8`OpUXqY(5Hd$Kp#6Z|)Sag$)U-&lKdf*xSPsBtKXAU|M?mlQ91-(rF5(Q;Kkycj zARr>o;F1g#Cf!2&sTJpRyo(Rmiy~ST#?d?4G}+~IFhKRleWpD`0z=ptcPpuVqHeg^ z4tL7@s4AQzUR`}d&*h%Br*j^tA?@_`AQL_0&KnmV{@~eB-$^(;b7{+2-Ge?>eJF_O zPsv3;IrX5hsj6#NuvR3J?kE_mIFi_Pi@9otvCtg2Z7e}*?Ugl>$cCHlG8T782l!b? zjfIdH)7rXXItjp?9n~W}t_1f2pb8e~Ydpp8X?`6l%G3XMN;LVZjRW;i5=}or`!N)$ zOPwBvP&#SUm3VnS+f6JKx^D5~{w|Nhj!^PW@Op!u7WnMkc>F-Zgm$UH-zQ3>}0EoOfgdf($y`fyXQ>2#_7qt*8elnftZ<9zf{ z|B_M`#|E~-`%}p-azZwgeejalVvfLmGtXc+ZLEKN3zehrs>w`2EsnqUmXemG&dO(oK_ z2G?itsLf1Yrg!-Bl3}Bfq+HcQZmu!tBbr?N96 zN>Q=Y=kgJNnXY{musnKu!*i7!kF1_c7*LDJ#|Nb_8!>k`VfTLFz|meiAFiTJ?lB>o zob>W0v;%48rub2{+^Z3_Jr4q>jPw=s%D6-hNi6Wg}2JV$4+!l!TTFiAc43e$^VJ}43AfngZJ!r!ye6Z;0Vfj=ESLC74_ zK*}{AGc+42a3C)c2Kb16E7O+0q2iHXewQ}ihg$srMuy_gQcmo{qUVCp@Yw6Fgf$u* zaI>4vBn++QKQgbBRkX`0T0Kf?lMgR1lyJW1q%17&O>cj|`cs=%>Wl6^JQ?QMXxzE6 zJlZap)Qz*~u!!T!9Y;;xk!E+Dz zFxL#p%DIza=S|+~2aHAXL%|3UA=jG>E6(}mC6`pc6TP4-En z@e%v263l^Ew>S!ith&T?3Lf!ebf?qDnJnQzuC(x~qQrZFy=PKL5>&T1wPW;Oy#A>6 z&dO?5$@aO-_D1z<=G?~bv;9so=ud=_LvVzgI@XXDB=?`@09dA8BS`w=?cHyZTC4v^ zH9e<bjj|9=f;Sg7;8~d}MkDAHF=i;{`SZCjO!K0%xqb=c4!v z<KWBiE3+-q|hamv!}yuj>wq2?kQJ5I+#?S>?3;O;x@~l??_7zc<(i*7k13~-M3Y} zB?A7|CDG~Pk%JH4cdJaejWqJSZj_h?8Emt6x~{7uJBZofQUNebg}F$;_xAsIuUi~b zZ;!?GRe`zQ(*84-BS(=-STpz42FBKWUJjMN*@O8nZ@f9jwD1HeScoLmHV-Y?kg_`K8~F50#laz>*wiF=>T=jgt~3QAo6fNLanD06$Y zW8S~NILf3G20F|Ar$9q^? z<Rl-?T%3^qX~6~i1__7Ot_S;DcrrIcRE)>YJ7MSGZEtClX>7f*NoJF& z#bR!#B=6ty2_Vx6$a}oXMtu5WOtK?Xi~^l6GC>L?eD7@dVh|tQ)-gd=!LXpE36)WM zvar(6veaZnHT-A`4_+4*f$Q7$yD;ThEvYJ0q32gghy|iU%BRU!obP~3#!7|bEg^hF z_&G|D>Q>;r@xnrT%>OU8IqH^O>P>&-{X}wp#YKiv21Y}ALS)Ce?yxTI)*!-ywM5`ac0Dog_I;M#r(9}S<{(@m_t+48vClmCxb5wl=X&g+VAznj0WqoB`_qnX_9*D_p*{ zB1fj6-F8-OT66~o9c>TR>0S1E3s~G`MdAks7B81Yot8viWN;Ajvgk7Mn+7d8RWW1nqf;8L=`n%3$= zeY^B+hn#@gr`3zB-d-hQGpu#&7nw|?neSBHjN-6~%gmwH*QQAxzFxIVz1$YF> z3$T^tUdUukpfLylqYltrCo7nxhy?n3}l<&f%CPm@OVy%AFj^F9tVrwK!8x4=r!olNW~drY-Cp+AD^cUY!-R`#OC8 z7p`(8=tN|0^=z8;EgKt!-C;VykaX4Z^LN?+w}3FZ$aF1`pb$%06o2}z{0kZ~ixP6fZ^Yilf_On3;UBk8=O zzwaW$pn6n=hvd;Dd*K{huB4O3) z0=#yhmC(;U$VREv`woUl1ocF7pBE)NI@ALlN>+m{2+-E1c6_)nfGeARCq09;I4_Ac z3c{+mlRbr^kOw^^!~@|1%ahT&YCP{LJR_UC$aW5YGKwj?`XmmZ^6ZjwKc4ns zl$;(Q8<)-p=3uFVQ=(e;OUif&{Hc2jCsgqic688QYO-v$y<^G=V&k0u3y(3)xNxuy zae2OzH`-{TEPD8-cU-taSWag!y1^c{&>)s*`;q7(Kc@d-9R~9YaA;U^vopxB4xZFN z^)9)fSV@cb7t1R;y2pFLUBroW?(jZ6`ZzD-aQ@sk$GE#Z%W zki$YY0dUhxCFQ99^~2Gr(cV9AP-~`sxR{~*w9xV-&ADMEb+%$ze!M&9G>X9@G~~=X|4xy58!0L|Y>&~P zQGZ?7)Aaxk9M>;jMx!;o^;^TC$@AhLukAli9l&l4`!EmC3s3!Df=)tC5~^%{B`1NH z;6xMeAN8r%OM|vHK6z|_8=&OD2WIle4FEaHK|BZB4yb0}XL!S%QUj$ot7PQtZf-H;X>8Egt^IYQC9 z;mpkPvY?i2!Q1-w^)o)RE7ptHOaIvY1wtx9xCC%@mhm3Y%!$iwS6%m#bzueNX{AFj zdK08}C~;Gi>Lp(yBkIV&0XA^Dy@)H_@o$;_{J1**yDM~dkDKHi-oLx3@GD(IX8Iu! z3Uvm|ID6q=aege-A64=6D;&CQ-k#~E9o>(q?Vf3>9mLXgL1nnnBd4}6lsh#2@0gw2 z7bBvlj`?lx_PrfYF=AddxyjfVu_t|6iaxe+FIgjETgdZ$HOj_|kqhK(40_#Klz=tt zyh-&Jh_L2Pj{zZ)UkjSvdRc7fA*jp`iYR)TA?#J*{!~ig-SUQ2ms3JHl&pNJU`LC{ z1QZ0jz!q#{U@FMAoX$M=@^QxeH<$v`U98iw$J_=6^h46EfKq4QzZJaa1pPxpUFnS{ zG&>Xrf6?ru}+4yeU3{)v;v7#y}tF z<2nS?A_ww#K|I!l~mzo-GJauJclabtY`qAC&`cxQy zKgjHIA=dbx8zx0zT$uNa=ePaCY zS5r!_7}$B%OYj0Y+4NuWpRrEHl}9rx45I8{q2yg8Nk^dqtNyq9M=oMW;jk5jwOa*Ofmj<^$FV0N3}^hGAbu+ zr#`>t>hv;3Uw~$txD!yTQ(+Q_Q%O|U1q+F1L0RVqazcu9%#L|#`5U%yTE`RO7X&K> zL#Ud5;Djfh1kXJEe|n#O)JfV5i}uK2%~~q49S2;-u?PRFHr%` z+bVdbxbgmbGkZ?|rMDH_LsL&rOT2%1EBm>ytwa}(wY611 zt(a>xHG@6pC@6`fNl+ZbsRSxlQ9*PPurub}L7(XA7zgnmw{>+?J1==QF)}ajJZ7ew zBQO!uzhYeUJh?-))t21k%y68A#Ooa&;8yuB(ZLc=xcq$4nm~y8xO^LF>cJ(WsNynM z-A&luG+H4=3NIT|)y}<`gc#UQ+in6iYfc!;qDL!3?!l$xNXAL#*U3Z_%b52f@%~;B) z(1*iUx6AbhIhF<2FL9^^qW*E=y9ZV!M}J^gOuV@{8qW}VF``uXGq<90K}_Kw!W@B|2NVSfZOR$^~Kf8RpT8ypass z{QWM#!{jFPxuMtzK)d}tvW=L3HxE9QwOX-iAv^5Vv=+y&5~lYYp0#3948Cx>j5 zuwP+pRNv@Tah7&iT3ASYcED1qbXziCBTdZC-k1RqxJbK4#PpJ4xu{p33BSvl!@Zb) z=wiC+IlBd{rhklS5mvVHM$SDb0v9bc>NST{U)yiQkm&q=p(wPi*i66f%s$(m<+AK* z`h9->dAIFfqV&dq3IGhe;u4_jZ!jmb<~Fs^38;SN#ajg1Br5?eT>T_)jc<+93+$ci z+(3`^n46=Bb4qC&h;z7=y_2ef>e!5NHX#%<_|S5MUgY%${D1n##$G-ZcVLjMq{T-% zhP#u{jp21}@+R1SxNKKT=cfK+)CATnS?hhPna3f_1`7Io`oRy~Z_?752TZbrK-I>* z0Gx1JLnEX23xMLWWhb~~*CW+F1;`mti?}oxRHD5vFG4T{IG0# z^Gs63_Fu(hwqd3~z#ge$~DAnO|uZsW{f+j#y)+zH%n}N zwi|V;0;@V{78R&`;8qoCByGCb8hgjKLB#*W+g?B_UHdBBdo2hJ*gmozlBVKmM~<(yZCk3!2`1Dw&({{ zq(-MbB$xsnQU%8K=e4x7C>sC3oJA6U@>LnBF-z#1g@>r;4X`G>bqM0c6b&^5nmhb_){fyr&m%rRMkeG3kYZZ!#q?Hw8lk0v|4OB?e zu$Qf(&3=LZED)rNp!VAJ&#FYxjaQQkYZsIez8w`{JS36_=qj3Guz_V?P`vj zSL4&FOxr!{Hjt6;O?s4=@AYd|n4rlYvkfnx;IJ3$p2(+9Z#^&^-;EFy^ze=>T%h4n z*Ty#X3+0~-i&tp0WzD;AZ5Ys3Lk$)Lb$C+Dc4tPi;(pQrzKAIv#`d+|W z1qzn#_b=vrVI*+*3y$)dOH-jz<7ONzlD1jLrw{iR7B};g4~b#ZcNKGld$0yKELwNf zNF0{g2pDz-2x2I}(Q(#oIGrc&Y>{wi>p&iQhA0hbuO#9c_QiTqty*u)I>c^V-AQtV z{HG=D^i&I0eRddt^i-I@2Uqv1vuDe3YCFuiN}0It_~fAlTTSE|iO+tzb4Ov#pRF-qMybgMactJ&gf zJk$Qjw(q%a`@@C!fHw=m(yHgaC_V9jLB~)%iXG-e@p?_qL-^ymPt(hm7SpQ#q<)4L z{Hd(HS)^Lgm$p-y*a*1)5~2{L3QxQTu1A4&q#ssm@+-M}>({T$`bno&#J`IrqMKSwE~_c`ytZ~lb|#L49H`mJ}X_j_FTC0v$AaM}8i9d4J6obnSt ztN_C`d{0@j&b<~#+kIM@Pju*^fsSmPG(nCiZO5y)hy4eBk$p7&PlpXQX`3}jr(idDk%9w^^Xsu58q0%y%c&x+w#uKG2Rht;Mob{o{e62r2et7MqbClYQTXN& zl9Y`fqy|z{h>-C|@0s0I_(l+e>Ll)8es`n*hJ;*F;*%9{e37*k*I5>F(lwbM9KrwC zS-$Pt5q?6@s63v(HRww5RRTR+*){0n0=Hm=GJ`uz zEGJJB$M(R&6n6tq0f)@8HF3$-)t7bmoZ6`>bh_4LdQ2?aPugZAoIisk(Yvd^vqM4G zwZcEBYf`w3_3|R`+5l@{G(7l_qkGka{Kx zuSgJbZ{8P5Mdd+1@(+ZgNUhNbzn`eG9LC-!mlq=Hd?GRJ#E_Hrd>&&u91w!j`-U)s z=sgaNeDAEzD^hAM&yo?FLwf(T*0uMTwBHV5ISqbZj6R;A6p2zVnr$9-p94c!qkgf< z{G8B7T7$|?L8VA3F`R;i@#D8gek>Zd?qc*zYzWqZt;`w1Ul;%6GG*gZQen=Bg3cK- zhiLTE=h}VuE}tx7c^6CG@s%rC<-b}QgAabvNVV<3$1jdMw8%Q!L+r1kPjI_sk#Oj7J57BwyX zferTeM`m%c-h(O$#uW*wVY;biDVXM`Tb4813P@y;>XJyncDSW7Z*!b_cujU zIyTe^aS{z%9pg&`n?O@O1)Hu<00`jpE62Qg_lvB+0N~`9XM*nwnRK@@KEwFzrJ=v9 zXjNq!fr{6 zt~WJHC;C;Mllo@s6n>FCpvTp#(ASGxG)Ne|8gCBw@_l|LgfA8I!%u!|=Pg=B0ZxVo zXENg4_;0Tu(759wJ|uFvEPW0Su7vAw6j-%ve4&2np&pdNHZ04r%6HI!=N`OX`}`Lw z8~4H64HVK7Z?uR?Xr{Qoc}aM8^Vhp~xUTK07Rv#aJXy&NifQG^9hk;-{|{&H8P#OB zw(Tk?MWi=HItWM=5kdzkg3_h;CL+Bh^db_c&yv_N7KkU9E#!3R*qZ>KcC@E-lQzcwmEN@im&~v&x_m0!5 zD*5(nW(y4k&cbVWBc*@cIo|{~@tVSo&;Rw|CCNIs0P6|sot5vYw#`TpLsk&J9iKsi zyq4u_nyCp@L`Nn_D&v`&!!c$u$&xha+R=LA1caXu#rp}&uLXfkSs4?Musju@>VDShPHBirE@&2ye(r$<&h>_Cc>O8k z5Ssa42;%^&Pt5;}b$}TMLR{2q0msjIt4V-vQNQZ=yKp-%=$;;&vE+#tN2MuRWQ z%rGe4&&P5N-o!uJ^eFL5ca;z=4`KB6p8;f9fa7y{AM3AV^6+blh~;mB#Mb&dPpk<& z*<6}w^2BW?(}jiWCy%$BsetQ^&lmDsqB;>$eg})q49)RLCQ)(Zhbj=lA31D)*GT;N zl6(YW%1qyMV*^Ud38P%ehfF=wo;u)&>c_pVii@GAKU{OpeEoXpnez1K$2)pvAmYw3 zHFL{OjB(iCg>r#(zR(xAo1i1~gg(Xx4m@Mk{42jN&e_`czq9dXGt_uroPnaT0P&3* zYL5=!=}cbXb(usyg0p2wcro_G+^vYFWUc!im(vkBU@y&rO-I_rKqN~_9;A4KrMHBK zWkY~X?&n^3S+Peo2w(0u?E*sE8Tn~hZAI0KY~mTsVQ8!R)?Hv%nd{~5kCo5{5Igw? z?k*cpfbihsy$z!#UVvHr9M)Z%zGQcr;wrOx&m?_^nkTeuQ(4!9mx0`diLL zZE7la`;C(a`J>HZ_KIu3$YD*#~lQ2ChQ;%PI8qF+hY-zPykk}j|CY7`7Oylh1w#0;wE z)PR#iR!F9P>W%Ot&z61)6;4(7t=@ZX=HF<>D0$r`1{=44y?0Bj=4Qj!=2y|4(yneU z=Q~N$yJsF6Aa4Dorc=&C5&jY8gWpqoLwZUAda7kdGYXn z_ic_X1+izQrj83O=KCPU1yXZ!t(=a3x%t*SwoTo(HKEV)_+E%%FCSa(AXyOi89=M2 z$+1Yvg7v03ys9@4>PJZv-Bs0Vr6napE3~Ze4U&`_NPzZWH5MhX32~F^zI-{riFs(X zWbrbq)C-d#df~3)gn~|md^WkSWDAxXKI}c#n9WID@r_W!Vc{dpVq;~)Uhic!>>l7Y ziP7i#2I+J$Cht|-^O0~f!^H)MZh_~2wdq)9#5F`5DPMk*I-FvP^U)Tu?0J7`GO}vG zRNVPU-v37#)4}|E6V6h(Mw3|aVn|i-WaxvSgUJtq6G6q`Ar>RvKxn)6uI@) z?>(_h##DiN0+msFtpH{RFq_uh*CCa?z@iNhV8c8=4b)((Nn263#O2vBQ9KN0Q-Fy? zwWz7_mS+VV*g-?c;#}hU{O3B%r*rE>Uj@7F@o#Ix-D!I>Y#1nG%#_FlP6#+uTl!)s zL-o7%HbjEpqMUbk-%E5hxz=FE2bnL-7lOQ1KUcM@Q&#w?d?##swQqYM_7&<2o8r;Y zmG4@A>3+jrJG_0WrlQb|=4WpIH+hGhl{|PcRn>RnmsOla?DMwmE!5AR5gr_z=_hBG zm&>IKAUrtpFE-T`icV=Kq5Jkz=Uyz_aZP*+5;$%wQQnbKdLl!TEIs^f=j03h1M=q zJ%H(ZYov1#9_MD5L{IYg8B*eY3+w(wX1}~gi;JV`{B`+J*KX7!=rsYR#)!*vHq!7j zRf2b}kcTZE->u>gA9P`qVi0k(R&~6uh<{)q)i2HGMcLH22 ze>Q8kU$~Po+z#Szb!WfW@1xk9Y$M{Cxx7tl>X|9xqB_W^e7Y>f6?k!9XVu9Kt%3S{ zq1NdkABZvfdh385>pT3CuU3{x6ynT@bNj_{Iv~}N{e=L?$ym8HW z2Bb5n>ow!nJU;UFfhuSWN35r_>WQ2z2%qU{fTL_I@9!lVa(6aJ*(2LmkIp(Jd>X1O zpkRm3;(F=ca~?e$@t_xv^e*)0_<6H~{$en#p&_ngzq-0QQS|KP*e#PIpbDz>Ts;S4H&a za}!C`P48QoeCeKZK^L<^2fha-=dOe2GIbOE{X9Es+gTlzFu%Bz#5d-b7tY|#locbr zkN4+vD~~A(vZ~~Dv;@0?xY``>7Sx#{;2uZ?;F6OdZQ}*=#71Jp1EtSh1qKyAwvwh& zuE`l)4P~LPYcpt#lt%T?hkcmR9oA+&{fcCU!&5{ZcRb+4!W%3JT}0uHi($;RsqDp@ zHtXshlMU;+n$(L$450?4PV2VCI=i`C@aKmIoF4m$Y8rSuMv?o(#g`z*&Py#3!7Az* z)4z$Ja^A)Nz`L9n=bsyY=Rs$LI%~)*I>s&bo}DCJYn zqBxRgo6EQNR0_5|1W6mM0n8`$u3cy66h*nQI~ih9!^}DJoacb;hOHaQ`IPEBEylK~ zu1@Ea-qA0DuWzI~lX-(f^uag)Ps;t5Uon3Hz1JvZlz zDHT}p(!TS2?<14ON}hLd4Iz>gW(FbehERbrR6r3q67_uAIlZ?Z)f^H>SNJKvbn^}hz}G9P1eYmYRrg`K;-m(s0v+I$JfOHGcg7ep{)AvQ7(uQf z#r=axW77eg5u3v&$AZFl9-q@W^i?7)YUWe12WI<~S z*@TTgxVd_`9s}vu2fC;)o`}c zv$#8xc1d_e!2!LGkyouFeZna|Q;HV2!eYc}&UtjLa#1N5FMA7W0o_)47i6vLo_0E2 z32xWfZGwlb#z#Ke51UNA@VGR8usRX{+@DNL=fFR!;W42UmxJHA&d>W@h|OY#PwDG{ zc2=Jy&d6cZ#w~PW5|l{}b@A#8U4}=_jIgDT9l2hM)S!`B-Na48XC4?@32)%f6)s5$#sY3U&3|uROr(H%OU9-GmLKD(nt=cmjTMncNdq&?wvD+bk}84tiy+u$h)A&1|j{& z(m_EveA$<46B^pL|7zwVnO}W}Ye;wr*%@{nE&|iGH)IrIS5F6N)aexXf-@T63nf0? z()l1!{F>Z(1r{x_JKB3s3u9UE;*S|mrQPdO%}Aw6<82fF<70!J6Thn|)RiBP)?5Uq zq8sq!2}gLrB)q!NGVRC3-A+N-j-;+Hg*5G3Cr8Jw2}a6Vfy$P*h}idUu;hGRjTqGO zV43399UV8M+Q2e_1eMhDD4oHbF7Uj2nyc9)H#A4qkrV9x>E-K$c4hfD#k}O_{~iz;3FT}rJV1cq~}nERDJS-LpdqUC$i%=y$LP+Pu*tXY@b zm+9&rif^-yI>^aqULk*P^otfC{`y|@l%u)I$(TW4Z)p^7Om`a_?+lRr=wozI?}DGj@t=zzK6DO&eM07_F)Bu;JcL z-wOrV{M&HW)&74qoIS?GB__UU#=@TO#?;aI$0(~Xlg@m&N$V5eZRk@!1t=)xj+f7L&uXGBXg4&jrK^ z@qn06MmnL(fyKFO&!1^#V)Fh;HqpK2K}n5IjI<$5=|qr!By)esgRY5VqGZLxM(Yqw z8Zh$&2RSQKCI`d%08%;2Wsskisl&yr+C_(VS5FUM{ioYAHy>t&1^tIpOhq*6k5g>e zNyv@?7~&?eiZld{Q^Q_Y_&yyng!UKn2aWmL`?$Bc_JIAJ^>%-WrB{BO-e=9{6Y~BF zP_8mEYz$go5#!7U<1^VC(6V}a>d|!GMGI}+=Kke-0bxb(R~U;ox(=Uw?c6U3RT7S* zrI{y@^0O6A#Wf*VWw^iVe|Rw00sv0`I)xCh%oBv|T+>`dB}(L<8yLYMcbb|w^BsQo zY;tpT%;dU5hS`cy+AuOi2Mb$7)0!mE3!+n^fBO)&NT) zxI6@G0ki_E*!^#2=0{bBntF6;uh**oc%iri%BmY(j(6EJq>g}=Dh;R!e32ub-}Zbl zsQiAnU!Na=XlKoGV<$e>x_-tSsX9lt;t{7*!qWF80qG1&$Y6?_IY6Fn5NHC^zZwYa zKN|!P3##M8MwK5{kczG*#`-JLGAFMk)tzK~d z`C6cH2RFi8aRd2U8BiwV;_aQ!)oCj!#~A2oYnSfS3=NF1=xbvbv*O5NnPvi`Tg^;# zqCjS*?={I6(2GI7GoI9Z*l>VpKy!tMfYymzy)z4SUN{L~S$oC!1C|?fI5$g|Qwj3V zr8D6JHyMk=3bCegW}_HD;3rQosa>S%OeOM{(<-pPqH&S)5cLjF*T%I56p5gg4=%vkJ{e%qQ&1*x~-0S!hwEaYK zS4Ln?`APiwC1cQl;vMl#RWf%;KuT&vAsErWG-W*F83;Sj;!J8Lrg^8MrSEh&CfayN zShbgoPmyw8|F$JC>}ZH zqtVLCJL)8yg$llZ*{27|&vY({{HlCQtqYS%749qzmEDy|amU;+9wi?yszMr5B}TaZ zk8Xdt7}~4viFJy<{+z<9af1+FrBmcH!ai+p+${UwEPaR0BJnuCpGTd~zVh<>QEWd&l>(Rh!88xVUS>cia?J zc*=J7Grtfd6+2Duj(s4FvxxNO%$WSLf0>w=5%FQ+g@3IRCkcaO2w>)M?R+M+c^rR8 zLuH9x+5W-8ck`$1Z>88uSauS4Ntyg&^-{`unXkH$Qyp<5*~CqnKrHjGcMVR<&BJpK z3K*Kb0wJu?QObaWHu=kj3F`c5n#JM)cFyMG`h+P0cUC~zm_tH>NV^CHRaLpIp`uoQ zuN4=cWXdfyRcgz7{<&$yidD^s0@?ENvf2$7eDr6d)d`)$w2fOZebSL@umjbZKzjX0 z8pzI5qRiTfvNG-(w*hy*^X36e^--3_q$Q5!y40=`>EmxQhe22;vi?(pTwr79%@h8% zf%i}8%BQf`-hz9jBLewWO5}%#8 ztl+2i($u%+_j%9gdSQx}N0e8cyYPTm(q}2VsV~u;3XJg+Pr9;3`s9v|M*c0Jzf(s5 zvPf$LA`s*F+rL?w!ED_BBQM^genBd!B&|&K0X|EUsq;8DY174-`uD*UO`@vyq2;zop9sw#GO0RB6XE zPhGh~Dh z_Khccv+3zupoGQ0OYxPnpi^cK@+;oOF=W=q+!+fo39mG>AB2XI@n4fOzp2d`D z8}8_#)BC2sef!qn2VOuP$SbC_TCuqw(%c!zn95_rw#|WG7Gg{X1pHj6lUmMMZjsQ6 zKq1&+a(z=Nn$WkIF}TaR-oh^UH~~f*_L#UDC;Y-0*Si3^256dAp4LGC(J1ls#%KNn zQS709c(MVbx_9KyU)Qc5OLhDlZ<0#hE8V8SIYvV(hX^B0=qJ`Vw&`$mH!{3!cIYzRHQU6{bH}FmR$|G^l!Q+aPfEke8zCcn3QH=Wb3rXCn#sj$k3qa0IA%OO^L`3ns_{TW_7Wk zq30h7kfsRmN$DwZo5B6Em*4brvblGB5^ttHh)hXD--9aP04o^OyIv|3cu|2! zl~41lcP8Hm8Inzml{nJ?Ki&9%YpIT^^B!f4oHt= zUzDH!rcQcnO!x@AY{cuoC`JE(OHQs4x#r>YI;j|x;R|@hhKTv>WPlPKvaqm#=fZ`P z5SELyLC{s+5^#32Pjl^hxm!^<)Qt&LqnR%S(QyL5@rbQ!`+v2m8wxal=vgHD9)>}k ztK;o7hk+oqbT5WMUa8jdEs{0Q)klZ+Xr9QW#S z9B7@=5q-UjPQNq6^r@2oLzzO8lyH4?JV!{`8kF-GEvvS;+M=qr!hTj_)=a0n-M0K7 zglOY=lvceXz)hbBC6ifeQ_|rI`KPC1$*+*DAHx@?eItI|Wb4jwlZ5^k-HBo(5EFG< z85rUIC@Hs#c|j0q0{m7D1>?d72*;AhaAL0-ijSAKlL^p1=T*B>2cspwihd1UUV4Z= z4W}&nGRS;`LV6TCpfnbBsHJbpAX`YMv)mGj2$5&*f=<7=Cl+mlUk2~~hj_pOh2Ep; zBu)`8IXAFc7vF+RBR=Wq*DM_+kWrpkQ^*qq4I}40My2Bc8}y^b@>!pDh9|EMHj7-3 zsr4<9C?B=q@H~sf-qvQj>WpWTo)Q{SdhzmOpS;S04ES(*`m`VX`rsMNYkzt-rL2rc zA!Vap{y>v@JyYbLZkbH(Mh&yt&Zjze@kpGSL|2uvr2FG`L-g4>bC)C9E2@_N9U8Gs ziT=!nuDe}W_t!B1l!6m$gn?ABsrLX)RZBj{=NS^GVl74BE&!Xd7RP-k9 z;C5EtL8QGCSEEZ|u*Ik1JYOxXZvgJ94y{KC^jd17gdIpRMs&G$Cq_Tk3>0v^xI*fc zSb3X%@k9pp{gX=jq8`Qg!FhmT6;&LPtYTF9`dQud~AFL?c1PB}z zy#0zLr%=0JldqmDB%JBL92<@K-ekj>z?T*nUqN%TuwnQaPmxxbM=)2foeVG^>@!>1f)`~aHX)G=$beuu1am6=oA-GIieTiv!{cE)hs=j9?kwJ>8&a9+o0X5)L%TRpM&Mv~m zp{b=MUU^_Xd*Ewt&;1I}{6EGIN;&4FqNSb2SU-*J1mu)MW>+hdTsuHVU*BO7Ws-%F zED+U;-Z+D6J0pgC+xWW@G^8ug8tjB39M_Gh+d`?E9>jzT){hZFOD>Zk z;*12hJp;C)A;OKirQ5Jy8+3^}x_O-^)cZO`UEW}V-E%%}Y4RKjYD(PU13{m}7g<4^ z(e#BX7_zlQ&Hq@=Bw?;$7JQ6g{^t;e^U~Lq#zka|nHfX%Y%dnAYF?f?H{+tB|LyeH zipOX4CQzXfa90xwx=2?8z(Fg(L%-0hps`WSEqC5YQCILmY`_Y}<1z1?-dm#i^E-U5%0kpJ~yWFgvgh6Dh%Z_DZ za!5+!hMoDENnV-TBaX)P!Y3dO(c*tZRDfUmTA1t2c77!^DoK3T*$(29v46*C56Z`B z44VeQ#_`RKVROdi$Q|4QYGSvAOVk5tlA@RS^PA3hBVwJ}ap|C+rGh%-@@yI-vV>2c z#T^g=&pd1bH7EY0C$VSXo(j8VTDX3+kWZSM#I|xP$Go~BgxKA4vt#^=;*_`Bw?^8; z$^Hkh7jE;<(DC%PfdiH0<((@nTL#BNv`4;Z4j{l}u50OahRDOaIb;DZnBopXlgAp3 zrN>U+8k>GU?)axAA)_b(4$|443a9HjdJ+}OpO(})U}f69Y0()>4%ud}sP5Uv8JC$ z6U@TNeU`5=s8B>d<{g;)sTApcMxM#auJE#OLBg!IvPDZ{K&%Tn$};PtnL(WSe>+E^ zuKyS3Xyel>=V%*2J_)!@5gZ!cf-1#qutv@O>qUyZ!3E_xatp7d>_?Z6FSTqSoYMg? zvd-Jv4DVSSq`%m!OwxQ$6&j@Ib+Kos>K~tL^|i5@`Yo}u&gUT6Ph)c{UJ3-Nihs@D z$0+gYaiQ@tP!{O+h<5#ev8EHb8B*ruR|_Bmllp9qKGY+S#H!uul7q+_*b4!+xG*jJ z!FMPA>4uOq#vI>REkz!~?*|m^kh*Tn#0LWXwdq(Ywma^mj2&+erF+ITCh`yn1dF7I z2n}%Q@MfMiBeWLD7?Gb3wN6EVl~6u;n#UIu?afc>3qsnj0xVZsLV*Cw_eb6&y!L}B zZO>ZD+m@ZsviF!4I|ST?%`h{ypUmjJHkl zD$dAfNdc}9{GN$ZBC=ZrcNy*}>|~~*AHJMSz4Pcy6}&$DEoD_XHlfDV<#FYPNp!x! zVMn~yw+CMzsP%B-H(bMSSa1OxaM-rPS7fYtdu{U~aU6Zt&u`iO{2Gqa;|)@wr2 zT`Qb1yjSAX?Akf^PRQ}k5(QQUOG`c)d=?U>bWlr;@X-w>no)6b|F3aRQbc#TU)qws zCZIe$0yHRA`)NJnQ({T7(jF^W05$mP#^ce_SxhsSp_G9C*DcTvAWOoK0I2fuHpZJQ z`GOo61yDz~#`N0wtG6E08%poT3C}nhyX}8+5cxOy=Rf#wUuzZPG3nXKq1WQCGJY_1 z$4!?O#EmR}FMw%;plWVDd;(srjHZmR13waH^Ze=)(Y`BU;@eMrm2~Dg`X}j3Q;R&s z0Rl+Xc;+pNmki9tyhvJ>?*W@n30YI-?S!xHZlvX3e6i$@RlwQ()||cJ#)U!nzuO|fl?*7k58rUaqudg(iPqfmbB&?t zamum;1C*R5W6b_UWjh%e%f!2KKQh<)=~`;;!?{C<6vXV3KJ%3W11L#ay?MC^n%p|a zbv~0K;CTe_rtUACM-D^T^aAhziB*&LhUe zwM^!`Uq&kQE9%1CX9!gpUoDyAqa`ApA6}aUooP?2Hfs8)MOT5l6$`xgvw*qUw+GMe zfC9k*P&g1t)Idu~7`c0N95nNmlQH1>J&^Yoh{t-v) zy+<4@(`u-LQu!}VZ^-`~zkvZDNg9)1woy?@B~p!HSUrWm-C0^AVGNHYzVC4fr6Ixh zPX$TUc!%=N*uu9E7VmW4ufSE!uND;Wsd@Vi$H|glumYHj33DGcK*7?`4;rry9GVywUoIluFh^qw5+r*t?^P^4E`jzcn-mwtYLU&(tT(g&O{gBRca0JU*jthLhY zBR+cV*VlG&TFie`Jrcf7Q%#AKQ6z_UZ-V-or{*e|evSheP;qqGjLG#^3B#E|+bQ1C z-aE__bpW8*9-#CM`G+ZX6Ym1J_{!tj$Ge?Q2*>*u6NVZ;-vQt$cc7@dPZvf#h)Fl2 z9_wyOzX#DafG==tViwSV_SwhRCyxbtbs7>DlNQ0dOE}{^)P7+vG-G}e*&4X9el7?U zIRZS(yVB>G54<<3E)}p7#x+Am7q2R21GWcV_vhyqvI^C5;uL)K)rc4sG1eEjh1$># zPL}{{8`;7_?K%@&M9Mqn&-WSQd^0n`GFxkL`pnIYpu}ad#Eep_azWec?5?UK+3@-a zGpD9A{<-~(v|pl@_Ts`7*c9;bd1femmESA?F2A~61-v10aH3BiQxVu~4~Y$EIw}_< zO}}26Sf>a~#+iVU588Nn;6TjCMU_QJXujk*oSHsP?l9V9&!AI-Vn?nWP!qz63uA0S z{a*F8aP>tIQxlD2Sf`pRC$_7%Yi{ZHOGlX(Ar=n?X2W&ENf)n-yI?IdV2l-nG<;pf z$;oN;i&UO62YM&4*A%n>Skb2S<^kV+hqX}Fm)}`4qp#5Um&3lZWGc)N>#B_2J2@QN zq$5R*-~FDeuo&B(rk)xzzRV{|z&HqJrDa0#vs&B9vf9mM?tnM*CDI7y<+v_4U;EE$ z;|r4``kzPbOl^kKH9UYe-bRQR17fbc1I0DG+rIVN;)2tIejz7k%>@vlJFXHO=-ukf zKd;4Oe5y{V9|2$2rDfHaTm#GNUh=pZK@%FFuKN7qiL2jzYAmOLRg^HZo}EfD5#K^v{#@nvamBC@W+>RoNqsf?7753=onpvw>Tb&M>T+Do(lU85GOw8-W>{{1R z2u^^4ub}_>hmUYa z;9mRQZY#&(3Xl{zHy4cK7l48et)ZIOL2W0-N{ujN_uGmYvoEZ0&w` z={EFeYgRc@3Wc24I~=z!U?6KOJ#-Ck)$@QXa21ZFbmZcP+|U zYSMaiU-*dni<%v(ybawaNIa|^J6I_O*FN`se1jn)hg`1p~le*ADt>`lRKpHy~fx8Z*?g~mFny}^( z^D>u%(>q;x0v3yJD-7C2NP|u>Do4AfpgLngle!U^rq!9Zo@h&;zW^pkvr{7o1swNQ zu0@XF61}-X_tw7nem3?GL-h%Wg9$bI14`Sd2}}D%wcJ|%Xyc#G=cK_=`{IH##4y3& z5>@mY`jOq#%J8g?H|4#5AAIeYew-O}ICRydJ+0T%{sPu>bvLx9m+2u8>$&mG*92@f zm~s4?lD1H;tL;iW?AY%S_s-lfC2%e{eDgQr>=^IeF3T`#DR<{@xYb3-uIc_8t zU1gt3OAwKsA4ZCq=-2?A?3Ck;Gf0_6sce>-S! z{dh4-xu>b66?yP>M7@j^zGJe7svT5#Pk_xXleo#zF|EOnJMB3w!xA4g-5ioW$6b}K z;0g-X@4<)>kLHDzsK+g~#;{Bt7FtvW`&S|krNvA*Pnth&t=>6b`BNDcq=9}(7DCSP z5b_u!o&U0N6$s!JM~w<9D*GuJSZkyJsd2>oDeV9PwO+hBtN;#_K7;q^E6iRdwM5rV znY*wm{0z3qJU8}Cn}5Zpr>}~)9|4mHtBy|4R$Yi;ZDyUsP_IF69H1Z$f4Ar%KHlA? za187?K*Pj_7q&j7{WGiz73@IKyDVBnKZf|qemk79Fb1M+1bQbtA`y|kEIKQO!}iA+ z{fWV^W<9Ra#=GNW0KW3UG9tDH>*635v}6I$c1Zp~hTaS(a@+a*?~uyBQa&A%0fThk zW$P$JX3V(#$dP96R?D2C0BmfxT9UH9_}{lVMiIFwT%A2G8*ztqX+ zcTw5n;|N+WySMt$G`mdlbaMp0KR$i0Yv<(bqz+IP7ZRvPgiNItd#0rQfM}-wT0`CF zKaG71p6GuvLFoJm{HWC<-htX!UWaqwY?6 zD#vq@sx6raKkTKYy#M?A5u`Ggjv0P4hF57mS2AJZy$~Wj14|$ar)z@OAAfnfjkG<{ zFvQ16@%{%F_*BZ%hSA`uw19&=uonvH0?pq>&B3sB5UPsLOsXp$yB*)+Ug z`;jTVrJkpooH8N7BS8ctf{wAbC+I`dyPSr!_`7y~jth!%eRfy%=PC`!QE7Da{8b8SP7D-Q)^QZHw(sVbvnsovs}KyL ztEtTcQp1JXC%i2Z$#@6bT?wR3#=RdWk#%1l$AaQ7JxTb*K9L5$h~HzeIz4!L_QT`7 z9ncP()?s#oVh1|L2PCQ=c-_lPJ--$Bcne`|<^7G{a+m}N9j@;TKPy$1C}*W@1(A};KhuCFk-|={yl|eO(+&W_#o0bs8qH7 zML7l({U`~Y+y9O-DJiLklvmD#%xa&%Od9}M62|)%-h*?NWL7$BERX@?--2F|WI}{U*eNTUXvk zCXodWFDK>BFwCnPdxTZG1BtrUrlrRE81+?snqh#i)o!;V&~CT^U`=9puIQ%HLG6Lr zD}c<-KxHwQF{{1($MZcvx3bo`3x5TSu@^|U2Ya)y(ju*$LG8b98YS!&jhxtaPnFT+`6E)nN&!UK1@eYo0gVGBTx;@3#-(P`sX zdsvPAyE3zX%2tF!jeF~+@ss7VlT46GDQjo?=t@et-w`C6ECiY5WauQKd5KYL=6tDK zOt}tW{`n~6WmJQXCpmp(F|4=nF>w7LZab|~{DQE4FBzVYlJcg%iauI@g|9Ow^rpcH zw)g5_CErqgW_ILzj+#XaH7|VP;a4yTH<)(v1)?zH4leKslF|ZBwQ8O4t?&=L)Ht!x zS03P${UZ}cM4DmZ?Rl z4D7n1J)A$*SPG*XRL#gT|5iPF|2Za!e{ws>G;qW%6%VnF*YM3pwX=5mv)E_T?-OAd z83jnoc_|Fe{8v(#Phq$ zKv(gYyd#X~HvqgX9Eg~Egjh9>J}gyIo9EzhYkVkYgxeG_LfC*Kb6B)TTp5M5KV3g? zOugZNuKBi(YP&>d8Xu?FXm}K!mo1mttS>CPj)=iOI`r{`f9tI()SV?71BAeS6K#IEi^gK>5;SbK4&l=U| z8it)W6#USB(COwdXR_ewz)7uX-z$Ud(cYyW9zb94K40zz@E8yNvDwD_A2wS|P~h(0 zHru*KEDtw$S%OwsnK8Yxs*;_-$3!$!Npp`nL;D46QMP_=TC0SItf(T1czl}Fg&Ss> z!WrhKFrJ|4;$wqks6hDPc25Jb16|k_Ap5K~(|a+>-ewVGF{mWUJew7WpGBnVRGPe8 zmGJaJnD9rTb>bKBx*B|6sW^bf-ULSK@G8Ne3M=36Gx2A_TNdBU9#x`pn>^#QdJf!vL$uxn+W=QY(x0kL|t6 zP+SMSDmw}eRK84h#A2XfdnII5nFO!N+3}k=lKn*Mj8LP?f6hv<#&Y11U*t=0Cy>k@ zj+L0LI@;jyZw|PWXy@3Sa9~0>p_}mB)o{rDTW3^SVuXJL6kD2EAu4$}qtD8wlP6B~ zn0w1f9{%G_BBkoQk9x|1X;?M~@^9?}8)GGaN085#Ls@U5;x<-MfXr2fT~7CZ&>CRE z!V;XvRh*N|5x?ExY;X5k^B}4JKYX@5u~$CZ%m_9g*~7hxd+&t!>P75LEuXXf7>jpy zD+D84<%}rN7J>r7M(A^)&%nY_ztT5)W${@4{>tS#aSu!$q)3~zh)1dd_C<9& zi20c&kQ+D57}xD8!w`wnYi~mB27&eca)Db@oT0@nAU4H5%a2~F!jAH7lr=QEKkCOi zQF#K;O=brRbElIOESij4`1~}-{Bow@M+FIeNo8pdVAQFx?@l(@>%fmFLqEj(5$22_N>>gD}%uo5+Jd7SF_oVcZl61<_N z(CeMWud;b@o=1(6`Nwzvr+9TPKL^2(9^a==XFU0yLXg!U2YdUPy=gQT7uQv4l%{Jj z8ITp*>m}jI#@9r}!iLS;pvab2qIc=hxD(qUkC)dv&rvGIno4hLq>nK{H?+tEHbQCH zD&d5yCZdc^)K%LK3B^3&O@xEUhV<7PNfU%Zgsq6h9s6)}7Z~3*iptciJ=8ek;p$|D z7k?9DT#=R5ERX7`W)K=Ff5^zW-Om&2tA@tnLS%q7brnouCQ3Di4)Zk@TVOaX) z`go=<;WQv&K47>4HgpwA2vjW!N6!AL;+9Q6;Zf}^r_6w?k2w?+3Ntig1BOQR7xziH z78K4`Giui*mw)5fJ}kl>fqe}Ad~El#9Eac6TrvJh64|g=Zd1CEa59;=t33dugHeFR zZJli>cr>xSNH$m3r{xHNMcG}0mEf8t&qgLx1-D>1;u4Y;%LLqkw6IC`xN)V!jPwod1kHjn4ah?5^aY`v-KZbh z;muoh9SEhZE_kb;Wb0%5<#GU~PbBgOM~7|zquOfrbTT#reCpXSw&yyXoTe}J%9OF{ zq!SzsnP8s8SWs-(Sl_H_W3)woV@qB=xE$~PqfIfQy)}QU5Xp*f%9(I<=Y`T~nsnUg znYyS<%wL~zltdZCVw{W|h*G|fg5HC16-b!xw{}$IjgQ7V(OJkE#Vq8$XnEuk-~$2>nQMYddwKcd)ZnDRz{-TG9j2UzKqGc5CG%iLr%TQDAl&@ z3p+^u++dD}0lL8b&`Cp309n7!(r^2_zrvI4%ml8gx71v(RZm&o=!4!0DdNV4aShwz z^!h^k2Lf!(Vj`v*o;?kcDl->kzz7w{f>)yt)jm1wDbSCpYxHuQ4Wr%;diDL@NG<+w zpbx24t@`d1@dDm~($xFrdrb<5YCVPok-sm-nc8|8HFPYEsY@WZuDYsrs{2qAX%@c7 ztbBQ&Vlx4qx0MBhw1qjGDkApXnPq9EZEV(-OXdQ1nnB@j6P-Wv51K)0b!o}&)R zfV)S^ACS|bflP=~;L}N+D^%>B>(xJO-7Ml8 zC0r0ExFLn_%!`_>;zpmLO>cT_0o3rWKXMz%~*s#cr}M4;KH2Bt`?= z(L^?jg%{;#8Of*KIk(IZzB>9r8<|0{6m0(fNsna!;Il+at#bS-9Sl+bEWFjr7ww;g zY5NKB(N%FkyN!*G!f$=uka#y$DRDZ~in%dTWHD8x?_2c=g_KX8abiM3bZ5iQ=o`~5 zLrvPITl4%#iGBxK2Rs9oHJRMk-B6nOTh0uM8~AZk*Q>rf-kAKYRgBmGb~3=8pxs=A z`2lvl$G$S|Mn}jHUFdb2x>4QQ$JjP(sbSVayf0D)AWQH3e-BI4=44y%*$(Fb>RiIk z@x?`PDalS?#3yMcbvRDMvuk2)Ya4o1h%N`$_a|Gq3kMYn${ur`E~z)P(SE_$bGf#v zG4F(%)x6?BtI|Jvf^0zY!TBq&lVBV;_6&8xrh2;L!2ns!dneS5nK=dfs4agx9<>%G ze*O?vMp7V?xhkR)a$=n-jPvX7frk%F0l$ zJ-V(69=AEW{YVI6fFQ)gzn_pGoMkaeZV6DT_V0J5=dX~hx$obrsI-``{y?-{!>io5 z?H2w4Q^SZ_2AEvj%T6-V1@oU)KM6-ZZb4wGz3UK1WP`sc1ow6e)B&V&RLoVo5BII)AUfwF zjlEv@a73`L*XC1a@*aR|pVjzVu5pzKeQ(D)l1ABgMCqYKHSo7}oGs!+5Zpnq9h87Q zdHJJ&rdx5nATgYXJ*#DF?5;YF?Q}5f+47gF-GPoy4!5jr$v4NC24E|`!H|$JROCVT z@2En)rfGnN7xS+`@Y(eL_dsw0D{1%P^0ga+!NyWWg~DMa^JcRk=wsU8I<3+lyzeHR z$43hNj5tNp4;;A7sb8GmuGU|KoDP4?!|1z8cp7%?Q@AWUEU!?^Y`XzhR|@B{Cr`l> zx~&%o@xf;TMS=`o*`ZOWVN6W?-%Z`HI3QG+8w=jtk6)qY407zZp`hUXqrmbj?BD!J zncDhD5d75w5H5s_YU}7guL89#{r(BmW*K;gJW-kvF1@=+eC=Rg(tEM zWOJU9zb-YYQEE18!=l?Eg&t!~4f?UV(b|L0q&SWz#}H%SiluX{zwMta07;dM@!GH6 zIBe+mL40^6<@3z4J{L|=7H%LoDLyNUNt2P~_l0{;096;?Ej||ii?cA|X{*nGl0q&G zA7)SyCFndZrru&_j3tPMA{Fa}MxPM>Ka{;?P@LV?wVM!KKx4s!2Zs>c z-2*i4!97TDcPBJ1Y24j{LvV+$v-h+2_g1~lEU*-!}?WMasNd{k_?_oJCUd$l{B?m4Un77agdOn z&mO&ZkmLzi|4y6xY_;qvPvjiV`&7quXR>DTSao_(u62L=#u~O9{WMRXM+^T$s!iyc z6$2Ew8qt1bS)wx($lMVJ12#QQMO*lQe~ z_Pv`e8<*t-CE+)X&y~8JL!Gv-UOSWjCriH^CCDC$ZT*wNCXcaJ^6G_HzV9DwY~)w9 zwSV!beK&fyw@v-non&(M>mPmK1W&lnl6`@xos;Vn-mm$JIq@(xHN{=?UacJK5wPk= zUur8Nqe^5O;Kjdo-QHDa>zQpo!EdCWIg5~ZeU-`gRC*~E2Dbfj|6WBFWV^(A;>Q`uDx%0PnTocV zw`U*qA4%D;&y9+blOm4=HZ~8=xoKsk83vVBIR{I8?K_ylt;ip$8jDDCV*T=Q z-+ma1Jk@J4HC%#lT-`OWtgh#6Ta9JFvq8y}Mm>=d6V*dwPQET4x1m2O3Lw$cs07jQ zy_Efa4=ZvadU1$fSpesJ!k2{nqJ9?2Lqeq&t9;-P{00s!Y#YE|hBc6~^%tA&ZY=4R zmfm0D6v8zJ)*5M|YStaNzzCCWwBxDVYnyuZKlU>&S)FoHK8JVbJR=oBnY(sSSv!yj z{F+|s`%f4&Hyb>H<;7*U-1SRpKg`>`l#h)TSc;!erY{>6i|~w`e#5#m1lV}PviXgK z3Y0ourKUy=-*%6P0_n|~+eN;NFf8#rDz--6o)att)CJnFin#25ZYAY|TNX=X^+j)& zqnBODL|y9=uS=x*{oo)#;u*7%CJea8qhSZpZD~O8Z-nTT!{FZM|AfG9De6&P(N?4$ zr|gXQjDu!8uI}Jm%OmHFxn%P8r1rAsS;PZ(ILxsGIC|puTs0nUKpL2Fbvdgq4);6* z1jw7}ZEQEn6bW4=X2bUQ?5vDy4qe&FS~H{@EGOHYc884ntqgIKG^x}q%dPa4HNxP$ z#y;*mRuI7ni@ra6u8;X)){D^lm`Fzc!~3E`*lI|p&|QFy4QtbOf4u#Sz$>wZH~Z&o zl?Tiu)1Lk1@~XtD!V>V}5}x7=FWOs26>^I|ML&@4t!O(#i;frd74nj{>+M93I^VTh zi280Wuh33;w;bdM@)`js4PL3X|JbT(bzY7gr=KPabUi<9{uG z%L2udr!d}cxm%|badxh#-SR#Q`BvPW9y^m8CQ}O7G-h=GqK>9r=4;Rph#$b?e5+m? zq;BMCtQrpvFG46*99#8*89zsyQF#9Wdigr)-BPtP7$j@L*dUfXof(>%4FgOYJ=NP= z%vK4b+Tq9t=SWz(%YJdr(TX5?UD6nS*2_E4b_BnJUV4JIdzrkE7;Cfw;8rn(A05~< ze6Qc)Oe8;&>BL-#IrA8C4#naWxJovKzjKZW|O`DEY^FJ72CZ8 z@zo>z_aT(O4Y#Ac+L4f=H$D)V+BRrZWYF`d1JIZIV@jJ^8Fj@114(*OfHPQeL7^a* zq0fJ@mo}2i2Sw(r8$qv|6pQcy99p3;MIyMlqlSP82>$B>$YWQjnCpz69TaugP){Nh zDlvM7i9k|V^TKyb%7ixNdIbjV2`p-AYR7a#O_E;QPjZybYD&^~2yTZt*<%J`$_D}* ze^~K#f!>Hm(-)u{`OB8^xO@OLp@}#N2EzxnwbbH1wBy5gR3#BEAHmBpAcmNx_$O?q{{%Pkmd=RSUd3Lk186Xjoi8OKb=lmhP zho~M^pt3?p?~`~MN+Yil9+AsrRC8h}4&(F#ozb+NmxbrbmO&OD0C*~gwkd(P12T7yjW#^bv**krkZOvju5435Ok z1XYat81D1)Hfv07tBRU0gNgu1iBZj$_@f|=%GLeL*hGm{w$bt6X>vdQC{# zB1f=b8|klZW#hfKfY}Yz!l{#P2YS}Ov&HF6R0Q|gHU`;6Qae93Z+G@b#mg|PBZ;lm#C1FZDR6G3@5r3dzj-3nwiwi^w_fx5FV zN8Hf-oaGe2QXhrT6E!HcJn4FK(lNyJ9t;YWKrb<#XK@UtXj5zU+laS4GllLvNmBH#yxgN`8GMw zUkVqnJ{9#BX61R7?VlDM>|cI#X#EGT5j*2A-UL~cPGK$ABm!aS?u5%Kl8sEpQ3R}0nbUOp6Xzcjv;B{p_i1QO{)KE8}pj_@` zk?FD>e>k2wY97F=);ip|j(M2Dj(6h=y`$=+h74;=w0rX4wH$3Wp4|ggsT1Hse!NJ@ zEQhr1ca$e8pbw(*w|=sy{g1pMH) zWAeRTBWUt|V1PS48mxBQk}HVdQN;kq6}_;(AzJY^Td%xk_SsW>|M<6+H; z^Uz-{fhJ@tJ)eB`Kc~blLj;}7P(H`q`M)Kv+1bXQnT1;FjC9&K%91l1kBOO}xYj0K zBMyS}NJEZW=2^$kW`D67U^h7@N_F0#`u(g@c5z`c3%M$WDbv#od!uem>p&!Cw+2 zlk0K+>a1|H)}@q-`?*9V5gxgnfFu zyRYnjC?LW2wi;)LN4bv%0SjBmWup&twsaP9c0C}Z`#rQR93<>GG9oGh4R0U=>bp^MZ<;gZF+ z@M_>Y%EPyBX2R}zVHdBr_G2#LcY6bT?X~a-jy8e+i+ykXf5yHefZX=X`%zF%Jx1An z`aOTa7ty;_pX@@VzXDgv!K9tNGj=+{OG{3xX+t&hXT8jucPFr#;BlF)KNUX59Vqg| zz^J7u@8i$EHS{;2<#ZCH7du7%1VZ$6j-xC=c;PK8`MBMN{LN8Q9@-I+UJdA%v&H!J zF4ILN6>GIK$f5UAfDQduT$wJSC&mRGRC=i`KK}%qr(wWXa+A;4z_Erxl2*$DY$=12 z?tq0Y!V!LYXko>WaS_Aa$n%Th?0Cn}Opki9&u%-$k8Cl|^P0Dy1INroyHxMqJ00d4 z9jE>ex_luGnFiGbNs~xc=GA~M9W86YirL;PILZ)hFQa)-!f5Sh-wvy8A_h?zo2rPJ z*q6#R=7yky3+>v29a@4}8NgkssIY5$(18~mR&W2lLCaaWQIgo~dkUrUfn;Uf`bL}Dd`%`<;`W5es{{|??~{&(=6o(Iy~@AmU`-kZgUDs%(5T@22g3KhP+ zKV4eY?}hTM)b}2UNhtOEpD}v3S&8NvKg6ixS$IVYU=(I9q0BDX6D?5xmGQPnZ5D$x z2``uXJhCzQPe?eSc_S>1ODk3FpHHbETy%V`{<-0ME&kHB_cZJMJ11;CW+$Ct%ZumA zaBnyqfR_RK%-yHh@+z|>Z52gw0X~5rglE?o`s}-hQippbOj~cafqrgsY6Bha4o{Aj|W)r>iFCj`U&n4x$^w2yl~ z`hO4LQ&nO1o1SVnA!l(?ZCs z-xD4_^Ytf&v7fPhpvgW3Vh}xzZ|vektemiUk40AcieLh zwq~^ni^(c&H@-eZQzh>;C}9)Qo8=q-R+p&0gd0DXJ1Ekx336)+@QaZgb~EDw#OR&H zo-2;Cl%))~QM*S)NwUjsu;C7^IN`!Ztt`Jgm$rICqC2Gyj9E8)(&Ou0zeE8FsrM0?jEK%g zYHRjl9kehhpl~~ZaULnKEl$EC_BY%vo|u^UDZ;*=UEGIF{X}eVe_lmhC4)fYYJDelu);+Y#Shw|CYQ6msw8iWf~;!i&YR zt*+I$Z{gi~FNsPdgjei9fFNt8O^;)u_my$#(>ts3%amR42RQ7`B=Wl#y-OtC(n@37 zz6zUG;Xl6|C*0ig=Z>GZ*UM*XSE{5`L*G9^I+}uKkvE5NXSJxbXsmA*k4)?`xm2ar zgW5PG3T%C~2jGLWDzA$;*VB9-Vx3U0)&&fSwr#tj*7xlF&Ez(M+wcE!c=~(J-ugv# z3O%K``cXa8CP&xy9vzmwWME`%V5tqK;aG4mIi_T;r&DOjQMrZwXdSSlsHFv$LMxpf zjQuve!TLZM3<1xjXyxqW^w~LV;o)q_213fUL)8Sq=OUh#6feEd4XFBUK#(1;mg!%K zArn4OCe~+y?t*T2pZ2MLyUE86KtY;TdTpP>B4meHRAI|_*o?$rX_jlLm)B5evm6)> zcfiBZ#66{xS|O!-1E;jSyl-~&?9qeK1#$jKvw#dWbob&y7qMDerYc>UWb5L}375-k z-pxS>N&qW6Q(_HF&g)b@*U<{DDVW_syTYy$nR8X1sWxJ)J@!P>G9)VTa^yN=nQ^I9XA6v4nMq?+g+RfPk_nB0P|A z2LIJW2LE-c^<#NQJg8||)qlmTMJ(z+wMC$K6S#_^uLVX$H2qEC@vG+f_$MZzzj+nY zr6ORx`o?+uRgbx$fP3kou0`9_696DV^ zYw+qZ{WYzAqY{W8qaL^uCt8q&R8bKiw8a}Vag7RqI-6Ub&9a5R#IZumjkb^KS55kT zDIO@9J2YEIJXo*TqBm!B?CdbR74GpZDEuBqe{qI1RzWm_{wTk`vj5H^Y_~;N(kk;d z6R(Ms9z^3Zv!&R^Dl{>!Z2$^A0RFCao)|m@RS&75BVoXS0}kUqsFiE>1P>XQ#TZDL zQS{{L*JB35b<7eO>`UzwjP}=v>B*2pAy!WF<+n}gC*7gLc9wJccMI_i+l_|3ojcll zqTPHXX&u?oO~y4W-x^6>r1f$IuQSNDKyyn|ze^8VPOQ~+QSa{?S+RFGKG>*dh!-hqN zIM%Nt+r(mF;)mRL1b&V3x)M*u!eU1E_I-gV4b1h*n-rH`nXPKuZ=y z?9&!JMU^i3n9(@e!9h3}mFV*bb#(W9Ba+cAD{Q!uwuK;9=+?b2l4zP4;r=wgT-ff2 ztWd#;xp8?xjtumYZesa+`dGUT|IXuqfoSBz!}3*&kftSNmi5Adp?`yvQ`@8Z8Ci{< z(OJs6wV;LbdcoaK8kxI#C~bKRvn7h0y*45Nz@gP-_RdWL4!8_ogE~G)clr`<`r7+N z=s*_t-N=Z+_6`K8$F!H^yh!1W#2z{<^bf^V(XGVWrUumHNn?7SB5R;7p(P99b0ytH z#Wx7*>cc9k_suS}7?f`=cSbq_e@k|(Zn5)35!+(-W?q9tU&YGb|1)_3dBXywZIila zX$BXZXys*@jaSOu4WQlb(LfNy+dxk{p$>f81X1{Heko`3#1sQ znE<@ntM{YN{wd&Urf*+uXLMUJ-0HhbU@zgae%xSZM9WszQ%RD^8L^B7~jAw?Uz|lLI>&vtdsiED@Lx#B_ls4(58Tr`s9*GRs~sNT)M7i<-T! z8OyiF)28Wxh=}kML@D9D-;XxWZG#^@@@@88{>*LDsy126%t))un{)B`~ znY=W2xU#C}i=YruiBHL58UYAQzkpgu%6fVsP3SvXdt5inyj+&qF%KO-TI7>=P~FZg z*Lc~xFOL5esSNkWi7-Ike$}CGKMs^7lReZx*E0!XtG{>6yIOmRC};DeICZobSY#`V zSqvEe#0+HwX+bba%YHaAcx0N`*+BTohAz({>@~AbLXX-xZ3wR8P_r8wLaf`FBUn|4 z8#@rDTgJ;3GFS@ily%j}S<0UJkxEMilaAdRtEFQKyYh-@0CUoGy*&8>;e=D?{h}h_ z%}xeAnQl*nntB50m)c9c&K;WIH}A;A{a8-mfin{#e`<0Q8<*dp6}wb?Qff_9fCkol zqZ;Kd8DnAYhbN_$+n}vjaxaNUFXvKN-4fGJdJ}k|Jvkn0p2Bhp!V)a{jrYH)Xt$%) zPuGW5Fo7d!gQ>45KNxog|A?ao9y@~*^Qq`6h{vl}a|?KhT*zRO*4EZwq^I?0P}M-W zJ~z^w?eUoD(i_Kr+?8nIUgT?;bTr@C zIP=S5N6ZE57|9t}lBl=)eMffpURHS#Ucc(AghZq6lHR%ITb3!clj@#hhN*6m)$y)7A%MPlXkYT7?^!aY9k zKJjiSm+}ms3cWUC?|n8q5q`170BbvW&d>Zc!T9z)_P1C}B3VMrH%O?2fmm~E@7WDW zUY$7*hHS1}Uy}7zo;*g^juO6Q?|a*axcL>!klir$X9|NqCc-vg=#$0y;nwZ`-u$QO z84Z`~!?L>C)=`X4lQ|_VEsd=$`?s^IKU_=Lqdw4#A5;kMCHy2KBnmbngnxejgp#il z`+9A*ppE|Ktt)29AP*td&rt@*uf{Jo67+ssoOQy`46e@c48-BS@bn8=ZS)VifFctI zTg?RNOxv4dJ6$&6e(|5U?WQ|UC;?dCJ+)4=_}&g1gk`90WsH=wHuSO&e{Ty$jK&{75cv za=fdBNnn;^C$`sF_!8*x7OlqZ=b+c>oVYw0+wzd4s*h@Ae@Px`vqyNc=W^n|9$v7~ z9TW3kR}Di0&wzr(TpP=h5Eo}ED$4o~ao#B7LE25jl$GCM^jZ75h0ViWAa8Hku9L;D z;uO(Syoi?!EQL9q;V&j%F*&yZe$C#NUX?DvC>7o*O%Cz)Q_r+mA$R{kMMCy2Qf5QTJIwdaDU2tMU zq&|$_-(EmG(FoCLqa$bofY=XAdRPoBi*{Y0VJW_CnscVZqQX-EU;ZKDqzmji+DgaM@I7A(y$_ zk2uUuAp8g|4b5fNT0(u{Z+?1i^X838VKBwR+C2

VDjLqJKgA(7~avT*#W9GfLAo z*OsyjO1qSnht})tfHnkIpT6w0lm~u2YD%L++5YisV90Sq;4?11-Ta;)dSa&&21#Jf zHWoYgry*6Xnu{aW`v#juQO_YAxulIWHiN}&cc=L` zgw|=B&+7*UW31wIxlzVjN@y2hr+y~+cI2gk#fk&v`;p-A5w);Z=FL-d%~^c!OwPcM zJaKmEPRfwjDlf$Ig~nR?F<_+IgR4FJv#mfkK(fFDdmgEq#yvgD#HG_$zw!KEj}`u8 zC!MqajfPx(iw!nWHqA$0>XOjtT?i%cp(rLMCMX(*7ClJ?4XpQju8M>ogG{8Rj3&mQ zzkkX9_^5T(hn!Hp9G(|GH637r@wyuo_r}NMfoex{DrDW|JdMLg$7Z8>X?7Jgm~MQP z`&;DTjU^7ui!qw(M?VL32(klB&qOB8VLy{hA zp+H4AN|m4hT$!Lmn*H{vA6PQFqZ=syuCCEJ}%P zxEBCxwT^q5tk;v9lrwzf?W=UdqnWTtn(vE4vWvbIc^v;Gem-N%zM=f(`LwsIC*nyY_l7femBMRrT&F&g_F#t-7ol+Q=bK3Ahslxan{ zdm)5qympw1Q)dWXGm8Jo{z&G#nj+4Drq0L_#!TnjCAG>o#!jT(4tCJdZLCY3__8r( zsp~sM<6TvVbwHw{Od;n?p4Vxj#~tvl+tLAAWF*eaU`vBu@Vq`V6Pp4enBPe2u~Q}K zOF&os)OI&Q4f`mc@_fC^Ns76f3S1{{hAFrM8VXL*vK#8iI}0aX@@G{^}+H9g7kC>8q>q`8Jt10zluM5U^aw7(-0@0EI%_2!r~{^$Lo1FJtGCJ1YWDZsfqx44wcw__LL=kz(6Ol5{fA86SKm*&`!dcGO;{-`Nx|;eN4ccvVVzE|lNZPC=1S8MFH5Bwup z!Z?9=e5YW$&L$U`iOp0H)8002_lm4KNm;Y0G#OGV==jQgXe zVu;;Twp!kcT+;xEE7iAfDiST`GNE^fPB=Zc!AL5)OX=UM_z=wvoc%NqxHn|p+;)o9 z>4=OA7siA z`gM^+81&Wf&6*4+=@r?{MVPDe(8;IzOXGKh%2PsgwGltFApHiN;qKnU3~aMjEDNbb z$;7rDDzg9krUb&%!S&Iwbf|Ug64Hg5<)hp!nNx*+S{Q`xwA7zLe`j_`3Jls}WeI&6 zP36Y-y5=<=JQ%Ltin^}S12{3}*kcxs2)yjibf@`3>Vt@I?@E2TvkFZ9kc(L}r6Fp{ zN?6d@eL|wj0OyfB>JV1BJ1rs-q-#c#D~isnRuLUsdadnb&4G@qbL#!X{g#ckO<%A? z#=Yj##aQ|(7RIb<6A-b|B@$x~%%4q7=$6 zm113iR;-J^_a58-a}Z|Vrp)nj@a`YZEk;I&JIj7oa}GdKdMy@*wJ#W6Wz;`PB!p}Z zY?Xi?G{9VfSfq>mJ1x zMXWdMs6e;h{re#9Ce_jcu)R>FmJziTEa=zlPTv^g4FfPk(sOcB`S|K{>C09YA&G2l z!Dx2vG0vdnW>HdflZh<}`=NfN9tV2Tu%V%|$2h4-sjaDInU06H)k#VgwdQ^i`R}(~ zW)90y$v|S$SG<#=>(^0xNCFrblJ8(Q9P$Sid%?X=%wOf?GvCk@*bqtErVDK??bD!k zU)Wh6Az!AHKh%c_c&2?E;c%HE0&E|@H~LLPupMFO zr09gz(6R55!aNL!D+%W3I|!JZJRik{U_EKEQO9E1MY`s*;`Zn&h0vT(jmSQT#m+|F zbW8;31byq97$Vn}c01f8NgM}!hd884?A1JM)I?t0n6inq1R^ zH2RmEtUCV`;RV-5z@>n4tYb3<-cI4kc1mCdhlJA?d4N6+uRuY4~fR@eLN^j`<>#r$grtltbo zny%XYa-k}0MdO%a-$~tcL7$h)`VmjHQ@POGi13S{5hIEEMYM#z1>Oe7u(*N+hivFt z{FtbH7U`|c@tm30vAlTRlg*UhGN|CX9gszzjzi^8gxvgo3M(P=U?}{g9y9&b9_~iD zR}lg7Q0cb~a#|TaX%L>2%W_VNSf5+MW~pOw8`#g}7lF zOytjyi#DC55S8dz5B?>ZnZgK2UwGa5zB6mW^%9-7;1*!P*dR&Kh7)W25jBEa25xTw zZT6r6oIX;L*{?WEEK{aK9`j_zV~$b-?TIM56rm_ z6-+kwk3!XYINBbMl(TVZ4s73$k-Rz^X^;`N)H1eZT8)#E8UFxd6V1u zO+nNJUo{y^LxP_J!TbJ}@73@h|EXUTEYZZ#VlRLA@%b z5tH<^i*Q-`i{qKw;2`1xu4^8NSxX1v92Vn@42_cUWAiz{Th2xUj z`)=ZO3i~7f&eNDjCTq1At?FAsogFzm9qp2S&1}6!m|R0m+CSH|l#RbHnt??1uO@3Q zaPy_cEZV&OigRFMFu2B?+E*sjWXHe}Jy=u9C&1KhR`f! zq-IV)WyAn^AH@D;Y6^kOPnvl5c7a4BCVTailDBLzpuUuEW7h5Yr9u0;6ElqC@eRE2 z%QEJaA9is$1~$vF21knPG$HJPecc_16@}uF z$BQ-r$S;Kw6+@58xH?_y7p*k#Eq4T@Nr@u5z`tJeqP~K0uxr4ASYk)g9tN2 z`;~?3$keBifUJ#xLH9_RY)~C?t&$>Owd>7i)$Z8gNC|dsm8CY04SjX$cr=hi;oDc? z9t>|KF_pq;U!GmTD5I4bX(z(Myu+P#u8_J~Wqa_9eKfWNS-_9F9{{%(*P*%fv|)1{ zQFE;(uZSDz!AtA>zf)uZXtE~vNeWbzvZP$+?1xrEYq9SN*PwFGXr_1}VdtSjrmsGH z`9{_`mBnwrouIQkF$I+9Eyu}=w3G1j{b$z<-V+vI#re7ytSO;=-NAmTH9I@qwC^$H z@p1aGE+H||K|&@^AhmDkjt+MitJ_kAB+5P5$|h_*S=0;h)Loo=is^LvKs4kmyombe zYevQ(LOw+QePYDi?abT^8d9Xqgh|DpeJ&KazDloLPZlCD8V`S-(vyeTr|kHdeW3XJ zyT0$+`OG>0Yg1=AX;oyBW3q*%{0V-n3Ps8rA#F1HPD zpLE?l&7XZk%g4e-uxm4Y{q1NG^Lx%nEfJ7eb*KKtYkR~~YpKKGf(#>Aw;H^^quQiL z8vQDjI6;#lyycWS6Lj8j=L2}{M5U;kk0YJS;{R2Ki`0hTloMuuC;()6TO*D;+6ysU z4gfnAV*Ydsce0hFqX(aphAmuD9W7HSZScuqxCYi{4u`Yv{SvocrRh0Q82F(PQI(E` zJS~Xe&X0@fa>QsB>$2~Qi5kk|u;Q-6MSS39v>JVBzBmi1StCRx;~?EEoHEjU#INpr zxo(XpNbHtld4sC!c-xtm`-RlepAjp2h^BgMb-I{y2tA2rhN9_=K2FAYMn%Cir(C2o zxt_71r3{VL;6Ep(+us*CyDwyL?vT!N*d~6!mV}Cz6x|IAg^)6J&{#6g2_L6Fk~n;y z)6VD^zA2+yY1lf?RJQ%5lF#xaMD~A5>PG_FZr)*3+u2cH9j=71q&KYJq;hFxn0z4r z*KgaMTln#vnI3`Wg^y#W#bOL)#YVjzi}4}HYmeQ*e#KAsb_Nx&cagPJBK%wgjdl_uW#_9Ni>7%YB^ zd6{SZ0}wA(=0Yo;`UUcB%SM5GO$_)Y8JLNVbeh_LEknNu?obs6!;pc3FHWhCDUg$bX9t@#6+xy(~8ZFS3fyYsWnvKz?KB`LP%~%UK$}!z-Y2IVe^x z9F&duyO)i$%aafq2Li4YVp-X~pUfgNpSuNqw0T$8uL{6GGQ1lpHlJ%G5ac}(cQK^7 zI>O5J)+S=T895^4*#E2|Egw(O#+4|r-1mNLkJG>Hg|DHceZY_=gP)Sia(RX2zWtWQ z6aL3+)Bj!y-OZf|+EB+V0~f!bo)L=-A3Sgifx^~Dd`pJIJ1JCvp=eCkyaC>SRzPhJ z8|kpS$h<=1tHJ(U`dI0Mfb@!=!ksr{8h%wTOMA|ks(m;4DiOL9IN>6auow)(mJ_)o zv-LtnMV_icmbrllpdj&_#A)mDYFbVerc~_KHcV8YJAUF{+yP~mf0jC7kk~aX*#E(x z(%l-yZfkUDf;x!79s%c9>xE9d z1cDkPypxdxgbtRzM`G5-hfyM=a!+Wdxd}qd(I5bp!&ktxS3-fhAKG^2DThj>jL%N9 z!K#z!hszgq#3UNuKPJ5Faz-2$?Jcvj71|yTEI!(v`z?K%MOW&0 za|shwrcd82GCZgHAwJY0!F?KF#*(p>semTTnHbR3M^e`TSt^uwaf++1Q(Q=k1?+T! zG5X}K^RX9?KOosE`10!!va`-3YK7{Jj|HV6*Abd;Njn`bccAzyOc{$@J4xiK{7OQn zxKHO-6uCcsypHtiiy{fBWNRWn-|-{!RLS7NOWgvemy__lu@k0E~98sak zZY;=pwE?fZ+-9TgXNS+CiaHmm)6SzP@OKQs4vpQV@aF4LaiqGGF&*o?wGSLaE$9*e!AUB z?+9!6x(;}Uy3b$rdGMq6CDZKJzh&76G<>L+3^2-N`UOklcBQ7o?t6^Y8H0BiA=O_D zxIBlXM?1q!pS6PBk}7bG-{Xdmf(jN_*Q!^ABN8df+-BH1E=c<&>a}Dt1dng_v^rf; zh_85z?0jA|_|~EHXR>GZ3jAvwyu-zYs&k~XmoI|tc*+lOM8Ajp&%qn5gF1`OR(AVe$Z_+ z(-bbq^9slDVTA}jgCVqZA?h8!JoCsah%KaC#gTl^g4L zDD!r!Ug+AX&j8J*o{4aIPXC$TgbSZ)%q?CmV2f41D9pG-iX`Phtxo!KxLA*3noGPg zGZP6eAj|8|2&Incz}5yfQ8VnjHQPVgElKxAfG#1L^TkgKGxs{T`(Jiyg04%l(^f1h zo$jg|J&;C&!jE_#Rxl=cE@o~$8Ak|evgb#So-e7{g#$r2EL!%$|C0Qs4FA$*sa$}& z0Zrw=e2u$zLLLw&(|RU};o-3ancNOx75gO>sLCbO`l`CWl==($bm}ciV%z{`m(+|TI|b_7( zkvlR=MT#P*MrQ`4Ereo@>!@@dQP-W6_Q#eKg|5Wrly816IrmeQ7aMX%2m?|e_qLu!zoq>km2|!y|OdHEZdXF>wrWs z27^K+L@CP;l=XbT1XCX32A_96+`AC+Sh|~~Bw%iAC<%v51^?=Eg3v38_e$;nIc>tN z^f$!S+74hLRkpvrnB5QL(QUSnu;KI(J?U`$c-v_gX)v+JZCssFTj6QrMdWnB{_*E{ z5UFTz;*+7bL#b~uxwWUTMAV6Tr@amZ=Vt(6@Lo1I50A@%az5*w42dP-TnUek>MPs(~VeXjLRG}dMzTFaAu!rw>nQ6QF)pwq~qb|RWmvHdNA zS|e2=Kk54>DoFSTpsOYXtC&GrQW6ikJq<{h4y(e;p>j zz5Dcg=N?@-7-|GmqwlG91w$w^e|>9W0pDJ6e-zJ9qFp2#7p4cFWB`&SQU^3Kfy9i$ z3GZ8#FBc~hi^yGZr6^pP#ZB>GtNe1vz6?x|i%j#1)!=2FhTQgI^{LL`aa^muD<20^ zkw?Z6p(7@lH&j%~JR3DNWEe7-K4MNzwe?vgcK2A+mj--TGo=!tBtgj8z79AUHdN+z z`BopYclUViei_*5Nds@NQZG#)21YF@R%~_DU-#^0C1` z$a*+OF}L2$>I=$#M1-}jMZ%8eHkGKKW(n3q!p~viy}diF!9a)OEQ1I1Qqj3J>rwtM z$hHr^)eTcVLRR_uFpiFnpyp}Wsw>0keV=vJW?jbnAn7k_rmCmetb-+my`|dX8|u_l zz6RL`6KTZ~Gn&qY^i7JgOiclwl;d0O|_{9tkA_p0~oSGt|J z!8ez)xFM9WW_Cw$F#4O1<_l*ehq}*{6Q{xhG;Tl!@Lnt?^pyZM)C9h_hFuFYARgA# zu#3l$La~haLf@!pkV_UBhhlb=Vn&A3ehhq{0zPF&5N|{O_H*f!T=1G!`aIAK+%5)K zYexLIr;*>=%$3WkxuSRvCIb=R-uKFK2_UL!=R>K`RHh>)GJaDPUzMyA^4`)+sl{L+ zcr2XKjebdw1&%tV-RlDO%73xUb(Ft6)zX;4;~EHktOmasPs_7E{vnDz7$reT)8o^^ z2cuvO=veD_TPhaH^M_GY){-6x;qEf9IVtu-qJ5DwfdM6e;-0`HMFLdQa$-$a%Dpn=xY$jRqMYn%s3KwsgZ#jEs&rKHWK542t-IF*E zqa1EcT`X<9<*u~y7@dZJ?fczXNmy6?R{fNCSm=#ypIeq#zpY%o{{n1-cenm_x;|i$ z6Su!uioD!!`DJJ$C%0O7{ieX>3c!bh|K{sE7al?#Js=n~UWCtb5$U*^mm4R-NeXQ# zG!TERQJ-tCC8r-1=s0(0C;XWc64je(mK^|;u&RPLz{~(9IBuIfuK{N4ErU58^*nu{864s^Sf!^we*Ge^(@Tem?7%Or>o+K@{sk(SfZ# zzo%BE&yQeDdcj9x`OG7=NES<-1?WR zvQ)jvcq;x)*`pq9!-sKo@Ni{=wMVO53hPGrQ@PsfpV3FGP*A>dqLT(f@Oif5(BQ7z znFq3f)fQ0}3i3LCirD9YYIB9YIGdHwGoNV&9C0T`hMukWx_~)<+sqw{t&0TA@nTid zU(||Zij%pvS)u*C6XE$<8`b!AjnffFZ}M@x##d^{!r-oW*@hEG-%Z#Zde5T8vu!?D zEA3n`-4Z9)O*x-se?B!~6N&tLv0YKE32T^j5uObd#ojKeJm|_8Jj93zPRAU(4sS7R zmocm4KN0ECA%Or^4oAqy?G%9HS(0)lC?5ygc5XX@@#Rx|K|_pvq5a~3%QN!JIO!V_ zySDk}6-jt6=p;H&4}AK*pKDp0xAd~gcJDBTUeImq$xwQ})Yvol6+23-0Lv<88{_x% zi9ouUQsb&S*g^-ARL)Wr+nC2oa)acrl0DLH9H6zV$Wz^ys=D=?$=I{5&qg1*0}R&Z zk3?Jr3X|9>A9fM^e$tlhtiJhemgm&D%A4X%=qZLs|7G-e`Gg%KN}p7S8}#>r6%N0- zU|9EATFlLc7*1~Dot0vo(9K!^8lMdR4^e0R7WMZ5}el zC59eAP+|z_1_@~xVdxsVhVGCay6$|w_dd`41LmjqnRE8qd#%@=J+kk!_ybgTFWc1Q z39Hys?jqg}y4~+Rz9~DBFmr0$#ijMXt8;R#WgL0DY-{;_o|m}@$>h()-4;4f)F{5$ zBah7T$hD_B)w#c>BQl3KbB|n%^{T=3oUaxh%%I5n#F&cTUUkv2VmJ|Flos2Js8 z1J$Of?O&Hg6(icse9gLfdH|&d1!kl8vj&e`)`7k2iS_oF*PN8ZNAL!*W($}zXZZ{L?kgkQD#9~jLxTokd# z^nT4pZ)iyDgfA{PJvm@{@C|1KU^9ALy&V!~DgfgFl}Q?`dk9>%qM zfN?jvvp*4XYckQzP3VvUPpXC}C5Sq-LoXy8;!5-a(a(^YR}0J;^BKzfkBJ;k~IOZg!AIdV0|pCvsw zuO4MngVrb$36i6@h9ljdYrdDu8#K{2(p}Weq+XEl8}Pi>PubtAd$Si5xCqYX3+g-< zqJy2>73#fi`;mpb{4IKhq(n_Em+S14{3AA|{YO0V-)qQd#wV>fVx;I3wL(H|>{t#R zrIDPRkfhy`O`#mq#I`8qY(9XYPR|sq9JoKHTU_1C1$Pinu;#Wh&a(`~AYk(x1A%rA zhCq@@v^F+=D;LP5?ld2oMG|}}o=oR0gu&xZtZm$W4r>p|zTbIT{<-Pt5wA(8$^DvE zz`@`dVDVn`SD@adGB?fFMQ7Y^>e%iPnQSD~1Yq%0^x$fzSk4p57bQe8Wnn0cU%IR> zT-ko`2R5mRPLlg1?Nk)y%_!gsX9?hI-xO~asHtiNZ#Emt7zbTb@;fgX-koNdc&f}y z|1?XbC+#C*SiDx=M8TitVe`cIJ2nu1K>a33N?j67uu+4DE4>Z%DmA zPiXoNWuJ(hY%#NEVrJupw!Jz^Q44t z4j}(z+#wu5k6-Z1k2lXk`p2sn!~9IjzqWl|^CJ?uf$t+XaMF(rn2jL$%94p(@L~>Y z(^)e%#wcCjR&b=I0L0LZ_d|(aet(`69c8C4;aolD8n6XgNM!g5 zS0xZCf7UL|xbN0?%g`)Mi$$zW_kq`bYFiC=&54e7QNno-QSq`@w_#%D{M4vHd-o`; z;?SlHn!dN`bUnVld#hT!J9yW!NTZ!zC=#~2C~6dXaI17mzz5z=|T_M4}IL z{@zktm>b^2s(*0!n{a_>8&z`ETCw9j(#@d+DyHHPW$Z~RqAs<-ND>l!S>*fZtc?;O87Q5o5;V>Vx- z#i^D}a|QEguq1&d)x}jn8SWYegcYmAVwPJ)e150Z*k6!y+xyeMLduUOx*>n(7ElX02kC~MSjHiZR5+@ z4tk3ECqQ?0epq|L)TBOAYAca?8dUl3`(BdRskx%;1%EvAYoD|zM4|lZ(!mEWRjp!! zh!*6HM9t>PMw|w!f(&7Z`hWpGzkqWobm8QBew!<^UHDt1btJFqHV*MWd)kEdCp1Gd zTd`JeeIYqZX5$CFiIFw1)n_X+P=J*#D6;lp&ArNO26As)goe||Z%enYm0m|CZ?+vV zC1Mj&g2T<# z$2yIkSQLwNkA+h&vg_{6wm-6l=AnBz1*%|~d-@jYQ?%w=&Gi6*Tk}QOkmD6mSL!>Tirwqes$V zKADb`A^v69G^D+xxqIm(t^y-``$E!MtZE_Nxcxq$$OX_FN)-ueO2+2M6i8G+oj7_c zf_K+>7@*vM1gI065k~z=mgtA)dg91rOjjRiD-NL3PokbEYWhpQ`8&&X{pv`+9Kei% z8|Vq;HMvR;@u{GCmM!E=vy#>-hutm$f=NKR46NFG6)V%DtGLI=;Fx68TTw1tvF|fw z=6Jy>i>P0&b98k$Irl4orj|SH=P@}xGW zqL&fb&YoIGP;S1Jjh|@~T4bB>oO{?3QN4saTN71!|KAe;{S8=>YmHT^S8V3UFvv6N znc%8XMKoTWWqSl6EnKB8Znrbg;D857-8R^gzU)zmAm3&rj?tRsg~+c@0xfq4Z$tWb zt0>!H_SB5uV28lZ8Rb5}L^HoQ?#33I3>7632{p+Yb!3DaU`~ODjN&o} zSHDYB&<^thXoqMk;gt-GXjrGdIn{toKB=OIb*R*~6j|R~$Dl{xdpoI>fxg{N; zxJS9zGN4)p|KrpTqh}LRU1?%|@}=anOl$Gtbodi9$G0Mln%h|3r1~48YWEik8Q#<=Sb)~w)&W|NP8`ec-0Wo|<~EMFCtKV$^Yn7(#}qm`aS9BS z3KHD&cO7oTVj*ve2|OcytPY)sn!;;Xu3UzpAfcsDWmYh*);Y-yf7pw|yB8R5_nf}E zNTiw+A&Ga6btyRAIY-xqy4B3xmH*`yoT*VXaLI~&SNXl#(HtYVjvq@F;uZ+uYyR{JEyU25p}u{_sr(bXg~P!C>Sp4 zSa#yLT_;UxY&(zc>IFL?iVf)Rq+a2>^V!(ic6733b>%AXnw{?nkJMYv*V&5MUZ@)S zZf;VEIvF-JqBlTwgc3=QHw6yD0%(MVh1Fj+JkG>RTxN?q^NdKj)1xbn)KH?b(&sBv zpAzC+7iaT(^e$V34N;BBA_Oc$AfJ;V7|P+bNj+}$C)#)^^pos+he+B~4XWwR-7zoIvtIliWyC zD=vt&y4z!jH5S@lbWfagqP%?c<#laALC+a(FM2N6%bb;pNz+gx&uiZ*vBamU;mWLj z;5#1fq)i9P4dSx8HF(Tw#<2`k6Jf(#*Tj@Vd-?{@TMP503fx&mPqJ8M?>hFCvA%XAD*b%IDdV%n%!rVK21Ur7blB+o4>4GH%+>&#_K3ARe5j31|LKRw=~ zW5C$st4|c;p`prinu%Ci@k3y`YTBpiAdZmszszoXo4FO`Ledd-O6vEtZbsyUY7^VL zu9K*9p~=g4h?F26D0W45Z=Hs83P%?Gj0a+)xg;;9<6OF)K;p z(xaFPEPUT6FXDv)f6bCXkS}K_htX3z~T7K$aoje$nU%7Jsh5UJ{-B*FkB=qAH_l_2YTlKPi%Ex#!q~xDj zDyTnZr7tG4@Al45I#lGXqVyLMqXoz%%UfVj(fC`P8C5hA1_;tzViVdkinO2{?_UH- zErr$$RI``8+{EtMRh>K`)Uaw`WL}$&qmOQ`-d^3A%kGMx5nah6C>w3QWn!3ql2`+% z(1s+?5=11DGWttozSUP|CMH2oeDgzd2%S#=QPUprVF_v&jmnZV6>@XGqf&i_digA` zd@MuEh*^0iYZZyTvjKZF^7K%EGZJ_@N0_t7#8$}y%~dFm;S`NpaRoxy!i?$TtrH{h zfEgDtupl8a;!dCU*xOI#bIJQYPMQr=Z!=_;?Hwz!(FS{>JaL!?-%SV>rDPX28n4y& z%WSaXHb?#9Y<7HKB?s-?zdo0MrS$FAR~A{lH<68Rc9ei+fe-Fy*3&Bu7ONVhRn4Zv zoXp#^umWKR`ZZJQiw`n|LJq-94Y2TC35V^R`pEe?_(j9$K12QXu=8S=S#)^VQE>Y% z^iz#x$&nays`GL^o$OVI;|@aVY7`E+l;jVb7g}%|Z0O*_7fdii=T}-|Z-QPDkDEH; zR?*a+548P5BdUtl#LaJ*rHb? zsv`*)>!lO;wdi`*@Uac3!T{iBEeWQMOyE~}S!ZWw?=qcs77M4^b69Std)#hmSbMlV zRN$}>uxknbIB2Rpr&0{I6)qnZ8%A+g$(-@A4yT5@k{0DWu3P~nVUWGK>eAp8RYV5V zCa=Cq4I?e6wW@#XJ1DFY)c65=IA<)d%`5HVfxvhz89SQL8oJ4|046kQkSAH3Z-r4|In%VUq;>>(*Lu!eL{P5#vC$N zm+-TjL|uZ8*lq2hNDvgu6T?Ua$*tewn~ckjB#ux8>38#4VrX^IjTuIwN)$K3#ImCf zC_25KnI8;;jw!rww0cJhyV^Xr7>J^!w2S&)e;;w#(XYk~W9PXjl4ESkXmoO;A!1;X zKXu_-8+s;Vwnt4Ra;V7+`hhKhP$&$YBOReQ$Z^8+RynH!=)`~vPX zH5Ck`5h7!{YBL_vd;Yd%Nqb0me-S7j8F!SEQRf zJM+u?q#lb9phNv8lj*t~dpfP2dFuE#bZ(Dmn5c4OF_Y`faeZPf)z(7ir6@dd_;~Z< z^0C3==uBIzJ^sXU4iL}zZrfQRq<-L_;Zd3D506LeN6R{aRbggxj`dzN%dg2jz0o4t znv`LVYVlI)9(fm#Dcfgg`UM|LEH8AVVADep)(qJ zdA=g&t1M3U6g0+}JOz<&{nJl9Ed?c6bK!~f5DBeupb_Z*Vwu!JTtN-ioO8jv#>yi0H%8`Pf8pU)RuJ4US6LSqLWL!dl9INLoCGTm1!W`q4oQGW`ocJ>f>g%y&%tIA#HjG zM1{f2!fh=?D4Gn1WUZbup(^}z*u0;`!zhNce7We&+hd<;T_Sk!Z?4t_9K3DVe5sM zy?=^+|FBSwHBp?U)tw#-WX6}TQ_MC5^Veq*KG>l}dHs65UqtibtD|Rd)C=YylfdCW zxjS*(^;q~z>B`|>|5$(iCT@z=5@*_T@&e%>WR*aUwp7uOV)(Q)!dy25DzG=_!>+h7kLB@d92qsD zynvCtnd~E)dnoH)Fa^bheI3Fyk6x)2(1vadTO%Ygk|2O6lZONPW?ag0EE&f7(U(9H$A4zt=&4HvUnRa_+pw) zF-WTZF^x^Db#;^N1oA1LUB&|a|45Z4DhOi=nGj4Je4C9u&m2Ttcm#VeR8d_DWqD5U zm`NVu$sSj2p)Pk1_D)nedQ)i>iQL8SpU5?5`I!JnV|uD&sveP^;GhAdvQ^;w9Iouk z(mCb^j7|a4zLtCQ8e~V&AgG%3jfqR%#hSzg_!>)@77?BS>~#m4sr*9O7P|p!@IK1O z8V0gLOw+e8;(^o6SZK&B8`1!fV}=m)t%kO??;=)S3oSGOIt6z#0R7?ESwoCJ-ZQ0f zx^Ky96rH6ckfLvb#Y9t<`Oi>5<$H!tG+n>t5MBBGCV&v>#e`oMRLSar>cKymn==&& zj+F2{NZ(jRXhH*(dGi)W*2$Hm8y+A{-A{)yh1EPiB=waZO&nMk`sen#(=;#Yz9?Sx z6F{{X%$e65lw7L4lK$(_b|XX_lub>_I%Xu>P~!OIJrd=8!fEYjh3*U3zc-Mr;wFTX z4H$gJnxzRVmR{-=dQS#cOl2f;m0gW;A6Yyqf9q;Hz)Efh8|~8}lDTqesK?LTmJM<- zXyC}A;Qk!@*eWe60l?XkZ9D7j5h_%Jg1M?%%WDth64-7-iDbrE7QVAMdv7I;bDi7f zqBWu)2Hxs()6%ZN`>>MSBHd9{IQDKFEvX+%Jopzu#I?o@X;$zSIXW%qF)bT7mX<2u zi80c);vw}Av0ZI-CenIyibM-$4dEVwIJ(gJ;5M8Jm@nC`@V*g(RNRfTw6Y1mBS%h? z4-R?eb^lEIOP@~i_ov^eThYA-F$QOsD5purJ6z#V0^TY-(a`S(Vg|H}J3V~Qa83g> zrJIX1vcU!qn@9*S0?d1N2wR*Nk6lHV1e;Wq%3?DSnru~3Tvd(u_fM;8bhu;gsfN^g z{GsT@2t6o*l!fC!vp>&8Dh>Crm!Z1t2iU75AIW&D%3Y3& zS>-_gD9ksDxJGH_l-BF-XN6QffR2Q~JYu(MZkF?V(;kgQ3R=H~vQ3!QuQD1tE-nXu9A znoQ9QU?0?M3sNcGFSTx15Gk-P^_gl@Br>?T1!32d;+nulMAM`5DhI?tKZhLW2s^!*j2O{rhs*uT$gN2)xwIbS? z^+C`3R6SZSM;Kf3b$dx0x|A{|Bl)R`q~RXsUWANA>iR+DK?gJ>#c08~gnoU5Vmv0r zl3%D17s%bV@7MWydGt7st)-v#qg4aIO!yC$&&O1?AN*}O7VnjE^~lb1R{G8$;m4i& zXm{eng@f^Nx{QUuh}gcx$jvS~J!DYT7U5{%y_Qx)VKLpWWU({)+2SQ+X19TYfhDYI zW89nA-}*TQcg18C{a7L#Abz7PuP#NAXsbPHp*V!Omm(p3hR6DWdoAVW?(2~0*X4$e zP}GZ}qn|RojoHSJ5{38m&ShCn!2)58)Y%)ZqD<+~oMFrIQ;VbqQ4K#H!01Z58HzHeW>Er8<~29`Q3J3~z^kc7wiWnrZTRRSXV@~VscYby{#e57`mft#!d0iKouo6>z3^kQ6v3Ed7f-~a zVs4XNS%HWzVUdSJ23e%N6C(*b$#eL=5iu-4`6K*cf-LgRhO6QH73>W%5$-VeQ*|Qn zKsYuJ=9%HkY8}a3VR&Lm<==ky;Mjp?c$YBd>C%pZ2fH-DQ`W#wBvHqy%<4C7w?@PI zj;B>y`NwSSiMypHj4-FPtOuT*>uRGsgY-haLT3p53^)sdT>c3AVqi2w)-$Ybjayd4 zQSF;Fcxz9x;2126JfG{FUF$o8cBakapg)#}+RQG+2x7T!`XrCd>iio!R_eF-`EOqH zbxfm_sq)8yh>V!si9&J5p9(Q1d;j`Wm_DgdUQ218v8L=Qqg~efb>YfaKV7W??DkO? znuq9$?;b~2+y7|;ZV^sf!|w`=Qaw$p0*N%$+}2jw*x>K|Wb{|(79)PlBpu{4Lq}4& z#`N9RGTwwi7wgNp(iFYf!H=}<0gDBdyNSn&gY{{GuDu7B;ToiOy~VMV)Yw@R7m{lHuR}D`Sh%~e zv}mY63R6K0XSDNj&5OY>_V$y|g4t>-#}pT6n5jCRtBb-(D{j_40ABD=;?1si>5H?R z$+10ToP7GpR=OeCSpXi!z-%5iIKX$`#cOuGb^v;^{IBksbgnHDQy{WvVnp&~pF;RA zRnONoSoteH(=q1`l162K-b9zqL@cj2ubWKUrTR>~BYwRP&7T15L6o7ct(}<`9wr}t zjATdX3eNrd)YW{&5{^81QUn}#|8_UI0!r1ecBjh~!&&LLV2&@ zKcSH4BvklS|FS6a+?mR=uNRn$tuHgj&+f-DjNSf9n{HaCat|X*;;4Zv)APuMS}{ur zoOt_@e+qS1YV{|D&#qQuZZE;^rqD#1=%&fRi(A`An6BY&uFGE7QM&wHLj4aeC$l^3 z?i)eVX2N@k``O2Z!!-KdgQfZT7lrmy6MC1>k%J#fe*$+{(UhFMUH8S%UA21KECjRE z<#T5BvNPZ1;Kw$jQ;_k>Ou+T$a^uWPRAJ7j8XB~yAy-`1%MqCV;ynmns4clWz{T6E zK#8u6oF9S4hN$oEvGS*n=qu#^H&Zx`od}!^cnzhqKD*X^q{ktl2jzUAQ7(7ZuWiBM ztyumB{)ABhij0Z8_574+y6-Z-vs#6U9i;NL^YMmgjz)-4#^Q1v&AmCWP7NEj z7Cm_EX=cf|t`*#+Myok9bzCT*B&<0YScsZJPfXLs%@)FJsV)nS#P&Dn+TGqQF=e!N zf7HVk{;noO7vArEaLtF_5q+r4}>kVCnc;EEollY!`^&|93lsmj{wP7 zk9L{u_E;_EU9;iymxq+su~F+4$^}Wv)+E@QoMB_Ev1Cm~SjtqCDt{B9-H%$^^b(L| z4~T}&h^A{04+8rnP2P>|wWsBfL~w)y&vPYGN`Aqn)*{gW_v^a+$m2)}dEn$mizC zohdbV);^;+&aTSO_Qkn}9N)iI&2C!MGenBr=2T#g7id1pn<|Gn$UfuO;a`pfAmi~cGvLc`s{bm z`Ij#9|7Muhu7fa>;8CuLVB*|`2|P;i{rwC4x9fdp{8A4RqxHk|C3~Z-)^^u?d_&*B z|D8OY9Ro{2_9im{9=`$$m0H=C(yEGW84)fAU&O}uU)BE>z=z{|+4TK=b+?S<^Y?nB zr`K#h6G>EDwie(dZ3f6S{%CS=IW$Qqy3EipI(*F>opgNf)eG3;loY9Q{Ym-g5lq1Y!snvdEec9!YU z^-x>Uw|95hG${23;-5NPaMtM~ay0x8-&Y+&%yHWda&@raJo@mamv0Bi+*Q?=3Fza} zq@W7%5D5R|$18x|^JjNz&9_}%1^mq4gQLQ)?1JbI__a-2?512*H18mKll;aFMz05+ zh>%R_UBHxWLZE0{qG}3%JGo8;>|S<#>aF0Y9#9L>&A zsxziErN*!nCq%O<(tgQ14U%pmyXi1eU~Cq>exZ8vPAPbM;k4J2xR7nq4#UM=1&>g~ zZ`+WhL(Sa-O9E1HnEL2{UMj2zyc??B>gc+|lU?-XAz8NFt)lyijvq#chac0Mmv#Al z)I4{DY7JC0aK2r8Mifsd)ndnW={qvnKtE~I##5%Xo z0eiar8Zf>~M2ss8OjW_*H^iQde75^FQcW%^BC{nmT(dM0R@7lyX%YCS@gQ{LZ&%4y zjJK$?ZT#h#uZGKCYM#XtAVRBoa+M)68LIWuczl;N9x3!I#omA$I2vb%}r#^+0Wh!UvxpY}>8ZCX!4 zHv?(?S8~k$V@AxwT?f&!T`3Afa~Ir?a%*dnaX5pj|0=kF!g#I1c>EYjh(*DB{-W z-p`T})S=fZY(MRVRGJZ3S{qi&4F33^`}YeU9OOgvTR6ZYw3X+}KAMyA+*pMRe%AGd zn}fPQ|63-J^$aQ26AQyyY6`V&(@*8I5%}XRpwi#4xy;r-Ys=pp9{MaZW$3g=>W=#l3R>?%Z*|!UO zmr9|p!Pfo95=l(r=FvxeEH%l&$~4Mi|#T`e*dfP9%${R zzL2qgP3I!@_uZXWF3#8@aZmUYdPF{P^dv(X;KhYy-fupFZZnIml}YiLr<)V#MZ}Cr zs*EI2p?Aw1M@~Y#GP_^yEpdI_ndA&&3>T* zpJ3swO8>1GFVH?vXfi;eb^;s@Q`wh)+4=OHCcs))1P1vxhRi`DOCTjowEw(9D zA)l(_3r|?HMkTZhyUpRx>X#ww8&Sb82zQEwCk}>9WJj%c%wWjB#cRgQZMKwGpADja z6>iU5s0reHDfBd2shtvjAEhjBsHZuZ9l9AgNgtK8eNXCD?niAfBNcDy7y5ev8%O-d zTV^Dy{0?&ngrf-m`g*$mFu@h`-swgrkKQjQiT(4y@}7QD~1%1?4-iN0nI z|2$x4pVRI}r26x+NI_o8V*OcTLC%(3J7wWrQ}-02gHNQgaqUzcn9;aj>F@Fw_z-sd zUj?jNVH}fvkLs1q(ldy^oRAFur!q<}nzem`J}0{zufr~%7${6=tiMr%>)^27;oHmi ze(z__9b$hH{UwqYUEY}xxetOxx)Dp4N;{H$P9WP!7TeD18EX9^g zu&urMeV6(1KWnb5--e#<&-MM~*6#}9g&QlEas0q=7Z9bsY7Yk~YiQQXeN~mVWeVW>}q~JOK5O)++ZoK*%?Cj2zcgzI8@?8bN}6J(-9bx})f$8`kQ7m;%Gt zUura24(}BEh%+wz{joN*g!93ivD?jIsu4C~sITjRS?Vtmm96N;=!)eTv*hgYm80)| z>cw_#l=PY^FOmVf;yWwb{u>2_XmW94?uT?xYRmE8WodqkCWa&biDK9k-dN0xAxQf^TwkrSw^NMzX1nmHvU$EdgI+})abKyhH^651GsoF5IYP4Wjpz9y*vTQl{NJFB! zqY^uk6?4mS2`}TY=xlF$#`cr@-^cSWKYr~=J8HnGS*4Q*7}DNHNrfk>ZUO9Zndswx zHG948!OxCN&skSc+#g+{RruXXZPh{M7Db!eu^RdFIUt{ZZ1-riT5OwR3>NVYbFy1P zI76zk_hbBSNE5#O%cj4~6nj?{yd?Fj?dJi|y$KN_`zpWY5*}yVeYUJt4Lhu-2KTx~ z-jcBF)+V!3jmvy=XfK8GdltQJV14VzX3@PFp2e*snK{D4p8|D}hXXsO0`tE;zqfA> zaNPdSh%!#x<$RsuY;l9KHR^nylM%$ioPQ~g>AeP_*^WFBXhUBE#OfqSR-&`ks|V&F zCi%-Ozh6+z@bjs%8S~qVgSK=Tbi}cw1LVV-BO1%>#MspC7-JpUiLVF@FtDKU`qQO_ zkA#{RJKOiBroUf5AwYmA*~(fyRsYJnxfIgx2j+w zQHLHm>gB4A(#g?%ydeCGdWq0}cAGK_6c+HSS+vz~y6}IL!4H(3YHP6zN4g{FT`&LZ zSHq|a*(GR`FJXO>I|E3bAE@Ic3-5t5CNMK)s^qqn) zwx@L`a|G>e8k;RZZu^UNWm4DX_el4eOWIea3;7rIgkzECJGYIHP823du5%N55`fJO zG`sn(0$aiET$cm`*{WPgqCW1-23{`4m@^!m*-M1EoL-^NP0W@_B)Z=Rm8}xnM!V=d zqc`#7LMGlT%VG z_{L~8azE9qJqXvcRdmwwUy{Cp<00Cj>!+!*0`8s<-OTbA7r%Z$ZEH3 z`Pz3FzfRO{%9fT$|6?&uTl+a*TNbqKzFSvp!jd@_Gd_2BnI7gkpYk5pOe#%YLgC=? zee#rgM^2IwBS4Z~E$&sW*)nUs`Ryo0433!;*sziKDKI)BGHB)r6T z?5K!?PMScQZXV@bjdKalv>$KFcCPTlvE=PE40>u%^3R%SNMldotfdn z&q)h4D_hsxdL?f!9&fa^`F1gJmYc7*)1>`bgN{RwBu}EhgF{x3c+JIgUzGr5$YqQ) za(c>M*?v)rCT|d-QBGjl#}jeGps4_|XHjn6567Yqb|r%CGI=x?^gLsz=Z`Y?4VPO~ z*ojYN0nV;>3PY_Ec_&*o8cJlS5Atx=r4z`(xkek%SL{<_*sQ}Hgb zKp+H{Jlw$(7iy{-ji@>c!u=+;F!qjtL4Lo^Jyd>4;hYmNfa@w5`bHo^m4O@c1ar-H z>_J11iH17D#!@7R3A zdYiF(qsrK@``L2IfsO;^-goRYQ{r7Yy54+G;CR`kOAlRo-XIah5qq!%Jw5c|5e_c+ zT*y~2oMt@&uiKq)5haLgM>&@9sbX9O0O^yyE*NNjW{wQLYd zmt|H$3*#88T+sK;H#FuN>&e8OLYT3*HdkA{%bshL@t8U`u?rv0svKwT4gpw4UJ6d@ zIi#Znl%TcOb?$;zwa3IGq;VqDcG7Y@5tVIMbHTD-3wtQGk9=3HWxpk<%TTBylzB}; zP&8FyOQgmd^Pdwx-(}Q}nEROhmkAqHW+q88H5Uj;?c9Ey+2UZ!AD%Pw7Q5}RgYQ{* zaNDNjh`=YlKm4k_o6TfqsxNFs;q}oa3tv(@u{|P}n?@Y}{^Nh=Bkg=6xXThtj3&Mp zixf8XFzez?#>u`U_z0WRokZ08be|jBV zlMm?9c8p}5Pl)afGnILVy=G%KidLi~tw`(qs{%CvU`$%Xy4Z_WAOMkknLKSZ&qerP zSgJZaJL#AXUsA6AvdAx!XQSyyOLM9cBK2htb1F0p#DVc40IK$GaXQ?Hu5F|Und&gZ zeU=+DesjHeaox>aSBCWag!FBdw?rufW7!Pf&91kdfX@<7Ybe7^iWP;@&_;tOdy=*bZCR-Xsz{{9-OSDZ8Y1;v#=Vv6^lpS$Zzc?#0b z;M-_7g|>vB7+4(OA!FP`JE=6xyfqufeos9OZ;aAsK>}Zo^4CBBd2}bKU;t%jV;}!` z-!Ik4xRPfs5U@HWqnC)ZZu!LTU41QQbqRBC2|ChSGy@oFbF(4cIk84HOq$i74Mq5gbjHDieC}%(5r%CHP5^ZGyGycnxxDvTY%B(itUdh4d3MYTHF7B&@k(OD7AjJ$DYXq#dP#U=B=v z@AuOQu3*bX;4_|#811c$mCyS0{BBh0*`hQt4;K>CRf{2_CI*nCZQ7$D{|u72yJ&u< zaAM$B-Sk0%Md+298h7LR#bZZ%TinQnLP`ca*Dx`kLe?M^n}^u-JEIe7ZSe0EI_a($ z4pz{6PC=gZH#d*wxg+===<;0DVc8SOzv%NtAgmM(_1cr+&7**0-e(jzr+9h zIH~`QuPsK0Q2YBjWVTDJ&FEkXEaFHysmR-1(w$)_VJ>!!-or-!tfR(s$$~~`##<)Y z9Vy^J_in829d@E5W7ou!adF^xCZ3RdVpS`NF6V0Alvjk(42&-X={%qIa<%3$g@{?} zP7a5Rka%IbdTcNPUJv4Jm${1-qdOX? z-9bSV^w*BA^Hf`r8b*0~dwS&!7o30sNhs)&Kqw(JA7-q3TkYID#o_VRs1mwzgY!QO zFe`?5eI!NP47ZT*p4BEo#PElnvte5i$9Z#QX=$fv4o6lOLGTsU{P!ie`93BoPORT0 z9i}9Bl3x0Y*ogF_M~2&c)U?=%560sKudLS=PbG~g$@cQGLV3BfyuZW*i=mo6ruJT` z>-a2jBv=jRwy z`}FiQ&(c#8&mfnkN*V^$+mCW!zlFiC{D4>*FFq{<*25uw=R)<@PZ; zikzI>H1z@3QzUncK=nYeK_b#In*&1GoqVM9$~|P?wJN6)OClinzi7(?Hr{I68;vlR z^VFaP3S&BT6;T&b^uoDNyUJrLgZ-0N!TyvbvB8KH7rw7YL(&x01T@uG;i8%_f0ONM zVy1l;2w+ED=G;e~aq(zfQe~+(577>GwayALoeXSWw|!8#Kj1m_OiLJ(kOH_+XL5-v zK&?`cqeDrcdj2d)Yy>0*N13Fjr#C16@Ka)f7FWbUr(uS4ZR_09jY|3< z^ZA9gT#$n!i`@Vd=ob=CwoRNf?334h2DMn=z$ex80QRj68B#cS#_YD<$R@s_hhJ=&@>1fzdv=5{(u8vMXHt*&PZB7=t zFHPuWL`#+4mWmMGvpZEV&kZz<)LJ*PeMnFI*kyIKK zq(O3sLApzn4iOnj7?2nmY3UeB2N+;zgdt`an*Yb$-(I_WuHEy$&f$(X;CN;}@qWeg z@nC2#Fjn>RWKHyAbFkqm48pQK$L0P`g`7gLs&c#7uNIxV`#}~d!T@iJK>REPLj~v3}-`IU{x!~Z&1U~*BATQ<%h@8TzIRsWzOOPj*g zidDO-c97abm49VXJk3P z{-?O^Y1fAd_}F?2<)FK^{y4WSvz$3PZ)0Y%N?KPcaIrGgn}G&a3NjQ0Mdl1~_;x;g z-b(BQInEhUhj~uNsRqUmhn!PgI)m4sIUU!mKRp2|0@>ZKrcLqX)wY$U^S@&QWlK75 z0<%bGs;_;6O*A<6P+a>gQSx5+;n^He{S<`g3@?l&lY$RIpLG(oA(+91~FGLgp z7f;udho5%x#0{ir2!b&?bDPFK;(k9x6lG_vYi39>ff1Ru)==TCPASp6sj4b0SU;O4 zY$v!%^)#_h<+z>TVFeYV*!1Do*93f0oEFzR>ri#&4HLcH3m*d2mkZ{=x-Tn(A6b$+ zZN+^nq&f_B-Y8^}^LuFr4D}usfnZ$EU`3lg;03U) z#N-%4<~^1b1+%hF=DWGPox^NC5wDW+MAk0IMc!{R%{;&lY|Lz{z5T`owpvCN&n0v6T%->W^(?E#FF)k%i301aIxI9YJ(_V-?q3Xx z%@nm*bQFt5B*G)G;1g-BOFd{XWj197Hv_z*R8oy9=U@Giq4h;F@K|YrhZoF(Jh6aK$HSUeKBq^QQeIIL$CghsE zrft_nw3K~CA9I`K*34LvES@|pBq`7Q-HlLpZFUdT!*oXNuK}>Mb)-&+$$g7`F}$Sb z52j^HNQ*k9&x7t8AWJ@XWq0duvixoI10N}q1rW~G#Wj@fw=J9K@||2E>AE>;zZ$h*@9J^%`iJ6fuU!=-xJDKDaPBpYVSo zxv-r@M#^hnHTu&@Olx-lD|5w`D{86eF1~~~$+od~zPVrLCHNxYI^Lq&f_LLpPAb@+ zSZvpXGx%tmNMD(G1XvoZagm3_d!H+{yRGmZi0XLQ1wt0^yWeY;3L}nnvoz#HX6!G& zy7lnt>>|#Acl8;ax9$K|VxczxhH+hKU)^e?x@x)1kc1p(|9RwYyKq$M-s;OJ;@fgh zhpXw*xc!IQakTfZAu{#LMx?8sq4`-6hj@vxR(Aj2C&0*$=$R?2-m1st(iu7{SU2;i zvAtk?Y~yQyanP;$`3iVjOf;TwyE?h6iT-gDwptMIwJVpPuSUe5n=P20VpSh`=zYzz z{npb}J${z#)_QpK1$J;9&Bxl%M`>mu!7wbnq3T>}zr-D&*Yg0?b_WxNppDuUt zP_-}xU1f|qH`~*Wxc0vVc42Zw{+zJYwd3t5N#+$mcG_KrQs$(md3JJ+Ae(&xc~$Id zbhCR|e|*X|kv@v&62vX@l@O2Lju+-XeX!fit{|~ETjv5y$kCjjZhBT#!SF{u!vI6G zs}^JZckpIR;Ag8bu@RM?O4>wZU`+YK3|zb+w`r{V?*|ycVxmnc0?|lb3aDf1F}`HH zvnC86AfG^SKR*Y2$X?PasR!8}>mj+CBBN_~DUZB)llqq0lgy6J-#r4M6kQoqo1Q&j z?dA8@`vEj4|Ru>?y8R7x@>^jz`xvzixBANqvQRq-x*K zDdNMJ)fH#XRL&f$i|z~@nbkAsc_$wsJCRjjm@DFp@OUl@? z+p=ep`Mo{myw!ze38DMhc+U9n62 zK_RVqDJj!z{g?3c)naDgAVj2h zb{Z$MAIWV{PT{L)C&5)Z_T*@N^QwTrsVH3e#P;?mrBcGKdmGv0Tc)h32qbKZtB7p) z8}sp6qmpnHy9T>qQkbIcXC~oG<{%^2FYlcYukpp0+_I;*8%}?9t9!=9rr-4{X0Rr3 zShlA{W4@bO!+i4$QnH7cuFE=t20dT;MHSOt}POSqZ4o>sTJAqJF z<{>HOr0>*~3}r$B*OE-;mSJHdgh9*(WVN zR((%NQ6XulruBfStfK#m8Qs!1YszVlA5ie2{KDyI<&Tf-e=jXSGpt zdxY+Oc!mYKb8fUCHxS0#LVwPv)TdSHs%cNwOMAS^9bLOvFiqRj5U;1yG+zA?%KX<1N zd*9sCV1Bmlm-zPuoz;c+w+Wbg3*l0@^g|pglIb-~kg?Y$k>~c3DX5)(xZbuQz~2-v z-nKi{naqf0o2;?E_cS`2uwW+L>0#kPGuE@V-;XfwISm;i7 zySw1(R__JXHe)Wb7$GER5W`B-Z5_B>;(nof z5g4!IvNch(80k*aweB@Uib^(bcBX09J;XM-Z~i4KF*+P{tDZjY<@KqYXIAI!X*5OU ztCIn-JVzhnJN>b59L?neyNe`^8I76xR7yD09<_9_)7W9oGV>x8Ob6Hk~oi^t4>K%wxPt>!6;^!((~&-e_O*8b7SY<@vty za;0=|a^C?>IB?I0b#O*1T{ib_EPjm0)dc35L} zil@~>IQ~8iy2-w-bJI0SYd~2%ju1!zmALd&jyRlWL@Oc=F?h<|XQm-6KtvQ2-HE%H zvl|0t0%1|mmP_03p^9bBp0ZBx??H&iH+IwUlx+p@G1>$+`NTx=X2%zSA85!bsWTyB znW{UDGrSR5ANL!dX=}Pt)HxHm5^Ucu3A6$LRk)dxKigP7$?Ck;g(|~;iUy?PHk>W6 znf!`U9!m?`mP-En3wS%0N>r0badWmlqv$fHriQ=OH}InaoQDSHndi^F8bmi{?-}6? z@e-mNN}4mWSF@yTCD_f6IX`4~Z|so~JlOwnGBR}XqLsT~@RVfkH%$*mC@!(i%GfgQ zSon{~LYh~xdsE%~18D53Ie?fw{guN0qv8|G!S#{k>5MRY-(1J_9r_+^A~5oto!LO0 zd3$S)=X)8!6UD@bsoHBW9m-NDrN#o;fmO7g;DlpUrR_^Yth{1KgS@q+l_+Nf@tvAcEdQ& zho`RjVGEGBiInF2VF&L?Tsc;#{rdi%I7Z-(9~+j*HZW5+S8B!P;1reO9~hoKAd_A? zfrFpyQwIWuAcJT)vTJE|%UrDMoIQ9eZEWvv zD5|Wy{IRw+srwC-!*?GoJm)iKiL<##-Wxlw=T83d$<&Ghc+3=ZlWgDmVMJ^w5ZLG$ zcz>)?)lsh6gO7$7crAvdeOk}*{`uh1(s#IjUh4Dp4R$F4wP9|?pXzGu@aLr$>oJ`Q zwM0qeqhsfr!pa2W;=Vs*pTOhe>|MUnR{eGXaX--JNHruy z_wecGeR!vbyOo5qTuTw42xqWDhGMUGjthrEbPjvrO!XyGW0-qJ*VD%81}0xu&7#)2ZW^U`7V{*TS#%>B(>P9>nU2byJ-X$tIF-Q&sW`*uV`|9hTy?u9l0ks_6(1G zF5UGN%604c2np0Pd|9f@=f2v+Z>i`e83G#l$lzPJeZ4zZ0V($rnSsOM53nuF#O?Ju zc{0@PQ96c@T~l>lO1EwSVes*@ZMqynx_A_ymFVVS&}d1dmB#LRSfN}zx$+yCInylF z({V&-}q8O%TSm0KW59;|P3nb(`*@eKB~!Pj|F`~~(J6n#;K6pumkh@Td0#HcUj zF{@?NY9bTmLINGdPmC7ew(CnhFGK$a&x7OXU(W-km{%TAsWkr{01EosJR=0|D+=bP z>e4^Tk@uRbpp~{h=m(2$A9Q*>8rrC3a$MM<0=_-NmPZzJ>FPBT(i`e1Aj*5B2 zSGmB~+If8qrq0VcE*};-P9&{b85VHyMD3;cF)8IM^gewqe5b9z-k548N*%VjRFA9z z+gEh{jM|agY2;7b4u%9Yo=O$Xnox@GiC5(U=p22qn?@fZrhcQb(`HAAcX_BchyvEs zD3`L$jJbdm7v`q=nkhmXTXOfnzY2c6SRRu{_kMkvo)@*zaXsn*j-b_(x#jEYJEk;r zBROXQ=O20YHrbUaW$JT)M4pV>ux;P{cV%8oRuKIQjL&_Zdq_WoTmG5naDuvG3`Jiv za2AkFh#WOrXZ2 zq1LM~R8la9>mK_Rn=!0ijz&C@L(9kbiDz>P{!55ICmQ>2>oQ-E{=j{_6m1zrDbiT5 z1}TVdTuKlts1n_z&j6ziWhyEve%u}JEt+Y%EPHxtqt87{Y9A2tXcCmT@Bd#kmadn0hOkHV%gnj_7R-TH{h?mF}04hyC-_t0u80c zEnNi-2n4UkV}Y5$MeQ<6W}V}|-{<$*yNJ^dMVD=ZKdl&xTHuyNOWf@5#lu>>rcF$r z_x#?^KW7n%y%ne+g^x`<@gYcP{L>p9O^@Z_?&b?ftoLdX%TPYBzSkJJ20ok)PobLf0gSJkG}sQM}c*5NfuaWjCO@t zRpLu3|UNxA~*Ibsy=x^J>JEBKKo6X%p zlwz)h67Kj82_>8b>Ol{aQk%v(en*y)|8zM?2Pp&+Rv_Xl`5Ko0j9var9k~u_#b) zr{#gMm#2>d!-o|{^5y5K+qNy%igdZ^atK?LvXw#Lc(n;dFSSwWMM62sK0a#mjojJt zo~|n#(qlv*D)TJpY&}!{m=FFr$ZV(yXk^HHJT%4P3>ML)7VC|aD8eHxu-Bw^J_25< z-?KC0lIKg~>Qvyy?9iqx240ZiT)SbuN-}rn&mYSMkZ_@9Op01r>Yj!@i$-yVEFM-# z!3bGU^}$li{%OrS&WwyY8#!AoItV4SZ>N1UN~7<=T8(_Ry3RNB(#sU&^?2c7W|Z)1 zc*1!1$S=*}Z;!(NIx0$@oN-Q;vWik4k13Uy%Vk>zrCPPJH=A(&bnKHikj7u>;p38U z__%xalF1RD{f%F=CnhUT#s;7B|&Aj3rL_hN9VsChDG25K*2O!^pCVaQgi~roB>~ z6K;I3Uk1Pb%ztn6&Ts1?eKS@nG*CINFtttB5pvATs#NUJUeHz`o&L5AmXEbR_jWyZ z*g+>aEeaqZZrH_X#LZ6Q-3ZscD~qwE!3kdW?ENrA!=?qk8@#b!ZC~8Z7zp2<< zf#z)|t{v_BOA~;BedebqxnJZxq4-%w%<@@8q_r`A>E#4;?Ms5tJGK{=nxhc~nlw#M z7kF)R@B_oaPMr@UbJbng3_D6{^tTND@L>i24Z-NoKN_l}!&Ej)C6li=CQHbooFS>D5)be7(s zd#67{5}8aiDDOcCf{PIDPCHzYV2h+4tCf+k4OC2We}NfI5aKG~&c6&7D|G|9eg9Hi z{5niUz<_n;D{SE*rn@gW=JYr;qz_c>q=gaad}oj;@j_tk-u1M%mfO6mf@Z)GkCBBB zI~N>ccKjIqbMAUOp$V{D|@y<%$p~%UjI-1K5K%H?jtLBYIQFApbRk(bAZB^ger|Fbk z{2a>E1Nojj4+VyCn#`iiX@;9zXGh0i{fPSKT^tu8VH<>nZ1qUNqpA5>+j^N7jEWFC z86S2gJu8cP;E4R?tWUS|-RH?+-F96Y+A!BQ2^^g9{Fc+UlLI;r7Pd5(kZVjG_S#Fk z<+bV?#L~pyz@4^I24=c*eoAm97=^3*Ax5FBGmzaF>WrA!fWc7i{K^1sI?j2_tBLjwBEUGR<8EhO0Jt8X>9pJ0 zwbdHkl@G(UwSqr&i(Kuf%JfY>@np)l_t*>=O|G3iLK&FJjRS;-$@R7hB0VkJSA{VP z(U`Kv^{|t)du}cJs?RrXAeT?6EmY4S-h;a(yVFsga@E$*z?7c?7K_%-Sxc6NAX~LP zt*@n(?64Bkx9!or!u)b#QKB9A(F^;;Ez+ldUe@JpE`R+cE{{__UFL2&iXX08-BL#S zh?_vi{Po(7hKBP_dI7W|*Q(;eIw(gmh<;=I#s_vn(D0_ZdE@uk9&#ZR%Kc~QFEGWi zMJHk`6yS>0*mU{ciCg8_(RK&cRP~tUe5#{J3;#%2OE*-lNoJd!_NIUN6Ig+Gz6ojD znd5JM)#^tAU4s9Oq=!ld#KL9yk>}0)*H2fia`)j}llvDy^|1C0E`9;zA6zfZ4HV+7 zcihyXZ7;hjwjOJE#J-3asI>KP!4H(&!iM~z8)Maa*v)AncJ&;n=15&|prGi#I*vlc z&DZK`Uqrbx;Ch8oz^+K_e@uFqS#qKtEWMB6m`5%v?xuT-I8Pt^A2)~Pc=}QevEWSz z!^1IoQaXl04|2_d6;k(>+qlbBf3vGDf^Rg#(BAVTWE%;YIAiPpD!>1ZQ*RxU3f3dL z#a)iR1~VdEHN2-vPJhMLF0}lnCbNlcj$F0mt(l;&(mXuYE|#3-H-C5vn>66eRHCob zm!;>{n0|?pNXm}o^S^Vrz2amP9Z|haSW!{?z$8j~cMcC5fosJ^igY}>+s|OvMLy?C z+dU|9)SG^Ws&g3q<1q4;o*)Gc#?6lHUEBN-Vfv(taF4b(w=fUX*)}wo5g-5b?!J{; z9MMW7(xZq!xoXTVsL?IF?CIsl6*nxWxlDSEJ59JQsK%1-ALI%qq%kn3irqoLzHNx> z%Hmvc64f#zj=h|}R&od7S_o9>pjd0Hhs%ykMf|=LwI#b@s5X;tfjb>d9G_0s^2k%S zqh6`FpLg~04H1=N!aWt+yQ;N_Go7+-(Z0z(C#+d*+2;54+-S?pto`D_y4<5j=ym`< z{G!iDIPJo1J-G@1@GcAm!Rp;owE|#Nvs#g_FV(*A&GcMwd-#%t2sPV2zLu+|>W1Cz z-j<2$HMIgYiza1c%f%H|Y}f9xoG0?qzIbxDU7k;4(X4QN6#ZzVP=mm&gaY=on`7P3%=;bH^L2?dxY= z=Hm(cp%*B0XK9t-qI@ax+FJQtuM6#6{es*`_ZhXk4L;LxO|G`>-~42jux*dOE6Wka zlS(RX7yz8B!N@6);H+FRCLNgsbY{9+`oEXft@D&l)p6peM){H>)&vn!p(?)<;L?6<3HR`Pq_ln z(aYYO6S2!FfI;EP6;~l_;YzM%cj{|>!?oXjflD?$9irR)OJA_>k`{6t_jFKELlT^8 zP`MLm(Sj38-K|s-7X$RR`VDhMjiKgc7bK#WA{S4;iqB8csEHaSGQ}Bgg(1^dXl}K&zIEpSM529PQ@>_ib2MSW@pxfY zo@21)sAW0;a=E+xiMixny_AH+XIKp}vP316rh)V^O#hkAdZ9V9vl%Qo1mb-fwG;Q} zmRwVlcI+6NJ`EmXEqZX z;}zxXFN?G<|cZN>Dj)okW9XN6x)_E66a8&5O0Pjx(zHM3OuTp{iFYY!f*Ae}- za}@E=w}&6B&{8RGtGh7L^g!EJj!*w3=>R~u+~5!#$H5GX63p7@EkZ+RPo;wH=afP< zXOUmxKhdnjM#*=2Fw$!a%Tf_x&RWJn2dzv`jCHsYSnw=gPbM`gUS03@n@LAFdoacm zvXIBUp2R)m>0tTawaKN*?C+8;Qe9FEQbh5Ag@dwkUwp4{0u$V&yC@!w`KD+R!ggfEoATQ3ir-O}gEq5k*$4*7Lg3ZtF5ZX>t7& zSghW=8}xL+^1DyD@Z*n>FOSrDIW1Lt!b4xiet+@0{py56RxJh*EyEXfp_kSlur3of z3G*l0)#Z{R?(QQRk@(gWcqMVSNsd8o{j9V1Y)#??r6NYFluB>TZ=Eiacm+iMcgx>@ zur$4qDwRK^`!EvSKdBu;@oM&VVx4rA_W9PJiUaDlnr?GQ1$2sZhUYxrZP)HlQUcD`?yDmTTKI{_? zVeZi+^ChWD&pr8E+FKY*(X2itNfx()Jh!&)N)rd#cmLr7;bB(3J-bBs?MCqV4aG#g z$#j`i!W+Dr3tgKLBOWS{O;Q~8zKZzwz8R~+;qS?=o2A;wKop`;w*BHw9VusBwP6o!Y0mXuREx{^4y z(06*xNa(!zAUFL38Ipy)b-S@~>GKT*x^-`aA!jkcY> zT9FyHgBNE&n@=K|dvaci^?Xzsxge-e@+N1uwG{N(TR%r)uvr7%kb9E-T%M{7r>(2G zs2q9?WHUy~^^ObStI{cic(Q_DdyRN^2QV%1E#uMVzxqXZ-@3mg2I1{;NrPf5Z&&Up z?neuS%Wsp0;goa}E#ER;TWmx{np;}%Qomw@U1Tl)|5)!KAnE%z69jL(8UOjM2Y=^qaGKC;sCT8x|XD#SD+ zba8Q!0kk~V<8vBepHCh0W&J~!OmTnMfmX5H-&OI$3fFD&x9!wb(m8E`Pb=D4b(0LT zl|a{`I@T`?43dw9vNpe=5r|lZo0C_WsI{}JHT$#XPR$gSIDPOAS)LM_XW{(jnrSZ7 zj@Nn8+uqFq$NpN%{g6x(J@mK6$&z#XQm06k`@Hd`5fwsFegA|FB-#e%2e7Lu-VwCC zs4}J3rRv~Poq&*w%K$=h0@r>AX}pTjpKi2$m&|EQ;*-zOM(JvgLvfeF`oMaCT3y}s z+3@(lgl)qE{HxX{Y9+|yI!!Y&j`#DCT)AN{lQwzjYac8r%f6d!8}mK-13uKhCacFE zCA`igJPB&m$q}Vp|D>4fcxm%l>B1eU0MpyB8gO%S!?NsUWH2uYx?NfN&<3`tzmN`_ z7feKe4H*sjf;-RZ`extAVd#;;Ug87Au9-Ax3`S@3&(i|!TQmuqqq#AJ z>;XTh%bvQn1D##ATH|-SRj_kCeRA#9hKEw{=C`?Hnh6W~#~=0o9NaEdNY&Ck3)n8n z%nME(2;l67%AsY~!+L%BI}8Jl7LXT9xIAwvAMm!W-T+WMF!Mo=n&NWZTgt^L)-wTg z)pLcG6fq=E&c)tyn~3VDvBwG4lxKKJRXtFrMIJD2ZSlZRPn-;-TB#O&cV*X)8=1i8QqUXYYoH0U3*9K|{_7e-%{_?z|PasD2+X{vu?| zEqb;Ok2#n`m0##4y;<~hXPMPr1Wb@?w;1Epe~5(iwbkkOe~`|whxb2j4Ld-t~C z+XsGylNwen0eD@nqIs@75yoQGU2)_X!qbK!9#9WJh>qL7avDL!M#rN z|GRb>`O%$n@J7J$4#uZLuu1R&@ly8fQzzTP(FERGSJnjVtac4Q?E@;;{xqCcSh9$$>&bPuP1nM z?L5)i`fq7+ohBz^3_>pt2iR=Gd*2OF#{GQwk+9QF#xh|iNf)r?Y^A1FSxLvue8+o# zS<;5Uca=nnCI1dn8NFchUkD3JymsH-b-y5SQ`)i(T#n241qTPKln?| zbJa$YMucvT6EZbl$b`HSC*kFbysXksNIxVZ)7e&4ZF)*IIoT;JMZJK@e?cB96=uQC z@dtT-hHA%8c!BKvp|fN23(1NMF^TqpUQXi&(SN5KD73a^`}g$0yI`CRSgQ>%Ut9S3 zda2%xTbT>FO33ok>FAmJS6NW!c=}Jbu80Bdf;x3N1f19>B^tABF4VlzC{(b(6Mb{g zf1GbS4)a(-U#WhPCSJyYZ(d!#dPE*{!a}Qmm+Lg{N`;8ewYz*y)KJFqYeU2M6$FiI zazbdoHepB?c?~e1Q!z(f=nG)$iDw@mWt&HtEB00Oyw`zOvVScc@KJ2MC|TlbcQr08 zOY@FI;CsLV^O16v)@rKIr}%t_{)el5CSeN#y1DQ5Rz-=|f$72SyS$BYNC@v__L0g! zpg1Lu&hDzrUa`0Cqvxl{dZ%3TL6ICh(*y=N?B(#5V}wkn zE-gpqYFUb8$c(v znRqCZJljK#a6R`(c=MsOr9gg zwLT&d77##@aKcOf7XyiJA%~=yWs;qf9wh2P&0xSIR$_b1){2Zzx0z<5@=ff_gB@}w znore}rE!_bgv-DT`+R9a`u#-Gd)^2rtFAiILtG|+_@TQO0Vyx4bTsxGJ3Q){;v;gJ6nm zM+F|b9AG>`r~BXt(j11%ArGX%ly?MTs|uZO-rm<(MY;)yIc4d%X0yt6QMip5{L6IfB~ib{2yb&dS;pmAzwkl=b`8W95`>)(X$+7*qHz<$Bd! z_aGLU4yWzmHrCi(G22}gk*peDnKT5zF}iHQ&Z1H0EI5)I%V|dpNxH#z7s*ducc}1WAxV26D>^sFDG^YnV65nN$-bw z#D1txs)6x((`jta!7mhKb#xBdSEt|pCm;kkT}L9X&&LA-SL_DVF3&nURxd5z{{IgV z#((_B5aF*X`wS__w1O_yR8fGARN+%d4QAYhgMa&u2vVOh;w+>+DabXWCJzVZ!x%SG#+ zv0}yMc<02-#jAHf(Jkw3N1Oappx{Zmw3wp#Uk(G<8z5s&mB>h6A4|7FoT$B?Zk2ea zl{R$ToFGD{n{|)PaIf7x_^N;AK744n`Th{q?|;EEDev7H;P^W0qR+RyzP_FrZhE}k zmykf8a1qoDt@)^&?T85B^6+h_!&~HoHY1p#ndX7oh)PtFXU4vf!1>3Mwb>}LU!LD3 z3osIE2V1TacIO|sA^NKIZDer);Oi+_jzTALb0rUo-#16Bfr)P|@~=N>i~666IEtqa z+Wj{l$>D$Vk+f#H+jHjeGiklnEqO84A|X?Tqy@N_*KFV8w$J^C@Ixb7Sd?`Md>Yt` zScY}`y$u6hEd=x8J&7`Plw@;&_{%g4=3a@$UJ5AzZjHO2+(V%GzX z#)UuE7QKvJB@ZxTy0iXGf2^y_phq5NpQ4;}YHjVKs2|wFTYrhOER#V$GO`HnkYoi# z3j-$Fh2sRIEtZg_=Gmi8nq6t|@k288Mi1l)V~NqU?L|(M?K9;LH}mZ=m(A#^z0bwy zn{`Fps~*B5!n||sOw@M;(?)r*u&yyqM~jpf;nA|4Ytd+Kura9=xaO{WDkM`i(vIRjg>68*1O?<>en}oi-%{Z6#JNxd@KupN(1@%}Lbz zzSNLi`Q&2MaZM5t$I;6Y zqzCwz#U?AP8+x;;w8`neRQ~?e4UrE>v-RW4{5?%RqfAU$HO^Z3(nA4rfcESmx9*85 z?D!Grp00>xSZ(d$I_-*4x7m6BoJ94Mc4%RlC-s@Iqp(&z@A%uZOPMbRAr4-js)6!u zd;oZ=(W+vp=R52Pt%AM0G95j-!gpCke-q54(F+CB0F!jum3}*HPQ#+YH*_o7>MkDf z&$$Oq*M;coR_w2I9-lNFm3$1}p;FK(B#!-Xx68okLM_AT)N@gL{gQ zuLw3S8EP7?zR0I2(Jm?}#1VDPA?ICkmbh#h0Dhy3B*|s?tj@;1IBRdsp}GjLWf`A% zN~1a8Fv~@cqYnqDsP);W$%p@Ai=RW6hneDwp2E#=fkk9LIme2A(8($UZLtW->yRwX z^SiC&x8rp>FsyxeDGv1+Bh_nrc#>uc1d0{T9<7E3F@c>J!a-7L>N5;NjnTk)?tflV zqLdl;JUW<&AvJo*lgO1I(W4dGYjU!)-a=;Idk?~&cKrC`RfX1=Z|+n~v>YF$*DHcM zN5AHJkwW8wcf;uCZ&-I#%X!0xaVq+i;vx5S^hQ&7yAiX-US=r_B1Ap65EtbePiV_p z^UOmw<;52pPzkj9&6#1zMmAd4u`WJKBH^2MVE*iCd33 zsxRUNgpS!Vqt|(BKlOTsPrVfEwOZuP?H+V#XizLud+)~hPLg~Gx}9Bux(OVHSw{IN zr)r)GaAu=3kprI3@k!^R2*I5@JQjF_q`$%gFwQh>x9Zp))68M)9do78!c_-6}^w z)Pgvg|IXXHRNLfBeKK+T{kM&`q@oP^OnsAkRipL_dYNLg+S+xI7we`!tBdEKL(3#COm&`LziZp~ zfTx*PJn=XCFn!w#@dQsjxcl8_c9Q}`97?w))Aj2AUUL6i331VP(CD@{Me+4j*dyQP zq;VoihY_Emd4XL-t*sv=Y1fg0eV=9 zXBs0&NyhWux4uDkaEG#pVmxjFZ>1lIq!X zP%Be^Z-e#KvrMVtrACPc`E{=ljwENb?peCbQ0h7B8%f8)s$!9ztO?C8oGE*%pARYm zvn=1hk3P@>G8Uz~Hz%PtgVdjm z|0YlmHj$g9r>E1G&8r>;sDN1MSdSHdF45^Pj>g9mWoczsFb~eA? z2C46?MkVU^tY`c^$y3npOw<&1Z0YD^%iY?v@|0u0={cVN4q~wiIdcnMO)q6eMfWE^ z-s@b7oJA_cV{)+7n{lkNbjEWQ(aRI;M9Tju6{#T_`*_=^f4Bek*{(yR2-DrKoN6}( zS*y)aUh)Gf{jYiM>mXg5k#}c}lph$`I0k0M$G^+-?@|EVJEgTA9^6)!2p5@~ZczTf zb}sI}HOJM5y2om^D{l@YoUO0r1E2kSYy2_L#jFNQfv5jv89vc=KJxo0qCDuEi$Xp6 zpV?~yMt^z{JNjpF0@ZD-$01{T(o_?>hrHw3XO;WnpU~jpx9tneuIo}gWv6G?^a6~C zG1RB(+&vXxZXI6=D27B4nrym6QZ!Ld!Ve=EO|Qz6P-JM7v(iVam3tkJN~0G2k$nI= zCKBi;6Zcfql);9)|>;*SWxwfz{KF z<9v_i4grX_XM5DPHXw=Fc+{t|igRUU)jAdOVLZ&C2&BKP7u|fTu45 zpQJLikVH~<#cA<`_lOhoJZBxmnSw}G@ImE&&(Cex5jB|*67s%%_(1{f`1nbm$}f@6 zC!QZkBHWtf8-k68?ai}YX@|-cYJ8n{1UXY|32wWGL#hPb1BG?>-q9JdEM`zD3PU_0 z-n(bVa;4;!KL9Ud0VPbt`MgmEvg^GY`TT7N+`+o`*K1d&rlQ$wLK5M23=Q}wy-;SC zC!Cfi9uEFeTT88Dt8my$n~y{9R6;dEd$pSy`@9g=T8J0O(lO)+t3`jxCMXApDQ9(r z(Tywh>f4XJo)#k}ZBZ$TnPfpU_B#g!$un-Kr0YNqen2_AX9rJ1`y`L$Q$A&C9%hf{ zqrSlgR`YuXS{|$V&#Y&DUin>l=u!S`;bF$2sa5olFc>03okwK4ny+Y>_Ke}}XQJ3u zY^we(PWCrp+d*3Rd^!YXxhh{`4#i%~Po-rgHb*hvCnSMfKv#L4%YndGh;vILW}#c8 zeYmgi=IgOGvUWhO!+u=x%KuxSvU2z!h#@bGUsGp`hOZ8GtM9|F^-%EPkrqj*zKcfJ zT4ZV`*2(L;-J2kY83~=fD>f8yxTGKrFwQRP%iVnW3}3mUv#Z!V5Tyn1j~S-F6=h4o za-n940^!p@pcavU=I zb{j#;^1ruF8KD9i5@OBW&!kAdhy3HK@|RBfH)596@m{f8n=$?EJsg8rUPg;P?!XW? z>BpPyrSPfS7A%pqYW81E45!KX5i;MZnM-y5!^8lJxOtKpe7i?;+BZ)7_!E23s!tnk z%=vFLH~2E~WJ33NM$dR z>M2no%-!Lvj`62$BK&R(g^7khjTbuB!Bgo-(Dp504k1pMT&}vN1T3otT+O^Os|_Z% z0y*(p1+I{9&L2>#d7WqTG1<}GEV)`VP$#%|QmRGI+VuXEV|}+}=GDPz7s6OkA<#;& zbmcIvAIXY@=*YIsIRQ35bGK1!X@lSC;rP`@N{eW~+hcZEk+Qw=+EQn6cGx{G z^T2dxf@-PrY}5l-m+6Y3U?YLZ{EV=|5&+L3u5<3j!1xyHYJ}XujxW=qQbmd<>V$pr za+HDm3%~Eb2g^9ykaoNGohmTdf$9Jh#J9}ohy+?)?Ep{$xTR)c%caoZ_qsu?LD}tf z4eS8%SL15~xmsm~h*N;zMt(@VXudYk?wy4$C1#;?agW^3aJ!V};%5w;;mYlvX>}d$ z<5UvK3LHHsD4rYQPg(`VXi#`2C7{=nG-@0`9l8?`&@KM>OtLIh)aUT|xoua2CyD5{ zbK+8FtC_2jF5Bx8$N!J9HxGyU5C4CsjZ$b)WNjs6X+daEL}iOCMMjdWV{C(&F(WFH zPzhxjA^S3RVuqqfvM+-%Gi0B^SZ2wL;k^6&zQ1$*&UKx0UB|^A{(H}RUa$Lp-rM7L z-Kezf4q1E))VRjOXHwMKdK;GSVs_sk7vD=w^ek~A{J$nFJv|(Dz1CCF-zgM$XH}_l zpO(u5axFaDUsoZ80W2`E8;^Qd2c^&C3n$(i$*f!o6}q#MVb`APII=?IKEt6thT#YJdx)Yfse zRDq%nARH837O8>b`>y zq}t^cDFBO?8@{c(vrzUxY}vaJWxe?udEb~4cjx3uy1s{9##@S-kz@a6=lAHdCD4rr zE4u6i8MrqErjJxrfVTUn3kPDF&U_#a4Y~AxzSVu5PJ9q(o`0{@5Hhso9oyppqz^%;+{oDR#)%TXp>2;-?QUse;=5bFPtJirk zyz2vH?&U(_K@~N_Q`n%|xv;Q24WA9Xw#Rm7;QWO@1TbGY<ZeQf40jlU#nTTmARL zuXe+Tq-RKkch$mAS1)pzeo6%}Ld`w^6unfh4`}Y|IAa71Lrh%({&YFcVXHqf44=_^ zF^AtS;Ju_3&)j|B^fGViU+;(X+Xr20OQ$2ocjsPfeK&wh&uPrc%DM!7<$W0Mw)Odg z%0R%8pEKxBSEkQy5g6#6`vN-~)S;D#RGJ1&?Ifwlmyv%Ue=C3&{co;SKx+|5rH_2y zdtENu&bbaTv(Ro%I7xnfFf!_4HFel_v^k2tkFN_f!bH1gMZrr0Yf>85a_ zcX!Uc+?UukRevOJswB|jhkNtNtuVfn{U3eCU-7Uh=cY(V1U&{F+4YmIh0XNSd`svJ z1oovcwai$L^xK2XSXUguj^bT{N&ymnhu<@rw=v`X?J`bAj}GHpnYm@Qh|Lgwjt+ZMjRCm#)5ZRr|a4zJ7TtY`1Hp{4cqf z?RPQ1?$D`!+q9%FV9R1KLp%e>?5;x_s`ZE{x-41VOa@KXS@Hj0FI|GYSIyt7Rfa~8 zC;XuatJTSNT$F_ljAvgqLLxRIIA7hkTK@;!eg6MH-2G4wMkFa|<}X(Z_xMoyLdu;{ zsHCqIBSNj>xGsyd)8A`PvL(5D^1~r!D`V)3|K|E zu(Ge|1MV)|1M4gEN5!$xgGDtzHIHddhx*-xjo+>|M zHgM_&jQ^m&$^jca-0>{!UcH~o9G-Nh-n1tG?h6~ERJxhw+q2hZI^zuA7ef|yzc>l= zi(FIC1syyyKTsVK$~e*EhY%Im`Jz@C%7}ngE7t}3@qsnTz~pu$AY(xK!l%G^ zu4w^<+%S<Wh{WU4y?58!W;4!35dl1rj(51ADzE>=-e|*{x&8P4 zj;sqBeZRl4g&Ys*k;}+>+!5>ht;_z;8$-n3z`X5ihS_&V&F9x6NIKU>b&NZpEYu(Q zP(h)U^X?5fu@;(!2}Hm2HG7$}#&XC+?egl{?<>(J*x>(K&DNVSqYDpjoZp;U_kdYS zOznv#X3fsh4@v)VE|6DS8DoUWM5ONgzEq&zX8cO5)$`RG519{5-)yM%tCChNHW*aR zg^>@x0lU&-`e7h}&K5u7>$dXn>31ima`}i4yJ0cQ@KR)W;)II0?M?YZT@u9pgN&i*>xL){H~Z2BH`?xW)VGChHc+x$p>@I> z|L@>=rCJgDU*KF*H$LKaGl+UXSLDWp$3=gmGxM9;aM4+%t%+V&HD!oBS;6JNAvykY zL}%~lusMKeSQRk*QnbGsF6)t?szR+w!&QMQyD$@PMyxbmk9iR zGtBvytNwM&*pB8AYJOe3nI-f|Pr9gq3jJwF9{FMT`I2U8(icnAZM&=PaZ9&e7gmT? z==^`MxId0pptUgE96@PV;Ga~SQ{JK`%IGKfs+N`lg8YI_Q7J6=~I`B1`OKe zjndcyy)&2H>m`kaP5x9WsdK~*Q*M12Ce*Tkb;MonWkP?s8p1}Fe%BcWP_L;1prl)B zSYBvY>v-iqRKq{(dU8|Z{m-Z7vF1U&QZ|_Gd`{b-`0ePZQJDk6m-75tzELhbe{!|@ zr1@`q9Zlo859-ZHK;f{PpXCr&R9kW;zz6CmUuT@ttWvKKLPKabw9S zm-9X3*^R`qdBfyl-o63qP0Tj*gHGi_`-fekap%|mi#bof-CI`w1T%Mm8;_n1oV!9_ zs<^VXiDED$_K|Hc0(%H5`*QYk>G@8`&`E4V{%ZG28)H-~g@jQu%q27=je`*0^( zI_1=5gomle)?!qYfX`TUVc?hf?ATtZ-t=kfr%elaX$=|Cv!@xt^U~J4Ge<6jf5E6a znnVDZtVK3ya*VOpJ)l`Kg=FyGX}x)GFT*{Q75{z&-w18@nX^v0cfQS7El=wIiETDZ z!_-%8%@#BMw5sU#-eoV20SiqyyF9#n{IQJv`n-MVL2DHN=yLDA&(p#d^AwCcv)hKI z_3aF1q~CvCQ!68(P1YRZs^Ve8-}&VgZ&$-nx^A8<-RO==21)i3JJ~@ z+kA6!M(pDKOjhH=%W9EC3xR>U5!yNuv1~W|!B3enH1#0)Uu4&(d$9p7JCu+{Tez4` z6uYr2f`MkBjMaQ?nN75!;&f-lc=|f@$QdPiU<^l0L;s(*Z;0s=y}ANIKAY)G2`L-E z&vDgPr*}}LWS4VaIj=e<g7k$b;&d19_QDVVLR zHo;~~t0Roer9_LU@I$Ar#IV0NfDL=EtXY5Z!lOPH8AcS^?`nMkwMyWtHGv?>$G^$F~fNqYz#W0SH- znv%~t4!!*6@IQRp54_@6Dfu6`rX!LN447wgI<;_BJ|TZ#LDi6P@c{U&yQNgZBM+yl z9ow8n{U6@jq3ui<9q zZ%~HCodQJuEoW0V3%?-}v;5@YC6|j7gz!>jRL-bTq}YoBOW|rzH?K`LGf| zxUK|6@74RC&2Q?PZwkM@SKDE-_w=I2+d3fV^l~YexhNwMh=hPL1NPj}J77k2MQ%f} z(Y5vRyDH?~wn60p7@dKMN}xGhQ%g5qQ3meFmnnENZ?;n%VU&2nBC?jbmx9U$v z9jODR0^F|(>3a5`anFJuZiOe%u-99&OfRHIsp5)@1G^2ItI3Y7NyXTu_u&X|kSy(h zK?~17M$+k#H|gfI?&l`cJdROYRSjx<%4iD@^kPbQ zk>JCtvLPAy?~jM2hYz|GDDBTt0ARv4Q0nY~v%HDztMQ~QPDZHH-ao!!+)M6R4SD_d z!`D7GPYso#R`ffGQ$O+~4E@%5qEBPi16MF;t5D=5+IN$kmpN zFy|}cbJ!m2gxU6sUyqa$;X4M0wQqAk@7?f7GxOG3+(FUDZSiUIpQR#vsKG1>okLxc z>iO&bKkL>kqLrOr^7#LP(WL}h>9twRpT`H}-f9KwZ8NDE+O47j{}9w$D=P!t+&2vl z=l`}Hb&7`JBu+nvtvX%ZT>m=V`7}m+ze||N9qnUgo{v&*NAz~?7yuT{9VPhuseA?; z*H@*#FKZwCpmXM9e-+%A^-A~PiL5juKbgZXJ!SoD$Mal%-8zuca%JCq^_zQP?J^-( zb_T-?76!h|Aa1@%p3B1bS5P8nrR;H;-G+0QoVFi+9gJ~cjOW+-o3W%@iiLkc@BZf$ z0jjgl`}3G+`9bC94JWeX5_!2FREXxrmxt<&T^=LPq}$lq`oDO$@OX3>x!F#o+)$co zPBgkA@ik;tHg+T`EHxwZb1biIPwlkecoXk6O0>mGcCk>;Xu0C--A0Z`&I$9cYS;#HaO-#S>aePhA zh@kTD7ZuON9B+_%e8)qe@7(1bRuRV6ZWx@F z|MgH>)oLYmTtfIuCQ%CaMIQFj7{?XL7O;u(klq<5akAgwK*8ngpMzazj69!sB1^E; zDvPRD`I#7O4X3cEOlNM8!P^KTYNK!jHZU^GE7^v4H-k&)O+T8Gd$8gD>+I+r6mBtm zqN%?~j#_`S39&E~$N4d9Jhgmaqt_X4`h@YN@zeTM&Uhdx{kvEO^D~F+d7Sb5c-Q`} z(~DDe_x$P}?zPZ1dRjYGlN;)GyAJ*`e{Pg50W^)&)C5(DZPC1++OwVy6WZ$B@1@r& za(-KU`^%dqORl3MUK zGk}ZZXUz^3=g-MO7gDPe<;@QR`|+@iKY8)#%FbWhDtt=(FRynr$wmP zl~GUBU0p3H$`QWJdIPArAsqyQfT6ZGL#b zbe%~TW4)h%OQb3@!XDo_8MpwZ?1Thai3{?);PrW=k$!fDK)wP%BRD`HlFnqT`^bD|UM5bmLGv zbs=c4ta+?OuW6Bu1`;MKYz{zte93jZU|W zP+?Qyd7E2$jUBDcj*6DJSJ4P?XoAc%&N3_)AJ)B}D7i$JQa~{-InHLTD1Fp{zPidK zDSU1YF9mwz955T}sOfyI?QCsy!u`gdiE)0DW6o+&5q(E{OIU&#$z4tR;oR}H_z?Ux zBxPlS>C{8P3Pb| z!WRD)vlcMbo*@3o(PyNbRS~rC;YTp7FWm9t6aU{kL0?CV4^no}o=q((8Mpug`}i^h#u z=2^Z9A7?aDv1>F}krbJb^pgoH=t{)WAd`_&M^%o+1m~Y81tv3LK71Y40=i)!_DRRL zMxV@+-A~_s!tGyv(a&|31$HTS-WC>ww=>sJTQf_Adq@+ zUsB!vP;isa=?@u@AC49{%N|Q2@1rJEQZQxYS8NEjZ&T|YU@P9c#&4sw8xFmsauanQ zf8$`z^r$$VRL(U?E|m*o;B%FUz#+YZu2Xz&Fn`svxzTH=D1v9pbCP;uqj744gp?%U zqn(voeP|4dhmAN6<7D5EfisEmD%9fKfqGr#bPks!g7Yl*bxQLZs3~$zsHzf&FAm2x z=?T*-RDBzH(bfD}D(NP#Zb$+tk164sej8CBKQ!Z_hv$$2&-An6RGthcve5W@% z@u}%)r_mYumfcPe$W`!Ks9+&)sx$f4wU&R#AOP`{;sA@COKI>hx+p6dZ8s62K}K$~2qlHuFyD?MnM zch|wIOS`t}t-`gUCt{D0VB1fBSX=BCmdE{_Y3#RU zg4msm#lHg->R(Po1kaNq-i>Pi>bJn51n;_b*YnZIT9+TmCf}5MIlEz}Bqf~qAum5Geo;N{ zs~=5@s({m7^%xD2hzdeK5oXv1VXG5ri1q`Mb#>ds$pYg1#1b**V6NEbVPQ!54f0%v zW}eTJV``z) zU68Li;krh=0{{S4E&+^xJi(4{?4~H~?iq;33@q&8FzyHCfC}ybh{yDh!yL zt#|w9=XTY>8U7%W*Ua-9jR}hvPBY(I-24R}(K%wgdaW+TLM#R|5Za7@A9pnIBuH0* zK%mGKk>1-m15e!V+2B+ioriR?pV>LeJ2fU72m4qiwh5!_G|Ez(hh7vMnGEu%S!q}VP z@2KFa62!Jje#Qx0cxzwe@)4O>6NSe2k{?VJ#>N^5Bnh;)U8zJp7(qKwXGS|(b|`9> zTkh1tVco-)Lje?j#Jl+IrJHvQ@+%pPF#Mm4_c9rS`3p0}dfxv&`qSs$&qLl82;O9# zR+TgcQ-?WA8X@?$@&|e^Chz83q(Zw=+gOWz$NTj(UQBS$CxusJwt09=cC4;6{hCQ_@|C>^1aY=%2;%zkJW^;R_gW>YvPyrT$Ss%6Y6b5g#W zqNil3C0VJ)&ks+cmu-k@uYM-}vch59>VJ2H`{BpA1G=e1A?KB6+i&7fJ#SH1h`hI^ z&wDf__Dt?;xrTHKo`i-#!V%Xj0UC9&8EvTC@( zJTI5u0RH%kSGE$WFb*=PQxPe zbco;9*Ey_Ry8|$g`^l$t%`yZEWb;EZWokR4&Nh^Ap7lUXG*bo)KhG^JEY1MSuJ@LM zf0I3<4h&H)F&DTFiCu_Srb&jS=!P`>0^!PIC@_B^fs%h=*VEyVLKFc zJ0|3jR8d#@c?~0(cBbv#Gm-y5)nezXi@iGGd6j7x?!~^~au0 z*762BD%%IYapo!BT$@qoOJdj_;+#GpO5X=tF-u4&s; zxC##`{lE43l$YLiVmqhf3r^1{ejFC^ahsbn?Hay*O`D%!nUoPSnx(zHkdIALZVFD? zByTqv`0_h&wNO0CJNdWvc*D^UbOOb;DURpRO)8umUzZF9g#w%;uK&sr~4{ICjF}* z2y{5gn4w-O`xUXHP8?&RKUfsC-4E_8l?E6hH|JkY!pA9%ogpckDJ7)(!rU1os=pGo zw|9BC)KRNkw1#(|As8-(K@lPcI!tX(7B26($08WEUZ-a>sOOt4xJ^yaW>OBZ&`W!e z)Ao%0?pZMPnks$+a)0xCW%>9w)Zz&_WS6NM8!&7G09Wn=4(N88ZI;3??|xYp)K7Nn zQH8N5nx;Fc$$O|g&XZ8|$RV|)d>*k=B|+~jScPd|>E_-Qo+Mq_BaI;GQcf6nI{QlH z!0+>jt$Mq^-m`%(&cxMQ9Ze?Qk8UDh_}p22Tzr93KJCLmGAnt2bPQZpL_KJzEl|1+ zWWmzWXoteOYE>TJK_{Zb%ZVN0K!!?`WMO?|lIofrmz)`dTmi zP~X3S`YDoFgc{kA;|VZJx*&4w_V8mYiB;s^g{a*OGbFC>tE#=%`y784zwEXvJV|8S zM+$vb<2#ZgNCL&Eutz-`(!X4FxG?*s@FC&K?FEI-hmbsMTgdn{RrlsoYxTQc zCAD~Tys*t*5fA-llvg1Tb4C6N;GA!;lDA~39cBJ@OX(&9YZx zW4Eb&!5c+X(O@TvCJJ{0Lp4gkdGXc*CkW2jyuMlz98hu|pA%)g0w>Rt=bJK?WuB&B zp3jqzeC`CXSixy3c&yq}`t_rGG7WT*i|DY-2f(mYKjVSEZQAg;kUy9zw{E85&g!Zl z@XAoJz34d3%cjGvKS29DafSdQdk0nGtCk7SQ+to67wq_*c1+{ca3NLKZbCziYbE6rX6h=KPr z6H@2Bls@?2Lfiwb_srp?J$1qNVU9U05*#t(LjcTjXihE81H;54;l{X>8K|KrpR27{ zs*l7|Dk8|f=w{mA)+RO^d<+cX@h)=t)-1{DwU7TJf}__&J}f0<7n~$}@2=wz;uzpee}XG%xh{&4w_QU_&2R&SN^zLxf<)~yJ|H;O9a8agJtGa!j#Jo{}) z5;`};r|HWR_uW!Y6f}d_Ix04MD!3SOxh9o!{TA$F5quc@Bo3Qmm#k9bJzANITnjcD zSY+D8cNWV!FHA@D>nx$ZcUQ4euvJ&VN{K{QqD<)ksk=XjQKtq-0wCs`#2e$RTa&<@ z*2RG*Xnen4@Z5v>gCC3~;+I@#fAzELgbnisQv^U9ccCNrh;JZG){FVFg0k9b(DAt$v<$>R~l-ktVM?YSb`jR$=zsz9_#y$#Px*a!KYbGW= z(l%X0-3L-Xw%}Uyn-p%G0d$9CPlfA(m!{xpvA{jp9@SZbmRg|R@PT!v?503G<9$*2 zAgcRzLa9=De599}QE6bwVod}hp|6;#JlEHA%QMa4OKoVGDUQg4dUn#ADa?7W+=~ec z;{=27c@8zR7zVgN(QBb8FGUXLZ1jC|ADlnR2UWn&F0?&sKgflL)Tdb2nFJN0Iof$9)(;&>uhGF7o=#ywML22o`2( z{5pO$`;(O#V~I|He9JwOo_)|EI*y__O_l0{?UUL{jSW z{|Uwu_*fea>l=bN_Jk()+C*1E3mnjmfb&#Kl)vDk06OD6T;eZfeVIJ}exXeic3k%X za?oHxW7f#SW<;-lJ=OA5m9T-;M!{-(+7pye7T8=;i)kAAnm%*!gzL3IkW+q%g3gzs z!MzrSfxHhjoCiD91Msdj)_z6#3!g?+r-l}~){)yt79zr>+EUHQWIQUF!r0GiY}He< z8liU)cX)}tM!7Am9|UVQLi>Venm4ZKAk7-Ws=FX>$VZ-&tWfEg*OcH9q-jDUQwEzd zaQfij=o1`eUYk8TI;Y*jI1Z!>Bz8|aedk?swdH@U>?9WAmXYh5aQd*u_Ti{q67*p} z#nOpDSxJ=@P>yeVHxJ7Yi0NlGtY5goM%gs8ZQ}~Qq^llD8gTP8@Uk3`Kfo^!8?Nq% ztoY^~RCYw#h`+*HTeumGvsC0Q7pUy?HA^%mW+v8-xG>J$u;?kc>jfjdCX71iB` zp^BT&;u+=rA@9WpQ)+Lixza1BS-Eg~esX0`cm~xWm*LI`7Rx44_xEWcNG@`j=yW#0 z4MyT>>M9H%-0D zDJ&dlk9a-_YAX?o%nyg;#1dEDY_n~4OO|R558lXWS$QRi!-E`qI@dT*W=2Uf44#4b z^6`?p&{*e@;^Cil%V?vhKQe+0%k^kup-~QFI8fLbzVCLwz$iQ|CV0V-tlx^is49GLRqNaq#L#Jv4Hi zdT+q%>!*b6b#xcc;>2#jr}sSNj=y+nRX?xdzN@Jq-9Efp#7Q#+$1QasMc~VaQL4-s zdj!T)BAB2@km+L^x7P!5bB;53rOz~_suf4vz?5*CShaOQ2xUei(YtsZ+uF~|&TqRV z#jbgcX-ATXw@=8bI4ezoj{z1bZvVdEkJeWXClLilH=J#7G7a`}>7?E^-GeoP9oh$W zKCNhpnusAvh-rk)^ZhfT1Ur{;#nslwO4lL0q5!}t!3rY|FU`~wLQ?iUtG5egW6W3d4fPC5}aUVPBg0E2}WST05^83|udzv$Im0US=MNUmQ^zJf@0PIEB9 zcIkHQ($6%zqV}?ZVO%(4x_t?7Ex|BH4YGV*=QL}cI8;19sUwNklp?m7mB>as908wc zp91Umt;49#?3mJHuHp>XXQpiVKV1{Lx25ro-qSLrlFSAaBfp#t;HCb4V&a=)9iGo0Fnq`M5KeBiI zvEKMC`#jJ10`7IPXgd5p`jd}n|I!Yx@bK`!F<=&QH>jb5%c8e>N7DJO$5`y2MUn>> z18jwNKF`gu)bfCeAo(bl5SwTY<`kYL&0ADA@4oWb`3tYmZS8C2(KRMb+~SxSZYll< zFlQQ^dJ;DRRrom4jwpQtm82Ah5=3%bgTcTz`=ZYB{EKj|1a46EaA}U%GHt`QeY+N0 zP|}oKw?tm#C&vs=L5oXUG28EJiwaqwzswN#0kW=92#bPXxIFkj! z1lX>zd)}Wuce`gm#tZKnGyf%qA{eZj%>ypvp~w|g`~1FNypz*IF3+HZ*TlaNov$^W zv(Ki*qVh>c6fDF*^8(24`+2RD{il2kqp+Db;O~DCZ=7yni?hbgCDnaw}{v zuA%QG|KJ5KFi?1AMk(Egg_2lOAiUExPd#xl=1|~-vP^yofPus2paq0`fD5TO!fyE* zrmqA;^^Ca%XfsCiymD3}p$yLRI#s7(N0UMk!2SC^6SgO?NGrB#Q(!%u2+pE&|C2y6 zFkEg)I_*H5;j=3fI#5u;9FC@yUPf{RR=rtsv}3XKd44{TF(gM2J{s357vC8|?Fnq(QIufV*R&Q^8M&od~z3 zFb1wpPwm_wX%6b0(Vk?~>P!^2T&gwc91RG@w(pN1wBwpjFEwO;)p4*FyO(kurb|_I zr7I3Cp;PQ-NGVb0aO;gF8V&mk@I=?Q;TD&Fo_h3)c|bE>z+_jd}krUMHk6 zZ7$y)M#DfJuw2mDb;+?}T2W*8L;al~wR0~5Df?oBM{kM^+sU+=xIEVLFA^)K@sQ+P z7Kw{@y-%9bL6!u|u(hn1mLZidWgE+gl}k=YG=ms|jwqQ(SCrW% zaoot%DE5~+dOsW$5mMb$M7@yP14uE0ec+b%H}gKzzjY-zBFy#C&y;KHv)4tB5SDGC zm#vnEGV^jzD7t(I0%e@a{65A1V3{o-gfklqwHmr<<+W1oc5!F_TYInbl{480E5b(( z!ED#zUP5-%`nfab(9s}Jl%;K`SEt=j?sjtIUB@OS_#rCXo^?CEw#!koZkH{8b|va1 z?nq=ZMhmgz@Z~kcx!RXcv`EMs)zoN=qr74DO;xPt5c6od6jUkPiL#d4umros2wteE zf+t(7#ZpoLYXI*(9d@8k_+c>j4!x$1MGb4K;7k?gA6%Ti$UGp}tJvS6%T1l2=$mtX z_6}4QtQy5iJ}+V~wep>w4ktkz`DmxQKAV&puya3@@lGD3V_EoCw<@lkz3!Yp`K22P z0y!F8dvYuz{a*gF2=P!L0B#0nLlrC9Gb#J3A!!ztey8jPMc#LYa-attl^pZ+U=BmD z!gFWXG*(Nvlbz){HnwPLdmg2&^HviXK7`()TdY&O8%#FuMAG{M4*<0had68aB^k3h z66Ipwj(`Qh=l1!_2yA37GsQbakX+9UWvueFzBj3ksvy(dHplIsa^0O+= zHHKJloF$>AF*b_G1syP$>i~CTBWnG}HSeVSA|WVH(9<4SoC)C}e4~G}z+Aqel=Pso zpOwWzW;6CRs^XmTm9E5LL~p>WB?s-$bxBCeIfjkyBNuDhNx4#z9;HA5z5p*NDy~+s zYIJPKkj;S(#I~S#S4ZI^OY5{kWC=v(>}J%zD{&)X<$|%+bGN<(Ws=7*?OJ|W>u?fL zDr2E!Zbmi{5cDZY%J%&S1wWO3Mq7_*<)Uq7HRL@gTYcU?SJ`g z(t1~lY7RX{vY3h2p$qhGt!;h7Y%M|2&D|ZBZR6I$oIwT}jS02uw>eAIcJZt#>v<=x z<6K>&sy)05hJ!T5+tCZw8w;6fx2W;hT~4)wX*yiT{4i!cT#eShSwuYqk)hl=oI7_h zl`)hdm&n;1ojPb4WR{acpOgS}{DB?UuDrUfPr0qXvxwH)X`;Wp_~skpUOSMm=zA&p zNo5}W4qD$UYzdeASXpqbMf1@D>5HJDXXbt7*dcL3iezIsKtY2U+?T{pqDzhd)_A{)*EFWd{{^54(QVy;UU2rn2e=P zHz=Xj={9wkWgNZ)yXF)E2j(9s zhP3yxFh({RM^gDxvW{5@qJY*q7w?DCZjcl025s;X6RJxB^ZsU*?l# zJ&VnQ;6~3$Cq(voeC%f&?c`xso2s(gvr^WFZHGFtZQ6WO%5;VPno<)ejLS|yK!xTa z&dF?}=oRz=9v?Q!N%l<{U^ExVGA@j?(^aIPiT3fXhk*#uX7{A4O|js+++D$rs;ii< zH}umNMUSCDviygghKFF|^;V#VpL5ZcO*V#ue$(ay+d-!dGmJl3kJ|NpQ0L;KYOKE! z5P<}jH>Sdm(hUTH1W6BZ&06@jFjBcMy`s~L!jS-dg4?$H^zC~Cx$o$#c-3d7?zVEm zH=fL6m}6han>wFrRr2tA28!y>zF;e?yF64%SCTPF*@<*&Q+9WT@zZDB@Lws#REzar zLx=+u)c52*(!G3N#%@7J`vpA^=<6NkzP4%|!7|&dnkL53Lg{HsQ+?ybYlKoY9ak;B zDY#X^JoU^Gd9#BciGB3p(!?5XqFwSj22V#rVu&aHs2tIFpG3>I$^sivQC83FZ0r(l zP9zO=4nj+I_EMNJSzOpwd__^qtAcM-9%cuqy3O0LP-qW+>a*}W)8^1!1 z>gwyTrEu{Kahxxp51Tp{-7=XM*@$z{G;A^{)0#|S-{?eepkICI=ZB6A*>ZUIsch^n zT#2f4-D^&ok)_&;HQU>n5k(v=;6pv^JuN-C1ASSAw&UYAvd)yQl0i+2AKoW3*o)I;hr3rfDZ z_Y==3qONI4<06W8gmJ~ON_aR7xF__YF_UaJ{1lh7O6SUGp7gz--s-7U^K2JtQkwCF!WZ=`coE+hccWPAn zfSxA6U66pvbt?W&gVDCcy5uNcPP{LoQfqg^7M<@`3I8mc3YgohZ*pIQ-?CK}kQ3a` zY4guz$OaEbqt4*0T8E44PI}dU`u6G4C~;6T$T-W+c2_E7xLD4>U6~a&KnsgLQ>c&@ za^1X1hBbDbgP6yXc7J8tYaJj%c%*{%Is&c}Xr6&Ba9)S#uU~g1IQ6lO=!12|O>(9r z8p}_0PdoebPM$=Nwi7O7lK~cR%oNTw#b4~JHBuM^{6B`t%Ux#l^#=-pK;;DW%Q^ys z(DOvWiN4V~O2RpwYq6IUeu0wk`YM#qWmUSg3VhUod(GT$2i>t3+H1WR8;y^RdSB%d zywIfJ^`Rn85srE>;^qO2!9!XpBx!Th7DV2`dvy#5lua;mJVb%tw?dqZyyd^^7RtaFCZw6T9jIiDrg z!q6c!n-aVN!NCr*PGyQ?*a8gdwS)Udyc2xbHeXejOv-Cu5fQ2mqj^Ph`O7-lZbh^; zth?H0wLPPKQ*jn!tYjQA69`pb?~e5I$+4KS8`@Ff`(9aN79~ijeOne3Tw3QNRMHP` z<#9t%yChQMq_ln&*%k{d(*d2b`px@muduo8X%C~gV@Rx%yUgXw)oKh~B9Br%uvFO> z5>l*S0h+(s$B7aTGZCH&vkYotJovl%$k0T8b~L2sIpvNoEJz|3z zb?7C!LgNJXzsR)D!1^Wh{m#Yyi;&ZfK@(Ch$P{LQOAl!{Om~mH;(>qs0pOl}HdURb zN6e{{_Ls4mih?vTK>|%q!%_2v3Jrv^P_prENhuvu@{yBkyf)>;xw?y@2JSIUfs31A zB&n!crGsiiaqL87AnD*ZcS}p}EKfFU#Gq$I>|J={p{LZwd4V0h5Snx=QMd@|7?W{q z$SMCph7NwA)HkF!>+eG&1r^b}c)MNR42^!)0)jO4rC|MmiCYn*vuFx9cu(IN9E^V> z;(6QcosN6StU>#$>oA6auFwy{n{!T`TBT}x{zwPyDo|cBwFY5T>)zO~cGlIgsy&d9 z1XXvAKO&XVT+^Ty%TDuvQRvWScfyh1c!oJR1GJw^>+e)ed9fe%elnorhx6NDJDnZM zJ(k(7KDfo3&ku}XeD-2@F$Oz%M9wB&yEJ@A=*tXMH6@Y4vihP_>41*{>tsPigvmJ;s6uW6((^`|g%==Cf z;OpiDT?LMs`WO(QoX5*q-eT_R5FoN2QoJ`4x^-#@4}usSL0v0=jQLgVIdEmK;GGVL z=+)I0Pu-s7%*B8PlP_h(-Og(2L_GR5ud&-^DD6~p^jb&;MBeLn6(eW<^W{&8D);e5 z$5a~3+g^3Okfo-E>`VzsUqZvO0zYSUvC3x^_^T$<^IF&oe)@YTh3%p5%H*1ulRSa4 zlVvjeyn&g&19~R_g=(3l6{tAP#+Zb*zv&C0th$R843p#ML$9RTTi3|+>ENk$xl2_F zC+6Ljpo}K6h}xnmn3&5g8qTLzvBs8S#%2LSI3@$A9K1n8QP|LqCh@hAXLIu=DTLDS z62wUxEVgB3-w}Cr8=a?jB!$3nXWowVj*^@a;W_Hxu=P_K(nfJo4a?HKCUF-iTVfgC zFd<`}!H_1W{0o)?3JB2rgTA>dIx)%})I$_boIQ-`6vfUqTpB#`AbR4KP~BZnQ^}#^ ze8{BcZcA`GHMm>haq}}#aZpC_@~YDy`%l-vK=#aSs_rv#yxp(C(q`X}MeU=n`U9QRlruv};Bi z68qKXnLB}cHV71?8|gQA_{Gp!E5>16oQRr4&%jb}^3EKwpj`TvMTU-swDIAuBX2Fr z@7H+Q`ac#zHRnIUy|i|M&(i0p&6m*6iodMq?Rd{Sj9O1}|r5LWisB>fDbRE^)BB-v;67vZVEw(wjZm z0{=bNr?f-QbAX4cd?}n`+IoQAhC~VbOZkyVka;Is#>c4JnU_7S1M7o8u6JdHoEVuw zAMQx*v|tpkRa!CpVW~2w$Wu!SEPTJ7NQNTK|9Hk`(%`~by}`u3r@obsegMza5MaJh;NR;4s!Rv} zJ=Y0n=i}T@c||A7iezySRFVB3VRQjJo&v1D9If|v4J-&kM(=Vp851nIUuCT6Gor5E z1?1WpFxX=u8JpzDz(%vnkBqAUHEFcodptYwIR0zLUHGF^h-O^ao6f_LuIC*af(r3G zFi?v2a>pVX!dAvbFQcA^MB43K=y#{6I@eIIg_ILofp`W4nt!r;(&HoXnXaW}+)}Mp z9i(h1ss`|91SCsy#P6!$PEVwyJUDvtUxO)#tb(T4;~-!^=V&5C4d&Dai5^-`#P>E1 z1~xA1RVW?E4S~299eIeNhI4gGY)q8gI}iG~obNQu%9owIFDt|!yleT^;xL*q&rnBg zN}=5m3LZD;g9`5fg9Q8p?Tqw05F-;=W08ZhwOl*5Slfx5Sqip&F}DGu$)xV`m*hNM zPSl(8lwe+#ZV+zPARQqkR~TlYIv03WJy5kb9&ibIKevyPedG)N?apj0Rp-sCrVvww z*E?KqXbHc-CK=mSD%?cyj+6G|mv#R8{2(3SYgYuI<%Hw#R{S#g$&I+pT3@o^SL+sDav1ncI$X|*3q0%oz)BOER5S`FK!SCo>)<4}C8k}=UQ(iTMlln8@8}a0H6*xc zYfeQgaMo-`!zh4Jezk?G#1-AbEJk@$wsMgEe8I|BELm8 zy4j%TTAGtdum*>yPAYm`{Ng*H1OLB=9PyoJzjCqi>LuPLkZ8;;_RbS(cWwuiT( zGxOb;B7!=OO~>lN{>HIv8DwfxS&^{d?H$1jMxV|-#63HRM~!k!hB#h|yNTIW_x~@p z-aH)Y_6;A_A`+@6McL9O*?I_tp;A1hC_=VD2w6tyUcW9UzYa{t?^k= zttSXd!rjcIbQq}hZbVN9FkgY_L(Dzb8^TR64kV&kcFgL;L`D$Bv}!)@-M-`JcU+0w|12U<0CoPJ9iKMlW-C*-YMYAS#? z#)eo=@D(9Ni}Yj~O?;FqIA3%c4UQkvmHWbF|GTX`HfIeKL=JvHQ_%%1ZqzBsnj^$S z|KU%D$=QYB=$qts)QAk%9~O9m&3VpiS54fkQ4!~RqEB8}A=$l%WAO})P}N!{{&#yV zH;)e);L#zl`oXSg=}i~<8uTfpa?irnyDI_ijqaA23v%C_F}(nGjg+!=Tn+o6)zxd= zu>BD%Bov08FAKNX2zld1iZ{Ca&hE>C;C390$U(ioKYz^}`XkpBml$(Hr7Jv34_w(% z-DkY!uPLgPZwA1Evm@T0QsS?fA`PM1Lqq28cDcu*{;Zi4J8i!j_421hn*V)f-}UT$ z3k25O^lBlW8Hh*gn$C1emZGqKcbl#ih3;USemmZC_eC}gSq%P33mOzW>5bW6t|}rd zfkCt+{bL+{v#ve_*V~Xf@77PHS}c|-c@5LhE2eikT~@&h(=Lc?DE(>CMA(13<#1+4 z8oQ+LB&S;ea!R)1Zd?e61`D46Un@4d^xn*n-1Iktm9ooq12s=$3+Oipja^@3W%%*d z%t2oY*Do{UoOXA?H)z*8e^SPQbbeH7M_%h6Zr>UrWqvg&OtPcyq0dl{5dA&cJ^-2#E*TqHa|UlyHB2e@+le zs~8P?@@zLP$wp>^cAM)v8N5!=9kXoQd#U_<{-d_9hc)3QP&E>#HZR}uZ`LWG7!Z)^w3xO6N2Y(l?3+n8_ z7Qi!MVRqvY}74$W5l_jq)>i zFCZeK;9{tr={T_AKNVWAv91>teqmLOaY{+qXP3Zq*PcY4!b=}It1WGbjeCowQ zAZ&Y+SD>aiLWvctO^dEd&99SrZTg>x9~n3U&si--f@W`*HYIq*8~EdN>o?mx*r9d-#c7%esL+!mu;Y)M+0(UxpKuqgXEB-G3~O`6 zh(OgpKQA@16F792b;1lj86W!MMvNFLD~1QYi|jDSm<-Keh@h@WppV4DdN;&qa2h`H_NF@+RnfC!9cY#G%>O3Me#TP>KX~iwQy6Kg@@nM3E?!!% zUyereNjVCDRX-fb3BOcLWl7)A+2QEt>tQPW>px-8l*Ugc2x1+&f?ZdG<&hJqqC^II zd^LK|c}7}!OmLR(dN#65$?~rQ&%8m6k2cWSTrRlW@#Yhs%}RGcc|oAnh^6)!YEgzA z`vg3f1Awr==+cq>)-AXQ@_Yv*&s8lw%KK^2<_T3BZ2D}x%&ew&SY5Wq4Ssg-ojaUc zuH~>lE#d`@9>TjHqY!EM)8JvBi-L%j^mV|wME5VYCZ2yz8|c2fWg%fhh7g?Yu<-c*XDBn04hpGtbWe{CM`zj{MAI zoRjYkKDvs`V8%8XVH7PU>-9rz;F!+7r=|}d0k~(d*uL>(*Zk4~g9({7`N|;WhANUh zJZQSR$}Eifrc&VUF}R+sV1bBi(Wv{zt%RO-Zeu=oYQM!QaWO5Za6Q?;XVu}q!#8sX z@YL{jCxzMiEJWTA zXHQIRb0`ry74@ROR236Gas9^9HD*qo4eYkGe~LnxJ-rZGPw+}d2eAyw(m5ypI}uxf z6ET3-h>XqpR9`ln{PkQ4qe$x(A)I$FTy-_jT%MAe0bVXhx$HRQq~c(B`xzDhvJ{_r z<)@q!P6`uvr@jCPN{L^$J)w@hT9S1k=A_T;Wky+g#m-nLT{{OM8IJ0H>O(5?d2`yn&h`%mk>ld^nslavw$$W@~Dae6ze7YjrI z>)((W9ifBuTr*69gXvn-i!i(d%(Pgl>52boFFjUz>qO>ELXRj(K-6XnKaMf}eBz%d zDCU{#*@a+t1brJee7iX&zQiqlH5)!ONqbMbQd7j{PQyXHvHpP`cC&}k5ycw6Z#hJ; zM<&@*jvx|=ayfM+UpvsUPIB!vM<1l!3p3pOaW%|NL4+L)A3Hs|g@0Q8>XmSM3e9LG zj)w?fVUgk9GxDnQ*AJ;Ct@;n+{cHk%zlDg@+g=x3~()?K(460q%-JMk?*BVAq;}8{BqP|9K zBE*DGC+aWu*}T1?W|K*CUO~9b9=>Vg52~&ODA#cQFvYvN{OENI^|+6mJX>&0Oa80x zIgUOD!9Wzh+86;j85{HAvZG!m-;Cauu+yxjqRZBuw2)#Hkv`m0)#Pw*{ucuIVWKH! zje+WOiPbAfH|8QRKu*Md{v%_4x(*UH?4MvLJ0E8 z4!+?m4)`z&V7_hg4u*-n2MhPsmZ7-2(%FW|2njNXS zp!LfI338^+h6OxBg#xsAqr3Ul0|ZAW*>XCi`|(}@F^Mz0M5`#H#bB~SujR#md*X(1 z%SM4~D35kX`5rByMbqNnAUM;M66L7#h3fg-Fxb-H!@M88M%Qz*#IB~+E492iY)ZaG z4eYPiW+PXwnP%Gst#&@t%nr))I9B$CK-m!0HQv&eB%}G?6A)v5FEXau_1jUyALZ0q zH#zp4c@z}VI#SYXtNMc)UbbtLwd5|R;~e}_T%P-tjk_x$2W72U2lHNB zaQ?9E&q+W%Dd_Gt*|JM-Ca@INd(Gr8*tj`Bz)N=L#@p%0;knWuV@q9T{BlftHy1jx~j0%F=L}^xk$5Rjj3Qr_wDy6x7D>` z`+~KmV)1R?oO;=o1|PY94cB=o(!am=j)Se+m6+Y?*Ki|L<71lqXP<8A@n-vChs}+t z84cCmI<;y0tP2D2Y)8U<;ITEX=jPOef zUZB#ORm1J6HbgMVQ7gOYVGqX1Q-;>7%AT3}{l$zzg!+1O@07CSc8wk);Q}{lb{MgM z_YZ9G4l?7=0V(+=n_|Z|<^=a7=0W?(#T1U}YXEuet9t9#v!U;Hzl$50o+tSnsRT>?>=+cX*Is^%@I86X z4?C95T8Z<4*e!Nx!QK>S_wqlGjDtFrEYYD>4t&pUKeE9qBL?cMRZU~r|x3k$sSBkjXDC8mh13J7vU&b|(~s zP0*(ql8D2Zkqj!t@Tp36HOe6V9;l;Y-1IR#4TzjJ$-}o;eRlm z?BJ~aQ?=%1bsK9bXw5=Uexm^p3&yTW!kXZN;nq8X$oV=OY91=`*M-WM5LR};omL*R zcS;+#mhxxSx(}*(cWjZ3gDE8|d@#GzCo%zuPvtNW4A0NjNQqeZp#?|nH-tRJm0xp| zyIutZXO%ii6hh12OEKx3HAh|k5x#hqx(eE>p9jgN`3OKEEFHv6Jb!3)DTWgrfWu@` zNMLKNlUOr|WF8TJo*oGGZ~@T}^64pAWzQ}0JdUJW;%=2PnN!2?>1(HAveg6W>K`uX z_WYyqGznIGsg{F_YTZG)2@hf)9*Y?NrW&yLs{nO8U%^Cni1v=-2H*HPi(G7EuL)1I z?;6vizil|o>lCG+^@TFCnaUd%r;YA8JU=_HbW2;O6|0pHwHBiJJY{|bTE!6RTi1Gl z<3~ifpccwnYtW>#ruirLBYatzJLa;DGU;F*ut3!9XHM38xV-k-z2_V^cC0%xBhK5U zF@VTch%e``x+uxdVGZFYXEJdmutlJYB-5v!7?ZSSs>V^jvHfJ&Wuc%``$z&N`Y+Vm zohDgu?W;o}bz-~$hSmdjb<~xnfYr^VaC{hu$PQ|vrp<8a?TkGt$2biKfvO8eOy(+q zB|QNE3ri{*zcq;u(=83QTd~Z%J`{__154jXz?Bm*r>4(0koy)F3xxHttSRx~?PNzW zS2;?E5?+wJsi_?71D~4A|74^@FdT8K_m=yH!Oz~>Yrxic5F~(iiPCt)7Kc+ShfeWd znUSZmCDB}ZPm4wlY$ zP!9vX9Hr?!#1ORv{eyzV904qSjT6MJV_O}2NoO+;=AI3Emo?xmcjjFr#lRExwg^hY zX3yfLV!pnFA1MyeQKO89&FiBjJJWz{YOK}p{uG*EQWm~3S)rOBv{&+F43Z;kP^ZVX z{U~)+V~Y@esdIo0AJX2}$?8pc2Rw zv!7p1T0ebI&xaUCcpLYV)-%|^7f*e$%X^i(hjuJ}{nlekF0Zm|dfnkj+pLapVEZg1 z;5dwd{|6R7=G{fOsC^sS@_ALTHwf=MO|f=0SnJ8gs!7DwYW>yWm ztOahJIl#(%Y^8em#H9a;bM#ZR3+}WF1_IGH3$1gGxHIO-^LE>Tq;P+=-mwsvt5w@z;cBf0I!5*f#JXV%*p2W z<_huP{n_w`xXQGT@v&&P{^v8&qcqC~6!eYqYtPtCJx;PdHAFmsn1{TSOhZ?E6q}W-Bcm}mDP5^19`${ zFp}x<6FlY;{Iwf{0ahOVYM+@jZW+#*i+or#Q4BvSY- zos1N*FefnUW!@6h(=FwP(R1*w@jp25X)|Db&W(0;_;0fCO!1~|t1FcHdv@HzBZ&6l zsG9ym4Z&)A9bVy{J*%HT6#iHvDC9MSvjRlWU+dnNEaGm#LPP|1T2_Bx6i1J9XtQT} znZ+%9c~vU-WCX4*GrwWpl)7rq>2=*yUs7)w>*MHq;=K05SKlHb!-+neUVA@_S3GQlo50TG%t~fXG5F|n z2Sw^_Z6;#TiwR$`Jas#x70E~1&)0Fq%_N!0Du&MihB)20o*1-zy|;Za75t%k`y$a$ zhU=Z$9lsXB$;OB~vi4hg8?ON`&ILuQwUaqDlnp&3*uzo6jIH|zV2#c$-DjwqMM^6C z9kSLHtu9IV){e$93wpw7<*w5k3ftfin8;aV=0Ovv;9?it zRo(;*ma8sBpDf>Ln-?>(wUYblaZNj&I1{n5QWkFtA_##-d;WF6pI7U1J1%zC%i_`#|4+Ub9 zSOP=bi5o|h^V%+9FX{U2X%sGuoVer|M5>ejCt&JH{}LN1KEoCCg*Z>XcTS@k9VBdz z8+C^TaSjt2C|T&e#xiFY!g?kjFDV1+o~W)y!w4FQm3DY($}q`!!1P<OmEF_fxO{`rjNyby+Le{+wZAM&6x6JWu7@Lxb8lBP74jDC8xqbrVBi5L7y+P zKZI2=?)!<6zB#>(1r5{81At@_K*d@vtd~)hcL^t<=@%c(0w;jjvTcnYwff-R$N3qA zDgeR7x)27rro&nSkYGiJP&|h)JefY=!l;oRIl~*PJf$*b0}vf-L%M?sd5Aj$m(|QH z8PC&z&eztfVwY`!!&T&5oMh%%MnDSc=0CbUp<->xIx4D>VUQupwj5PEw*Jo0ON%4v zV)EoQ{ej}3bAUO?$H|Y-7mE@3!wsS=WP?|h75@FHOE9C*yt+#InG~{L3%W=Cf2nEP z&Eu!!7iHNI{xmg~#m5ep79R`4O~ra+zw~V@jJ`f0op0XP9~jf>E=OWl zFlheOv4f{_PG?u8kHPi8Ai>{53~ia~MT2J@xv)Ms#h>~2D{!bc4)LsL(B}OrAZ0a8 zAoqh-@|Etn>2Irk+C$Wl`_30PZNu8ruRfqR1&ur?&@^|PQ`NS(T%xgOecorx(9By> zHO;>7Q&0L~u@q>e@+oM!4jK2si}xLw^*!GDsL)i5sJkutZBNexJxTk{!|UY*f8?+@ zNVw&cy5~@Al_S%Vxk~>O4!PO~K~vh?fviha z5QTPg_^^FeYf&~Ka%(zQymZhAvC6xt&X6<}%Y2|%&tK26mYf981Qq#YTbW}+`PlyF zWl9NfE`eHqw`bdKcHERG?qj%xk0B;SS2MfZtmNLlG0~tw&{TNM;AfyOFG#NrJuOY` zj?Tl8t%_lu1iuzQm;kS|^p^0GwFjy1CDY-RQ-y=A1D`Z~PMpBw-s1FEW3d!B)UE?O zqpPPpn~9_nv@S+$USbt`+Cm4lxo{X!jKiZCWhs>Ah1WyNNuxp2*Y-Z?x6gZ(CvFM< zuGKzR=4mKNWZs)*e{$9)GdUU~H85utv3NVtSD=O9^cpFt{q|ZwflTxvMxG?Zy?O>)c6Q;A%-E+N(#qiKjwN!o*;$^h zviusEZ0Pfa&r_-ZTv_=&XHFiMEjuAKvgO}@1wWtOlLJ}H)e!i7V;Z3Fs=c%3}IUuOt|WJSmr6xPVmx>H@HzPZgn^uG&;7G_bil^)(-Ii@?JItg_gS zP*rFP;rtD|h;b?Ru}(f#_SqUXe>ZL#A%2IbBA_@dzaGM_M@w|P9^qt43pt*8A^8{1 zjBX%VOD&z^h=mWAG9+scMfF}QhJ$U!j)tK$D`wwB00R~Oyx^kW@-SoV1wKJS$Q5$_ zXG*o?IlkRNREzm%2Bpu>i)2CGl0f9|f?t^%Swg&U&Q?nNDf}C!Ra!amf|cM)x_xm2 zuacabjv!A+ssWc)(9$*YbwBe;Wy#+a{b)GW%LQ^8(`Y<05d20#2QKFatp)*v5Pi3P z*7cO1IqIFgS(y58T-EzqR6N5myW0~9j(OGi4%E5Z8~WQ2IYFDXE#j~t{u6gVwi4;zZT(D8~6_|Tdg8pO}JW!{S-8@ z%dst4r188%*X=Rj-40lEz9Oy*UfhPyxq2bxnk2uFL`UKR6GJ8lEsw^JeK|*5`wnT( zOI%R*GRAL$wph{r_Hi;7^kt*I1Qt$=)wj3;8Gq3lGY?TayvfqODTg%Gmyz3)P06VW zl;zoJdLMK1t~p(fpPjuM6(Y;OCcty8oa)k`Tt(Gd5WF)iMA?3i;nj3NRr?>G#aGs2 zYmo(QMUu8_W93Jk$+q z`}^y(y|cpI*n|#n1KGJ0I7gwYp@vn|z6(nOo4|)4@D=e=RoS2bfNRPoG+8 zb|YyIdO`2w$AJ<$u@b?=bm*((bJi>bWVj@aFVuS>F2!W>w_DAX7jnLf+9z@NtIZU| zg<5qcXyN~D@b!eFP?8t#q3-0mra9_3-ez**S)!+-E+U1+toJfcJBCVVO#*IUfU;L) zx2sly9MO=ttZUTbwaLYvK(4a6nD zD=04mn78EC&_{ZirOqigZMLfh&mJEPeP7-5jJm~H_1pA#b@azus-Enqt4GT>#Tu#z zrn9l$ltF4H>x<22YWM)~dNg!$3(2!c+K$<)D4HoWmIXCU#A4upN5H+9-bmbze?Dr! z855vWj)5LhwkU8^N_jz+bVGJF&j?WdT#;+LLZ2kwjvBcOZ`)we@zEt|*jq>rz3Gto zcmfu&a9M^H;jFqr|?8$7EB%B5GcMwBq=5P1s_RM)rJf~uxfbnrMii>pg`u6{>r$C zZG(R3p+Q{>Q^5-9dc~D2(6Z`Er!%EB55XJT#BDJ-yI2Tbms?qPBY26qAE=Y{nrjI^ z>wmU&q=tNQOL#-N-MGC-LWjJpA6TRED?`}kIjd>7f!DVP$S5 zyR}X-ArDKAe{N2fc*H!r78~YN4CFGp#v)umjTzCMP~_j6DNZ>j)L^q(;NrxnZ;!PV zMjgl+){@$?>Ef@eJ>>G|_J)MS+y(P82E=;h@l8xjIPi8)a`H&!@p6mSk~&Il><^TM z8^DQ$ zQGVsEWqZTVI%NmtezG1+Vt1O|`q+yx0h6{iF=cUbXdw7P>R%$$T+joZoyyxQZA%174tDeu74tzX`F;~_e95N$Nx{?Q5Ub~28IU{GTb+)Wn@FupSI{GU(_@8e1F0%U`BcSccS?5DX9f~9?#FIy`gkcq zuSNmwGR5C9OEbr74TraS#}m3wj3wmaboO<6L)@-eIqn+9YPoO6&4=ff7gi%&(#-MSL0qV=0mHctVWlG`K=k&yXqvNsVk|-bw}9|JKFYZ2yx*~zsg{rmz?Nt3INcP zPBdK}t&?dDUO2n3!c7m~G7yA_;~jW0|mO^tD7N!n_ z9y>d4k{-UbDeuGG$P?8|>TeuwK6L==r8@aRvOjE)x--xJ$XrSMIa+75&*>z}W+A3y zk(0>CU0R_au)`t|;1C?6uL#VhzHagv!dfcTYwPDF+mYr+G+PfVBVEeZ$4xx>`yG2K zn7_UI?xXRkYSxmWyHFiV$KDb$mG2 zb@wk@zc1n-cm_O>KT&li>Y6JZ%UXcFgr|2{ZCSUpC3LgnM#6dX+23LTMyaA`ErH^u z^WZ6%RbQfEe0<=1wsJ+`@uBs5%D3DQ#O1ZrvjHgf#nqB9~q_YzQC*wQx z(S$q$!kK^-n|s~LsDSCqs&g}c(SE0LsGjv!!pvIYEmepza0a+Q_oQNxGms&rP?Ft~(&yJ>WM=_|2rI+9=HJF@Ll;S?d93(K)Bh>N*# zmbv-q+FIXL7{UXQw`ojo)$%6RtXMg&f5q!Ba4H%n?pA$+9Q-_=%}HX7OtV8VE8FXY zpgnlE{Jg95(t%+THjG>x!@|;T-GKMZg5eCHC`vw6%z9A<|7=K%f9y$Ln@Eo?&V#kLZNfP( z6K^PK=mJrk6=I`r-*99UysR*Saz;G?aBe_K@DL(OL%IfVA#qo$*Slf(z_6njxD~af z6jzYxHwTB6INtn7mGU-sZwzfM7nw6*{5C13k0Nz;^Rx>$?9{rl3VAl^CQsZluUtq$ zi`Nv7%zP76ArDSk&Ca;BI!1^ejzU}hT3JlRwWV6GhD3wwHI_pen&WyVyusN5xaP}; z3?Dt?_}2yvbM=LDb%URqS8tj|a1J>aUmmWHtQQEIllHy>>^l0E9;qc%nwkyT)4C+A z|4*gv?n0Q)_T?oZ%U!e8Y_iqW6Y8Rfebs=LrPyeI5~QFGbj#EDiG8d}SQL zH39Yms|^IlGim?`kbCSQ{%Y;jHw=?o9FrsKNpNrO!jHVBV#8NTJoK#8NlT6{@XZ17xFd(dX(5<{+I0= zqmqRoiEuErdA6@Sl3XRR}jUw_veo1SA_OwOgl@N49$ zLHSGY{!p+^^!5>ozIB9fEa0Q*yyOOSG5L1o+%RKun66Ks1y%U1Z-Me~jwd8+qCW-? zrC%^Q6q>iwo#YkpP4V7~tFg~spQH1OIEy6{8yylhN%_v5;{Gt+a&KYs-Zq1m_6Ft3 z5Ap!c`6TVTuua)Yl5@t#k!m0WEawN;d=Ol55GW?;s|VC>8@V!u#?yG_3fV=0Y#;TC zfe@gIj|;}_YBgSY1g21X^-(CR{FriCi>TD*z0>y2U^qVZ)sU36A?GspTjfVHH<$c9 z;r2?Mn}RR*AKms?IQ+y);t~rZ`}5Qye-e#u_iT2jlFFfynL)KZS*V!N?#gM=V0*p^ zvB^h7#F4*PaQg5Kwm0z{qRys2!u_#v@t57PDL0y1CbCG%%mlBP``mh3t}%J&=YNSq_K<_6GaMj`6dw zhhdss`xUE?j3{SYO$?+TK614G$1^dRaVKH+~$8sUjMt}>k zJSkaufM*bbwvUnk@eWY6{5Mt=_495t?pW^qC7zl6nN{Dj3)7WVL1sCNQSX^uv#i^it29loiyb6@G`u)ZVvx5{&>%{bjTQ@iZh5W~(em061W-=V1g`Y`6 ztKsUfy6SiF#p;3Wns2X>)*krH>i3$RKOV(vgU6PympN8&Kb$YTG$VSuS>`wXa&Ly) zDf-9wL8sGk#0Zt%E&qUFX)O<(H9?=fq2JN4{0q~1erzx9Oa?#tw*d6rd}&sxYh?< zmy*A%_{IKA#uWQLkErdas}TD+yRp@??nccTdAH}jmJbfO2asYnPNt4C)+5#c$DOy+-daz`9_BP7h#Otuivd3!%Xz~ zYvNvOQ6$$#ZYEk54TB&TMup_aiv=RJ@>7aZr|p2Css0M;EBs{Bb0_~2<)iSsrGuM1 z>4O9mJ_-gt;p2!gehWR}19__Ov+($sV70F|redm$ysajAY1Kt~9xH!m++{V)1z~F6 zRKU_efiaHH%2`PZ{?KV=UF!Z(Y7G$ll991ea=chff2w``D_ zkq70TLTRdG;QhYhgsXq4o`d(Acpe#_lpopYULg*;?ItE_nuA2&MhVF_u2axGU0lfm}n9fQslp8x6CpMKcY#I1ym^iY0>5 zr5T83cAN8&HXjK)B_KuprV`78G`Yb&o@yA#XUAWpJjr|so_gFce`vx7_WT)giX4!b zT47o3SJV3I={Zw@J-OL&R}MVxU`^hkAtD++U)P?YK*;cWCyG0#3L-t2c%%>T4piwiOIR`g^`H;H zdMEace^qhWXC8fg1Jvc~ob-2Sy*@) zya2|iwl5DV%64*Za1?Bef(%Vuvz3|ND!Tgu7cmA~S#={(@t~x&gi~BlvEq;16Cv-` z8a2`TeU^Wi1$4|NtVw(=5N8MB7Xp2IF9W=S>RS6?~8r{NMZ7C z-NN7ZBBeZaq$dx@?mHj;$WkR%HN<|(egpJcw!%W}-UhV!xH$4CADu4mCcE8NbPV-s zlK+>`N>a27ejJ}=x$<^&`K$lZ+-yx{<<7)(4m(qseeyDxz?qL?q*`YJekIm@UcZ>& z5JwASg$8RT&WM*P-87Dp>JeY`R+*SdlNL2A%L2iEujqb-rFC885sX!}BklHR!rnp8 ziH3LaHwvYf_y>%^dMZ$+ahajQ1ZtF_P8nv#xvvjeFq0?#6RLFqyx$Gghi+Z!5ElNt zd5Fhb9&D~5d@a;is8jUJ8@xWJ-$3YNHY(;vU*k7dgGip^slIpFjDljfRt7Q=x3-!y zMg7sRj&meu##`x4xRt+D|KN~`0MOwUg7_#7jgxe%PTl>!ojlvWzBC3|ztSmUZGW~* zToQ^tZ%Cy)$RzC*b^mZ!z@5Z zTJHh_+&ZPTU(?DOFT_Q4>kHdIJYhtySBEV1k$tqZ#WbsND_AyAymgSY*dNs&k394n ztWJty>o2XEsp%h_O4eMD*}s_%WvEZuAQxLGLNB^cK*z)Z%9vp#EMy|J=4!xbz>Btq z$Cv1i0quH?t)=GUlqUuhr;-+fpuxr@pk50wehVa}sOOc24#jSI!yfDMkVCzzco*U5 z;0@t<4Q*isyGxA~IlP`^JHIQSgh#rR$%T)d9rZS;H=%c?EP35{i@J!R+b}wqoM52WOn%EdY+XbX)!8fGjaiZvn<_QR8^mg)Fk#*-NY+29- z4_isW?o4&cNd70?;mN1ogIy#BJidfBMZ0iv&NF#BzI3qN4OxHPqs9;Lx_+f|vRqiE zwOl|va+H%esMA3OL^N5t#M?Dt5NqP?7m31L$pD_vq-Z?pXuW2TLtxC`>)e5g{K#-i zP1A%U7U zhMnWLr%>sw?KP?a$I&4VW+X61)A*W=;+T)3`Q-NwYW!$Y`Y37HZx}pZyAEC%2&89U z&NU^!(#e>-uW9vk2j1o!0nf(=6VX9-uNG#%XiG(n{wItx-+JccowNyWQbS*%mm{us z&Igl%Ez92GCuOr++R?Puz*~28TZQe?(qH~pVW!B{g$W}i>lOa4N zGrDw&=@R;FrC|=6@iu05ZJM2;v7EZb03yRk*iT~0l&*53GfHslkK`v25qP|E6({hAU1 zm6-1>xtu!aH~c0F)IyvW2F_Lk+r3T9x2wF#@A@X3`%>iC&ahsOW%^VBukr6LFZf>z zHk(mJoo)I>`IeN@k-?cL7~;>bNuloTR|9yXujKho+lul_V;;-OJNR%GERm{anGDA_ zC$7>E0_q!_TlsbU->pN8uIX%+j206I)S?JDoEwcR#*xWB(C7Kl=lyVs?W||-CfF(X zj9V&u{?NdrV)g+yQ0mTAU-%ssv&h~&0go=|=6~DTBmefoC`raGB< z0GX=FMe0cSt^P*W9YKrF62~=320$~I?!8THY#_ZF2FBNy7h67Yx$>w+qGyh|4otA?2;U`Q0?9 zLj+lG+68zs1lz7UJf)7!R`)h-6zmzi8$YH&e|zb_-!zv#1H{{3Ww~c{Aj^KL5$89! zpc&Hcrqu`j-G}G>ASBJJ?R}DIuklL4{X!w@OQ7(;pS9$2i?-18TwtbO5$Uazh`K_k zg?CWmdhKl^Xu5Y```lQ-)Q9F!j-F&RS{ESENHW&ua{_@6I>p`$T~|%p6Jtt%ROCSk zhFydPHlHY?8t3Ni2J9d(@=?64z#rR7s2w_TbxU}!mJgRbeZtJWMW-%v0eZ&qliKP_ z2Uf&mlEb9~;m=+9|FZ2RU$ZLAi`Xo4FovGdt`p|H> z5!u>IV6=i94KqwsP1~&?76tCpEkMT15XsNS2egDl{rlNF;0C?cc6t46;<&L7L?8=^ z5bCs;4xD#fh2;> z0W1wpW`XnL5_o%!Z^~}rD3cH#VgVhg!!nB~Azbo-q$l!*vfE~5G++KM$q!v^qbvNN z1`euEJ+R%MG+CPD*qlxmSoTwu#}3oi#`TTh;;TItUlI|EA*oZ_QKwvp$Zp(ZTT5Pu zgT0@~(*aJ%w*aV-fv@2f=js)|l_opl$bibR@Ja$d)n(iK9{EO{6xllli#;|zS}OR4 z`L%~Jl|Vs~=MT+BPevlew_Dl)U-g3a^~!+9Ua;Zan}ie*IsG@oIPf=9L2R7}^Mw^# ze%NME`%jh?wi*hb-yqZao2mf#_<;aX1;BaEbjLQ99X*Z{hD(8Kz@2y1>fPq*p5i&J zbxWC^d&tfED2)H7e;V7MXE~l1x;?`axujbsc$51Egw0Eub+7vs>elv+74gE1xNXnt z#_rwndw25J*H`kgfP|UiD3aG*q`vEcp%3wFIXv}W=iZqt>A>LpytAaXaAGH8< z0%}0_JHS>jIxL<6M& ziMkymGycgg))d}3Bh&dw2%BfCpNgE{kSZ~=M#9U1dou86U7!60`Vl#nZ@$lCY}@zn zujLq%v~lh)RL}{D!od3ryQW@GAbrxTrX%b4+VQRTIP3qQJ+Wt9FQ)5xlFn=iWwx6v zP0l6_hqoU(6aS=q!#tliFkYkl$kV0pDsU;^qkj$>;&1;H|1Q`8lsN$!=6p5v4zf@% zrahkifgY1xA{|3b|3>bmq<%_o9tZyA;L-YW9O}b;ZS@;d#~Y4W3Sn8suV8Q%(1{KM z>(HJ0ZM)43)^bK`Zk5UR+fd36$#RUK!7&ya&&9AgN9EID4_H(9t4CxT0Td^$QQBu; zEhxxAE{$|R`)z`gv6SN@z*DCuCjp3h;M#UF19QkcPpaXwe%s=^mg*@H-Sxc45ZJM~V>nKpLSHyu>v z>p&`4vxIt$1em7U-<*oO%xp01;oI2hy?!TGi;)6GB>>BGP=iphfKa&4dEfZT?$jv* z`t14%2yHxS_LH_5BUHySi30D`sFl>%6MN(VVa&q2n-rdxorh$Ro1PI(K_Kk`4HLiW zH$YOQs9VNr*HsRSM)B%Wk6T(Fp=7%0KQAq>hvgP@62fcJjo~=x+$VgJ0|Ovwi`$u6 z=z$z8wcJ7ef1wc|7iFS z%|LULnCEa-);wrjXPu`;OHS%f3z_kdB)P{FbCx_VZMUwg;^tQSeT{so0iwn9Zq&<- zR33!c(}IN(V^$2okyj6Tfq#AuI&YjnB<`FRv$d_eWgn$yn8%FiW?_bI(o)2{rpkb} zzv72atOGR-5+-hs4X!25Oo{WFc{3h#sAn%Xw<)|qpC%wwtZkz(+nd8#YslpoW4z$~ z)Ka~3R~h_KF)M?`FWNza&waWC8>NLxOMMLnd@jHU9|oUf^EakPQ14JxiLcQ=QK3Dx z;Jqcc2{xEb>^9agY6wktX^b7)6*xysKCUK}Uqs8}NvV4p1V89>hp`!!nnNFGR3L`{ z1M~&6$7Khd>cRwb?j7J39i>h}F77ke9w4Gs-!zO)S8N12Jl~mqyhZ1Ehj#G$RVP_Q z+xM!J6PuwW+R}~YDiMN7EAh}xUt&TVViW69QxgSk9O|3f`+wT|_IRe-|9_EmE2
{qcUkuU)U}dcCgK>3Y6UYB3CuPpq?@JaVmFW80THBLxE! zR{(*9oz=jkc2kE1?DDbd;l($s`*4%#WoItu>}9b11N?G>X}h{Y6cNeVrkM>N&RZk{3u9+$m+k;$GgPRh zfjk}cec4N{fvr5PP34oF@A=B#x@z-9IoHSU+U$b$?Rn(`BFmbX+ zYFrYkiuathK6SG|lz#R~2X1QX#P2FGJT5xUcp-2#i&=Wa4S=iB?;+mfF%NmsumyFU z(*0rO-_7cM)YKlfH)H=#gk`e zgcdd+Xho^5>RkpB?dxS&OKl4|%t`-tuQ?}fmDc4ZWc_aT4G$K(geLXnx;3jJX; z;7`FauF8Lg9$7Z&eJuTFl1b$EQk{TrpX;H`Al$35CcClM;4=tAEjO7@MdryG?I!lu;QIr!qJ&p@^uOA|plZPU0=x)k1Ez#>`A)(HVD$GI;5} z9^P5jUEItZmU#BD$f$N9NIpVHyYwh>I?0lCqjBh?BBN%)(PX6f28oXwzn)(`))f-L zVGR!+?(gdge%|m|S)UgLneAhG?v-v2=*FAB*rW~4INyzF3Cx8FBS4E^t#fijk_=cgL zV2E7#eT7$hPcvn0mWag*8cLyL@ zt>5y%MS3-Tu5jv-`**@TO*gnT!+{})l4~J+d(O$9^TUO^P1)RJcp2~8X{7o_YUkM_ z(U#0P`h3^16w^JmzfKJO;>*PakAYi%9=Qw7kgo?Oe>d85E1;QLzP&ZR&-5E2pPmyw zG#{dmYZ67NgM8Q#P_zNe1PdB_$nLCa9JcWdi=9TN(I0+hxZYzx>jQ(zvpw9EnEFy7mY zXTO)rwEzH7)<3Sp#7Ug-4y|`Cy6d^rLMb!&?xeHsjjtK|o$u7ks(@-G=KOb-&G#!~ zCky3JijPRNxyhigdF7fnnHX~L4YQO;kb;z(W+=oRtOY9S%fE}wRRa5s#chT+f;`LJ zv}3<@dQmG%q2MZr>drf(`sf4oz$2h$m|zt|qtA?Ij<_;}+O*Lo1o3rx9_MoWxBIws zP+p_5W;w~{q6@rDrzdvTc0QY^s_9zl2!Hc*=EcEVGY15_f28xpsRZP+J5QM5E&tv* ziTk{Mf65*g)H?(=g4em91{yWdwi+=(=v&=KUwU?nzVtgs%wKSHZ|~d!Ckh}<^U3bW`nZ{0*UQ^7|~ArRpCm>8Y*y-yMItbty9D(N7?ojpDgZb9!Aeo*9w6R05|xh+N<66@mdd$1c= zB#)hfaV|0~NbY;m{*rw{j7 znVVd%gbzaIa7iy#$E~|YFYSlylZDAdC+%l4ThdsX2B0Rj3Yn_jr{cc6eb*n#`)bX5 z7k@-dky5izgX4!31VrZRxVttiBTDI&xn;qt@u(@Uy~$UH@7@iD?+!6`GpD0gX6Ij3 zD5r2nj1)@YkQ4jqaPH}d{dL20M)bU|YzyZ4T0|!!HQ~JKq+q?&IYWv4aUPAmlYD_d zka|?2IsxyztX*72uSLwvL6Y5pjiyG`&^2>3OpEr2=UL~-_avIB|K{m0-}u6T@!723 zAA2+Z?qaES!1}UHS2R;|m87#h-=o$1u=It1lpg>PFn4Dz^H0o!3}%7(0{bcXY%DV) zB68#69y5-AP(EA~@?KQ&gj;nP%AxaZpi_e20fJ5BoYEY7AszIzqmETwX4p*^Bblf6 zFqFX3MQ_>&qIHk=>6FZ2?y%tayv$bB4MUqiq%xsv0*driiLQ>O1^Kv_@~@q!sl2K@NVQ^A=rBJuJH7D55#EW~FU> zjx9)|c%I470QQax&PHv%|Sa^+??K$Jf%Cr?lK_Xp2q@BYhg} z7rRf*97-i*_0T);x)G0LLpQ~Zyfj^ZEOsKh1!^z$=A53nMEuK)~)#7Ha5?pfdq2^a`i0e zxE1g|*R+7mh8tA0j9HjwE5*p*ZKA&OD+s)JtRTmGDsltJwscye<}aFF15SJ9Zn2?w zPW3%h7tOe$liAXknjB}hkfiZmn)-;(<>S~nFTAo>&MUu6HhTlMfdW{EZPEjF&q$}h zsL}PFz9@0jZ$#FKJ=>V4o}Q=s$B6udHRjMpWWV4qRed*bvzHx~3@d=#iBs|iRS%b$ zNNZ5N1n!M?mA`fbE@<>N@B121I?6W*d!JVZ9|2%R(Dy#=X-A|W-E(8Da)BIOmmh+0 zzop2Uo(tdoN!!%5;#H1|^?UxF1K>ko?gne;Ac4s7Z+%nJBHF+_pR5L_JM4{aS5`nz zyBU7c_F7O_8vG3VCXairz_DKq?1?c?R?p8V+omNFK7+%}dPHYu*!?yp+$`Hsg<>Go zZu0&-_#aY!=#k?j`M(E}SBHxRC}!Zbj$5dXFA7)R${;a*(q1n@=dA20KA{dZ?Q^S7 zN#UaMQqu4rh=QB8~JrTjdlDWhtnrwP~sfT(Lh#<>f}Su(CR zf*5{*TmcjNsa>vPAp6A)nz(f|xD!EY<$BW*k`a7;R!uhg@1^V~Yh- zj^UY9Lux%_7i8@ri_)N*beN_lnIp@WibQP_C zzGA|qClq0xC{*!Gf%XbTem~fk;s=!_4>oN|2`I_KBFwT>8gpcZ{JY;=X{*g+)T*`9 z=9$3N{$Q~BxD-E566PN8Y^P@)u}K&oSfcs=q2#e{D8Hh1wxxeRgT5lk0Gu6%Ka>0Tanert0`jhjYv`xpGbpWWo^HwNQs~X=P-M-2+cUy(2d*5V*1V&r zWA6gN1$3@@&C1P_`cQ5BP9@Mhiwr+JZ0&gsMGum~U!FMz8b~_1JJ+k;Ibf3zcTPt^ zY-|D&!&`wpYi6r_lq+>?_ffhvXkbG#h&UNNZHK=8U_yYjDJw<8_{TuQ12O#+vxe-g zg@H)^r+-*U3z&z}6WY{#ib0pD-tz~Jr)lA+lhzKPqy^A9Lr~ZVG`VLUJPq&t1hko?ac$wrTd8Vr5*?lu9)p_Y@AvMg# zt8#FC2MTkGT!-2`$9QSkAEwdx(yVB9Qlc$=YmKSh_2Q9c*j8}y4}nC z$!v_@%mYyutB?NVy4{F^-yY%nc}ysV z-c0wSQo{8fpNwwrY#CFW_c0#iT% z$H7Z5`8PSOM<+532lFp#Fv3hr*CugLGKsAUh(N5jqww?LPaB%Zw2X%$8TPXY%z!%Kvksn>k$wp5^Y~Plt%FXgXJxm^5AvGE$0*9x0wTe^dbNBU-Z%6g$T79 zKuGNJY7Zv{kwGLLd77VC0T{ST)Ys>?9XW;FF?|E&-_ir1T zOy+TAKeKI{I!{VLQdI(zx{UPar1xA<&|Q!P>zmX^xf%%-USG>Af0_<%#~o-2cwN3PZm0 zkJ&naqNoJ_kLpb)H$z&5`+f4(0gp-o_@Fj0ONW|PoiZdl$O;(BHVFV{t^#is_;hRJ9=?pmK@Rm{O^g0$xBaFH7)Vk#eKVw2|?Q~eR z7S6?M#$L6Xu}1dKe>#{3G>Z|uUrlys=@ioDX=jx1q&nu_D$Am+FRofFP1*i&ypbXb zg^4G&VjJ_ikL{VTf=}R)SkpJG&ONnFVZPFzz&=(JX$w5`KiKB>E4cxdYaOW z<5=h&!{S_mIjH>xbk0zsVt%zl&Kyz*4a7#iiNfrhCr|^V-k1uPk)VTxbY^h>3n$u( zn^QG)p#RTYr!9}%EU^mCX@EjE;8)AQgT@r1-ZpCLXfn|gcb6ce)M@1hNt`7R&N$@cGDMSZiK$o?FMOLgfCV^a>vGJbhKW^XmQMJRA;Vj&x3Y*f_)wiTj@)Coid$O`Y zSi-dcH!`+CZuFRo9%pJXgT+_!ALv8S(VlH~*ivr?pKC29F!pU605e*gZ zNPxp?vvb4a=M$#_IM|F%>b>w|KQYB&8DQ*8#6{-6qG?XaF?j-Q>nG7Nxtk6Ncnuno z>0uw*ywW|tv~(tIy;%w!Fy}Grg&TpW0e4o>kceBfwdv~VrtctMXaO}A=r-pC87<8B zbq5T=eTm7I=ffs7Gj~@E07xUzy|!l8ANk>+MkkkfME3%XahF^6O9+)xTR?S{NXR*jiGh#Y74TmW zg_G1Fs;gg$;)iT_PA8;$>qH*nQIeWLpsQd>0{-9V0~k5&y@{r4^hi6@R{rC`)$Z<)jadjR1iU`KR&JeE?B}yPH7?4;y{WB@O7(2#b8c%cz({zz6`R{h9W%O{0l4kcAOT)wfO`$bZwvQ)sq!0dfCK4;ha)7(o z?^4KgNf<}?bMIW7$+EB81>FH)pNmCblhWh6$UQ?%UGoIAQ}6XktNBkarF}`}JF%3*{NN(ltC?3NwE$HUL2VmwpfV@6n1ikNkf+e$V`QRc5QRb)G8I12!{M4d z;WeSO7gE*2LSqZ6Ow)FzFGGg)Pigo}zUHf^Zw5*XA)eU|B$de3?i85gFbxn$6w{hM zQM5(B-x1rr7^$qCJk8?*Jeaepf0)-}{&K0Qc&oB+oYkox#w(gU?`&w?|h)2ivXwE6{u(}|1f3(GKwwKo+*79^L;sO zK)0_ZlrH6DVw1NhT6`;bXCVx$55_+m3e0iRHqpo{v+F?QQ>fpw+9l@DnkTK6ZEeem z?Pf32Torxk`&{L2->=esm+mNEi&)fL4*1m;`L_pVV}f_%icy#l7CU`##ra}Z3G{ka zpj0s@**f##IEP{Ccb2p~o~cr8@Y6D0T};#~YCJM;-wF9NOY}wNHfH_n>B6brt3Ftl z&X$uQE(kSrOl#Lt_L9qh_RuLKQhEDbObVOrTScJqUM5)EKAd=Qlm;IX{j~wgNa_^O z5OeHgJ0y?r;$(KS9fl6T!JwU9T5WqwWm3jVgRWQ>4pA6l5+MXnfZ?Ps?KTyx!y!9cl1o)@pfN7)$RZ2dUeN zxfeAb5w=h?DI^yue>X?>b72XM$NV{5-ury-U=L*DEF__qmE>!@I_5czRcCj?qgeE^ zJ)nfst0emfj!X5QUm|xCq%Blu!_i#6j!;VbieS?J5^TnYWRAp-$tfvrL1TG6?(#8m z$BF4#F*pE(15|V9f9<>AzJIiT= z0NA4Dt0!bP9k0EnC|1$I!5RnyQQ^v66v3PA>7cY3smA{210e?KVm|q`I9ciUVUv&n z$bQT`Eo@9;j<0Drl>*iI#0zYDO~=#2#GIp2TC&8{V;Dsz1jqJL)^t#hPri@cR#yWW z0zGuU7^zEOZ)&XkPOT@Z?4!)>f#G#M@{{2{SVLzY&|*(b_A`3&sr7TvAbgdxL#r{D z%ttR9wA`Am=DYT0dHF@+8{6$|#tLXE>`R{c#i1n@qdAnCtd}3?V>#@<(d5eZkT2)~ zZwvb0F56>2EDkbXFF23$JX!qmgzg%j?B-5_qs%v;os4tx#8)KDIuNYELK zbZP3#YILlwDe##c-wmA#J_|Ysk{7}z=fj@DRY)p#dwJ8s8wq4{Ceqy!N}pWu$naS) zt{fv+zb!B%&8J;M2g~^X+0Nr<=iC9MRRDKa1O@Gq%;OTVk!-8 zYPV!uuc)zqwg)<#H^ysEP`BaU*>X5D5S>oZo(X?w%zUtBAi7A-yw{HR$2HUXx-CaB z>g4L%bi3bfGYeb$aL3^VyP-*rn?tHX+-%hN6{pz#Rg_qMH|(Oi{qV@lE9-;=6u1dL^>BZgLd1{7zRGJRQrb4X z=++&`fomPblY~tU-`fHIaT62A;H&i^tp&D z-fpDn>n<=Kzt@iBqQ5ra^EkQ6`*_cWj73x)gU0V*6gwRr6j7(JhcS4I)+BPhF=MIp zWlIv7@QrZpOk}?KXizVi z1>qSrQ4>t$kJ0aTh9o5k$SkaD@yDX-nz!@ibu2PuCclg70+JK_0%!AHUSeX)=| zNfl@g0kd$s6Xn9Rl{U}`Qh#a+)5&~-K~kKV9u4M5Iy%t5IIw(Z0}^_@*Q<mu%y0byYh{knodR&hG`S7s)3iv#;z{Udx;1=Avamzqs!T9nfpi(&t zw%DvfKE&dDwu;nbxc*dm_X}WgP~W<_&xgey+}S%urv^Zajodp!lKal?*?(KF_Vx;V zS2aJc*?q8P0;hsVZN_R%(nM^1N{fp*?6q$SMdGK`U0Y3{MC*PGL@yHHO=ye;^EG@> zQ7w|kh3`yhH526c*-7n1T*c3p238dWOz%>YUCBdNEEzF(?a!RQ<{bW(bwBj;`J$=d zt#pIzvFvBH?Pd4ROM&A!`bRAt+lo-#kS#Tc`hg7@-I8_OFye^yvrlOPC6^uMFQ=q7 z5aTvPXPAM*yUxU#YUM<#JZ)Yd;LbyIUapz^2>trU?%_7`jDLLKjdo69Xs@PYPkZJ1 zXBLNFp}#mWeDL3L1lz@^J5naE0k@)myuDDMSMT91XCpULhBFy{G>ZA^zm@uOMdN z!5SJXHy;|(I88j!+r?q}7wrpD&Ysy*D?c@J+#3gtN^h=wxv2S@dfSibuSL(6FrWGe z-Ea#kkWWrT?UqaD%x#}$<(bObg-!@(CM{Z=^mXM-h2lv=2Slf&6 zPjtW0_E(dKA90Kssym3yz5Tu6QxW=(tRkF5_j!pw|I}y$zLQTrYW0@vwDfGIr+u!I zv?OmdIUb2DPIxtZ%pY#;U-KD)r+y0$t1f-8#I4EkdfQYX@qj9afg?Cl*SnuSNIn(W zdxGYI)UPq=)0}^Mn?D`ewGE2F<)?<{!%doBXwOgHsb%zTW3!h_Fh5xS?&#%=kBHME zJ!LkxCU8LwSF^-fIZgQ9?r74#vKOQ^0h@coqtHGbA0amUb=#_TS#&%ot#{Dv7N!ze z>0bTAIzM9O(1W2)I7`1oeJDa#-o;$7Tg%P_W5dE(GrZ^3+t~}Zmjgr5KX$3&4m)yA zP|b>*d+q=2jTDQ~RH+_BN@{p;$eZx3E|mQC+YsqvXj0Ahj~E<2Kk|rhXrv0?di))3 zrmA*(Z5ytPwn+HSO%uQQZwd-z+h&Vl_;xzME4;7D1yKYkbZUh;WzAj=a_@C%L+WRU z!qLXC0wbPCT}wC=^7{g78e>JB4Xw`iar>$ftQ0MtUjz;U4)KQ_aqr6aVPO9h%Ex{4 zej7^7C|cz2sg@ru+PzW*w@#ihK6?x1KmV;SCuOA{mLDpNbmVj}iH~nPal{{~TM%!? zRoeGQo;cwwriZI$r})>t)tIIDSG1;h!B;7JHIgg-`F{_Jb*Y&Ck3IiCH^rQsUw&-v TFBa=oU6jk0tS=HTc*Oh{s2a7# literal 0 HcmV?d00001 diff --git a/docs/ovms_high_level.png b/docs/ovms_high_level.png new file mode 100644 index 0000000000000000000000000000000000000000..14f78566bf236e0cd2e686d16549046802929674 GIT binary patch literal 121586 zcmeFZcT`i&*ESqLP=kmC1Sx_AM7R+ZLWh6~sPqz$E`s!4r6i#kumRpG0s*8+OMsyC z8Uzug_YM(|&UX$*_w%lIJ!?Jxe((3L_03wWGs&6EnX_m1-q*hN3@?lfbhy9* zU=Rqzb^qR76A*|Us2;IB#sOST#*o~B%W>a(mH{9T=h?$Q77#j}ANZ0bz(nUZsI*6D z`LMlNZ-5n3s`2XmudFlX5HZnGR^<#ZX8iV8-@Jk&h ziz7U_I=FHUH{q#Hoo3!d?vn!!l`>w7T@(b;h`C0yRtRZ0Pwi2=b(-<>{T-c@pyb5r zNuWvO_e+P%*|iLpxxJTqoDqQ|1YNZrKK8%exfb{OhGoz%YkL=<$G(T#EXP0^f)l<| z-o%ntG&#H(>IrXK3+Noo&)`zpjmuA3@TNbVb%C*5@@eH}b~gllTkU?d-%7-z;pz+s zFm_Qh5G$xJ77d}jbGB2+5{ThtaP58phDn|W3;svw|GVNXu>snca*+_)A!o8NG|koL z4a`*A*kZt*y-hM;P&BAHtcUvQdP$(!Q4|S0;NI=Sw+8jyHlRC-;+;`RbcVZSl6dZnz+|~F;3PAOLcrlSk5U3)Gt9{*hM!6V*-y6&lwIr2q z;(EOlfaO%sLaMcfdLQ;=l?@0uQoau}n_00{SFqE`)tgdm^##)o+|2s?k6{#8`{aYG zghg8BdQnD6jEVns$Cb@pdlF+6Iz32f&CHZNn$g(umtEtyQ&cp*9!dT*E$($Vigp}RE|HQmMS9;w2F!PpOup1tB5>v_{4{=Hs=6Y zAP!{#>YJ1AqLzgoDoT{(uuc|5e9DC=%rQKtEuJNW(00+w;&!iI29eLM#n`EV7c^t+Fm9(ua1=LE|!)W$bQLf zt;qc-v2W8Dra!xP=j3@upk<{0;k;@Lnu3;IpO>Aez{0d<2}@O6w}EbqQiTC^kyzT; zkXPunf4Cde8ha8Vd|Aa4GI7ySz!a7DpiE3RnHPcoJ@-n_yu_-pYLoo_+B-e5icB6+ z_Y0V;fTU-wv0UXO!tbAFyWjsz>Ml1n57S)w4IZ_eED>*&GN1Aq!I7@79bH~qlcVZT z9}~L<^>jEXKsUO{F@Lz1OBgR@KI4TP@l#oyDqruNb!lEM z$aKClyBVN|8w~_h4t^-7U*liI!1vEtjNs6$XW6gD2w(D=7&5IWwd4Zv`fqRV9w?Y( zd+?a&WxEl(QawI|XQvvnq!3HuqCe@%G!{BIcneq&7L1=Rnhnq_9QqnUM|n^Pb$0DA z3B^IbDM`&=D-aI~`tG9MtdPxFwd$w$gIoOAt2S&#sdKE&o3a`XRlp^{)0yYQ8DKDd zGlv2$d>frl?#&dY*N@M*j&@iq+FU^0;vUZ&o(do?%j&6E&2a_(EuUqYvXd&?BY7ll zVJYZphTODG*w19o=L4^4P0WGan8Ie#YuR2&bd?Y)(%qdlomT2mj=y;UPvj2K(?;-gskTJ8t^J6NEY)94P##F&1Q0XPt@WR@CT&d*A`lp zP@xX{1sz|1z7t0uKl`0mGTRK6m!E{l8w*{uf25qVSASlLg^oB?DeV?<3k>0nrI?Ai zNVVm_bKu$Z?Rr;x8~9EoYTmNfZNi+3WmwgI@p1Y1fOXKy6_`^#Bx1iTXI)7Cy(AO_ zI+X}u!pJi`74KI946+Fg5{iD_O`)2m3YWbg4rGEH)%GHJvoR%7!#J@=#ap2}zwm&Ry|5R9hEQiScL_-rcKx_HJvwta`%xeCBdx$6eTlf`e{rv(W7O`Kjxe`#`b4{tc;;dmAQ}T$}J#!4c<}x&*A_Gqh z|0%XiNmBV`dvqK*tZLKVsUdMJh$^cOeQwnrYGH4>Z#y{kbUx^b)cs0?RRTx)b5728 z9~Kh$EMO0%yG`k1Uei^>rFYb*xWQe_mT~6*BK03ACHIQ z$Yho6H#c9*zQa_Tp_UBOoZp;*y@W%+%`N>7yr=Z|q1S_rj-gBzGobaG^%$bu*s~9$ zIA62@>^E;BuGP8NT^=`|H}K%0f+3gP>D}o6E!(4rEj!Wea~m}r zF{|_t^D}sAccn6Jx5<7skf9!k>!+^w+(+W;_IkK5^STq6ewiZP^eJ@)5oR$)q$MP< z@QuaUlm{&a^fNvwp8-XxABhBk%-l?SiHP2y-~`FnEk%N+@#X@2&n2FkE)gYEEzg{d z+~?I!3zJV(1Ye!w2=|S7Uu@YZFRwThPfb@!ZtpSeG=pVa@MCk@UsYQCQY$a|+`5VN zT8XvIsdsm&-*HZV@kuOKt~QF}!r)@?tMUP+7I=}nJB|;3uJR6<)p_N$KpSPiR&Ob| zL2AUneGkv)De~Jvv-$&B(pLEivl~0y<5|T*JN=gKAbv>VuGE#*p#DD?vcicW9=+Ow z>y}@N3vXAJt<4083*8g*hSSJls~@6vj;1(grTOSguEzafRCwF&h@tFD7D;#D2>8|~ z$mdRfO}l}xeJc5AwX~#i70r$DC#8r^!47!@(dv33jXw8TeqqY$BkwG|1S~({B;eOi zq;Tyu^mPaQz%{qw`)!*do~&&n9(|}vtiR%|=K1hkNgQ!8K+~>&%drKjkA3jX_Y^zV zn`}|h7nA3zBVgLGtI0fd-z*o_N9B@?5pN{tzb8U0johHm!S890#Pr)9SzRwAw?Z3x z{n~*w;9MI@S@p!;%ndc*hgL1Aeu24>`BL1U&1MpDD{`nLtB-g)#E#_;6cTMm6`exp zlPqc=Z$%ijL0mD4@TfODa|d5tT;cpflczfk%ej%6zhjzz&tE*j6oTFz@k`!5|GU?E z@t#^*JR7L*Wg%cAAAz=#WIHLa2K|kaiVlUV8Ln*d#+61>8a#-+k>y5(y3*tM=%aNn zs{j-9VZF1*Abexod7`2Td>$pn0P$IITX5xjZMNcT5OX~08Gb}dRI?77(=(W=I^=8x z+l_6LdpW1R>Epyv41%uZml~hI899fUe9^39Y<*F_a0aU7#xRM{yK}&2!cRQzGV|&9^Hx#0BCrW5o^ET{0*E2T40Nw))GAZq9FqwxM(E6f9n_FBvl2zR{3UtuyM+lT}dnGgQG zrdM-aw-JAT662_qnZ1#+zO??%m^zMQZCL(3H~*9Srw zsziA75?X{q=2x}NcBIBzSfp&;%qcI5$TyJ0L-)&OpB&}u`$FF|3=EiQY?_Qsn&Wua zM{C`@<^ARupKncH3rFUw zbPFOGGW9vx{GlJ#dHo^~uDM!IhaF^x_J?T=0)EBXg1sK`qnZ&i9ZU`CT8s{rxGVTt z^Ul<+oP0mVPx=DkpvhPla_->%nOdr5jo4?S{#13C*`OP)g>AdbYay7oqDjiN3Lk z;c5j-@^KVh3jD%%CGWiOFzM(`ud{y=vbJvOi}<Q5zK1peI=^VW*8xDIG~=q-HBQOVfLNSFfp(eTF*Obs6{C@5j6u`f@y1Bq3k; zr>y4|mTs+|`SR^{Q^Fd5>VEa7MO)MF5B7P(+^V%$(nV^D?aJkMgTT)3M%sOLOXh7x z!oT7;YOm!Ds+`khe}e$I9*rmrSonbKYTtR0dZpBTcs*HB`P_7{&tzS-|@N4p~q52d=0qtgelLB)y)bvYulSCt#(8q>`pl zx%{725QSAg_u=1F%x1~uXqiz3%ZvLam7~k~c?Y~6NhjQM;KrDrpO#Wysccj*>5jSO z)h2%XW!b33p(~9e9FBdlQ5VSTa#?q$)CGu*drkhi3Zrk;mY@X^WHkY1pE{qqBMm6izGix$x9I}5?`sf zZ}JQ6+hHZ6MTZEA@0}Ry{fij%b$eJ&rhKM|L1(rd;e8Kg z*yBM@t4M$4(4RSy3!mx;d4DT0blng#ZJ0aErXj48M{#(~88Ntd)VE)K03`@U z?7_=cWQ4Z0A_;lX>bg7QzcMf1p9V|-$nlUf%qpcdl?R?JHoi$mObZMtdFAJ)HYSQz=}W)KpHldT79R%U~5=e#v8cIqEFK?e=2&RXbs7 z;)X?@*0lWi`SwIeQW^X~ShxHzGLyjVcxIjIYjlzi5&wtvO1NI}3vQN5Xfmz_Ks2|f zu24z#Lh^3RtNoM$uP&9zbX$K4eDb) z*02wh@Z@fVqP|vQnQK8n1-)z-6 z81be+Fp%ATHogw~I$%wMC+Ngo&%q`ZIp&T%Deee5Q#%p%l&6|mQH)e|b~kRDcz-75 zbd5&g+bjps!5?6myMp(YF#WD@%!o2R@Tq#eSpYH=bm5u-ej9ZJ7&2KiWq~<#BR0I7@gy>}_wi3O?odf%H z+wB_4iie3Iu6Nj%;*NJZF9E;5FyhejBtXApfAYh29&ew4E9(hRv*_kgwm(UjE2fHb zl0|{#*k5F72fLG$*Wc@JfZsM+5D>2lX*fPcZJV4%krMj z4$tiG7jE9;hI*PjdBlIE=zbxz&L{U7Ecqo=T6IUt#9(nst&-T8`IJLIP*Cu(Gqm*Hp zsY#Y)+{!}?H&2}7lAM=BLJD8u?JK>LPHM}kqmwGKZq^$nUQ& zc%7VeWcB*T*l7Mj^H92t0T*C>>zB#G`<7ZO?vYu})-MsVW2nzK*uE(SjXgKAuOE+H z>SGS*HaYEi@$(d4?+?kS!kuq{9;a#wx&F!%Po3b#6!vxJ?`%qhyTX)+r+KQtG1HK=Z7=3(;NF%!Bo8q%1I8EnmA3jKN2R&K?WXH~%z&t~Nk#sG118Tg2fdki z(O~OUO}J6dsv37o?`dIX@-_!CX&`utzguNPh)gJ#?NP-#vt+HBWTQ)2->4~Hu&LZ! z8D35cTZ!V%Y?ZC0Whk`gnl~ zsRVR2#z&<+X!@dZH7~9GPp=3aKf*lo$$0Ty$%T!QwPBa`jF2HBlXXBBw_;l5Pqc7Zi(D+>cYuu#^l{}JCRU)*E2&Tm`aEw*yULs;In6>^Cw64_ zB+<3B#NTG2@Fz>A%miJJuxfxn z98uT5(*=3a#XHplRHerj_OQ4 zxD|wagg4_KKP{-}hCpc)RlkHe*qc-fQkXTj<>P;qG?%2d6x<2dNe>x*1Kw=`^+Ayp zhC_5MG^j8B$^~;PcM*?Tu$FG+xjyw_>ba9m3mn)7my0IK&tqkyoV5<|lIW&N$*9)K zBXh+{*6CAS+uEXDi)mpy3Xsz3oP;d?rQFRnX3nEq3N$Y*X6F9y`-6^0ZdY`S1xR!$ZY7dAgdUQ?TFtUx# z9y^;^s62qHG0mzD(^q%3;oXKuIP#o`O+r&__2b^{Q_WX}_CBtv{lT70ma%sOefu}D zDpyiQ!Tn8#O8RcqiWhb}rUw&vZz`V&@^jWs7f6`2#NWB+= zrxn&EYOUQ*xqG`A-B6BrN`BpOLQYT;;6Q=$;}q;Z2H&zEwWaT0Usq?sosjJYTVBF| zAaj47ow+vZ zvRXQ7cG&vj#G4PhiTu$21F<)U8UFG^`g6o@+b6KR|M`U;n1`#cZn1fW!$KpB)fXLt!{4*RZ)3f;TQ zGKE=kLHW1Fa+i09LFRBG-+Pp+Nos!jZ-74f7^P)~7pM0nM9pMvA%Z85GLGq|1xH(c z3AD_9@%i?TVeIHRZQdE4?tr$J^_*lS5{*;OWmIAi=G=EW%l6-NfK{yISDxDkF!vbE zuk#I9+|Bhlj1=z1q52ft6TFbsyhsasL8=uBIUj>y9i}~uiP}#Jl?tO?*)DST4ecVk z4}{OhkEXuM4f4I1m7kav*OeYBFe_rjsvr5!gh@Y{Y4GsW8V4wEy~d&zJUp++ImYrc1Ini>>Bq0yY~bh(sapBbQM$~%c^nMug0JxABe&O zE+W;H_81?$aofLPt(Cvn+I&Ci?FDe<(66JEgHv0s6jzm4oFUaa6p%_U`I8?kLR)X1hDMf~wkgFqXAS+=O(ZfMzJqZd9MjxVID3wi1 zg5ONmVX~Ky-%Y3NtS<38FkypsYI2#^)i`cwu!H1OSTz87$=~vsf$S1eWdSMv5KuUF z3>FSD@;nMq+lNK8{4mD=6zR*J9h28PIU?Op0R|B$R^Ph;8co}>!6}s`()=(M2a4-? zAipyC=9KS!9!7U+KrL|Ro2cmFoyZTj0P8o{$hasQKy=H`&21erH8f1Lo!R1*;sRAj zvT7W*(&z;GD}uikfnE=L4LO^Ggn)ge3vp0(0AX4;hQ(`o7$>>*5-(886JA&@e)NhA z*7^6rmQp!c28rZu&ww8n9rY|0B?fPPX$=etLjNr+{6y*qY_|GIr+Il({bV4H1n}E9 z$aH_Nt1Wi4_}Y=`Fy?U(%fAEQOogm^$|4!J%iT;3@%U@oFR){#ybm7=7fRwEXzvVf z%NjzvyJ|g4HCIiTe;?+VkQ*y&742vHPX3pq94B5}9dpL(o|A#-ez^XMi}g-YEL@!( zIcT6dqX#G=Jrz`TPy`-INvDVY{xJtOf`#s)zA3R)aQ-_6UE{$SU_Qp;JXjmTx+lyW z9tnUcfdBOnH$*$7@#a=`p@R}wjQ-sS)Awa7Ch~PT6iaF%l=SW`6sXgAj_q!HURQ$a zv>z)+S`@5HmC8~qpdrB8C^ixMwA(r2W$kr#Eb2u1oTTJD+Pokm*wirbynJBFI@V8g zFn^iw`NNRKZ;g@juIG+Kf`AgKXXNYBTK|*I%m=kAyxGK9ruz|Hx+wskoLQi+G6U!l ze*l?=avsJ+42}T2Cc2j$<|}Yh$=l3vU8dWy7=Z%q>6Bx^(P8w*)Oi^+}GBlb(GZ5lCv;EiD?7n|{B29^9Am`AE z>ll+DUQDuG*xwmAj?BNW^;3FP;ju4r7d{?lI6a^M9dr8^msvCrECA(AXpE(_xsP?V zx6~0d;0Vec-uhT??=_8j$FsJUKR&8+8m9kIgdIL@oW+0g`D|Q1u<`{_zN6b|b~xjj z!Z&MxxqM!dpo^AIxBHqR@;fQbd@|o{$B2LNc7<0z*?U@H$Fl<;+&O}{SZ0k7V#fsJ zh<={(uk0O?c`~57GxJws3%>87qxR&Ju_szzKM_y!-SeQn3^d-Hw zXSDBwUkTi2vAgE5AM)d3?z-#uuzfbC!rRwRoPl3|Hc0IshII6_BYi4-%!+(K9MCNH+D>f7S3sXf#IT+>E{Qa^Lw;iD4 zQ&s?mAPEOB{{w={SQggk#W_@Dn7+xNDVslJed>Q>0L@-$2trcXr8nTdN z!Gn5erGs)UMMC4htmW_Bxrws33YH*Lgu3CyyDrAhsXpnrQs3Aao_2JVUcdcThPTmR zHG?89bq5ghL*>5`74Pvg9UR(KnD|_%!Ef6ZocbURcZ;SECb>l-FW+C7o zbClGXWmG zfXVe|R2k0;|7%(HRoM8>ncm3xD%jHe=>ktzp_>OkV_ZC2A0Rc<=Flj~_f{kb_XSz! z3a+Y_IJ*L3MFQc-XvjwGPgwF*jwFxH^Vw)g(Nvoa5mB8_;6$(sE96`X@(Y7Pm9(RgB;`x*UZ{+8#8N)qwmLz;hR{xqZ=Q z44STO4O{r2{TbFg1j}VPQy~QTSpl=W8RO!rWMO*{GAeX0VKGwM0Ub>r#eEV~KonnF=#%&ns2?ybDh__Q2<#4Ub;tI}toay{@aWy1 zVq*>B2Qh~Q`@%!8x$m2MD=wB>Ms1)11G3e^LiwNHHmGJ!6}DCN=gZE1lSkK~3I*lXGScsW( zuEglmn}q%Ed2F|i-276rjFJ33P#kXZW+A#rvGf}77#02(_Pk@|wShgef&2;~r?mJ#kWSacmYI13=<1@NN)VmAlL=O1U zidPeweZ}7U=fD$`^B3^1q2!NPNz=A;?Z%)aPnWw6qh7{YXfr=&onN*M6*t_=(9IN> z?}QH`0H$E#oejPL-ZIyMdW1oYHxlwkpW_+fvYmonW4r5oCG!W1y;Z!=(LM1eirvg$ zOZmg$DMNS17Zu`1D!}UN-tY-`p}L0(u*n|K;b+2^z75`dlkvWZ!a$^3+O9d`{CdHg7-(Q z?_V^kzJl$j55@P$;-_V;*-ZN=Adn<9i{#}84CbFs_)ACyH={8wZP;mmxox(uK;Ax| zZw2%ma5eI^$&*~|fs9+BkLNthP8Sk;@}i5erFED5E}-p_Vr*xA z2`5BF`REc<;LcJvj~Oe_t7t6(y(mQqW6r zJyKjl>|g5x61bNuq*9P+KtL9%v1$N=24;by|5$4&#gM;EV5j*XCGdKtd3bAr`~U5o zV`B*%u6{C2`#F}?-Wdn$K1h!bGWOnKVPR|J`n4~w4wiJF?z__=_Oiii56S4!hVK>S z1R7Dnd3~aCchTm$i~9aoZZv#%yx^^6O8bTIt|`BEVdo!jF7EE(nN++2>1n%4*Ie7H zZUSRhfjYHOPx4Msr!2L|RnLTBLm3B)oOlwR$soSDbMmHJ8o8eQPz2%*5Lz zz!yMSE=W^onB~eBrqdRo3Z!rX6^uu2z00himY%U^QbxNi zT{2+%bSPsU+V%dLsh)*}#l@$vtbC1;&`#@_$BgX)C@&q+tG1WYxHqFRe2aJPS|OEA zLi!*r_CrQ(Lz2?9SGVGQB3e1#S?zMlG@PQ=WrWO(H>}!{ZWVkC!%T~)tXG)!6ZoX9 zoDws1uh)KyjTKbzmh1c=Eeq=FAa>V(XQ%`aPXLh`uX5>v#EBrr(vge<=dP5SL}Hqc{A9I{!n*VJQonqczj>*R8-p^6&;Mx0 zi(npPGA;M%J2+&c($Wf7_zY$)S6mnn&TJjt#`wEP*l|#+K#!t%{!a0Ia+8uN323oa#NaoJfJCOV2xQ!*IO&Ki{U7Nu68k0N)= zGb_6zbx|*p_LmKj?oHHb(mA!gk?N-5u8(Re!`l{ip?O?QaQA^?$r4PCgiv<8ddz_oj&iCY`XZZbJ5{_Qn_t-ysh&4g=}PU`^2T z4>@UJ>`xIpDd_zRVN2QaddjrT)^0(dWB7(rKsRJ`Ov7bk zscr{OMiSpLP)+>a&RHRXdmDl64EQxfT@X`pq2x;!Tc9nVCbfZsU=R zMdWq4Cc4Mtd>!NcBs|Ne4u*9i|!6T7i z#x|n9uXiz8Q*B7qhHS=L%s4!@5?P#k{D#$cKpLYmGys!90sG;Z=?{c$6)`3a%lbZF zxFI31lw3Rk$)pFLI;z7P4QAP?4$jFk)3+3cP+%R}n#bYRa|K zUBEcPI|I8z%EKrDn?5wBYZpIyU@6Ys=qGoyxeRh18ER(Te#~_@r)(wa=!G)<2x0bb zD^jEFO#IE->No$rEf(pwt;`kO$^dBS#zGOZQ9emf#aUL3iX)IwGgPvm zoB7N^PsjvD4#v?IKeu994u!E9pyi4-)1$Als~G}0NH;!=6ZMCVy2-w6lFvlkLg6w_ zT@vCsc0=~@x92S+u-Cc29zMckv*-*FGIPc80zyP7`-7LZc1jq~;OA^& ze<@OPoljUWMos%I)<`#}PqNg%Y}yT9m|A5RQ%Bp3gH{ZG2rHZ*wQ$O;S^Q}S1`KSx z1_+4$GjbEfnEiD6?qn~Am_X`qa(G+HRpxtH57(NzazgBv@G9UKjNzqruw}o5YJ?ToQX7h{1lz^#* z(DL)gyb6eWzuy_2`)F!ZtWFz}o7&sb6<5CQPpp-mfVPp1Wqr9W*M8>3@JaT#5+b(oGB?elTtF+rPnj&mJtsuii z$?hnFVa-$9MUYeK1;$as1!y!Ire$-&11D`JfT=B(Y~^`F?`6^wh1o;KzdQ*Za)89p z0KoVoEK7J&ie_i-`!rt06f{IHSl%ld#L@Pu!KVlfzpcybLA& zOzy^p%zPEP>8L}FI&0?@ys6Ftx%V7>1}oB(Y>yaMFKC)${1IV(-iT?FNb&!g8U2?q z$pMLQ*de64d~`z^s=O_?D~Kb`b9^g&56$|n(V|lwLB}qPN;U_^ym?Mc8`83YLYrLZZ77!i2?qQGJ?p4o8o1EHRpX5X`C(Ciay=mJ7 z|4anco-+~HY$ZKP?$iBDe$|pOwWz*V+d|vkXYAp9n1BUOT*!I!d6RatUFaQ^oXJdo zbsKBLQ^Tu2e<4gAeAeG%v{brQ*!X!$J>Zx950S&;#rX?92y$A0@uri#NF9T+`s#xq z`}5jYGEIsKflGa3q3acc2DK7by0oI4`@?L)mh-f5)-}Ekb76#VQlr+SdH}I22k!&y z*&glhH{NFK&uo3D=3$&;$L!Wh)RaT(7emGz;-+%6UOi66drekNP0s9AwRj{d&OL7- zT*uS4azbrF7m~CF)ocGOOj)S!diHgb}o zG}z&)nI_vI zjn~ud$Trp9wi|#l|Alrb3Obf(*r*P^=~(cmTEA05Y$~KOYds z(E9UU4bb{a@n+XB{6V*mXIn;Y$5nhUJ`>3lE8n;(!=q2eWW=UqguL*{!Q^KfmsxD5 zV<>Zl7-Fjkqs&V@P1n^NyEuHrIOZ|!5EI=8SPt_&`ffEIlIQAs0`?8QVwx!_SFi8? zQ!F`!^LmY_pJ<|l1f+V(|CMbiNA)S#$(0cNi(*Qb?u0swxX`eo3p0dyWWskpC#960 z|19|ztdf_#EU7Es73uLSF=r2AwII?x?JZwIu#@Zdh&x|C-}AFQ7CKS9o62$K{0R(e zpxIAeRH_y}ZW!q%IXdL@5SFK`-uggO!H{kXzxlmid8zRwR&GcYcHgR$TMZ^(7j(=` zI8YT%n-?*({mI`>;6oQwsjNQ|^&2AORq69)&4w+9c$G6QoaC9uY;JEZ&VCG-T*dbw zGw*n-Y#);npAOI6D;)|K+&GFHXHZEWvBblJDWJXv!1iu$06?#$*Zt)|(_SNPvmVWu z3GLpMSiN=<_mQysmW#vwC4(`*sYI?1x0CNx8{89M=U|AQLjpMMG20EuAk5T~uHWr{ zh``1Zu456zA({oeU8z@<{5n-N)IZy;grq_^NDwY;fk>Obvx$FL90~`1TXlJOMSF@p z&FUk1TT)IhXOxC#?lIT!U{*P96stSy>fV#90P_6Lvp)oLM&o6*;%{ zOcEHuDqH^265Gc1@PrnqmrpI@m@q0$0Q7BnW#tR}gfcR0^9i|ArFt)|ba6Jh%Vz`e zbN6&D-AAc>1zOFw!&h{$PZw0*?z`iBqwqfFLV}3;0h`UznDpp37|#W_K2Dw4zlk_d zhtDNUpAJcj-}#ylIC-8jz*%6Jv$M-~TOS)V&`L08HyVEZT~I!Ggcf8V5jj|O?b#45 z2=NV-t?rfBqqQfnuCRWz+mzaa@}-0q8OOc*kh@HOV@V1a*k&})BOg*bkv=U!dJ`Xv zE^F5v=7IV|m9lU)WP9Y)=D(5jvg6P12Cs^+F;R(Lh{@z8Db9TI>p zYpUimp+TLT+B@d_`MZbJTKx(J`;r(mAFr@GxpxS(y<{Knq0h@YZ-;klpg}4~V66rq z_d<{s6>1IvzdwmL_d=Nw*TH_Ay@!}rYmn~A*qtia3m4xVQ(Vok_P5B`xAE@zQyTTaNkV;@MZg|JpZ2b1cBgz|@v;Be_?WSYP42pktxj8)Y4W69(uDn5ubT8m z5g>ifB|GL^PrVWJuK8*XJ?M7Mebm*k*GQ_2@!yYm?D~is?{1ug^nP!}<)0m@rDPc68dk=n_`|dX#P$Gtt?LOc2496iv@TlVL*10QTxYkpx5qT^rc2zDy%Tp*npTR* z@f9gtE)O>#4@L4<@}S-Z|Kh@L{dr|i%L(0|jWgB9(cd|B&MP^dNr>mOf7+%R$b98I z?^B&N^G6*_BQDG-KylSIlFell8U3@Kc(lQwE6-xt&obg!d6nSqK-+glI>UvGu-XIn zBh#OvRB10wEe`m=z!d)ZGk)!22`oKm0#2SGHMr| z3i#WaK2NiYBeo&aPnfWBE9j8XJB#=(>yD%9n^8HVh3CI$@5Tpg{k)ns+PD$)8EG2e zwqw5}@s}wxIzd}eeXXn&jK`*yi8Q#0Ob7I;0R#Ax5R%fLtCMDB+PtFhf_4}cr)ekW z($^COiu>Qgsjp`)g2OtI-`3?Xp+`@`+rKjzB<4BD9rN2+{N4Debr~f-&G_a_DK{v=jnYGKz!?tI&5-EC^O8L zchc5XV=sC6-ZVOoUL;QIY&ao4?=QS1^Y6@Lbgk-aqf{{~Ths@ zpF8Sy;ZTRyN#OM!6_To8;X!BU@4t0CkxmN|&-_0kC~ktRL4kohjoK_@KmNX~+tbc% z*v4JO^HDft%1?g(EZ5V|kK4e4+b?#@Codcv1{Y=!OI^%+t4ep+#VlHXBEku=86)z( zSkX)Z+tHZ`5D&W{T%|cpLc9|lGbDk>+70ldWc>8c_U~_&@Oy^S;|(3uD??ebh@>Ks zwBZEc{FI1n*a}Axi>YWDrxym|I7B;Ym%dHYyS9Y(fan}Y!2wDfu zkjkmS9@XYogUh_@*Q;{yhU$C6&fl6;t{O076D(34uS@jQ;Q{V|52Y&b=CZ-WO9hl0 zLXoJwFq|dA@HaCqFb?Weqjjjjs>IkYBhJn!y06%D=s+p`IZ=8%b#5Xa89t3WBQ50C zIPU7pM@L_wZyFmM8$}I8jye0ADeT9De5NJ@HrK5v)eHh^jWjsO|Yz zwgrw2U)re=C`ap_t^F|&8^on&ffyMg>_oJMm%x`6$-Rbq<3NZXIik%jik|sf=YpHm zw7mC}E7C$E_z)qTzN|e)h+9|s77@niAwqiu9wYF(TI8+Wh+kE$33s~GkFLAs+clJj zPNz|Bi^2}vE+pLka{R3Lx9is3u7gIt3kny5voi@cwdWgw(D@dye_6)tA!cB66wj#S z`pxrJnM%SVJ@YK2^)r-*X^TjP%f(-?oosD622lZ@su(ZXP+ChA5PZAhw_Z6q(Vv~)gDz4oR<6iA`H z;a!hRpL~cGu{fTeCF`}587-z}^W5V4)K(gQ{ocdQkY0;hjrf!5?6v|I>(f6mfED9v zUOLflpMX!K5vvTwM(wY$sVP(#+C2HRn_Kl2#~A!0Y`$DXFtkX;uIme)D416GQ3a&K zQ`&Stj3a!W{Qhw=fYLR1NC1BU&i(28B&zM+Xcr22?IkXWIXNgQ04}uIneatLDv&(G z>OyWMLNVf*ce(`k#=pK^lPqPJh(rqrwQ@_JRNL;$(NvA_o7EO#M*y8XoDH$~ymV9m zd>qO==uN9`Sdh*)IpqEH8YLc5ei|YkS8HlMM-wD{{n?VA?;^+2Yep)Nau1_ee{5`c zQ`1{cvGbMQy77r|hFkGhT4m$kRE>KIv73~>)T3^9$@aeH8g2QiZiDwj&#-5E zKYCq5%n4KX#QbITAt8XzY+2QR(ML!)WcV3U;BG5dRrbPrkBODN&sR&|4&fw%sCj0- z=Idb|P!GmNqlu{#p3*BqYt8)Ls4#(}sJRJaY@tqx>b?0HPa8@Y@6H-3&b&TRmkbD3 zLwh0e+~%(sZelRNm?aJKUI4j*)kwqlE9fXc$UW@@qA!H;^*G}OALOs`EM-u6wY&xhpq;W z1|ug!2=IUfT04>!XPckq4DgNs`ZreGCz`ZAy&nKvyBbHM) z>KAiBQWo3_(}AY_)GrRLu~QoRRC%yJXD`mC_KPcnK@gHQZLdZas~AG*f&JSQ>zahw z?OD|vjnX9_idw=5Y*0({@m!Udz7%i@TFT0R=jr(RF!`ca@@}KuJbuP2jj_R$_O72^ ztvDgPN*V7D@Sczu>&!ddWZ_30DA@U9&xiV`wfYNhu*%o8XN5CqXP;>OTZB;v2WhBK zcB^q!+#pwlZI^s&1BLR-;(}7>4Q*Su5&pUSy;52%79U3 z9GTb+3%9Z5%trpV@4whU>e9)Tk2loXs6pP$&Bak%}qe&vk zVvpfYpHgOP(!JjFBORfvj|BYq#DkNrr-4kz;|+l3U$wYK?g?ocG~aE5eJy-x0P`Wi zN#U2o@L{NCYQwVyWr<;7PB$rvUJK>>wJ$HmR@(eq!_8*M2 zde$8DAS5JY14sZk_OEWMBjyDOKj;7{e%~fRk|u8W(Q~gHx_(G=kAb_aibB}4mLmGX zPgird>hQ?7^+3Vs4g)M$?wwKPISa9rVBrCCkTP1m+n=U05SkHe=Yaa9Aqf&I{-772 zN9)AXbTL6gH-GiM0{(%5Mtv^pe^9sdWw4v4h-I9R|Gj?lAtTWB^J=Ci_6P)xlgu>J zl6UUnS4lxBPXN{s=`GbL6u*Ws_8!JRQWCJ4OAqL|DMI^o)C=j30bAh_jkW&|ZSNh_ z)b|ArK0t~fMFDBjL=izL8bo>%P>>>3N+_aKLk%D$1f*CHP!UuRg7n^N=%At^y-AIT z)KH}a5|Zpo{Cxk}ot@eJh{cHABeaeDGmGFM4ixfeWYIN9bF;uin*$kcD<&(gIZ zmGZJG`i-&|+&Nwy?kr6)_gUr@U5O9GS~bL&BkKGXzJF?EsBMFA=uYo?O`1l9x+%5f zjR4{cW1^Q0h3TPz*133KBz({{dOzOb7Ehm)d((_cje~rmACWchqsJ|WMi__1KI@_q zMQ!f})Ao~g2W&po!KNPi8_odif3`s5kE3%NaUYATr)VG9Ba%BMsDJ|`9+a*FyDK*e%f^R1H+Gnn) z&+ug+43g+TT(t^x8#@H!vWZ@olz`p;)%PPY1mVXqvfp{VxIUZ;wNdPmqY_ z&z5z8{lc?bpb?~U?iXn2YtfR=lI*wJ1CuO76%9~y^Qj$wpBaZ`xy{`8=$I`z$MwP< zN!$x`ka%aP&5ht6Jg~*!Q&e^BwwBM)BUhJ9h&hF)uE0i}iVLrC{3T2XF!3C)TFP3i zME_M77msVkL2E#WK`E)H63v}UGlU_r4b$OZ5Om1%^ValM-^Uv2;RD%$# zU>{Vj4NQXy%Q~$UP+40!mHpC0bfL0a$TLm0TmGBHK!JGaZ!0nJL6vE2?g+M4konZK z$axP^apcYKgKDOnb{3cP_4M=xF&3|Kg1#AU1W{JjHESLfaJ z6P%1vn>NrrM-<)n%s9BN)j%D3twz&5#!A=6wkyG--#<_Ds;A=ZUr)&MS@@4&K=W$d zb3?HJ$s98s`^f>BhqNUkMhN=nM%?lQ#IH!khi>^K2JUm8Esm12gWVDG-?u<5wnx_m z3%(X`HM-nbmfBZ7AXN`bZnZDpQ$J<5^T)5|a>C-~CMPmu4m0$3$>AH}qD@_=_FnbD zRu}u!s<8z#TK0M|HyH8@EGLS$1C!j`5noTlSa`S;bO^3$YQE~sw)x?(V>*h$e;Uqy zy)qQ*M;A=g=v2H|`y#UM>iSWSZ}RKp4NvfUaD({xP6VX~!AGiJAe-8va}li;6%DWJ7?XVe&4$# z2f4-Ep zyk_?cSHHZyCKVk6uaRMvt`_mogRMiK`u(59f1Q4nm0!`wI|-ak;@Z8qOThL-hf~l+Jd!>!y?x$!w!*np(nqg| z_Lb(bh_76a!8{t^rZ3l=@8HvZ*nz^4#g{c*o5wHaUh29m+ot6^WO-8|+S?_VMjYjA z?fa~bv1PlAo^9`@JfjmPm zK@L4MW};|IpM#a@<^EKa;oq@R3!?!fOb&2yO;)L0YLGs2=W(a5A5CmRhD>0xEDc;H znV?0=;Bu2c7U6bL`Y1G#-3|bl?GA!g{F%Vgr?^w1A~qZ}A~v*h!T>-VmHO|!9@A4xZQyC19!R?ea*vIC zMoR+#exIbxN~Y*sRCOuuUv)75_x|G1uqsw+x;pFINg%7x{apaV5vZQ&cYK0m6BPk~ zZFoO^w}-*a1Ej$Q>_zr!HURj`&rMQ{jsSp9!!6jW(`{MLz;7=WzQEs7QvsXr20vKh zprr*kHf3Z6g+)?%Xhc$9cdFw}K(3Wrcd9?(03+W>_RoZjA1Y{=ods?T-xMOiv?E4K=1|%cE+Ua1bp%qurl&sdY+DaA0kh8R{#PWo zM|WUV1PtB>B^I(PdaDqOY%b;eM=V!ptfU5n4@Ahim#ncT3}nvK+Wez*K#%UR3H?(S zEs$@T7$j)_9tTrgQ7c2aHgxN7iUK#x9;K|R0~%?bdt3dY$BqFa#ugDCD+~}i5Mhk# z#drpwrm`gZAJ>BWeahp+!LblK%yaex2~22ZWzF)B;V*(o4;bh){Y8KqU3X5gQc75< zB^ph`>l+x5?qlGe4}?q))MS_Z>xIl2)oxXpRDec=f5V#+T{d~U>0eEm>8d^dmFT0Q zUXWP81#SH_wY?F6d!`BDz~CdW=L&cZ#E$_3x|8z%gk=3WSl|X0NH732KuugA`#%zt zLtceCKzPA=R?oN6gC#VbZTG;AjtJa2i)(cQXA=k)jR+uuWg>$D1=4XGd<&txwHN>* zE{PpU#e;o2AkBMpDqa_C{cmj!;H5=0a)lxez=$x2`1!{H8o>I-ffb(?xa}JQ{ggmR z&`;k)I~qwx;W%KTek=k20C}wgD?tzjh`>|*6bnFFCPhHT1OJ?sA|y|eMo){1fb#>4 z>N98_xmB)485)2GxHbZSV+0d@Ic z0DxQM$n%7j23TbzH3CUOGZMp@7Rs#7Ns>3RV=BN=Oxn4Wv~0jjx6f6%``a0n^FN|I z$2myy@3b`TFg5fS7*+q)Gw6*@G=zAc8Kg|o9Er>njK>6oR_jKQA(xsSL@N64>qH|B z+hZb_$$v;qR{S8?IckkJx3IOI&%?=oCtSHx>9T#xn%(MHUDFwb?>bW|eIecs{yQE% zEO4oCbu(fxyO)MBQLc$Cm$u^72o8m(IdjkNjz-fMLlwI&8RWPKxH^FRp`{P!%+KJ6 zK{}sEjq1F+*In?W;M9QpXr8bJNEs*~0EkpvDCn`K|FDKg2>2wzhXBQl`uo2po1?t^ zf5{{FFYMv}bS6DzkX<%Ep9^1>Z?kWzTf@e(33%*-yx!*pN3QwSorKTDee z=-GugskaX7>FnI(RJ%5K=B3ZW%(_kl>c-NfwDuG3zA5MvRxB=ci?{fblQ6&MEN6PR z?~a`iqVDJ{Ugc3%e!fqu$Dz$z+>PPS>E!ybrO7P+&`h;}v5Cs5NlBZP=KkMH(g5H~ zk;u!vu^#F+mV*%VL>m9huHnJYoUEk)=K>A6Qh_W1J9=H$zSe?2tT~T$W=>q7rxVCp z@LGoMAGoF45En7HwYpLrg1M+|_4GrjoUQS7syHx2iB99(mp{CFG<|%rVI`lt-_v$s z$8Yxfy&G+S&q`K|%V1DhC6;%{pIsNF7_xjcez?kblU?m4*|rwapTY*%E;oDU1<(PR z85Cz~H)ZDEWoWlRcy?R6xq$+K9MWkn@auEq0@}0pZNH*afW=F%+wyHT<|jY;a*3l9 zoI~b12-d3f3e1ae&{F1Ko4Kg{XgM@wiL>NAmN_1j!Zi zc%BOm7USbT9Gs?xY)`+NH%fQ?}9Blwxi-$|aeG9h+m!840gwxAIj7QHL z@Tf!Gn*_J|tt<}Fc^%KD<_yJdp=1Ilw5#vXOuy*dX-!E^`G5l=Tqv{H*Ap|43o*2Q z->sM4VcP;5t?LTP$@vRCDdI-{i)%9H`wSHRG863<62q-fPDu&M9V8(h@vynSQh1!4 z!dVPsCZ2O=DL?I|g@oj#8wj^HhmhVqhIp$o@@LkTWD9BdxFVQER~PhbZ&r_Au(zVc zoBkT|ESDnrW#a5C+6#6NI=d4eUphDqjm!r4E+Ici$)H8>cFQb)0ircC{Q4$nZU-&x zKtx`az)^R08nzoda^Li0q{DXQ++3vmjG1g^D>+O+D5PMknt^VsFtyyg-HGmrC!Utk zLdfts`Y(qz!H_#9k*(s^J$VAg&=$7y!(%bqy=HxMEee9QET8Ax=9+w=jxN8~XE{H5 zpt>bh4=+nu<&nlDE}sxsRZ#ls-+e#3FH}KrN%gw~1>u;03kN8c4umnTt08s^o`>qw-~oCu?6i)EB}0snW5OH26C zr(1PZQVIL7@J{eD+mI%N*B$to1I^ooFs3CQBU`VK(sI1Ff9qDQ)DJa6z<#KGvLk<^ z^%P%OT7ogF{+!l=kIjV-!clDIUb2jze+DcMn3u;1Xg;b~wi^u=&Zy2!2%ZvtU`$d` zT_~yUwHFkO>1T70Uut2vtIzM=!q*=fr1nL%{IKW>#7+TGw_=d$w?y2%rH8WxcUbXS zpftG^R9u7|)IX@dRMC;?RWlv$BL6XcaSl^qDIj@({k>Rwh4+=Y!bkVql`|6i! zdkbMS&7L-}e$`@n{S>_z=m&plkpS^mSep|$PYT121-(JUt1mwm_;X`a30$T#S0F6~ ziZyu#0%K^7b|iU5g}Eo~6@~$cjHP*O%hMTvg(|Z}iA&K1gXE+e{4G%8*j8wY!ueePjq*M|- zbh@3ZGjLw`nhvX?9W@sLcd9#OAmk!XVrsS!Fyzr_x z6406X##=ca<5>nC`g zdS!ii8$>kI>-%YXi{M*!zbwB0(Pq2ie(~kgrjBB(wg*m+iqZ~ztkH5Ayff`+gsI1$ zk&5@};bQ>f@C9LL!$gZZx-dwyop=XWEoiGSBdvePI&{Yy>-57O9en%*oDXYB;Th`1 z)wvx?--H{Ut}P8`a?EE3bB!bl8*Vc4ySruhPUUN8X~wywbKRCWHip?hF{V8yOruc+ ziA>M-FMW-Zxh!Z_#AWuVMumxXKF2-J#*>|a7?-EG$?YMYj+;}QXAn~gKVoF*Z``r( zZg7Zs51+-Vj0L}1Y_?(&z%`cdkOr5lyXc=C%6RwHP2sb3Sn#FZ4ean)?;R3S_T`Q} z_H;dXvdttj$k>Jbt`w^7ujo72$?zAVq&cSN1qL|k6q6NbH%Cqp|E#!_S!$-HzArTO z6fx}l?bgC|^Y(4tH#d6UVKo&#C3t!G=Xpbn$a9!2@%Rr#0y8;VL*CNG0$^y=fcMXw z`9y_VHRZ!hdgl9v@9x^=u1J19;;0%aC}oyBnqML@7bV?@%K3CW?Nl)>_ufaGaTHBq zmd8CX@)mxnK+(HVZ(4)fCZY4fWxbBq4BrR*_CnGxlob~DTlBCos7FWz%x+#SDt06O z?jzt{2&RS2ZjJ{%RBwW%5fw@XNRu{EOl|;e9)5zq{#2^gxk^Pd(5LrN+ zo{TAg!+&ibADwd=J;@oQGE8* zvu+f}%w4vJ$L4{wqMRD131{ ztcBLBJ0^V2@kIDt4t-l+vm%Gq2DYOv5j2{!q5mya3JG^hSP%`mhrX@>nKT#yG&kkTKd17bivm#r5~J+ds_K{#r)v*;y`Nx^4*a*s#{5493&0$! z=V>(n4YQwhwfNPG0Yk#q0)`C_;pO*Wci_WizoZTKP#9!V`E<>u%A?yK_0MA-dKH%H z7M^cIbp+n1h1T7!;0`2qiSW=C;SGqP*SaEDm?P{0-Z>oR2bAay-*>1!gDXAtcwOm3 z1-Kkh&Qk*%%vd<*%Hu}HP=ayXyi%6%fM3>w0>w!UrD$O{PA3cDnO}R>4QnYe`;8EV z-qnL}@Pt%%3tlKG-uB(ZkkdducW3l$=73waTB+Vp>*UL-325Qgxe}G3s@*McbfMr! zKZ6hGs!x<++AkFd#vr7;94sNf9;Zf)7dyxcx(cA1)*T`4YZDQqnUS}DH2Mgpqap6B zdunZV^j<@*Hj|5WMOSXy)NfU5a|bsupi~o6jzfvT2i$pJt^zSUW#+x{v+aY|?tJIj znfo5*-#rT6mm!l1WPMk%==1DrU`1|zYfmd%vi(xBNG;P6X;wO-3x7}1i8YTlxmylj zTQ?ZLFBf)zERLzztjVoEm$)mC@aDs(&8d=U%>Mch@6f?*vwL`-_BSC`?jNt&ky8^` zHnBQ8q`!);+`6dCOFZ@t@ZY+@W%-DGEH=ugqMtyAPse>*+W-D03Vq2TC2TepB{(T0 z$Vm!0>clcaa&~6PKM+=QeFQ($A()<0oYLK02ZEomx+@OG?>xLaM<<$v&>bWG0+MUZ zsz%qpMyJw_?=0RXCLbp2Pcpy^rU5P+b_)snfI<40B?zogz_RUDWu7rJ?+nI+??bf0 z9QsZoi#d1B8|M8mv~?njy`9ABAHwsVO)K)jE&ivP{l(?i^oE5}syd7LwWGcln2KI} zy+SO5G$XyH#(qZ1{TABzFj%=%^dg`c?7bS+U-S4N_!9Ptx9V$#p+b4`&j9SnZp4a7 z&yM}nQX3DylR}uxv{J6>=HA4~pjjDcO$Fv@SJ~tM*-vNFdRpzP0$YeaW?p&8ygV}pC*;$NVZ$2RSV5XbT|~h} zlhMD^bu(MF5**eI6+TI&nx`aL?{YU69PT(dcTZx35z8O5@-TIZ^$IpM#a@2pgTC1) zd8TOlV&+00)uZQCAJN}66hTZ=~}$=L24{PNuqF;;~5@&}WG9Or)F z=B1+9tsfQ{6HB4^#De0sbVY)Ji|q0v9Jmqufo&cS@tEn{v8M|ex%W$K@Gu+KIT+k_ zZ=y;|-b=MFw56XETrO)ewPc^Yqfe*WxI^k-96QZ=bPlf~cw#Q|+@!j~u239hi=|7^ zBS1z*mKSXmU_~#V7uWeB4$c!>P8Fq^ncYzko;kNFUS*#wioQtt@>plFc{q=!v?<(Z z^7M?zs*8NDL@Ekj@Bgqisrb#HiucU|rlF$|q)n-tDmOy`{tNr{t}V;tJM3jp+raQL zb@g7cG8-eL6}SUrt)aDxDUrT0x*K;Y)yD&Yxym$(JjGDMd%b08@~zGsQD+6eU2>oA zt$k=ZYj)>qttt1{jWEF47_D=4lZsE-LyE=?J5jh(2hTM*+J8E8J55c4UQPZUs z`?Nsw-yXiRbU-Q}B?yi+Ukwl6DHW)_)h*~PrMO!j-sbgfeSfo-mw3gY&}^FPu0quw z=u9VIWfAVyxFCt=f=kMl{BhZPG2tMo=&gWh*ebCp`ZS2g8~(-1;PB=X7&1_3laQX~t`HBXr}h6eJYw9^tGyh%)z5hpn`C#t(CxA82{Y zTfdIXD6f6J-#m^Voo^m@P3-lEI@y)Z}I|=8fW+U+0YuGSG{@UTTwUy~TQ(1>6*4K+)x&-k` z1ht}NL?F!TYU$~Q0k?f;mvE5+!FQL4p=yw_td&7~ua$MHI!ow1AoLU2O4_C;wIUR4iQ-#6#`HR-8@I(xHvi2V7oWwjdp-+&dNl9w&Xe zxZh6oVGzE88*e>RDGq*s1m!&J#ys~m(RZYL$UJq8@=`;GqV_N}NDgme6Z9Yu%uq&< zZHk^L;=3FZ6aRBf4GPqW(+o74Vw4x||BsX(|9@;IRE-)1;r~4ezpO-SSfBOcvtCsO zjoCe`AM*WPGn6Z9R^usZ!4uCBIDrR!$}lBS5st>cSrdzmrM5rHx?=f2Izy;PqwcL1 zUi|?o`OY1?F=ck3@afMFzUsF7APNRL8W8~Cg+xcV1w|u49pgz#i`z5L9=p+W_K5Dk zlO4byC@M0^N>SXqoukzNU4uEm4|u2_3{pPO=;EL~3u;+GmR+T4KUsB{o|aG%^ip#{ zK+cGYRK;$wQ#2`51YArsuehlQCMd~JBJwmrI{?K70HjhAYPv~+HGryEkR)r$wmKqD zn_?)S0#LPtZu%g=e;ED&fK|FsG1$=1h=7!OfB=?M3>N>Y08I@pR17xMM7VCpL4ea# zfKe(y3P^*l7^T&vPbr+O;nLE8FjHGiksBa$frIv5#Ys><9$`sU0^5K#4F(n}KmZls z0cf_ErLy7lKa9FePeVj07_BKsNx()(RKz2m!iM`)622bO(ioxE@Rgrd15{M1lk_wqAXXJEjc6);%G5guV38agY*hLcjzxeCgmKV5rhY9(s{!`1B@@jQ54G5e zTD;CdTPIAZeV~up>$Yc)Jrt%gaE96ibvjJ%5R_+t3f_B?)`j}Hh7gSi*dd|O5ffDY z4^S1}?Hsf&$0^t~#4ymWM=B9y$rbp3#efPs<^5^w~VXc*b44G5qDFg)d` zAW;E|>a%J1mto~{U`N#K^u2Qmj4LZ}>3`?d3Z_w^B<{())~AfDJ{%VXUH2=YMI;Y$b;s1bts>`&3ICkvg9aOr&?r-W%&@XirfpgfP&EfVY^4cy@EVqzVUB6%FubrvLZBfA;{r z3}m7?PU&6#f5wu?s{bKiq}2c`Wazoabdxd?22`k>m8y~QAA&`S@(LI@SH-uE2X^u0m%FV7}5V909ByeOiTZVLH!@V*pL4JJgJRx^XvT&Lk$}h z;Pk-l{{c{kqua^e|1i}23u{!ZWBvT?Eu~Q{;}nb z$6o=4aO{S)dr@wMu~Sy|gORFWIRNq;f0Vr<6A@UG4GM~efhm2QkVjFm8bPQ*0rRll zhMBCy&JSZ#3;VF5#Slf7%{& zEP_%cNbmpAOP%VnX$JNwQZbE7%3X{M-MA(oymGA{Hf3`#HS4_zfifms(>oLTc*6O{ z6L%LcWzz1JcFapDt8pWGLcMn{FCy&C*b&9n zJuateE7X`9YRdY2QHoRHNhNz+G3n{E3Kd?2$(YvCpv^s@<5L@(s1}Uccd*-YaAxV=od5D zU%)2S4B7DQ4O#~myl2aYf1v+{MBQH#22Icxq27(Bd2@>Z;;K5-TWtm5%L4G8c|m)k zh7Um}U?2TC2B}7bS?=;f|4gj5^?!g~IWqg4o|5hTxGNz;3TmYaY3mp%Sg_Q><{@m? z)ye`VM%S_}fA)i`;`_4Wf{4a2Od91Tv@})BcnQEe z3OnxB^mpR!lvfiD{tPpZCVFAiHereT6N+RwylrowuF-ow@x^}Se!xwhJ^n)(uZg+P z`RFNyyWXzlU{IPQCYU)9HW*(*aLX z_=2D-W{&x4SzTCDLoDOl+;dQZ&kSuQH@L*X7VXQ`)u9>|esiX)wP+24)#-`&&Zdpo zy9U*r4M&=t$vo-YvHdbdVO*lg!AZ*i{-Wgpl1&DdWVGk{SRESM?N5mN^4Ks} zKoWffxz9NHCFL&9S8dLCbzN-Z z+b)qRm$DN{!MaVNjKI56IzzV@zwE&ryfgE@qdT(J$3y4wc3iMc)I-h`Slz_)Hsl3? zVcW>J-X6tqcjAXiSp3$7O9_`-dK1?P{tE$x<2r;{@TQUNQb{A^OK#BPjn;wS`7M52 zJmFO70_xQId|%X;Pew9D_eZXPYl|mSF|hZbl!V%J(9S6&v6M@3*A-$&nun8N>Js4~@wk<~n8n8Z(oH%3ub}>yB2W^57emw$ynb0t0N`l$K zrX$WWj~umb!<`CDWD{EX_GPN3{hcsK=@B#k;fm)Eam$UWA;P&O?5R_dcfq@3l#$Tu z;d5D8-l5oA*v$3un_tj(XQn*I{J?$beKr3T?n*<;PQ+r-rCRo*iayo1F65i!xMi8W zG*rrZn>w>dQW3Dj9p{V(FOy*_Xv3$jO?h>QGeYQM`u zJ5O_>rCH0Bcbm%cUmwgYe1g7xp0QKT%D5AE58#ymr}Fsd4nxzMX3ydiRm=~pd2Zbz z7E0XRi9$QvE4^!I;UriBGm!dmhCUk^=Al0C@IZ=8=^ZI;+SgJ3M3} zqLpE@G-cl5##{X2EG9^)Z0z%~X3(>yV^CpV`6;9!QWHfF- z3P@!6b+l#F8<>=5)fO4w&ECrZ8$Iit3#uw=}xejmu305yniVr91qS?;38 zOXWvRtIfOHIGzu4L!Zvz=7;E9)-p_z>v}4j^2UzxIu`KSYiFWe1Q(RD69iQ2GdlBz zbN3@XtJcOYdkJ;{N-N&SA_}5uedznJ$i}hRhv<2|?l(nOcN=q_+oXbtzvb~NY5re! zfA(Lq_>zn#dx*j5#b}k?i^(8sDCzJ%k>x}g!OZ6};vECc+2c20)L+r*Ygzw4A z1op-o1h2~BWf%sID2w`Vz%REQjm`RE=1us{hy5>iMa>IE$|j_5NNtZS_MRtaxHvvv zs`t9Om-(Py(LclhYIo}Q$Jr6Rq|~?P(|O9CYqQ5Jp=m7W_0Nl;=rqJu{2$JX%@8w& zmr_ZvFIee-G`|gXLFm!+(s2B1+c?iBt^#o4;P-LRx0R7b#Kf6-WP`AMrFs?{b+8pW~zJAn^|t;oRZzRF>*Ibo^`ep_jLmm zEvZB7b=Y^@d;fGl+u2aetSRlDVE$@U6}DY&f9GpD&;FOgxsBx=uiW5Y8BObo1)9j+ z_xX`NL%Wg3fhZjmtO!y~hoDCp2M$!vb`1AD>#W_1V>c}BSD=T*lO7JESFavM&WbmB zhlg6NT5_e1c^hW-ADWK$l4?gipRm=ETNl=|@nr@*ke+ygpTbd=?YJG3F~tE%s{xco zr~#4ul@>Oxw-qR7JhsQb96qej>h>d~(C(x4S?)#OR&|_E7#>us&c2?bVb(afQG9Ui zk1`IGC=IVGS>#N3+@AyJPpv8Ns2yy5Vfg50;&W#uk!j7095T6ookWjfIXQoT7&8Cf z7BQ+*D@dc*&_1q|cbqJFoM}|uXxpvrTlcL7HBg&Ri z7pi;1H^MjD*0M(&1$>WWI1Hdxt8vz3o2k8}CH9Ow#OoTFy|C%pbQ0#b!v*!m;A$fB z){u4&3nPp!f|9v?B%+fC3SZIcvT%d$w@mfMtJ}H?^UPEfV0!cO+}~7OjMcR>d&F07 z#Ejpq_c6a_C4WFl4pe2oQW=MRL+Q0!|f}INjN^!+}-N46}X5LzdSu6aE1_p|h;X zmCXO`6yW)6nOtznGVx_XPh6Heq;&D4%>9kxv+v5^^fd{zs!UmKx)F^?zclwT6BmqY z`^T|)(4D|U{XSwxUjOO(jM{0QBD?bLut=M2^(i=OA@)X0#C}irsZ#b4Tp>p1Y4Hbv zwf;Jr9Y&VH)`5TrH>U)t3ADSsgnrYGZlNLYgF6Vip%s>}y|y!YsKP_+m$4ZPFv(&V z5h=NHR*Q!Ry&QsscVv32cDVp}OA~hJn`M_jWwtz%?$R$Fcj!V!m#Zw~TzLXK)>;+! z+YU!!ta$BKHYpp~!DhI2DeM$f!rgwg!Maz1%Vcq&dK}# zmiND@5SFPg>(}XTlH3m2H7ht&3jZ8fJ-GCoaB0H%)6>*|w;%f@aEBH>hqSdE;dLw1 z?N}HNebD^2oqoXW(d8v1HZSf?+JGQ=MG>@eGdsQ&J*K z{3hdMgr|Ha8Cj$g(-)kb;N>3pe|RzNlG zV)MA&qn+33f@X61zvOt`gtMMzAfU&_)V9j8)AB{erFX_Yn=^Dn32yj?;HPTk6paHd zxKJ~r-F$c)lBY{n7Kai3cfUzA*XFT^$7mlGmDN78xGxPDFmJ#==ET*SX8U;vnW-s? z1?UY#-P%Mw?iEW^>irh&`)?csBz)9C+Em-y;>A)ga1xT^ha^C;_|4&%;JH>PJ*rP&Nj?w3(YiRiPVZLoLhM zM4R?9rlsv3;m!-`u;Y>@GG3nyEbitXUm2;-c;h)Mji=RBm?1Hx)n=vXw|V#Z8NMaT znAAwuG7ry?Ue*vQPI5hN`2O3EP;`p)w6OMrjENMUT*yKLlSe7&H=^&{iDPbi)@;Ld zh7Y0n3)K_u5D~89XlWh_Z5$MkfNeTPR70j~$5G9LOVvKGxG(=xQEG5sQ+r*W!~azQ zSv2__cX3E1y{f5|x%h#e?$wFRxXtsJmO8hb22;Y<@T(5pab5~l?%55`wH;-bcLrw$ z#>5&{S6RygoUO4QYkQhk5B)ll>Wr7_yx_xOvCFX-`fYrbCZfzz3kBPVZ5T?sTmNvq zNnh`%%%yQJZ@r3mWj{R9Y(ce4& zRxlyn!Kl5JWhe|=)>5!bH?T?>)(zgYj6<8vYe=BB50?FpbOoNTwCf3eKn;kQa?apcB2?DV>s{Bzy1OFVBRmq+$v zqi_XLVTii(F@9bXMfU9VhQx&A)N8$d^9|o6^q_p_h-Ay;a3edEV{Sn|WPP{-HI{?$ z(1Y)z;SI~NI)NBBh*iC}%b>8N{#KNabW~48DvY#jId4Iy=;f1PO82y&yd!7u>CDYEX3oycs%b~@R7;v7 zboEn~S+11AXa0o?;(L1Y-kUqNNsML}L!gtIR@asop||jBGPTCUg%t$fvWT&p@v4x> zMuUI%dkoI~kx>z|L$X`7aYm{z{4(=`g;SW%uFkW?LQ9sA>$&L5U?2bwWgg+z!4=M` z?MSl5z~@C@@Q+Qy2U~FA+05BngX4m-kL(Iev8@n<^vmJ^`Tf}r*Cqi?(|)Y~Crd&+ z?v+!g40Wc1v^`&r4c70sL}eeZ_u=TvW+}&R7Lz{M);Ldyf_(o z{OAFAGofJVZoxDA;g#AL{S|?1S;MKhCm)JR*t*mad6O#&-qJ_vH7B_`4?5D7b~jfA zGtZw1R&_^#79h!m1ithqYU7{GEefy5UMLd1$-!i2G>zw!4Vs}XQ$04D<6Hlx$X z1&rdWS%CpGcqtvjZKBXKFAm9;cAM!?vquAS&a7ZKMBpZ?y*b~!WT}9T=&txz_X8y3LRTmicoz-p-Xv6i?+P9xjj8Hx9| ziVCmX?us$6LB+ENO?}U}(Be`qARJzRF>ICm4Y?jWro49HlKV?`TjCCJ%J5aa-9wmygsQi zpsdmLMNjBCqIXXCy<3ES!_H^nW?kvk_04{$?dn}a{tsX2_sXRB#w|3atH8hxa7gH{ zO09Gz<)=1<5TF-%Xji}VSp(`&c3_qD)B0Vz?C(pA~qi14ki)m$qWurG*zs3_E?zEk*{cc`-FW@O3LVdX* zRg-gP`BLZ3Dd-ZcqP&OE@YKjG3I;@!MM<}7ME-bpbG9RQA{P^eo?VV>%cp75gRv~I##1k zhkOsn%FWbocc6#As%4y38V;&l(QI0`pE?cwQrbCHiAXyfF9iqg?#LY|ewr}Y)=xeh zwVsi7^OGtK{0_Nhf9>juhcd(Gnns2Y5?^Z6mun}qa)qDkV2n2i2I|-vfo9Urk2I)o zuLhKZrbSVpsZg%E%G9d5au}%14nG#5@oG8YT|!%qb5--fCoTVqY-i;{8tvGxnoo@1 zB%)Nrf`K>K>lVPk%(7;MWyf-4sj5U@i~y}rs6N(ra!s>Z+G_2Y0DK1T{Ic|vH+Baq z#if8p6C@Q%syOfDq|I5;MMs9R-%>VmEQ=*&cnY(hs{dui%dFb7JN)tzv9}2 z;f@PWA$fYR_QrvoEV(FaG%Lx`jP6#wK~ciWgC=0LN}V`)X}H(> z(LEn>y?u!2gAboFr5g?hV52bKuG_?nq3>#lu&^|^&k)++-H0vs9Z+7$l^{>p zfk5z1)^lhkFoFnNQjK((e){ez*1^Y`O;0e=)C7`b%8dHz%Aj&Ar9X$GJXlNDG=oA1 zioA(ojj_c4JQ{tDx6XZwSvNKfLi-^XfZ~>x2|sSff*s8qxq!Uq3JA(i1D* z=2vtYb$h4bjT$)`DN$#QY5WQ(#<67yLP~4fSy-^20kMpq(MC}-nr^DB2+bA{n3AOa$AH}w{#b(8;iM5uD7+OWe$=iU zVI1U`{T6$ldu7bIc&{eZu~OZvs>K^oLRuOQZ(HGgL0TW$;oqX4>C1KUKo3}<6j*(W zcOLx+q%F_f7MKCM4%3L+YB+h3*FJx<-V5@wQ8{p~>fjow#{h|5Yjbti8wB$E17jZqYR-1rpYu z3Ue~IZ00_z+=n093D?&(wU8gVDOI{9T93Pr!|MCh8O{#aAE991Hrkb9mwEE?*EsFh z(iM^rkCxW21Y#4!>gg~mW*l7w%2F;*yJ>8E+jG*WaoTwVXgU@qfox;*E7+w~+PS5d?hPRh$Y&~4fyMMNq44lM6e1?Y)jCj9f4iKd8 z*F6xxbGt@&nb*N0`W!*c&0&7UXm@+n zzmRR8&D8b7A*=sHobe_7S<9dy*!kB14}Mz_GkXG#Dj1ss8m2Pb1uj9~l7r3aX5eeG zLAFi!^1|itUDcxMHimcso^#{RHNpJob+ociOMx{v!AjvmLOA=ZT>lh%8fNvX+KlTV zIIFw~es(+#{vI^3)mlFx(7hZIj)XT2BY2icNo7)QWD8-dJh_}kqHJ4^92HXau}|=89^oRvm1tUgkR*x&{kUa&ATwHlYFJWM&OG z=pqf5q-{@FN*Tm9$gQwt?6K_M@MhF<{eHVUJ-9EI&KfH-2#X)go!39?4)AMQSxvZ^ z*YS6c6mTS7A7^Qz;4lRhCkED>K$j~k@ORR)mTiJ*f}O`JgVr<-HKQh{q*c)Vh4U&l z$ndrEJLP1*nLt*>p~3XhOwHDSeB+?hAvx$~Gpte>yDeAd#gQpBB%Lc0vT3#c8Xp;C zvLJNs1n-p_Y=71C61Ku+C360r0FuxF#VJwytiCr>jTZLQZlNHTS{A!`>JGb7ET5rOV~z-@ z_u@K3>toVO?>Drk{)k|LTq^4>&qN!i5T?=ftV0$!hvUHDdvINbI}4;&ChV*J((qn- zVEFgSRcv2dp0bn>AWiH)OGMU|EZ_-CJwBU$ zxWnq|H1uvAnk_W6+WDPRKJT*r?YAbKs(@$?W+wNHPfd6DA3aa2-01Xqbvb!HkCuaj ztUvgu{Yaa!O7tiZJ(UL+hX0u3e69Jmj9mKGG&Jq!w>UV$JNW-%@6E%Z?!Wk9QdB~l zTal%W5X!zUDYA=@y|Qm3%wi{^q>z25Y}vC;W{e?a8~b1|hRVLo*bRo6=cDiUzQ6Z# zUC;l|^*q;a{xL3t&*yyJ@Ao<9bzZM?C^eu21NNcX z@@_|43hxezWdrl2Tn4m}NPWLIv!k9`;*VmDQAp>q^%&65F z{Wq&S2!`Dst5%8YW-qc@9s!}X5fT^(C8R)l!w3TH7AXYFXU8BxWL$^)v4@1yfec0|nk{tAkq7Ez{EFeVK*<+C9ay>x^k>?cMXV9=NLQPcAM4jfZ z2dkvOPZPHqUbIhQJq&O?_i_XX!Gv?z`DasCn6zSIIectNF%uMARz+;Qf)%dtcBz6h zb>6lQ*{1@OXqDfhc#Fa4es&nE*>20q$$Lkc z>W!h|bZj2FDE&Hv`@{}W!}AQy%I?WF^1gs5D#P%H(+d0K5UJxqDtex*LBRwHj>7Y^ zOw5@!&iiCMPHM_S$Bg(HKVpg;ogX2UnknFiYwBU%v@9VtJ{SaI215g7mvdV! zX<_p;6Hlk-^UF7#G_{>{bYz;VNw*fnkjcmkv)VG95j1M_qn<(qS?ZcNH?6WqOsVNk z>8$tmdP2BUSbb9TNZa?@uDp%dL8Vb>&&l5*M-)D9la<4VFm7#fkK`4w0X3)ztY(Y| z>W@{J#uhNM)#+54Ht(%kijk*GQ*_2c6gL$CRVj1$Jw4ie7-wXfIW$Os5aN@AXPX3Z z(-me69OQm~A+3#^+E4q^(?DpebL&QzNB-#i;vq=?yg7Dr9W>lR{7p)!m!e(|Js3D~ zX`yY)toHeMRZzgj>MxJS$VJxg(NRBF^Ug3lEShFjw0_5Zf+lTWGE$=;N4g}fhPmjf znrY^n?%m9#cjuSKk4L+NnF0Ku`isDUSCH{hIx7oOp3*6~(nU8#O(V6A1xd5) z)e5^7o?5t#^q_ilKTdK~mkycD4zR(^vzjMp-EGT&n!xWCLO$Y?UC@&#fI#Z30sw3B z)`GzU^Aa|;f-cK)4)do0n(BELTJV0I`H9uMnHsjDmYk4VNjqOFxvkx4(T5NCtN`-j zMZLc9@tG;E5qql1MoQATa^=cbZ;PsVQr;ppFY?uoeDY$u=x4uYE`VL>td|v~uX)YA|gFtw%kW^CT*Nv=$6=q?fQm zG#=dB2ozkov5VYxOmFGtK4Z=z_95#E_tX!la|*CV}u+u3pWK+9ABr!UAdrzWOKzO zfn-yk3(tBm#3}I8(;G_4f2pMCEasqWy;U|$jxEKsNU1d4Sw1E6wv$jVKtdJ`Zp|d0fmr>)&*xy5Sn}CcSWRV%JBBr6oHosD_$0E-%#B9SQ1# z15t`!Z@Iu~j;FpFM!bRw0ah0WF_2!Xz&tg?pa(tY?Ks>ypP}@)hpv3(sAc zWW_LfiZM@o=qHYxH7uoNH;V1zYz}{{yQv@=XTGWEwBKml2;oRw`T=j`HlfHu_?+;1 z6f({0C9p&iPZL7+7IU^WyP3O#({;$!TTHry_F5&`x1Fqc=WACHtMp7=E>77_6@6Rk z)u0TLlgMxdIx5%D($YKH1>OgO4_1g$|E!}LS+hO^jNJ|eohpCZDa!EG0J z@Z4W3bdk@ORfnKVX0Xo+8!3R!BUBfR_Og3kCPP?V>Il*#kZZ0cg#7jUp0n9=c*!HD zA?^gndn`c*2=4t#Ts2g(LT_Suiqp8)*V zGqCts7TQW|<$m5qB^CBEmjZMha{2`9>sN8qu9cj{(Z`beZ9N0@nSW^6r(x4#q8?J= zRmF~5)~4DC9rO#zz*)xk3hX37P#sR@#+(Ig8iB2%4eEV!blJrXOh07^1(?^NAp7RH ztB!Y;05B=M3-Fww&_cWEu|u&e_x#5K9~jOCzoUPn|Kn09@auG;q|m!SxcCX6pdY>X zYX5M(fp?*2|B=_+`|r?KI=Yj0kEsGb173XJ|IY`Hx&;BR|Bsoc>S6cw|6DlTy{OZw z|9Q*zz3arneKxe%pKo7HuD708kUZ){_2+f(78|Hb^0BqXK2k99Ft~lx92@|pNZpo1)+0`}ml{VKG_MN9*_^^O!0SV;a`^WSp_sD_J?D3L8}gXm&x6Zu=c=D7Hj4HdzZ!;ee9}o^hFpKfgy$ z_fYKo=E;67o?u(LT+9L*n)WVuQCZM)BFn2mNy&BgFkD4&nJlN<59x3Er9p}0A~NR+ z%Lac15S^nXR()p+HSe&S5vx<%P}Sog4D(1iwB1WtFmUJEndPey&ON#^7MXDlAY#0G z>LeYV$3mT7-3qxa&z#%J;=>&O<-@MSLA!Q-HJ3j(@t=>b97!blBM(swKOUBudl$e5 z!gUj+|98zDtn_?zbeD%fHrJ&fyjo5Eo(G1m@G|y9{(pUj=g()(JlRaXSTcox9WRJF zx)9*LZGp=^(ov~0DlJQEe>`1ms7Cqpzw2B%TJ&*h`GS;}vsl5Y8Xd#*|C#t=Nd*9d z%2TZ57LBnEz@)eHj>kjkLi?TFLGjisNC!=I)R4?d;+!I5q-R}km#g4oAM>=_7qy;W z2ae2lj86M~WQbEXq(YH)@caUGaq1Z>J?4Ze65o)gpB?m~>9(KahcbW9V!6tafSvef zd2Jn|d4*kH?FzMMYU(%Lsq00;sCFdbfDp2FR?cC^uVKPUIS^K?C?8_8agcvOKZ&7D zKhZ45{FY9(O=4qU?}?O~wBVhOdmFv+9P#r8X`AoMvMp&rbr1!Q_6M+@O_H0}Q)Ida zx6Iu{qJ?Wcu_B=dS>LfkP%sjW^)1RyQYr}1Q&}`I7jAHNb&g}RvAkv?tei-1ViZSY z*kPZE{l}!Lg4I9=&H|l;6qloUgqyRLQ(eT=x>7*yGoHqHpWKI(zRcRNCC(DN1)kqF z>rBMXb<8&fe|Xj7W5eg<;wv+*;IK?Z+hm;Ia=BGx>h~-QX(K~7G{4#KnxrL1J9God zv{>@%r*~ZxHs^Yz54fTiaimJ$@e0I%yAMz7g$Mx5u=T`hiat% zJ?+Gj+%*GWWbb4B3_100M@k9-&SQi%TaNUzw=6ktiTF8w%%owK)s3a3NeeEM)2_uO zSXD_p-3u&YSn$_W{sqUG?ZyRv;Hi}w-TM}}h^W~6)llsc1TFZ5Xk=`*xGFpU4Kse` z>IYQYoQ8SBtoqn+Dq&a85*6OB)WahFX*Kx#`};-}`1)W}Bjv!S+HuUNc&hs5ECN!+ zQ(aXZduFu8WTOhw{92@G2{@%-C)zsH=IFQnn6~DI=dPo@#tB zX}j^Zk+xJ??a{iQIQwbf^NLJx;~Axer(*$IFE^eM{1+)hwgqKgu`{3K_^VfPi^C@V zY?7lhhVIq_g4}`e@laKTe$qz(d}%Q>G|9=v{e0N|!zcVEOB|T{*&=Hh!M1Ymna<6b zPslm{lsct~Zwi(5@1|w^SB_tV;jO&3HTZgE>)o7&IDfne5qM8}B#?NZ9 zP54azv_V_~`j9)#djx0yeRo}uDP;s1q|xC~nrAdMy3?H$;6K!*!nu>|q~aR4z1k$) zMP<}?8|zUmeHm|h&~vz5(20>ePMN@oNrx=E zBcO_nRhkA;>|puqY{m#sZvL~3UgZqIJ2f@9ea}*GCMtK#v(CGR5ti4kCyD2*Wo1DX zW=bQ650@P{gU2Yid98LP-Fa#Qu0T;H0f1ItznAZS%&P2f7Kq>EX6v8g9^)M?MAPn; z^B{Xq0xbrp>Cg45gA7m?Kl!hs>}T?|yqiBHem#@2Na0`Z`ru$Jas@_P`W0WJA@(5!lj~q z_&Ba0_BO_R#I#Qp1CZBt5i+FQB6B_>HDjb)D{apdS3v(M$bBD%pfeOcPk?ZU@8s-H zG;m~)zSn;e&snD^;MJNKyLITxYd%hg!*POU76c~`ZEU&e zYAd6?MMUAt#B3{zw4!I9gl`>SU0V{ILevYpGqcXS68sKLT%7KVY>z^=cqMP#Ig17L zlZZVxZcPR?#9k%r=LGl$ckNZR=T~0r9+6XVA+XEfMMn^s=q_;%e>CZ9_}HXaSRJ87 zSizz={ykxPd!y@+CK+NjGSs1FJYA_-eyg~Wn-9;pA`Cf<^k+&5`p#oz!ERzAzP(b} znR`f%rSb00Jg(`mUNMNG$SRKg zUj!9b&P$syYWV%OG+*f~^#qW`X=kcDuB{EILRNKWOiMf9|5UZbXDqC@!xI!L=gE{^ zGhk}XNsGpU4H|8 z3d^sFJNm7kF@4$DJ2NroE?u;8SUR@MrWRhWRF~nc^RuXi2T4}id-V;vfbE1Rs?w&d z>jKkeT!)b&@XjJPStFLui((N^C|l7Rzft6lRvh(_Y2#M>_vhtqO38}5p(~@+KMsOA zYF!3vJZ%LNdF+>(HyI*VoqF*v=jA$d9~=eVz;%n!@o|UjIE z3Ke85TzeAF z4xUa>R`TXr?kt&@<<=c(XIrTEJb3um)HYkgDvsvCDp^-@-@^taKUU}d^UXPN|BZq2 z5#u_S@H4)PE|(8x)WvtBmm^vtxjki6OcRmk!j$PZfgr_s~$Z&`x-px74za8+~%NX*> z8pGe4npPb%Y4Zh5o~FfT^Ol`#jOC`hOO(Rue?S~+6cJbz)((s}K?y9vrZ~DIe`6nTwdvPKngtNEUq|_qDV^#b1W=)PY|CrE?1BL%23A=}E+sOexZJBR$ z13J35-6{hSks4`^*g9~UyYNqN)}j!z9{aCS5$jlwP`E+!6Mv{>xW2!_A&&RTU*Q$l z@?%5o;y8}@_ezfGwK9MyJW6#5p#nLwdBbNgW3{EqqVWC7CpDbvP8qioJ{;~ODVvO$ zq`UhOJuZ(<7z1%vVp0mGDIa{FlEcn$$!YkLc~h%bUayKKWrS>B z_apsEW_!TWA=O;cZ zdlyg7#qO^#n-G((?VY=+#8To-V*}=j=h(A7S%>6Ax%m^KIOLS7-$s){{5(KuvYe(h z;HS~Nt{<=gjetB3Ek)-=8C(;Ln>LX9bWbiX{h>AbU-cIIN zD7QhiS*7UTMI2Ip#TK97p4uPrJMRdlXB>aY;t$%bWCoJAC*WC_F77E+D< zUEfM|X16KL0SC5WS04KupO0~lnSDke-||#|*~o26A$3+=r8oU02_G)RCc-#Wa=aU$ zD%h3<;T+%l{6yz~gIj1+c8C>pr6$K+dFxFrX7uHJ&R3%8*e^2n3P9J9Ajl;05f?+gmhve+r$(oD?T+O*6!4hy(1BO6r&Gkw#%EjNHzW| z+7gjDr}BcmAsN#omsI<3;~RR%Y>wO4#OyH~7~gGNh#7al*u#QQ zjkP~*F3SeKdc5^n+kB2#l$}`ahD%4YWs94X1d!w5Jb-chn9_`)?s0)Eep|NH+s&={ z4c87wA3=ZawR@t*z*02~eL24Sl9%yoxm&IlxP1+l)vh(`zOMS}s~e_$#`zWQUtV$l zeb9F}TDjAFM~P|yQ_cn)Iw7@AyezjW+*{Ez5V=>OuR`|h&Zy>cyR7*v>M`raCA+lW zYn5hcmAupKIQSmO7<7>O)+y9|U#ghr$raZU`s7IiM4I#T^kz@oDvi1WO{4qn(9Q{l zj8r12>vR$WFU(5!z*kyzi6wni^$E&TD^H*mZ+Vm&!`pm0j+Jxz&8HO^RZRV|-r2v- zmVpi+)}>t8iV_re{e5F^2#&svjZNo6F<;u#a>`rT`_(0#0^IAW)bm$zDu{M(IoJ7# zpW!;1tA$eDGxg|>CS*;Hm!kXMPykU`{Qa}?-$~)*hXGjL0b;%Vc(%K<*cHN5jR0N4 z>yS=|^H-l1-UZsOoK`=<*H15Lh)pn!av2^itXsF00^ddrO!LpNv+m&y7|tQu{bahGoRdy>V}9-`oJZnq|?y?y|Q{;LceWe9oX5$ zt6kt1OUMeI+Jp{dh37yz^^4@Z|GU_E3O0$>6e9IBaL^*n79= zdG|>-T=8;O-E#w2#mdh2_G6iC$JzayW1$ZymtcdPbGFSAVKo(9 zS8d_21>AcVE5kp#o@8yH|)Wx6DEJ716($wuf7&u zWcTKN3F;EMQy^2Qp+EEya%bs=JIQyn;5$L%PzHxzEPl+(m3x|n77y<ZNO>}c=Kfnt zHD|k)U{q2;J^z=jr$i(N3IqUe?k~v~^C5%EYQ4u7Z`+H~nDnMPEh?)XN zJ`M3a7qXqhYCaufo)r`6f=~`Io7|Oe`rxg?DbA?35A9*I5S6g=T2*Pg?Ie6wW%)$v zKEqxftWPIqPjR+(OnjI!K;{iW7$qyBZtcm2xApbbzJ~^qa(QF6*O3XE4gm_e5*LLJ zbS9mLgM+jfEq|C;M-4ncik= z6a~Ny6yuq=CiSh{n#wCd-z|;asIllc_-(h}9MOwDaQQE?%pwN+EyFTWd%8c;hcm8W zIHbt8#aj9Z)oL^Zfqp{#c{hWDih(u#h6CUFgST)`3~z3m(uhJ?;eN5Ly5G#Ex~1T2 zK_;aTw>Pkm^e8>u;c3L4BLVRqlv+e@k61rNBuZY%B%X*VkW~=Z4v)zo1Ec%YBkO%s zO;^^$8e+~I1?a&5^NDW3w6<&di%WZvmGtw2bK=hSYoofzeYW^}9+#TXXRb8m6hDvB zFh=`laB(>-BOuiqvndYK!m(OnnkSqiI0FK_N^?}r^Tsc zQ@CE&wf@1UJpfd6EJhH(xk^-E;;iMP$X{sD#CtOekKU*pV9y%CdsLz_(c+IM5E=DpJKd&AJtGCzfmbFhfm;)AJb_4w!T)$KTo9 z8Pc7-<#>C18_Qhvq|$PM;khWli?NPfCJGb&$!dPhRvxngPx#(#`+mpS3;kXqOEj=WkTWdix)qxYPSw54}LVQ zv=K40@=BWUUsg0`k=?BQZTe1mqvKoRCh=TvLsBTJk@qbyw6XLQZR0)wz%M|PmA36S zej1@D-4fz2rhdJ|Nbi&->B*OFmhPF*xo@z_?%ZG%HTkaab4A8Ib&L5)Jvp20K6HXV z;_by}R%I3M{yweNs!c+r*RU0Jpr6T*ZQEwwm%;vH_{yfZ#!4rYq28(`gG7jP;j*)E z|0Zmp2f}p-6#JigPVWy_FA)lC-uT&6-y6(2lwDB9Dmn5cJiI@CQ_=^y;CO+WwN<>x zpZIuOYVT!;NXJqzIE#ir6CF0KLtsb!pB79_(c8&ofqOSM;XErYKiZipMLvAEz~7bo zsD;pNYM}JMU&NN59nt_HjAM&(%9KI=BN&-P4yV6JN&}pW75}V575HgGBptG5+)-gC zCkw3Z0vmt793}u1X5`%IQaxpCq;@_R%Nf-F0@)SfLr4?Rme$hlG=_xf)01m8%zC%R zL;BJ=P~zAT3v&y zcaJ8Tykiq$c2o>8@8=!@6YljxRw^n}QdXmB=bxAxUvCfes#laFOH2Gbh-c!BGYHU1UeJ6yCvkv_(TXF0_X{kf2dSloOIJN3FM)H^)cAvf0f~-!yTpq<1p&K`6J++8$ zVfj-TVmf{B;#6pC#rYyuHwK!C4pPmh@DNUYWw8JwImdkQoVRkZYT+^M3hAJ@~X?+EY|; zbzB|6^-qpe)R!GlB{$njyw~VLNPqY1nG+FSpbL$^nvXrKq0Cu3wP~Cw*~OCDsiGYS z8-ivXoGH{D@RAR9>d6HM6YAvJ1}lh1`D5IcxWY-McgqO$NjLHnSmyb1q2;BQMuxjZ z=*P?N+!lEtJ+{7{a@0++V)`KZ*P)a}t8%%s)rRf?2P`OX zU$*u9v5K*3ljL&i>v$M3ned*)XpZ{@%Yun8(;1%I%Q-Ah#$VrPD^3+F$lurA_j zidKe^@OR2f3(ec_C_{qPEtaOGx;BTnL+=S(U&6zDyLMN9yN|FJ4GZTs$-%&sK2SEjQ*%t1z*~)yPuzL?r4_Xh`zf#eFKX+-%mpXkJ*fvNdBqR>Cx}S;4xa1kiI|+rseruNGQ~O2o<)e3oLHPqFB+X)&q2* z^SK{?0r}%>P_dYB1D{4as)oP}6hJK7xRvz2>4re+6ke8;uNmazSRI=(TJSH8b(%&D zcWRpe4BCd3w-TMGonNo37kXq0}WxoO6AZ(lTWGZ+g>6W3+K;;FM*ID9^d-{s9? z%ucmhb=C7kcVVt;&mK?)o8ImakLulky%gpJ%mhm36G@f@8D)9_Uhkkni(s1DU{p~$ zuOmKs@-=C>0sonTRU;Q;dK=cnu{z(U3S_X;gNAoLZ=9nWJj#R9A|2J#VyPQN!#?8< zLqb3J=CaH<1YlO+Tn9A$WbbQG&2SRXRLg9Xav{k}8&)PB5n>?+5Yo3&M!hr*^kQ(I z6ukGNS4#~2Q6B6Ft)0*mKifgz13s73DM${TH5V^hrvUjj*oZ1$B@ z2K(>LFJDH#%MMIzB!7D@abdQ)oj2i;rf%F@9-@jP7T>2RQaM33*o3pb2wBM#D*%U= z3QyrP5&zT-1}4G0L<$;hP0E0IXy44zAsf`e-s>0sA~*PmCDYsdEL1sI%I753z2`iZ z?x8E87zdipWEL}EJ!j{cM~NA4{Z1-s0NZ+$!TXXy6RLC%1UaDORfB&bd&qEd4uX z^ErEIhE~&rT7FWDl22FWX1?%fRRFv@XhUmkc=T37NqI^9So;-WC84ko21{};-m%%Q zZgtd4T&^{AlPg)w^Rhj)tY(ulWeN4852Hh#yuJ^yB9y`d4=}bLl*5^HVE>?e^5G=x zLQ^udpuB>}F6*b^H16WG79=z2VGj3l|4XHD+&pyKOOP-%)c!*@)Ec+mT41T*zOtLR zSt;VD^;QtK4JYp;rhleecr4GK$#ty3uCI=RaB4s_mrLQ<+!uEe-X;g6i_(Qt`tenR zYB@fra&I8jyppEdQjRPm&V3Nj)BCeB9TT>;@r`=* zzo)xu-8X^SoR-=8@3%>Q0Afj;Y+2M1Hp@rZFlF-Ah6H}u8TM>?)|TZeRyyTxY-97H zQts=Jc5-aU9_EVGX(HEu(4K`e2$ek49iWP3M-?epGgxf;1j+;D5Zc`DDF$7!X!(p; z*21anGV$30P?@$Wtwp_Y>n0Tr$en@EtI_uKG~qnB|0%Kl@(hruiOXH6jPZJoOhy3O zHS%IAgi{y@MQ|BoJ-Y3<`Gxm;(o1VMDMY{d58%6I}53zm2QC4W9vUf zlI7=ot0%9lERg|5KASC#ExxDMij;NM=BK9?fy&udn0+q%y05c%Wo7SI&xi%=*=M`(lIvgwIQn|98F;k_E& z(do(#3vTGm71=O+PpCO>5%Q3G-~`H0fxPnJo=Zx_zVwKM|Au{WS@GQvG_^y0(D0{Pd7a0hzu)iV z4AeGJdR`zaIP#3bjG@|h;@y0BTOvLs$BN5MM# z2&T2R71QB4NV(#*S%Mx?9%3!XvH>Am`I^ zfx`b*gcG z8BJ)(t~7T9%R7Fu&QZqoDa+dSTbOa_A_}Xu-+nBt{?y~p9lCj4GYxIlbWrVXgVNZj z3&Ls`n^%`s^u9BLEO-fYta(iz2inK{L%;(AN_DU}i)~?VrpLF6oion|Atq8a@$WO@ zGVg!qR{!#utWp8NIns8{;L+Pa7gP38=q907xyffW=Zm`2ik?jA7s{w9kTQ;aAt%Hr zJBirwffDX<|BT4w?6t5s=Pb-oS3SVZF*qpH!;m=-9w7K6#A2<@q$`Vu_zINn**wn&5nElqRZZBA3uHpm50=Te)RH(ek}aQc@4iJZL+|Qn^#dN{o|q9@ zgzA4eV6bSW*)E2?_8%7jOw5c1C{IrE;pJT4oL@M!au=|J1p1T>#w`mgO0rDo9TC~} zb-K;hee?bAcH(mv zauUx?$36_mP>)rxnGD%xmlzq!Fi_$j&U@XNY;Boe^+`)3GdSMdF=6EcB6bp}Fu6KZ zroS}$OJtK(2;Ox8KAEZbRnd3pvE0IioU``-ehyJ|XBUp=Q1%~elx{lR>|4Ii!O1sk z8CCT1Ijp+x^H7|SwTwAhci2Oo|0BK*m-XVifxx|iDsB-E6(Tf4kFr?ScLdk;?sX2& zDJx`vq8$+7un&ASh-xE!9vOX7(4K>fE2wuLI`Htmeww7#^8l7ac#hvuhH?YKCDZjA zNlKLAwVvTV2r`O{Rj}$3GD7n}n)lXoxwkLzL?8 zYxsBR_|W+AGHoft5MFtruNwYeTSXF}gZ7`2+?Df#V%*X46jG3nW#7Vh$P0{nEkItp z^i>p9Ue53CNqpp&!p1_G_@dWN!~PT^Xytew${0WVyzC=za{x3>8ub-L+US(h6UN?R z2c&@+Wp#x;l63kwkbF}Pc4`_0W2m~)r&Az!@q@_32G`G*4A%kJwTupvmNeOR6&zN8%x`*v% z&R&KNw%@cE-ub!pzUO`Qvpz>bu1g_Vt|`^(G1EWAIsUF99iub!KN8dbUgbPkAoy6G zxVP@L_*?Sb#%o>wF&6Wf5VJEn+2%J2R9g{1K}0VbXP$(OZ~;7p08npJa>zu2+vMm| zmEme1%L+_hgHG;EsBg|z*A&5*j7lY)GbqbCy1Py;m(nVZ8CGt-e&bZ3|pX zBH*o>JX#t3NHMeoN{CNgS>UV#z=BKMZ_BC%Q5d5V9cXpM7*;?VcKI;M&S9cRKiN64 za>D9wT%MUu)#W4eoqjDz#dM!v4&FU%eR?+1WL!!Yxw1j&@@+k3ugFty_nYaFujFlV zS^R2+?pP;TRGq=on(ZIvU&X_UVr4?yPD~5>jE8K*gZ0ybveULll!JqK1hVe5zUwRM z%ZUVWJ{f;%ScwKO%*%ibcvIeC&fRaT!k26l)KrJ8vb^l<^03Idp|0mPs&9(6FtK}_ zJL-j*7=9+B_I_nwJ?2G#-*5o;;&jbc!?MtY&|Q|5VI5@^m-w=&dcEWTrq|2$poF$L z$HhmFt}?i}IiOLDT|d9K*vaRi!u!rmw`00eY?BKNl%KV|8ce&ILu;R!eg9xG0JKva z&&D0t)b}d<#NNM%%!5aQ)N3r6;0-pG?L^|~vX=P4(1)ef6J09Sa}AJ9_a zygf0l?^W=!DN}`QiVEP@Nkd=s;2x91A}s~VDb`i{eV%c!vvQx`Gj!^9376W_v=SU< zUPU>o?g4?{L#Rh(LgB`nm8eON$v%EixUfs#?*>t z<3_q(UGH;eFS*oZL_W2PO&20QP}gQDb9b;79Jr$=%pWeab(?Z!aWj=A=-s|A(Qf#~ zBLHU{WCXRYddhtCnA6f=mAsk(@^p#};k>xK8!jz5+@6Mc5>8O7E$i-<_3JvpkzVmE zz~G8N#CW|EGFS2+&N-bb_1az9ErcqQ|89^Pt7KzUk|U?$x$)Zml6n&iYTn7?V6V7> z=)hmsAyYdvKVB~S81Ph;J=$`34HEPnk4)8#aqV{X#?5`u%C}%Dt{*M)P!44zWBI^e{V#Jv08co8D1Wbu#V8HH?}kp zrfiRC8v6fpoc0gDDnEAizv!cBS5L!F@5`_)Nr`7VW`#AoVUoAu5Y0Fd12>s}oInpL?zQRI{BAevg=5`V`3{Hb~|0n^R~?mH`Civ^Cq$BHsbc* zFUi^eek?cB;WFr1`SH9oGpNA`N*V=GNpRd*IqJJj?}mmln})~P`}2)6d)Esyh4NU_ zZh0!A`xSFWE>AVy-XllH_NI_0eXsgacO{xW4ErpjE9*f;#NB-OL4fCNR(IT~$@-YB zdZ~9rKrv58w;&IMo6th~fmWDR#+lbv{$t)^ii6lN%ZhkAWhG7J&7|lX?vimel4r4W zVwSN7Q+0xBIg#G3?>R;qmLUwPEHgHNTiG*yLBW0K-_Q%)4%Kd&)?M;a?q^P~))IJ{ zrsXR=<{2y7UCdC2=Zqb1z`2h8YiNuH$5TLJauR$vJD+~~Ywns@hp|mAcyvjc@NCk4tA>O0usIg6(87{&bi4JXqTTzDqrVBv z!UB@%+61n~HP7w01)ZA~LDJ8biuuphTuqKtxA2-XEnRf*$GZ!SAdZ#qU`tfWBaH#` zJBNL7=gDAns<6{f6WJS__v?7WFgt2I_pr$w|CPR6ImS*!Nui+B(-65d-LzdIFGr4F z{?BfYxeqXRSZf15b5ULHiV2K0JVK0{O@ooGh1cY=wxjz#$rUdJX4AL!R>QwZNLWV`BO@Fw+e!8~`p$wE}u|FPr7TL|QpMO^ep>e)xh$VJX#z~B2 z>|oHS^s}xrov#DX@@-QSw5B_&^PS}jm_qB+WDlG+0#F^(*)GVhW7Fb z<-+*O&XBae!0l@d)3GW1VgZX)d*a~lKju8Sb=J1$InvNK^4L=vwduVqiM7Hd zy~+XiBb&Q>ZI;pH)fY^6;pQoIEd=UhkS@7i%9&f9LaZw420>`!dON@Fo8&Qi?JF*J zw=XKLJXRf#lHB-62(B|)?fgC(4pmfz4aT|eBCk^z@La7dOye2U(}a9&TlpmC`{DLEBSzb9zHoR0^ysU$5W> zgP!o(Rakow>ibC@4Z0eiVwDJpKNsN)j0RBsFxDwL6nDs)XvOV4Gc+@`h!_@y9TR66 zmW&%r&_SN1s5j?o-Hu5G47H0Dl9Cy+c(G~1|AzTF@}Q(CfaTPg??5{bPfr0gFJ5)T zwwvm_y#4ci?H2bS!CYZbJs}%cW|cn3I$j_L2pvFl#mj$2E#>ANbt1ba^Tb$vBenJ$ zZ@$Z{t)ZDQN*8$s$sS1!HBfI>=hxR`G$q~mOU-VS+o$VF1F?iTfj%kHK2P)%D{_k} z_1LHNjy2kHk39D~LJ+rnrlVdM)DLiDzuFNOr!fWNqTJT6p%fGmEhafn^bH;Bt}pZW zY5PZz5M%=*K0Tv)WCYdrkATp}i)uKq@j67vS^ON!*oyzeQxWG(6{Si$iaAC?m-Q2Q zm2DMl_3cBi1GS<8(HL;ZJh`(gd}ZoMrhMe@yG0jA4siu`tUSwLf%<9sKl0+Ue<<}P zWZm)`7^iwOnnzekkNA6$&$tF_508`cP(9K>P0Fd??q)XReE90}#QcU#IC8NNkju!{ zhJDjv;Sni+h1I%mWgy2MHeToJ%wv+6HY354i?)W%VJjXV0{v9z=;TiwNmT~B6qLr! zuA*?ZaaN1Rc{{iLT8s_pMn^7zLCp^8vNG?mceDleJ2VVmBo#B4sodVgdf@PQ;y|Zr zt+%1erFqc&D@7-&M>N!WL`&CQ!3941fvG4&|1cI-j@1~msHvWn1_4c+QgRME?JXAr ze#dk1VwSy+{{KS7A4+%DclGu_9hrd#XIan_+NeiN261-=qcm@`Tq62xg<&7a;Vvio zJZonFBwOtax@H9&InD()`2vI1zh#yv9M*8g%1dSv$89F&$360_#DFd(fPVk&(e{w6 z^===>NPdXS(f{;o@a}S$yS9XA?zT;r?)P<93EF)QdDExTZwV*NxG9Trj^bzG$10~} zUKhT~X%9sB+R;u_P1BESX-Vb=%&g)HY=eIf&Fr{*-im|w!01#4&LncjC9F`}E#li6P&dcjE%>q6g>E83y_ zzjH4~K`gOZlDpUteEL-xUW7g@^Ini5Z1-L@uzC}A3a9B29!a2Y7o?uKF1de~Z}pZY zmc(Mpslp>=ZFC4xA>#qUIu9LP?)1@V&p@W3oO3QxxlkQ_Sb$rLh*>9BpO|hy&|T-VJNXj7UQ>1z z#BXfEP2|Vr5r7wQ3xn+=&zxb>39=-&u0g zw~NiZ_FAr$`D-vGp(Z|rSBfZ^7I>iB*%mM`SRNqyvrk1R zcH%H45wis;&WRpNT@6ZJqI`F-^`tI)`*H7kx=s0vHNm$IKgCYpSyix4BvVgsthB-a zHvX!9HjbP!DT?#J>_Erd11xh+(S;rWx@=-|%v~5h#5T22H#~QW_QJI5Hj=m);2iQl$4zK%_~NE;V$dSEU4k^iJprBqV$5^StlwKd`^u4|_g9 za+i}+X6DS9GjmPAjuI!AQ-h^u>w;$bE_C0n)7TbOQZISfzA^L#h{XHjj>cgGo}9UJ zn_tz6F1hcySpIGbTG)N0?NUc;8t>3Q7tB%tp<-y9E6m&ZeDgrx z{ePZIet-Atq;q^-Rl37)&!94a9(>u}3Cx|>jB80z8w9`~Ja43Oyqw+ZxH$E7**Sos zPqU3B^N?vBNhfnQzRLzqH+^N}MPZ!+vra&mhx+vi(g%SrL-BWjml?x$2;(3RIFeB;o`yc)i3toPFD zJ?C58nzwhBeeO%UyT=Re(d)us@UwDz=6p%JhJ|sLtSsM5X=}JaZC6I+)o)~`zB3L} z{E5~DxmKqg@$&N0oSOP9ZvsJPy5${&r8y1NcpSo5hy5eCHoVMzh1LfBaC>$4Ny3ups*77gDJhue>8y7TEVk9cJrra0qDb%!>!kDpdYd z^wunV7YQ>}CJ(7CQ^s>UhcQ8{@FmmrQFa@#;r>^Cv$IWvLC(Tb^jw(Dxb&%xI5pv3 zC7m19)1EB|PE&|;bogf(o7!LYJ5ny^4cTgfP5R&Mge}n6DbF$Jc@x|7+p7q+Y`aLp zK=a+J8%^UT5jb&qPUM+5=W@RK4OLT{i48GFnU;b3@=cFR?U#bae(3$Ya1XzG7jy!6 z^?3xdP(DtGLpE6js@#0*Ic`==?C!Ir%U|R>#L_#aTBRnINLvzJH2-QTRo2e`dtYyTN7qWyhO>pwO(5mlX1gV);AfsTJteA;Cg&$P9X5QiP9=yEL@t~! zuebxe$P~{-3Oy6^ZWqJI6+1bVb6~usrKT~@i%v*(-;8HDpK~T{ou78b-7I})TZQ+w zc>Ja*VA>_Teyq+Jh6ic}w!LBr;qLPU9t;iYq@~1Vw`Lg+78sgMVdASDZWJCe)c5}e zbuWuc!k_4nPn=XPOovKWGS=W(R#`|FI86T0;EXXf&aGf)Ks3y)ecjzNxna+lJ=;`U z5Ef1%`|cgjS%>=d5K_CJ%71gEl@?i^JvHky@tm!YQo5>HF0-Df@^|AsuAQyQj{fUv z1;Cx$41v{Lv+3qR1io**DBioZ?{)nPw(Vb|X^viTNy5vYa&EYEV=Bbn&0mW8i)Y2J zr@E;;TbEMHEm~xx}98{-l`Gv5mv!vGtv;+ z=+N-8g~xdQB)++b9@rHemH0hN`fT84)|FGUBU%T^H{w^bbk(t}Mi?j6rnCM0|c38K_ zW=CuZfe)h+X&!sng5B&kS1MnzlByq?U6K;#tZBunKDL!|6s+Y;SE>hrxc`C^*@l{k z77(!vj7#vtva@c5zkk?y*^^-0@+y4lcGO`k9m;N^;MX_n9AMkW%8C1Idae6&&jj$C zITSO0DvMk`92}dgN_Lu`|8sseBfV7|wstUjpkEGw8sD_FySoVy6;3~IStjxN@)TUu z9OQlfswZ)S9CMQUeJc-U+^sCW_~|4f>yj{LXZ!lc#|nMC6P5>{^}l(LH7V=63lnHn z>xz)Bm$Oot(@u-GI^U0YWO4E+9hI+h z9%;vlZCivK+US01L3~m;H8Dl52cC{V#dzNOpGK(Djm<01!*B`4*Xb&@Rgk4a0^vr^ z@38aPv4?N&cwP9-zPneVSas&FIDo5LoS0r_K)ze{Z+?K`r-5E97UF|`l3IZ9P31f8 znSe|c%Es{btK0T|2Fr6k%nL+0^GjvJ6i#*frEXO^_IHnXQ|~>R@8~%1#ASZ+Rxu&V z77;9QMV|>WRtMNi4wCVkdTU$(nnTX-Hv_Jt>6s%t6#(d6V70CHQkNW3#kX>7h^2HI znMp?O+CxQ?ae7I{UvSp?*t7|DUcOe4i@q+{ljL~2xzvbim1y0Y(9~J3zHr_4P74p| z!xRkR=GUdzKr&u`r+M2FiXP0|Od+kr4mU)zwZuJU>CfQkWXFj%{bD8N<@)SR{(D|a ze8FGEe9(Tek^$C8Kp0GSC&$ZBzE-OuihM$_(yV4i7mpFWGhf-#s-fsX&T}D>u`|aZ zBRuCOR~gVUVzS_Lk@@=2nF?$7A#GSpkoY!T;q%BKFlIo|Te%AJ3oP~`LVD4?M|+8l zR3%!k??D(a&2F!0-hyP5wo5<9y+I6v6Q(nX7?R_&m7Y_&9g2W8;KIV9*vvfh55uXO zE%}FCmmC*+trM_p+DI2;!xUHkNx}IEjT;|G@eV{XuHYGfo;&I(W~BbwRgNn#uvM?F z?$(_j9&We6`5B{@%*|7Gb(kX5=aR?$ho!YNHNS5m_)Pbs#2V33Ca4%t@FJi!gPxK! zkBwwkJ6h9u?Q~`4?RUkCQ=7kjZu*#AO>L|(Ly4!Z_^=el@l?|I@~dnF3C6i&DCdpe z5`_X8XlWLC(qu;VM!o#1pOIF=dCmnKP>ftKvBQp)M7P1AaOV@(lFXqJ)g?mrT-rHj znijIk;9s*)QjFua&5kE88{6;>F_uH7JYiSbn?ch5T&V*`NJ^jR-ehD7mp1xY7r&T# zV)mOU-unJ?k1%NTaWi|YFNLSCrJ&=s^e^{IFLnAtQOe$ko zHkNU&!BYm=z=VAjchXG`b>w(RNV>#J>$Xm*zKi#zoSz*m!%-Qq+cc}`yE1uc_TVI> zEQpS8RmQzJ_fbIx8x>f+ChvU=Kz>1fwb&MwHCpq3DP(JH6SEpaXto|pn zDyo1WvD+Ww^V-*w=+5lk3ysG1y)tDsF+ zwlZ<;s^~7#D93y@+DVVnPj6W|ck>NI+n^urecISrN7FHF+y;RE>O@h zgV>rMufL;}vH^j1UJ(9mcgikTr!x83;?YZT`>D~E?I|k(!^v-5?$Uy_dNj4~j||9m zW%CFFGPh+5{Gaz#bT4It_p2Pum;K%%)TKTwj_|{*-ta653Cm0%Q>1;)ruUXKb(*3% z4DPQwNTXuk;5yyhuO< z2m5Ls73^`Dz})n~%*XZXx|)uyn?k~VL^{}m_xdYwC8?!VZ#Ye^n|MgX9k1~Te8hnXqx3q#}f>~V^NkjXJS{OsrI*XHduegh=6cAd>_3q#T4nmoZ!gW zfx8-*xfW;O$*)XU4w{`u(2}m6UuPdm&0e<9?5VEOw;?Z*`r@--SM1bNi1%zDynlNy zg<^}}>kC#21Aby#NqeKvH2tT|dy`$?Ri1#}tbIy8Yy^&|9-N&oY3T)Dwxyp1zalSo zOy?*f5~0a?Atfbc%p4XOAJxdb(P(Ww?UmC$8_np-O6KQMd&&@2+h!UWj_#O5M(87V(GA>JxhwjUo7yYoh-M~8!%vKp++{u!v4?+0 zUSAlqGPJq_U6=g;O%A#R1Qn!!NJ*@M#RDm5iD)!T@?zwt_#z|o=gOo$FU$zAI{H(L-GxP=zu^-xt!7wtrv zEf>@Krd6{@K9$~zfTpdfi|Z_8hf_DSU)AsC<&>EehLahSMo+DQ$?vx?5=L+Z1&FvbT8aio402VV$Wm(vWgc)v+MVH}L;fYgUH8rD-P$?=mu_F#@x?OJw) zeS^26dyoBC-Mht4}>4E8cH069v`6gM3F0}QtJ^jnFg+>qv$SZ*kH z^K4?c?*gfo97?qdB1J~}u#}oZvIxsaOs^xl^~uJW=E|`Jzg4zX{eJ46zmhb)qd@>E zeqC<6$^19J+#dr$Tlk<`G*P|;+pMSN=@XMxHQtz-4TG`hYNG`0!1XBxIAEQhYE>J~ z_(-Sw5Zv#30;iAO2p=9FwU-2m#jA(qKOSu9?WJ#k!IRfnj(zY zlxCMB1W1;uU!4Ld1^fjccdAr>kb<7d@{>{JPXlRmJ(#aIuK3N{4dF*YR1MIeYNHxm zu`D=pthb`{fQoN{qiQ#=y%yUCcTUf{BBvYTy?&}xME8y2qTqq@rRUejkg}&@+JK2_ zf)M|&e`aY>LAH8D!S9MUX|BGNcRV=nK*6ISof`rlTH$4nFjKhJmS2uNEmaYSA7f#0 z&d6+*;*sa6k$#u$O>eAtrkx~!Q-INBKsYz1?_EJKTZ<{@d!9fbKje|ILK*+Fy_lgY zpV_sj%|=~0FQy%9mC^J68vp*f_y37={q0@-*og=FWcbD8;s$1I&-OX;h+&oxu6{#A z(r(Urmcq4a*3-)U5&;%b(I?Us*j@W|*ZnkHu(IEJdU$&DOGOhbXHJNL=@3v^iMz)c zn1(9x|JUD8zgU;vy8N~luH=9F7EZut<{RMNx|^gIiM`eKCgZ!a+-^&g&qoNo*RIgE zC39h0zSjJo$*H}1do8(#mHKb6{kI4AzL9vf04SxMmu3K7d_?0MEJLQ&|jcbFE8K(sLrgBrN7Z#PRMiY$!t@L zhGccR`Dv(8WeeFBGhkspXe_t5y^tAyFc)EObA7SC{7bN*@;42W_>qMBe^4stfj|Of z`PZ>_$o^G@L1;s?Q%k27FXZ{TE?`$R2acAgeJ&rvH-tblJOjq_B}ie9-j9RM1IL3( zX@hqWeC_J3cu2wF7>a+*O85&-b8lzPCxTH_E$%o38khA;|Aufmn0}3vY`*&YcWzU| z*+QEmRNz|4h$0DK-)@`8dE+$t=+3{_wSl0R1Fq8ON)G)$C8i!QhvMm6@e>3#JHEi| z59)x^IUR~yNm`d@4e}yf$%U_3Q@{Q9-U0x$S;wNTfhA=i=*dcH)!xKQ*vt`&K; z`pK>34cpF-rBC9Y!-KcXkA>^5#q{&B<&U@QU9xAZ4ATPMEGT6B=j=Tg5b^k?stwiX z3a>=cTiqUwi;u)$)#6wx1OxBFn&I{K@t|WHSf@gw{{R*A-oico*wZJv6S}t4OBFGDUvHjFy0$QPDO%EGp69>z1lp!~>Sr-;^%p#CWqghWZ>u)JgxilM#!%&^G3=_j?|#?) zN#oS{4a@VmZjm(JKmuG=oO{!d6DNSN*arZ1fl_<#ze1Acz9Q$wS$&`Qf#my`Gh|53 z`%j(=4RiX$yAM8^w`|W&X)O*EoxO+^^I%>I`a9?zO8>vKbCs>zpjK{OgZrQAJzEN` z%Sa z-KYBd87Kt` zfd9?E^1lb|RZLIq)4%HP+EZEK0|`B3{(TGfl0UvJMvwPMOwY6EzxOu)x2I%4ThTe@ z{MV#M`yv+#HY_0dlHF;!f6ovL0y#l|IrZR>BKX&tcf#NGgVhy4k?Ml^CBXRz*%=^W zv+e;$Fe2^c1lzg(btPZZkdw54^5lP4LytaC%nQYI_dDl7=RY%#(k2Zdi3u}$BTnu6!)A4@MTiMK6>2v&#RGRSn)6=UM*%~kQU!X$*q4L19vHj$SCtI?}ENq z#?bw@!u-AIUFrUo*Utd3=)sAPT4nhEV23?C(HGGX!7^n)GnFPeWTaUC_1-j0K>;EHz{Ak)K6H6a+JY@hWRe1{a zPL=rI^=$wtJ#1s5{pM38pc_hJ*4?r(nSZMRZ8MWGbZ1;VH3hC4i9P=L)cWSX{-lUV z1{Lk`D$@e>WT_KRwf=`yHIuG<1}Y*Z&uN|jmX9ZDHaGJGa`^23)+DntY4M9C;!-D_b2%hSy;b@@*rj#V$`@iNW-yA62wg*tPt&Z^i4d}Mx!|X4d%pm#n z_tLhR5B?3v#H2?zdAGEO0x-dDXQ5uZSmyt4_!-{MfPg!u8~7et5v0ce2m!TGt^U^< zrN2K1dfM{2^k;_bF(9M90l!gn05hWJiz}?vk4V7UmX- z>qmL@)jx?$~T^&kO|BlE!-ej6POcQTfzK10fL$536K8WPq=+>SZ*5l znluJS8g>)!(aYBZ)^kmepZyHF1phj@@aibyJ3wGh?V7#RzA%3n6-sRIJpLy0uZ<~$ zfj1l!UR}h;pA&@vWBk%Z5rI;&A;sGIOWc{+mErg&zVrPj-zT=Wul3*w9v{WqOMB1& zV-v(Ur z**PJ|gO^E)4(Hup0JJaIIb{Ez56Az*0RKPg-}Tpp5fDc+RXO zZPH1A>)o-p0D1{_{$sK%rr~$Kn<^pgIH!5J+0^kExJxXJ|L?9$50n>d=XHWf-yBJM zCl~S8Bd9x`rxWcR7-Nrz~6irY94ILR<2)74Se%p`k%1i{r|y` z2blT)VhHdJe9)7x*1ImoWAe<$;5J+zE-nK%fLp^HpoN@UGu|LQJg_;TMACVDa{K5+}qGYa?W9ob}S zG+@1y*t1Gpzl`i{5=rt=go9UwRpCOEi78N>y!77FyES_?2fJ;A%wz70yB(B!pIH~4 zQ7cIR=tZ@#-Hyg2Sjt?U?H9Fs$Vpu@wsIsb>!{V_m-^@`-|;PEQox-Dz99m_t+2aF zQn7=lgJ9+hBN89pNP>3g;?|E}=M>05Y9%9{gN;pJkE-Zi2rSJjwwQvF6!Auv{s&Wh!D)r4aO{_nRL6m^)b=9`M>7_jmz4->dnz1gWKYxvRS}t_hOT_VnB_7d4UXPqCu5 z(ltp<+$wGTqhi^k3Kvcfm-A)@D8RQf$Y)UgYB<#Zd4;55d2Q#nLXVH7zL{QBq#)+F z0Qk}S9J`f>P@+?RaLfa=+I)LpYZIm9eY-8>XS*;yS4f)eP-ZD>N8&w*;D+@)A0nyr zI){UkAEIJKftV^aGqbMZj84mh4u3h`8*1AadQuaEH0DHB@gjMuBYSai@|^8^EY_FS z2=w~KsLSAmq$m93Q$s`$&5R$S#(=3s_r$gLcs6sKOR}gfsJ0oAf%cp`VaRF7$(Bw# zwF#&Y{jvGvygezv!arSD++0}8oiANww=mkur;0fWxj0M?c>>bp08El8#gX)e1YyF= ztK_V!;Q6efBfrv14T9Rk<`$tC&V@4e_r@+b66|+o1ihh%uyweHs8SA5*VQ9Qpr4ZBbyUM zuls{KShmD-?*-;Gk^4=^5^z5=xA=7;if=H~2QjnG{p)@-)6xC448gQDS8YEEJATY< zziW10jlkH~#_gLr=4c}TlWew`^k~1w3HE)U93i~J&phSXOtz@b@;8gI} zreG!w^z(hJB%dWna2AqoS(}D>sGjF7*Rf8&6dp9jcetE}L!^vHS}&hR-8>6A_sl~D z#6Ah>$~SOl-3VkcBBw@pFxBKOUUME~cL(3S?VaI~b#?ebp;mSKmr=MU>86-z@jSZbecxwN?IsHg zW@2s`_d4B+!^f8kHSO~z3huk^DMcAU{QX&!Tg=BaCS(EvqiRy-9bNIy0zq?bu64-G z7*U6)g{C0ZR$zCf4D3FFr;=Y_ZWF&S{si7hPt1SI;tHX>kG)A4h=oM~DK-Ev#r@@D zD+^h?X!_ceiMYZ_jEc40V8t}4Hznop{6RcxNWS4$D+Vv$dpoQg`>K5qMjxdwCr^qh z7f+8v>o?QZM7AzJE2_jjQ1WH8hzb@;sj!ZEjXA2XP8=3E;hN*45KjG?n{ALU+61@drKMqIJ&IcI+1lC0)>_$FvW+ zZ}Bc~)5S=;u^4jJlr_m=@U78F5cB4Z{3n*<)||{^I8c7d-;_Jt&+xSNJ}}oLq^P5| zT9;#1%!i=>GN1bLY|`7140%0eWyAY-v3|>rXAj$A}K>-y<8K_Uw7^z9yT}|)M(UQ1J%_xPC7U_l4niP)wMRTW% z!1WGm7Vf<|WcGH5U3m!$_min(t0hD^>Sh^|QpxnUDEr_$D&?1ZTMd$j%te(+Ojg{| zeD->lGsyXSY6RfoVVBxg7m0OF6FdT@IOf5Uc_yqN`|oljBcfsE1l3E5qfDpKKGqJ9wuTLm4d zQ0Ay1s3IRKI_u9kL9o?ALsPehQS7T#ssO7f=Pv7gC?7rM2?zG9jTX(DR7%VhP4p4& z^lfy5N}-x#+#tllcgau!c`iMM#a>RUM@H^6%o164^StPR`z< zgqn3TQw|q>*Z|~T7FXzH!p`5QNP9k7)pyOWl}H)I+Ir1Kv`Kyk#|#XVmYry)3Jq3i z_|kCoGQOS`tB_n!;8e2ibzj0+pJKlLk?$+xJcCn>&OK>g-`zdQB)4XCwc^SOq%L5x zTiSfXT$$!i3Oix0>J9Xg^-0*p{^opR24V0f4FLj28cTWpvXc+dFxB?X-Tq1bXM(oz zwW980>Fa9VY@Q##0;4o}S=uw>mTA-ic;-nSyf1?kacAYKoq{)u)g33@d@DAtO8bwFsoA$= z_{s9)uD49}?YCo+(IYdgV(_q>f_PyHxhv!neuzp0Y6wSCJT;5U+!kh(m{ua#rnqcx~7J*y~^(pAWd)v%x> z6e=oXxWV3KAZ}@@zC_91-r;L;ara`sF1mTeyuRr6hc^805{!gtsqE}{FTmM=-wGb* z=m^yK(jfBzx|*3z5NzPLH`@YLrUxn8InY9QHv@5L{>#U)a0|P%-J#l;8Z#Z8y37ya z)DCCe$Xkd@=g;5{oBVs#`^O9ZurX_d)cdK{#&4$I4yV-?6P}XHP44 z8=Y|NrdB1h%kMl=I)$iiX{1#$aS3L{oRyYCY$a0?f)Y>qVLCE~(6%qAK;-M=cS~P` z)jZ=-UG-y_ow16Ccdw*`$OAJ}(O@+Vh0oiLdYd<%J-a=fM^**)Hpydc`A0MKJn6qbl`HZXkuC_j{$q)~+ z7f6)3;wNlsb@>3IaD~GukW)aO0*BPiAT(Ku9ND~qkYh!%fmwhCGki;x%T%!t^2^M` zR(xP+)znALgx%UdUVgRi5o|m!l4eNIXPJea5%)sWazxYjY947hXj0zJ^t`?t2;xxN z;byTmwX#l%^GRPutuqM`#IRunSMo9mSj~GdbN*5rC;4zB{iE|%@4^09an+lZP{Dnqe>1YZV;W82xpnxe?3-=c zLY6V2)l#GXu6I0g%?yW?T>sibG#ozKx>1xq%Wh4~gHhP?!c#XHgdB3`>{KrO_M8v2 zyde{@v)KLubab2ecyZWkR`@3c5m}N=wYVuG_WTYII_%sX?$HEWn&a|6uT1j~x_ovC z(}+*bgMUmnGQ$UGce(ggFM7OTi|JVtZrf1QXIZM&L8Hz?7nNQcNBPcs_ZsG04coj~ zF%UDa!U)B%UY-gA9uSiLo%`Xs7mXD2hmw%BiyfAxJnye>e1`C}p(W3Z8q(>o4XUV` z>DJMVc885dck*;jbtVh^-c3VIl+BG|9|O1{YxYsKXOp>77+7fCjmce!L4lpIP8#E2 z>zJ~TC%KW>8pO^k;Z_(Uy$_o2ut;a^=~zGEt{%-5_Z}|M5K-pAO&Xn)+1;~>@Rrg+ zm6S-5s!dg!yZG+of_OBr*ED<+w{^XQg_Ms01xiPc>MqtZ@vTJ(4QBMmUf^Z7`EqKJ znI=3<&_Q@;T?5lPBC^IlH8H_DZ`i)ca=D@aP36R*y>=X9&43)c-SG@oeWDwc&&=>1 z*s1GY?)W0mvKPH_GQ0%|oF(xI;$yjw>d2TZInHzDM32ebLnx}bHv>2B`d&-NrMZUs zXPhe*7hW4mOrU?!^bb{A2dWYxzdc7XIXpuor#QEcjTGwl8nGqSwn;)xeob>cjWIV6 zTjtCZI9TN2SC`br`Xa~IuW+KvxA(jOm^1%AnU*{Q#RtbLGFXOtE&63d5b<6h-Vs_%BPIo1sOkO_5$vI)QG|nuYjJ(~M6(bk!GHdPD1R4Zv)eY*Z zDcqCQ#0bp;R5by3(lFz#n2(r7kjj1y!|z-N@81OU!V7cBG&q3MHS1qeIvuz9U3AsM zbK0u?Q^a|Bnoj%!=FltM!RkZ*0r98Gq`1}Gb7T9*4RixuqEmPKB%qUk?waQUBn3#0 zR%`?Wu9>GTC-Dd)J%WQ_dXZx-ro8p^(m%p7>zT=Qi5p1e5O z;8#ze-(2E7Mfe+kD=&ZY9K(bvo zWqDX%n7xNoT+ZS+_07bn#ullg-ofo^4Br~2S5&Aw2HM+ZTs7}wmrinL737C4m%_!2 zI^M^&VK<331Co(}X4KLBt)F-AJvha0JwaU>x);XKy)f*Lg_&P4Nc|LtvL7#f!k#8-tW-Eb%OzBd<`RRAA$%Ce$R_HAk@ir6Rk__FxeTVN30-rkqz5Njdx$q8y zT$+^K2}rn}TfGK9AO@!GAP?9kcf~G8nY@)_GtYmh zmaHbST}vV96dUO9xX20w``&!IPCL)M#VkBGsJwXy1`%HJIr}Pp>L;5?gA#ngVZ{uO z3;Kf0@$PJY-FRr5@QPPrce3z}7s4(rJeL$edK4C~jfHXf`8j{_wH6;|nu~Nt^lU@RtQi z0mcP=y_CVK{#QF%6w+@U6a_}ot_a5)}sit`)l z;pG%f%3UNl)&`hMn@IyNhF>^}jgxq3)`aUA)0zrC4ruOOU}gIV^t7b*%S$hxk8oFh zs6$INv@Rf(n_HT94Om;k0YxVO^YzXfUY#>CJFY@hyKXHYfJ z)60sMLcMK2J2Yhpyqd{6y~_FJbD+0zv*O$gHsga3Y!W~CrpM%q$mXM!_yI}zLg<2S z+J5jY7k*R3x_dY6XaO0u>a-RQS>mE%YG-L}I!rs+Ev0b~P-^$eb%`8I2uj@)Y_?9O z;Y#gWwrQnY=GWUK`fAPl`=~?w!q#(z5fKq7t>(4Go+~>3{fI@5^=CY6)Nm0+ zpsqL0^g~COYg;tcclmYQ@TS^ zDd=><&8RJ)JZ8DNDM#E&TJ$}%quAkCvbrMOB+n?9_>NAfqI+ph60Qhx#PS~Eg8s00 z#OqtY+9U1CiDOPoW4NMB;=uq*$Z9nh2iHDekbq);-x9^Hz5e;J5SjQIVzzp7N$;e2 zJfK^*L+WCL()Ujd%{?V~%*E>%Ntup-F;yef{$8zf5B1Jcc?lpgr!DtoS=0-t1>u|i|Cj)sP^&U%2URa zK3=Y5+0d(@OmitE9_TvPy$*X+lh-I4t?R_Y%V`8}!uTjAOycZkwADUt4ud4dT305U}61^7wlZH%8 zW-PZ!{>oq4@Qt8*3}#~oX?d0p=kq`5DDJ-eI6FHejJ-;$fGwo#C;1_u8Yw1uXw5O` zvEDp*InQll=k?FBwTO&Iwu4038z+}8gAC)<=^5zuRam1U^Jx5&ej0%_ayB7hr=wZ@ zKkUvT+l#p=9mQ9EqL(vDC#NbM%t*=ahFFiDYq#-b@NQrncY3-TBM$<Qlm ze9xk&PByAllLCBx$a3|Q7V>~+;Q9}R@6zori?@VL zex-P$(lNqx)tG4IO{-D3JqhVFU6g@*e|DGTHeeyEi&d)~52m|pvbadzf-kXp`w;b% zXvBH9V*8H^M{d>A9ev=P5kH)ebz9e<0oFkj{N#_6 zXi!MAaFJv|zRf!jbI4xj;`794)Xto|M!q42G=(x^GU_|88W0nWN=J#CqDm-xuqr!| zB1E*|)0{4e zxR(8^bu0EDw>Ht(%XMKa_+kuSGj0{Tw!AdGmDcuyk~twEjDY0OEMMgQ>=(U!?oTH~ zx46|mzBE}}T4G3&2@bpKt&xYGKg1!kGgDi=5gUC!@lHPtx59FN&F3y%j%Z9Yy%v`6 zA2)vvWNAOd4D(3P3F8n?-kik0*4-MyBWDx9>>1wiq2@CCBzl@gQa+?t?(ibL19!0s zwwTR*wffWiFk}{<5eCIAw?AmHlE$1&nD(yJP;CWwEKRhVkgx@c-XpqR)lNyFS_xab zc&VeqnHRJv7l2#}P`x_idG{oP)9pPdT&M}%R#Xx71DVIyCuot)_cQp1pY4Y-^LKYc zm6@lEV;1rOP15xyDF=7CLl37;nnipihwI#2P$Y~ku6d2KN%EVsW6$o@gQ4h9pZJAp z?9%7sLnP;?Y~;$C-`dV8`=1GzRL}6TuGNyDNm1a|BckwkKQ`sU5!A?@AePvsY6kpn zQDrcF;##ealCBqT-&{@4c94en@7_s}m=)SsT6xuA%5KMuboPuAJc2Ki$HzmbLz^R9ME zv)4o@TBvA_6Nf9{Q(`{^$k^R{ikDx1b-N`EjF);8Y8p6ykiNp?@=@yntnsF<4|yIN zpVk&D{+T(88%{*_bv&N0$nN*b$#MfmGhgtFIB`ppM1rjf{#PuCiWB1cQm;t&!cDy% zd=8BNK|>TCbwk>nfP>h-c0r#0K4Xz2-}2XJ>c)FjYm8mYbWqQhf!I^Qi8sb~k%<)urAO;Y zT<{QI`zxW0?2Y?|-aZV~YIwL2F00&)M1Hpm;iaYD<@7p6MztbREa@&$ zN(OmE(q>_@P=5w-%t7FY*?gq40>#5-Qx}1oToHuTb2NHI=ig zCGljmHeS-75lCpkw&7}e5S#pK{dknVuRAPJ<>3yQ}Pe8tHvvkf~1i3 z$UC^EJE$#ag#BI-J+=8bcjZamI~20oKW~6n1H+l;e}3=9!HINEB%|CFi3<5|G(@eZ zIUN&ySf?*>fr!2dq!dgP{yt6u?iqwaN!@8GdlBPe_XijnZ|`O;1c+}!HG+tn%)#D! z1C!^UUv+nLBx}ftJY*KTJ0NQFFs8|@oiSeO18r5ys_qAdjf0zpb2OuE+(F46-!$zYRSZP+!WkO1mo5wQ_~zu4_OITJU$cyYZv)XCHz)JA-@YKuX|Bg}-IZsQ zDuDPHak2bCdWWbpM~YZVJB~ZG6uKZ@_Rb7DqtjnE@Y)@bD@K6fJj2l9*1etl=asLw z)ZE3wM45YzZcdH`7X)3bbA!S{2#wdCy}|7|3QRX++_V_-Se~Ox1H5r=UMzi%~jsLL=KIj$NAk-m-gcg zz-&DHDk|r67h}RSERB9`b#ptt9?N$$+~eh0a?mW^pycao`<2Xti#q+T34@uQr{w0C z`A6G#QtG0oeu>RSixB*{Ow{@`;=`&dn)oT?HYI~=Tun=+1?bzp1|x<)!0E^H0p+M( zdM?8Iuap~c)uUCfUur4vMdmG06*>H#{Q`JaMoI2PZXX(B^0|}t^`L4k;i}inXN2Z}-MTV>g zFAAmQ&=s=bgY6stsCZMK%PfkQZLB~4WBWv%L350Ta?*GVf!C7LSCpiqB2&N>w&5qc zt=aP2XKY||-kp#zlSjpb3fpSkqnP1Cs1TT(!(ci1^Xco(@wd@3({&&L{3$h&M_eFL zJi^L>O+_zEjRKbD@<(m%>okv3wgfvvYjq(LFB-2Q;yGpkSvd>-Llk1*&uo!PaaAw9 z%SczGyMO+d*AGo*xZWW#+>KZ-law%iEOR9`~Z${Mmr$LK6)I#ves!KCU zNY8t=W_?%!DxqOWcqRTDeyM)L<`o+zxqMiuWBmqb{*QT9d%S;6XFcuhQtdBKIe8z; zB=!yo3Id@j4gy6I_;gRXdJ|;Z`w*28=L6m@OTV3I!nL$MyG%}k+kXgT zN#vSVy{$c-rh&ix)Ycl!*Ij%5D1E~rk2{A_sqwR5n8F`k@+I29D-St|#1V#JW+nCw z8}}im8=6bgW@#W3>y%nketr=)n=(a$ZGDiR%%l5Xwt}0&=&x#H$ApDF92tIQHGX9j z#`2Cdp<=e4uOKUqd zy6DP2C3{jhCfAtB!8@}^g;l(i4awMcswLtpok+Nhjpas-3KXB8k8tW}YToUl^n;5J zde7lckcCaA2~cbr25~%&{EmNr9(nC~Ul;SkMmz5Oj3fkYlX!XK!JdaKdrKUsc4r8`P4%GCg}L6K@Via5VU+~K6x#O$sty5s=6c+%zE}b2}G58viK*{ZeOp^&um% z??mJJqDT52cj+I`S5k}VbDP*-cCi4L?L7T;es?n!tnk-`yvft6m z?wh;=kZQ2J_1$9MHZq=RIu&Bg4a;JyiGiv_SA1qi?gw|QJ3Gl%n10h^MfJ6%tzF&j zzE&^EE}^2F8y>%HvGLiy?efBMzhlpKzjpe!&qXHr^Pt7&iFd=!x7QH){z&cprg5JD zX$vhnIb-$VT7Om(W!LTmrZ?7&jp&Hf6+^vKyNfKF6{?X_`Lwsl+LN~hhE=SPJPHp; zy+Et?J!u0vy$7K^uTtKhfAMQOI#Ax1_$r<@TmPp?L-l{q^c8+hhuzzBcXuOQ(m6U6 zP!v!aq@=q?gS0eC3?)RQV>F|?yI~;GT?5AU@;uM){R{T_-22|weXcm?a5dZ8Ou7VG zJEX|L_ntU~amqE&JPF52@rtn2->HlIDkV&6P=LmicjepM>?telNfEFgS>WFLgKQa; z&<_NAJjV_P|DxsboPA>hLF6>4-K+VkI@Na-T6+mk?mh#){^M|+6QNT{ag z`br*2Ll$*$%6VY@a^B}fg=QvZVPFlPlcJ8PQ=Kl*l${zV#zj^}6NBIMe0XO&Ib#kuh>?nZM2l~v(5P*@P{|Jd5$ajxQ=v)h5XDa9cfC8dA{8 zNW1D?X+ps4_U4Eaa+5Y1V2R8HuC~iyj`7@pV+qc|geDJ~%Qhdc#Ewt_97_qLj>=GUU z6ny1NbDT(ltWULacAgg*VPSdHKRMV?Kc@vEoR|H!4CUQVNZCbx>hb78j`$d4>SP61 zLZv2_?QsrscgpxDsCTK28lg95<@$MCmCh|Cw9}#yA=pv+Bn{`?GZ-D2Ix@=>J$mqT zMz}OpC>c%QC@rlmFqkR~V@vtcbdMB8a&GI5lG2>ZLvv#Kc}#sq==p`%#Z!%n??4oz z#t-C|`irw5i@^YUhA;E@r`_$Cqc#17=35Dt`%?(1y|qD7!>qkv*6WhJVbr#}UNXAG zz^C=kD=~@aL(J(&yljgDGX6Ex{fI49V^b%sT+zqUxm`}@+01!R{E_COMOf}jm4PSG zl7QULV9!71GM9?(po`Bn;;Ltw@PfaCBNK}H91LuQO<2K(zH2`uH~b%H^1O!IV0XPB z=uZQ`esn)1+F3h*dDoVe!DmS#N1}*mFb~GD9y1XXrL`p#cd>d?g8G%EvpA@-wKcOU z?)vEpkTz@m7|*Eg>$B@k?*v-BVBZ@Au#}XP+{|WTg+1T0mXgLUfC$iaGzx3DxPf;E z;yNUU>7%+ik!8uc@x8Af`9#;04`4DO8KMW&k!7#LFef^FUREji(8C`rh3F5{nT7h{ z?<;X)BH2W1Y$$pR?eccmC@w1qW}y~Ag?q7DW|5)i9ismXV_7QMc_1+R9;Ga2+ANBT z#xj+@@ay#CmL${Uqa?ubh^&__IpTiBG!L?D=&vK!2GPdUdzAcS(Q#r34mJ&%L#oRi2TEf@0R=?anx0;;3FDS> zs%Zz~F8O51#Xt7ZCZ#`r(hR~6SqL`kAh3|t*VLB6;R#ylb|ig#if87Ou>EIxyvjJa zouDP!A9DU@=Wd{S5{}6`k}y3(*6YE=7{(eIln~I)uV?RHM@uB7VHyTC1DyINJ&R7F zYUoRJpAvb~_WqQc}#S>0nq09YcU@)Tp4tBe$=HR5YJv-Ss z*nU4+d?bX_U^&foqb8XXRCaL@?R@lP(v{yc%zO9~(kJ>NEj#5Z)DQ4)>`%||;7JTH zWc2*RPv>IJ$-raNCY^+zum4(FoJQD(G^^TCS;vILD_iU*IOg!Xusb&m<<=xYOtfr` z06($^4W&+v1!MpeveYq4FGAF|Rl7fB-k$W}=^JCBxjJja0Kq2_%HZNraa%9ce_A|z z(Ng(OV2UTF!D*6&q7Qv%I;~AqCcygKU~Xrd+cjR(P}vTrdeHBpmO) zTQ1LcO@cXtv1BU;ZeVnWA@Kca>jwU9+rxZ?SnAjV35i`B@1vqAY^f z;r$xl8c&LH>Wjx&d@!k`tK)mVk2q+)LZiJ@9r(2|GJc~DA5Bc~*4uV%<_gJ3uiOwL zRAWE0kt~%UEA`EI?(Q;_toMT?MNM#5POCX_E;R;O15_?9qQbR~id|r7iau9dT2vN zG;>*UZ}pBMN}pNB>a~F~2QICq&s4n_FF%^o?}CqI6EU~kU0bL3+oqGo^NEy69ux-o z|6*14KTs$o8n!nmgexCfdTCTw&8JbZbzV9BZqrcY(&{mg zSE(~-6GG7nljp#+4ExOYN<^`XtOj)Ik}L4^md$}j%JAFyEFtMJ?&vuY0~Slamgk`f z?#c^QDU61fC_g-pv9w&k#$zRSnSK8e9oiS zhepAS1|*FLgR;Ed^AEMmUMvm^KE8EYi3+8?^)S@{WBhd**B!q9O7%DR73zdcB^JMP z`J<6tXTr)q-c;jHDt%N{4A>1;w=PGG@GrQ230M0ml84m9?YnA4+-IbUItC0r^HjV+ zI!+6R@3mwmwLIQXPSV(AaC4M=Dj#3-Y5KF6#L|~WS{kV`iYM%- zX>}hFKVSLsa=PEZOMTVIce|P1CezJt=F6!lC@#93C_)I?snTd&U1yGNb=dGF>gSuH*B`Zv%z_c^^_)O*0C~xz4s~eNx9%I@~>CdH?aT z%aT)VV`M9@Nf*`~PYiqHk$LC@#=OCf+?keIAR{1_-eA@f{p~jpr?&Ln&gGn#w#(5B z3gT8~#KlD!^+ZQ8zfRNsxy*!+-b;|hsm+KgBGA2uU$i@L7ZPn!WCe-v4k~h!WyBeh zi6sFh0{$gSN1B>bFLDi9XM}gZe#6`@2MUY<^!*oAc8AZdMUPz1^)?l`++T=*3ohwd${BO#rWC@06;)DKyPKr8 z>c{&$NZpJwG@Em>#G6#7erk0;D|EsrAvsRw!io^ZODko(tY2$#f*qyV!(yR2YCn!I!fsWLf%D76|^`C^sZKBbd(I2 zD1ntAgSp(@C9gJjQI?4c&b4OrZFcKEk9D%Xoxg9{JYq>Hd|VR#GFeTlEB0OCUqynb z5^!LHC=!-r*Z4Iz7GD%FC~;w!2d0?6gQE~JiG>(I%TWbfSX7XZh1tl;BshbwhEo77 zzc~zU(N*_OSsA!|K-`GBIx*?jtIF`IUf1qx3 zbSD3z_~4^YCFkqj=)2|H*;#e8u+ugb`_c0No%??n;wW=7Gb37#)RxbF34_7jdCIR^ z%8$S~Z&qxR0$K#M*u#V+Cx9+$6GNFJ5og#De6&JO!puIKl->PaYCqCD&%I~)7txl* zzlNKrP$K6Q?kGq0;dgsunZ$^TQ4x@QGJeM1=vMZi@Vtee^ONE)G2)l#^-+2l_>nO~ zRR?xw(G8c0U9uW$>)RzXBPw$BrJp4l z{548eX19Tny0tt&6TNKd(>bx@(d-CM+zU}`Ym}Z2o4+OO1a%Y)qH5*__h!bf7vj!w z66+-1#HZps-@f+~(?m!&d@nV`B!ss8VxZQe*L};iSWXPg+6q@fjOv@M>CtxS0gZM; zUT4l?$=c3#gG8Cn+fa}>9L}|jPOtBEbS2+;%PTNtDyQ%4c>N0}m2{bqBxFZ07nHW0|c@rko0+k%AJU>Sc&j24K+!7wI@ zA(fd3j<&SY{PZY3mQkjY?D4S+ymcHNqFaykQorAc)0rRNp2a+KgpbmKgeZ*-4W2qc&7D`8xml>PD9k7$rh@d|>9cEn z406BB2mqUxG1UqG6bX@|9(3@l*GG8pQ9MK*qy<~7J(&Db>e)q@@Faqt>v<0mFneD5 zs52&Xh*|DMK7T)TgoBkIQDt-RB)UyfjH7vpt9P9IP>=*&jSuM}$UMO|Ovik7Ct%X`)Z+>N&RXw=2U6;7Zf5N4JcfE{?i*_%Bu80**fN>w!b>SiaKJ~JG!c73^wA_D0gGmp|KO01m z)(OzUUGRsP-Fh<73g!=HONY|qK9(($%%s~&W`4&DOa!XCwF#(=U;NYPN?U>bp`~>* z*^?%sMJ_msACl@5+Id#}R&mM`+tjn3JWHiN0lWh32Q!{GBd!6TR(Fm>wuCZn3yS_z zhw{y!=s&+%gP%SrN+BBvmQbhSK87!5=@_z0%3u0Itt` zuMmY{H@gi=vyoCyf>Uvk7#yQ(?``C_nBfb7`g=@xFTZWL!K|z#VD=T{@cQ8M$=eUV z;&y%#CGN@!IyPp>`fL}JjRk!w>IR^HJ13?mKspm5fp(r14H7|QB;A&<5xthFAygW+ zwx*K?k1TJYhExfQ9>CO= zd7)6(EXRI*q|%2Ni0Bf2^rtKfFbuzDo_U_jp7~z|q+9OWabyOG2x6a|45$bfcm4*8 ztku)*vvKD~X4tYU=0}9bXovNW)?jeITL^0@O}-mO7Nlu|T$?O5ZcRelt}3qIE6G#W zxuTl|aeiDpFyo%R`FMd+2>uV;K8R?dD?Y>R+S`e4h&M@EXA)s_XkG7%z-Y%X6@(3a z%dCj7+!cTto^x(}5`ZZR1woTnWM`k%*^ATTd?Cm#2m;~V$`4Y2Rt|dK_Fiv+9Gj=6 zAc)(n(3sUXq>nbe*vLmqpGGt(Xm7s!;VsNU1{X;$% zIEjEopdj?3m#FHo=fo|TVlc+V5xe`npab!`XWr9x92;s#ml$yu4psihUU^!qll+5) z8BQ|o4Q8?EQ)^ciEt;5j;{s=8vo6)cZMN)vageG94Gf{V79Xm7Zh~yuo~$FG%wq7z zmiSdEiG-E00TK0o7~kfHI@~^`A5AHmtP5TKiHe=jc5c{V_x2}1ZE-Fa5L~TnG)DSl zRCJNWk-Wu8;9!(^DS-TXyI6aBRjOm`@MrMr&P#s34b<=!U{iCXlskWG{*LJTZK}-O z^=C+7ipSG__Qoher4O(P4zQdtiYW`nL(#47dKZ9M5s?zzkc{WSA85jqUq+(d@wQdh zT-Nhr-gE5l91T+7$}{+F6+6c|6jZK7{NP8vB1H<7Fx~TbxR6L>G*;zS5zXKHr9V2m zJwm$PpAoVL-6x9Q|D$z$@5oE-s1m94pTZ#hzXFl;3-lITWL>si?g9~l$K6^+C=)D6 z3|^gl6&pG--+!4sjyddg+B*JwqC_&t<$Q7<0}(KK4+Wi=Kq}GN9*>q2rgyry0$hy0 zSsFe`Jgpj?>tWdK`Ot!ipEpMRZTgrF2|n6blW!VxdJl)wwmv;%O?MoaQsl~(NXVjG z|8SV@d_aw8mZVw88If5s6rR#Mah`2^tnZRH=N`iS-IW!F3mcSxpQy<+4{c0vrE955 z;bv;*e1fgKZy{R69tyj*FX|yuFtSvs?9+F#s**Q*nK^9I=e9b$aoiu3iK~n@XXll* z8J3GnGqP^fiiqjA03V9>iDhm@hJiTKf>a&N3%(%jFRtASn%UBPl|O;01tA!pmhrw; zBBgqTdKJKAlvH78=O37sigs21S^E$UCw9x3eMG@>pfzd6o_)xFH7i|e>Y;PX|5#sE zgx}0Z4W5oZGcyE7J8x>VVLClq>{(*}Eyx5~6bFxFp18Lx+74)+;nb&v0MLBi4d@N$ zlwRW0w`1>Voj6Kuy?Xh4xIF-sd|k{<(@|OEePlV`tbo(wkzwA*D_CsBIq?T5$o?iR zi`CfJO6oiXvf6oK7a2-R^0|K0uQDF=Z1-5mN|vks>4?I#^t`%@H1o6QeI#5~$8DHg zY(NXpmF3W`+s$@R~eI0-gLk1r0B$mlx zQCzpYrgv;=O}$+dgR@fQ+0APT+2^oj%XnIyTUTDL@vcjS@x~N;C4U+NoDR72D^5_* ze_aSgL-0s$h1uV8)xmjT_Fwv#-lQJ)l8GQ40=ZL~ttfb`C<`d`^*-JEtO1uZvpY@L zIxVz6g-JaBbw0%ZwCnRev5W{Ty+zjn6DZ@#W~MEoO-6n%uoDiQcAW4HK>9WC1bgzL6~!natXMvHD4uq9+(&@ak`FD2@6X4mPZbt%!Gkh1NpZmDtmDAFBB5hU zHu8t1q>6Lh^*-|!t`{`Fzc06JhL*+H@+5P{H*x1|D`S##ZzV_MEu^Ao)Zx!u&^_8vwlAEhW0Dl zdt2McP`g&y4C4Xe4x)-2EJ^$tYU*-P7#H%R!YE+X@HV*Ta#BAllKl2OCa znS1c3wt^8ee5)$KblJ*LGxWvcZSAP zn}z+in5X-1zNhrLGCyZAh&`Xg7JW8Szo`#M;qK&|xE5 z2D4C+1oS6l$bawl0&Kb=%x$aHZh@zDNucUy{P|pOk4hDgn9Xm|VF-u!mS+2?4JJ<#1rq}_L9byL=MRDPh%ol-xJe(ufqrTW?h^d*B# zE7~LRpDwooPy)9Ja+-J5S|kr_py#~Fm11d*7aHoqIzTMY+s!_rzna%&;&oJ8UQ3wG zAWpESAc6jv$gN>BhPGD#n{Q_NLIN|H%j;7~H`;=C1cpyheoyO)BsYRCiR{wM%QK6} zd@+JlcVExDe8EBu-6rS_%{!E~djLgCb5UZU9nv1k>4<{P;0#-TnI`JL{V1H=PTA`| zUkkxR5#xm0=m0RTYzH>KHlTc>;kF)pS<;w>_8?NG!`)=jtBVt6&yxUv(v1oB@WjZH zk&IM8v^)fGvHCU*vKq`j39F9&a?U(6LRBQ8xwU@H_Uk&b4Q18oK50U48tWV&og9|TqJGe%|C!1$Nc67BM1Ls}vwAiV6AJ5oex^WGk0vc#5d8B2nDhQ= zSKJNuO$uky?KaiGH4n#>{Z3UA^>!X^hj6JMDsMt0&6{trCl2N*@i^wWA0u1l1<*+3 zB|JF8CDGf61(iT)EWV7!VGCf2kJh-ywQ3?nE9DGE?O+_{F((pBx3Mr~)#o_u%+w|{dxy_i1Sg2EnmLHOMlCoc$iyN|`kbHMGmgmDzU z491C*)RI0>kIHSL$UK+4eAJ_rvY5!N=8vO_ z&(;DugOTPZdyL?$(1ZEy6#P$n`W-aqgP)Gs;3|R3N%+X)aeSobY3Z!}%i`ZwA zITOpW!ZkstRP%i`c4~a)PDL&QE><*IAZqn9SB!^p>mht>k3rdl=~%_B+o$;RzfJQF zaa^?x;Z@Q9q!!*a{jgVxy82q^C-Owd(Wfv#Mn%N@K4zvzSdgpBn1-;s_Th>P@pX$p zP*L>h#65ZF_2mx^fyR8PR@4J>feNw6Z9m0k7{TxWg4zUn5e_w}bpER3hh0?Wr!}uHPsv`i&4hx+{yx{lBmdVYoOhig|PP?MYF1m94=V@;I zzse^)l_<0o1$oT>wMyq)h%l{!5MeujdQ5=$E%dYZ8!lHX8q$;4VW`NA9xl4l3;3mS z7#m9T>MewEnXwD7s!5C{Boh_mHu+5UJ=AV{AsInT0=D}v zyKEX~D5u<$Ipu}xphp9M&-Lc&^;(Oq3`JyXBrt&1?p1EA*Q6|gq^Vz3j>6NUn$&&` zfSqF7sQy-j@V+&XX zj3kQiA9!d%41GeBI9th*)K>o*+n~e0#eWLm=frp#y;!2w2`d`mbXX3KFzSCpF7gpk zFzi$_IrTPiKQ$%#CTXT)@jM*B;?gzXolMzM@@f>YS1mcjzEd`QhD!P<gl913iw?j?R+%WLoApsXladx)Suy+VbjUH5GfKyU+2 z-$xM7#;s;+QViH5qx|o%U z8f)jee?0sEtw$d?*pMHBx#N9b3lC}iE04g)_aVf){2xCUhBBW_Cf?c4Uu5wja7)I)anv0^a$ zRu-X)rA`kMtF}>8O#0_<_-~VWI#4P_qzjkoUG*l?4S~#rz;RE_b6L_H3@|LXq zej&SgH^$4OyCt^mYDu8B^(^$=wHiWu^5jDkiTrPl4t^=Fk)MMd{o z!07q$PWASVUa8ggpdSkW5Z}`S8<0)cCFmMwAh1XzhgC64z6ujq4b$XMWT~dY|FhcH zu7yv(e@{vw*p6@56i*r4S;G_%{ia=OSJ~^3L}0WR0x%B`cwbp_uq+btI>Di?Xim5K zbDammz^mqniqqf6SKd(E#Q$IJdsr(QFIk`%iQLB;?wB*fH~DW?8`cTaF&GFaqc2Wo z)W(w|lNiorRFJcpsgE#^VY&3L)EV}DCSrp-t=~V#+FCK-Lw}nW; zDHEss9vFvIq{vDiIIpR)CcQDXy|ut`QH*R#8Z1hC(sBP{qWs;q{NtT*hj0O^$DQxA z`!05{1(43Ak8{P2A$%(Pvz0UfeN;zBt2N#M@`grS`+hi>Z*(%m(Wv6_(Y@VXs7hGk z^GsuD>L~MR|@)iTkDN+n#`+W!mOIoZIHVd53)PII zaYn6a6-j<*mGbF%U7HxIHv?tPel5lo)?BvWxK;yZ?2^^bt@hS@g; zpVI$Mn4i5|*_6u1{|M%+#Wjt0R*LZhEi5Hw3_}H)C5Z7ijJQaN|F>o!{1-SK* zg$Qlfd5w{6NvV!f#1eX(emwA#VqtK0Wbzq02zs-GYB>^-9(14AC|X#hR@z9wuF{zu z5=-aG8@S&_B^wc1>;Lv+{Zs=?v&)^9!TrPr_8ulDCKvcpKL{L@}fWe zrIf-ZwH4}`io%OkaoAwZ9|0;2xzvpC+W?Cea@`}m9Q8Jz7R_wl)a)I{RFZ5TJ%YNM z@_^5Y;M7-I)KhXY5-|W=87wS#w;~E4YN_X}A^o<$j&s#=a(yuHQA3upm^6apWQVcI zJj!Lz`CDyqlI`dp_ovr-+v$_Pc4iol!S*X|) z1(u=MivwUPRWRvi{KMa`fxyq=;XJNa=w*(zHRZKh#@QDVuuexuZI)Lj%a;#Ptwb%1 zY%~n)+*+d-J_`8-HVH?z|2_aE=4S#UN}V;R7diV_G9%~ST`jvNj{2ZZu+mnEgW-!l z*bAVP)YM@04ZAc%|ZN6qNN3wZG9JvOoVF1CI-sbaGMcmyVx)R z_i2vLNdtmMt&e9UnP(Ropvl$cwPt5>d2)3GtMAnN!FD7lKvzTg4U;uhxvVhJMQVYj zplXujL+XJAPNi-Xd-39P_zaD?1ygni&Ppv5^97iXXnXd7JX3NEWS(Ihg7& z#q9m5os#*E1ob3de-5uVQ{iXE8xGmcB|&$WyhGD{hskZk?8Q64%6BzOkUrH~B(juk{O}Vub#}(DIFy#60WHuSUL*~Og8Csn57y|mK zVaIJ)s#g&&U$FE-?0_-F?u)zR?$^Lx(c~yBr3hAnQiS+i4JWWmZ66Hh!@8Xz8uZ<4`pY~S zKVh~aIk4l>@Fe!D0tw7~0Y^#U77&N_<}GEbULot@w|%sq=r< z=A0T#gE1T$seRFP1|IY^$!gV+Bt>zGE74cXuSYu^dhQ`&!!Zoz6fMu zz@4R+$+96c4fe{SOoeES#7o?VW7V%;0E4>w;j?GoB6odo|B|<-pGC|- z#mXx1BkcwU#cND3_D<&Uu@iX*l_X#HuVq&D2nLu;_ZEWXxPbh}{^VT6d-G3EXEuvr z=Um_YM*s4zNtdX1y|ZQEu6VkoGyY=G#M4_ZdQ_WN^_vSQ%|gg_wgMrscUvXA1DZwS zZOJGxH{k7mdF$NO*$_^AA(D9_v7JEXLQVM;KkR}H^fZv z9bFCf?x>R{+K5~s!21lu-_dVUe~;-&*$Y|IW&O`nDxVhSBzAJp2X{LE=#jg-O#Gae zI&YVa!X#^}T9gng(g+X{RPeE2=s@}toURJExY3w~+&rG+=zSWQFlB1cGw)uTs%od*-5n?CWm% z?eo&>7hVS4lcri|5&V_bSiv5^e(U2?M=OYx?VPz^@6d)CcX80q@UShvQTmG&zdsin z$1>BWBK+B`JN!%u-J8ivXuHv@jsZs(M2-PENJGgN7t_a|+h4m7Nva1k*F5^!Dv@L} zN~4i}4s}I3(02anPGE@z754887o!{zRO4#=6e#HOuo6*hEPAi?_b|#6(H9~U3Ylg5u6jVdUcyd<8s1pkQ!hthe1HGZN zDHrzbgmfy(Q#DbIUT-*(v)$IQ$<>D+eFos~VVho?(4*gc)7nP@LJ8J)7h>)jigons zMab4e;S35@kEc)t4L04u?yM+p=P{r3EN$T)C0@_3#P>fk(a{6aI18z`Bt_I5DE)>E z)>f6EFUCCxm*1PM%~2Mu`r>Rej|IdobEJcSk695w~`B@FS~nb%vu|i^-Vv8$XsTNDM1Zooj{6J1!Yhm zzh3?q%p`0f-Lo(-LcZJ4oaBX>p;dt!wrvhsdIzEZ3v{QAY!r$71ITlO>_Uvz{|x<^ zl`Rmf05XP%yE}LBIJt9|ROEi$@qP2%cr?t;3jO=9&n_N`${Kz&EmqY)OQwx zvv@fM{~HcyhX6J_))$`|Ma1!FIMona5qfOG6 z#`H^wM%GqiLel_3GB`yXBY`cs1m|h+Nex&FIi0ulKRTjQy+k`*l7uM6IzouotLSx~ z7e`_FrEDb|#B6UJK{5+0kE^t6D$;Iu8g&LNa}~%0Lpp0{k@r7wK|;wts*=Ju!izmH z5W1T`Xs`*!2Yo}>{rB<|maa~1bDGkg`rly#$@@_nW(Y15iTIK2Om;eGQXaAg>O*FX zKu!gZHm2tN`+JO-SE?!!S#$jtTiH&1!O~#YDIIW9NNiT2o)%91bJa1c_Ijq0`sFQ= z3ChCEy`|Go=*cGeuM`35RpXhjeSf^@MT%rzEQtSPON<0__n>^71l%6SLA@(YVH2KE z;CNyzAn9ijF#(s<;}8@*Dtfa61MgRrD`gcBn^16|Gcv(_V99m6S(^=&fpZI#WR%r+ zF%aKuYzJbpVvuHu?|smCqqOA)xjQ_!H6xq%p!YHnurx46g`LK|f(?cy;9Yq<_PSXg zS)scRhy+<@U-WvcTIdbXX0Pky&woY1AcFs4my+;R1+0biYoQ?;!JO_R7R+5ETljJM#(p=ru_qbLG9pMGjniuWv9tjbbQn4zPb*aW4Vy_1bIk3x)B$C&*B@)7CB&m z4GO#IGe?w%0?=iXFSdlMf7SlZ+zQ0N3?L<7UZY3etrmG4cayt6fRK?qv($y{#fqF3 zngGl}=2rYHev3$C+oSmP#Hhs)&yJ1#)S~iWOI@qEF7Rb>w-c6lq9(%(LEU@zZ{~L2djwxxe(XF}2X0bUneDa#4ND zx>|tG{rrn-PJ09D-Rr6sFbBN509Y8vZEWm`hCayMfqh7Sia3VaZ_aTLrqH;*lve4H zxC~^uT@B%@9qSd+9e_S2D{nGg2Ls$OPD4Jk>i2yMeSf=&(&%wa#<1<&UDr0NJ-HmK zP@$&7Xpditok1B}EW;yrdCfNVVJ*x=u&_JHyhO@7TU!pc{T2fs7e`zYnuW~W>VX6m zJko{224fw{xWZJ=P@ko?wV1`tZ_!8v30&fJcPU>608Twsl{$;%G|;$5An)h{2L-9! z{&VYSX3z~Zo1WU}*ptDRKa7TTUh~b@Z#funo(NZ^nTIGWcvbQSyrv3<38_i9oes{ofg?h~5W2Is;4Oemu*DIy|q*{M85;0k(@NMWM{MH$DKDCiHo&~ zC=-d%0{(yNc}@M`EO0xBIn6v5aZ`a5&funqPM54zzvU>6cVEo;$+lhawU2n$f&*A+ zh=LLrGV|H$gFm#hqvoiF%vrJ+12kE#_9ZM;?cEVL{dq#fpypxLBK6MXq(JohDYJph zqNm@V>uTL?876`2Up7z8l8=bEH9*@ar_!Zlk5BfrFf)eSvJ`=>yvv9yle}fbU+z+) ztjR$Xw5r&|Q=)$sps~vS36qTN!c@xUn;lBy**r29SElaPD+^N;q2FH;d+%m`tB5r7 zC#K*a%wRp%6ckZw$DK1Jl+E6I+SCYnsY)i~-31-Li#ATS>Gbr^mq>U!Ug&CxtS-G- zjKM*DGi$h$f<=<2#C&bODofW?Ja;)RKlA-HS}?fcgSV;wedwF~C!f87=HMoCsepzC z|DBxhJw4Z+5U-Exqc;FiljmMm8bp%{sqb%so$X3I_j$?JFqz+Fi!Y#|F?;5joKS4P z;)Hm-^IRvUo+%~oaTiI3UktVMouD=b$qnR-KyN-${TNxJl738o7|8zZ9v|;>4=qwY zad=ZeJ>^o_>b~g_#_A8X*u$3^3cToWJMY2=S^ta3?2(BPNcUZ`V-b|Ha)VB^olThR zl*kJ8gxhcO^Jla13Yv*3C3eb>oUeo;!%##1-ov+ZTbhq*n{dws$g;u%F!cNx?HTK- zZ|o53 zOO6Bw=kDKR=Qp9X)KE=7f#iDZzxYSB&W}0Siz14xJ5|)@_kFalqHCUIhz;gYKLQ1~ zKd;@Sc|>n=D;l})5?qU0*tfdOe`62VvGSz30wOE2PXO-;#r7<7m@ zFyA#r3AEub;3f#x9*M~pbhU7uY7f#ChQ|e(`Y-d96-ziH^JBNyY&3R$EM~K6xxeVL zz-LTIj=j{6La@TKTAzUvp@E zxncPOhFc9P>hV5u$PNCyo9sKwf3a^&>GY{QysLUib1<#wzI=i9Z30qQ#kyeKzwmLD z)%v$08o~@|qofyjA{E&sEBorgFR5Cq{8qrSi0j<-iy{2YZPq~ha_)30Kcxh6>c*Kc zJ<;cz4VJpxf;RtKn7p{T{#?FIVO>cmb=_qp^|VyGWUU|PZ}j-^INy8*tv;Wio-gB% z-vWmoCvsjNcRW0#ywT6lII_)iiOXqV;D=2(`3-&!&2rU4xn%7Vtz0|r>z z2zyE(Ogd*cnD6eiE{JNaGoy{nL2>q*`H_a56%LfWP5gtKNMTfDpaZXYp#2Mk`-17`X!x(-Au-JJLGPLQVLN)bY=*$srCGH4)`>m8> z6^==P5p616k>(fUGt(PAVhcK$z#w8;N3*5)%YvhPpX;ZmPN4lE-w z#&VC;){?6w*CHn&R8EdDbw-ZVWDuS0I#y=UIsayIZx0%I<|G{8+To70J%XeY@5kb* z#FN-)s&1VP{QAfKmWq|zvh*aDl#oq*>S7yvuPB3ME>6~`*J?G(2I4Y;!@ckY7>yE! z!L|)1fD;{Bs$K<)y{brV%I)d}Rrc|;-Kvr%$q=9Ar%Xbs)g#oi?-+HC7#EXzN1;$X zL+H%o??{V}#fm?7Pspo5z>Np>MtAH^=Sk0F8J2jPCIL464|=N`-i8uASbE_yQlTMx ztM1zm`g@h(<|}&ZzhxgNqR-b44Nb4dOyz(N3pBH$%o-$`Z3j=b-c>#C>=m2zhoVE< zH=cK_`F@H-1$*NC5CiP%I+b|I*KkC5HT@w$IU|h--D|H!!}NCLy+U`L!MB2Hv<*qhd0B`Cqjcee|vog%B-H;cS;7K z8Gt9fHNUmruSmC=k46kkVSY#GZd*cSTjlYIQRhR5)YN4 zvgO2VMuH#%u7egl+jnuj;QZurzX-SHzsCG#W*Yg=`+~Ry9H`Al?_GWiIpfmk6>p_v z->M9`v$zcOcc)7WZdz2gGw-5VKUmvMD^#1lPZ+isS9Y?&_uX)z&vQREcUoUC4@&IE z*RzNoMffsz=F1SfL2iG;FfOmAAr_oqF5%$qi^r`(bYGI)&7Vq%v5?;b1u8o35AH%# zTd|`i?#S%-|BtD6jE?jB-*98wX45oI8XFCohHV;SVrydCw$<3i#LhH!W81dQeE+|5 z);ja)q`ym~#|PZh_&W#Ns@f~C7_ABZgm^fo?7H1CT+A>hbNVxCUd z1Pma9E}SDF7rq>H5V{(o$@`1w*h)?F_wpt1-)1rH&}MC&cim`h@T0j0`KFYx0;w@@ zfU4$CT{M(iSL3?#PWkZ3@U8ZHZ{0m-2judT1Km!UQQg>0a|NENz}vXO*yW{(NCO8? zehB@RSSx(TheMJ1`jBZO-^T(5vx7?kst4HbX;72>vN;|ejYROBS3fX^Eo>J#5vV;M zvzS%nmrYl7fwmPhIxJ#-iNwJBn@Ky*pA%ZeJKGJ4{)7VwW6=JnTLgRmee66~yQbcD z-*?fuF^?)tmD@(6t6njPrIz^i$~T3?SlKs`vHU1@5I$`uoFFCDa0Tei;bab7EML}} z6+HiC)0n{@&e9ypW-QDfbe*tfp-+oxA_WNNOWsoEv~6>sB=gaIsQ`fx)KU?TXahn|qO9fEKc=}JZ|_|c!wsoUl<)whc} z;Zt&obP%C9jYWQkwn#H+3%p1sA7;g&Zhv8OF`(OF)oHwI(X(d#@k}+hz#k#yIvmzc z?m>HSw831NQA)W6t^HyuwLAf#cTPDE#C_;0Wg*f`&WRN~sPa=Up%0dq!vEDA?xB;N zc>eCQ^gUa@_=}BHp!)oy{Rj-#ob*u_bK%LqUv**%0A@*a=>K2PEk(fJ_tk^o6P4?3 z^;=NW3fgh$`sJy0ka5RbL@21h^-vnBmP8RO6u%w2HI(ju5lc#xVN{kLVs8*# zVvGX^X1h)!_lOYL&oAeDzrH1#OklGp@Q$fBWZlN?Wa?EMoymYta~QEzT)qw*Gq;h+I8 z7nD*uU7pqau%6j2u%;6_5#0a}W{f3o(zGv=7ghZWoh95v5MCR<-< zSK()oHdi+P1(`-Eo;e|yAhOKUQzi@! z%>;77_t$2R{Y|mTX4T3ABfnxyS7E;eO_0-{fzI#EEC481?VK1xe+}M%+gt71)iUci zY5|G}fRK=yZc%taTS0lhzV;6n9E|o$jQF@hNsR=z5_zrdzY~=Ot0PgCVa#22-5lXA zJhT}4UofAXgt9E$Q{P&^-mV`%d-|%)H#)%v7sBA}{@AX;c1ro)0a;ECqklNeY29R~ zrYu7$lU`w{6T*1B0AGb2vK$u~=}-te4^ z@9_33;L$4x87$9N3L6rH3yAG1!}0r)xmuBUFP!r|EQnjY4dJz8`S+AOS^-hBpFa$C zu_pX}3TiF1Nwj(Sqfda=^+g_$VF+|{)V6NjrT zcgSr;Ppb?@e!BElVjGC!Qc!tzzbeM*O-c}HLhDk{|%&Qhim z75c~yBClU+;jQ1H8p3KygUCx>MEsTc0xI6?nH2dB+#5OO!nFUy@s@R_5GBI9YB_` zj_h84MZS(s@^fTh5_9fGSDV@dbF#5fW91A`fJLH_dbcQU&k5CA#zo)X_(ifBGY}FM z%a;dO+f_H~GGhktyWuC^ZnVFb^2KFU@ry0|#_fdq=U8qt)YVu6t)dl7%Jp|OWtUgJ zZ6PCJ(Zx#?jbxGhh2jNHB(phfIW#Ox&}|lWW6GI?klmdTeYZHF;;;h_8za*qT&xkb z7J`bZwzxqso`!5EE$K^A6EHqNxL73$#e2DJ$hlBF@qh0m`ze z>e{C&xZtFM_{(O~=gW;0IPVfABc0PNi}nYC?51Zm)D0FZ04ZHOQ}hT4D12jM$+ zmlx9cnVuLdVC4YsND^E(A4~j*NLn`wE%f~b`dSkcihE+UAA5`NBe&VEy*>laL^SnR zIx}Y#a;`?}E)>uS)51j+x1Ok(b!q?7DU;G|$s-Ag0kSS!*XHZLmSbHBxx$@eicD_$ ziyd%Nvg?AK$W?#p2LI0bEc2S37*J{bBP?pl3JIz@Msw*8#o5a-52-505-kH7N#?(q zp})1Sj-o!U`ssCT9gRqozTAl2o##L6%wuf_L3@+noyMJq9=ehXwqk`RAC8HS7u;>V zEjLv(w~>e&;G*8lxYJA{Jm|{5B@RGDH$_M|OsBPCXl|3du{57*q+Q3ok!$CpHGpVL z-n!wo^?59NrvPJ7PrjnurjPr>2O54Z-qh`|SM(|M!(OOqyPjs%esVOe+qmfqa1L6K zW<-{d*`lr&D(-H3CDDEza6zy^4D){Rp@^LlIo@a|QppB0)qpcRo*O5@_y)6eQWZV1 z^F;WH^U-m|YmrOtvKq0RNECvLz@zpCOH_R9Pm~6_-97iKN=I4)C?dZ?BbNrOVCIiM%-2XH?_g-`9%! z#1rL#ipH7V99&2zJ)O!MCPaQ_bKoh-Pua&|kVZCu4fbn$hl4?g^$yEj-8hu6`q9%` z0hSqgOM%AuU973mvwVR4TroHABIsaG)9_hb@V1{vJ9+kI3a`5Rb2|(wQ&*6u52^*! z)wPkw)?~?Tk4ee>&|b-1(InLC(Jr*xbqG1|j(8=)JEIANz~&7~LU3L*S(J2d4GNe= ze*d`xJ*!5IBng8p=O-rDE2_~PaYvua)|`=wvuBOLfjIiR&Qam+b(Dku>MLghbfy!t zO}Ltl^O|H7g?1F@Bw6I=^1%InaI8W$p4Qx45mvT7GIK^FpT%TsuF}D=;1kNpKAQ)x zP@I4Y6MXl>brvO172swvT%)!WgD{7WU^bO|YbmuG{4*3oDz+Xg3|v5s55DZ(DnPFj3I#LFR4Cg7d|g;CR^$ISab#hT`@Hh7cJx*oEY*@wYAs<6V(;9YYte1u7Jnr53<$NNGVi5kCK( zv==%hw)q`OOwN>Q9rzu&lWXR%+lacDN+pO2+*QC-yP#%bsJ>1`bU*%zfDNwbpbCT0 zqC`xl;LIbs(CrFbw&}mXOHuVIt;kfNF_5D3?^SI=wl2pAHZawC=Xa(Pv~NBgrIxoF z2#v17OG&*LEAKz$w%OJhud?mezI`b?Yu~B0Y1=9yzU%9o{EaoO@Ks_1*J)x=c%=@i zQ5Bp_4S;qY`Ef1ZI5U<$54@_kSyEJGXko1Td;!RL@K(*#C=SzGCnl=jQyfWgUJy$j zIqxz6qWQ+Q!FA`rpem%Wr*t#2J_M8Q~@RHGs9gy&H=H(%(x5l)oCvCnTi1ttU zcV2>LF?(27FnVown(-b6M#`nLfP>#?qya#K-+mo#LgLvqIfp9^@=m7oin@O9og%?EM7=p~=r#e-w2!HZ2Z7O~Gwfx*^x`(2z!{$gmNDN-qT_}dJD)3IU|ul&Pv zdAUHxd+!3@N8uX>#0`#p>)Sh{S`ANCQJI#vu{LRRTKWgHdkWBB%*z42)$ah0O&(}d z^m{0i2y$co>!oQGIDZzoFw645lAdk7v4Vl_>BM}b?Rs-z)$?`4!C3Y6v<`c|KZ@Cj zS;v)mbKoU&TFkvV*G{||u>x$4&0%0-aXJjZOtGU;Ds#@f*ONXzLho^bu<{QiVA#CWqrWtbl6A6S z8T)-^`IX66@Kqd|iH@Z^PM%5@JsBGvxtvxj60*5-bzS z;4K@$zeq`ZAO7@)Y@TroXUQxH%Zg)&q`fJD*(gj#qsPzc=|Dqa^z*XXc`pvqK8D>j zx>o6ZBw|pINnX8q5OKgf7I;f0YA~;BZSdEJk0AP=Ey0mU-C?4C7$5W*@0eiDnTT1lbs{B*c~r}Uv<1}afdL=2BM0_`@A*&B~pv& z2kfAtK-si{ElIA4lOlrS0s33>lPNoOX5E7(gr_k(<{MoZZA6LPy8f}BS-_TU&XK6b zLmOj{ncG_8=TNSYC5J`kf0d||sMAX&`CqIGtA7lUQthK5+g}KV0AE}B6SKu#Z;IzS z59BCm`DsD(v&GW+mzp9Szoq93NGEACO4zfA*hwSB33KNwT6R@Qp2oAO8`Nx>EEX$4H=h8)CJDK*<4_s@0~l+!$8adf<_ej^+R( zso4L5eOV(NJan$rM^d2JES{uasdsokzmbhz_KO|>v`>}tQGY891j@@UY%4H)(=4Is zci_f-7uQ_4+^A2G&;cHd{zpNgeOM!?ngbdVx$JLH3`N0LNS@LniquAWJk(1B{jQHH zc%Xy?j#jJX-iWnzQ_=AFdy!y!@&m$t_=sQnygLvMIm3NqIZj?0-UsZ#luFYWB&8A< zPUP=>y*K^3N3;e7hx-AXJULuwp^y3Fz2^P01v(Dsw;xuRMB+6tNH)|wuiFb*&XupO zcn;ZjnRU%!$C06I6{PUIMAre%!}b;Nl3`TYz|zDR6(&0L>O%>hv%_3;UA5NkI<+it zVur?k88phR84Bx_;1WjM6+s`7zHGQ=Z< zPgM6oNLwsRgzkD-N0YLj#Il{H*RL%YEgs*;sfPW%@j`j8+VbyxE^7yOG{@`FU~8%E z_Yt(3<7Z(L@OqKStDi*j2{luvw`|7NZ4chSv{jRpFb=N8~0xZ z4wQ$@a+V2tel!6xz`}tqt}pDJtbeA*Gm&6>fJvk;Vjd`obZY-AtazX2_rl;*3l0%S z*tx$ST_Wa^>{F$EE{FA2D&AxXm7R*1$mE`SWB&y3=0=MAqnLtzIc@t*X}}z-n$J)0 zJTs>A{y3Dhh7U@dB+2um0se8aK&AGE_1n}x=?56-k0f$hF~tKqTO>I<5t%;KY(=j) zKk^OIEPhb(v8bqMaM=d;b`>V!c3OXP?=0JdI1ng;v0;Xk<8FeH!fKU&7IL`S^rtu^ z1sh>sw)!{B%L7N;O$X*u(NVY3^6Jt`^ejj(~4pGY3!VcxL5r`lrfbBFU> zs>&Rh5>Oy9DW=5-dp(WkhVQbyoC{$=mPI+6ixj>@gyMIHH%a-ot`Pmt{b0W^`oMl3lETO|GN(GQ0*VxI1-Juu9Y`x4{**|2WBEyl;g#0WAk!hg9Iq@xIksM> z>&JzOj?+7Kc8Xb?(`vZ7(ZJ>V)1lJcN7y1M9O0}x^yw(MbW^kuBE{qoGQA3;2J z_>GA>_b1KTA!XZ8_4xW&;s4<77mbKMN0Bp+0DLzH7wua7-Ogfm=1Hu#ELpyJEPr7e z>kn_o-)S?;z+w&h{^J$ zJ?%H`PD?bG)>C+)rhhv{vFNNLibkv))EY^GJ&f5}y`~KN6X*VCrtDp!d32;baz6j( zzmiI1^CL?mC{r#g$7JI5)u$Ni{Cvd*+S-}+Z)WQIR@Lt)O6%RC8Hp@>dyT`tov{UK zczRoTU>XscSXeYbwZww98@RDToGKl>pUOl)9{e}9Qd~j60>-PesHf5@T0g?ts~$N{ z`-=@1@F9z5LA`7EU-pjyxk!C!=IQgBA{?cGqx&$_2W{IOK5H2IG(&akX7)yvr&5c` zL4%i{p8tO61ApTGK)D|9xvGOd)U4Cjq{UdSV7o@uQeGpH6hT+kFkq#~KL`!q|BYI1 zNm9x@{|CQc9Z)lL0H`W}_oZYKx!0*!#@^B2olHA9QQhno)AF^{4>XRL?g~*uCLhX+#h&xth}?Xfvg&XeQ^{|i$3-YYPiN{ zKXC@D!02*_ay+hSER~i&481;R4A2{Dd1B1@4vDXczamhDswCA_CnX0l+@Nvlzj>ID zHeY1M0==)N8oq>S!L&{L)q^e(RS(iE;cz?&LfCWij7X~-M@G~F!oDU}%KtTN|5tdV z*j-Nh3ElfQzh~7!c-4p7hA$G~CYSO$!z-th?mTbvR(%7;0QO51KF2eWQ?1~ujjsR! z`2EUP15`;84390|uw-b&?d(UAp%q+l{x<0wfT;JF)b&y=o9mh8n!ar7eRI$MP6<}C zw6*!>Y(wjzo?;+&asKsaLTi%}FNkca(boIqrxc447s{K8C<6$L)(esDSS7 zcZOBp@v_ByUph!mfsI*PHS#6r<+ooS2qvfXk8}I?5Zfg}vm6&H+t&jl&{4CQ{pqx4 zzKn@x=w69$Hx^sxBeI?ckAF$-3zFx#2#Jx59fKH^Ho`Uxn3}h{95Idc)?YQClhrp1uKpW zthu56f`eSf=h;Jlv!FjTVoK!IN%w1;VoN-)jR5sxvb^_XwgPS!Ws2w592D?nxJO7b z?~5qa1vU6rGH$=yLnvA82&QZX;Z6`d$^>aK9A@zs12=-$5H?c#7${`oSyfHTS$C}8 z-%kNzZG$Md!mtN9eBw!fpYwhTqB*f{}(g}W#o;Gl1b{srV>%G6v*$KzNHblg0QL`cY~h1WG7+wq7#Az+9WfkdPG zva$voYgM8mU?fQQlM)h3UX(y{;2xUl6pGNjCkr|NWQLpnjE}OhLQIDM@{KGw*7pXO&|K-Qm09p!LeCdu-8wf7((c6;fE zPvi9EMv-u@7wIv;`ED6( zjHM^zxjZV|<2wtpfn*B{E`41UB~%@W)V`s>B8C8;*_ww2E-cw?e9)hrWApOa1CAUs z4)K{}9~{VVyUPaW(xZvE3Xyu*$&~^U(+8!z+vX3C$2 zVAal;%_wjlk;!1H6$Mc~mm$w-Wl!K~C+NrR7aAzJm)reS4`|d@3f&g{6@PDsmmKIn ztG;rTk>VFx6ZPmO(2uaY%UnYGoji|V^2Fs30k-$7zF7 zX0|@`Zc!Y1$1x!o7c$OR+#O~uwZJF21^uzh_r-jG>sVTgj>ac=`F_`&Hv-ygwwgc| zmnEE}kI@3F{N^rX6*HFJsou`Iv%lhS(Cr3;$6AsFTM(@QJ>HR;T2M9^vqaGphx357 zKSGroX{HMTq{@#tDXb(wz3$ggnGF-w`S^6+b#0uKinlpwMji#J&7i37OVSi2_$_8u zS*giUCA~{I%4G^hv*40eAidL4$F$SFUAclcprhq316Rhe(`L4J*gW zsx~7*aD`j_&s*`Kto`F39t0{~Ie|}{`(k>tUu~1^19)i`J;qW%JO5DO^>+t^1w}=2 z1)k>=DR_zcNBi82eFJFWCSrXYMAn7f0 zx64xn>KVWYEaX0r2xwXrA%V2h(PvL3BqbQ^g=krkkbbB03*Z3ajK;6bFF_+_Yy9B8* zEVuBz=S}lHZ*qXnGNfOxG8V8a6o4V@$ZStJHWrd>9&D{2`?Pf*6b1Iv^K6?Fp7gZ8 zG@t~jIC6`Qdz_e|9kVd`VGFjGu>-9+=GTkk0AvH4HOyEykrCf$`kjN;Hsi7CaO9AX zL_YpGp`HE=iku*d%^te%-DOZ^&;RT~zCcPFlDf11;pHC=sF4(>2HO;#v=8?cOe%p@ zQ_sWHNWOPGQs#yWL}pNP*~3s|BBJiVNEiR|QlN(y5`B-`TC^jK8BdeM^oS+lHN_Ii z79w;Nxu-)n-3ymrKNT5!uuMWQPi3|puzBzf+Q{i5wf)eB7RlYVn+eLb6BDJw~LG`p=oJbCbE3~M}P;RNk85q#i%-7WDcUcLBH z3aC|G6@^FShr5@naY`;0VFl3{hg4Lorp0Fa#8rmIRS5imE|mD0N!kv2)vcT&C7(m+ zi|4nZuJ9LKzjGKmy?;EYa+iloAfQ=r53TJx2(%nr%;3IYvfw*EJT{Vym0C*4exllb zEj+zl^l9F>n{z%&SRtr;J8ja2`kd5$iD?8K*3>V0+{2CKwdhju4mv#z41ym@z~=^C zSF3YaO?#VWth|1 zd0DVZGpkjy3JO(30=4wzCnVDnn}J)oJLqpCo`9{qHE)kHZU&$nzBdj z#3)QvY3fqxfZQM2o9ReTemmKf{!g^ga@|wdUq0qFck)fTl)^p!_~C~ewe6Y~;nePJ2(G26WIo&0ecnvSJ`xqHPxK4ndbCn?}J-O7b1064-%*xlCy>{JZIRoi zZ|dd^TXAGm8d@p}l2-JRDbl;kUGSH}i@m+Ud}c6`(EDO1@HXCavp}hMyJ51#Xgy#r zKjyLH;aD?Acs~hTt;*KFY=>sWorgH&~Op z_3(lTwrIx~@DN~|4WW7s7T z()28Kl3|0K$Ns+2lIQs28MFe?Oa)T+dN-Mp7bs&nILu-44uQy+4<=?l70*|p`736K z{k(AE)u=E6vikJsd;$58zqgLIzcClV7PJHF#b?4T+;VeBoFpbk<3DNMnz78s zH)34K9=@=y8KhdVJ_zlJ+4n~fUDB}}ni3bu>o~heqs|n+sQ}*1>YC9b#ydiH2y^I< z=dbo33-YM`!zBzE57FRWifuQDSTRtWyPR3l6MAES*9mb_2&oitxjgS^vA_hkTW z)$Lwm7x!=L>p&HUX{1Ak1e>i~xi);1(Mb`DC;m3!%EN&8j%S{qwATaC5_+t62MoFm zk0nVW=>91f^uGJ&Z$Z7Upeg$hMtL98?RIO@CQiFvUHc__K$MI%ApA`_HF9Fs!Nc8{ z7vO>%Gt-QV!oy*OR&4xPj6NV^_6TX(qZ~;<`1v>%7qPrWs%X%BBNOl{-RzMyX zMPBN@Rc?Cl$|MPGJs!YofIW&xC)&*P6MJ`zKm?)`t&O(OklamDQCAWDrpC%GQo?g;hSLryWzG0Oej7)G#OVT93nc?pBwB! zGr5eqFoEk1z?)qEVi|xMAw_E7BGq(m>%C?nkIJ}BECRjPkg+VXl7A2}OA8NR><7fe zTh7igV*0!NRszunEb3+R;y47yUSaXV%h-K3|iwf7!Vx@V#&# z1D$+1-i(68_$urlW-0%4@KM6g^?GdI$@2N_*THYXcP}GyJ1e+tWi!B(U&-iOui&J( zp%u80HNe|G--M^No724mALKNe%cV8Y3HD{KM8qpvkR;C}-;`)||wT*y_PMA5)eO-Kahn<6Od`Fr5ZdX;ZaG-pcS9jJPS?rZb0KM0-oW+M| zmJliL`U3*1$PUzPzEM@>m;4xu%m})$ahmE3PDSC`wOhT;k*fnaMKSfpi}jrWa7~po zS`R+0;)qs`!3HEZil5tLG1&#bomghWNV5ou<7=o76Zs(H#n)YVVReR8 z-;u8ChQKyg4_xNZxjiP?On($Dxt*TOms-R(8wr81&@&gUzGEwrET}M#hPZoiXV5H# zxc*EJ85v=!wn%phFmN;QAzDk6GB13oO@Z>X%KWh}zVM-7$J{e4h%VNWFVu0JNe+pU zs$6bWYQ=+Tq*wkjMQkf|6Zw4zIt6L$)=d)bv<{C#NC*DMkDJZo6IcA>fn}A4=(O~g zxrhwer+dW?;YElvn8kNQIw1`)@#oz&zCxI@Wp6P~Orh8@JxKVM=6T;?3}vqMXBtxw zP#-$Aqpg&f`>M57^>gFh-K>6asw%>Ws|ZIPz4aQOa8s~j{WBKg4!6&V=I1O2KB&xI zh9b7B>C4)xP*G8=~;N2@)3MrUKtgN6J@!U-0moQ>Tnuviw*$?|8ZcZ67}%ho$8E z#qbWsqbEBvL?S93u-f(kYMx;HkvhKxKe>}g*HT42h-ijjid5yhq$5hRaPfktO+(-= zIi-*gEjlStG|m4Ayd01NfLiz@Wcl#&mJMe}5`8$LF{J13T994@6MY=05Fz#B#1WcG zWdvUv%CT+7E%zpnD|q6g{1~qqbYvu{EOz1b>Q#I|xMr3+Sqk3OR8jtGMBa!Qt>9EM zBeENcYVsGSO&IWQob2aqa(gN}&Alu(nQL50>~7F?kX_yA-xwNkqLAQuR{^md-K6}# zFk4#G*Bb$w*ZM4_f?gHV`AwwCFHSo}<1ZcL| zy8n&qO$S)zWX3+-@@5=>rk1LmHKrWCb}w6;7wAxl?d9e+!o0*3PnhINWVOkNF8f47 zk)A3NN5IwTyW{%!Zv56!1ak##?rrTb)`7#|QszESV->t%K&F>ydFsG8bbKlKHqX!8 zoA*k`CQ#)H&pBl}$?~~-kRmD{(M<9$`lCFKWH_n7v`2FBxn&-_yqpF@W696H0E?ptOC9&o9U3d3fgT)k zGiQ@cGLa-5^svM&5sb*qJsz}}l86^QSJQww05+2wmHpyjxC9_m%M}BrXk%tcyzss7 zhJSd!pnE;J-!KY23yHKK17F;7euQ`4uM;fjdZ(n_Pe8V`+mRPNtp}9B4N9UKkYram z7})(RYRyrIY1_S*&uW77PHlV;g<9M3=WS||*n0aTpNm~t$p{g=E{Y$UD}bVX$2d4H zV-t+(auLc#z@6939d77B5Ou2RSLeOdH=4)87v9~?k(yE~TdDEmY;I@sCv$)A8f~0JOgs7mRHCWfA zlNkK%fmHT^F%-kRH(SYzNk5l(GtcuDIV7wMTNh=99t-x}mOOH#zOFpPntXDL0xnIljW`8p#w5!K;*21r* z5)OI`UEgCN1XrM+yn|!*8}WXn^_H%`7&N&A*>$PBSxUcsFPp?fL(++cjwjsX4^&vg zys!o&@X=~$0rnLd?>$~AE3)~&Y`g0J5(`ODikp;Z7l}$e7C6z1 zxvt;^KHJ#$G%?!u(*S`=yLh7LeML1Z=HT?vjH~+vtN%{B=ARa%eYi?tVjFmWA_2y1 zd)yz!hk&?@90nT5ZTP|Ll}i4#N&0Rx(p2{6wJek5G>~{RgGRyvfA_a{f!Y?o+RhVBM z!5^HG+0FIvO7c~G0n2Jf7FF_4!jCd5e*&v5Ww(zktfszapShL8)=G>vD!&P}Fk@^_ zXiS^h)fX~5cy<+XYi?3rD9dS+bqfO7PTS6~47eZWG6CCH3tf_$KSi>LlC%7Rt6*wC zJ5sCfsXNnEmC6HSI|Ib1DV5mgjQ|Be(1zX%(!0Y18++R3D)vh>xJ%G3T2&RD=AO)K z<4jLaK6^_DI9!95;$5R+_NZ;}bRY(dc>v$Ag0A&+fLlN3{3!ymo5@3f16yPgr$(Jw-+s)fGE=iaZuyq<5jPpNI>8N&TolM=fe4F?!5 zM@jMR%muO=M~W;$`r}009$zMePcDc!Dk)vo+)3ll9tvW+2k$!Kj8y|J*adqOF$R`Vjbvf)s6L%P`OS*npMB03?%a@s$o z&r-jC8f1+se+!4h^9wA?r~G?AQ}=2hwe4Epgx5PgoM9H`Zkbn!#3s=C1GVrPV3mW zlsMy*HpCA;XPBYjj{A1kLSc3ZD1merTIdF86I0{Qdzx#f(57$xl=H-7E4o8kYUDwO zUcomK4z)cgCj7S{Cc&2jHybZbXkolIz_no&se{GOl3v{!>Nj(N6H$d1%d(!GW) zVrPvGA_{mF2;~nF1V&&^K`fbZ@p!Yl8cuz`)ovzENPKzWj2)|ek-ez(v(F0e@lQ78 zsHy)^=L|xF1N|--P!*r?VqWMI1D9an$`M;S7RK@#F;c}fHr+aIR^$GxQxAMbhFd{< zp{f-Bh6H+{X2u!x3-&^EUJlF4lotH6?(e6#<}-g%`G1o5@MmQgTVdz_h{OLF1@8AI zwg4HZsSE+44fC*OYoyC1NSz}$lJY4jX~9MFRcTEf#COri9ze`+B1PMO;!TS0Lz)&< zLBS*S?9Hdiv3hVg1qQI=S4n>bggm70>Ex z$MQ&oKU5;}M|we`kV7~JZ;tFn#JfPW>$JVBo5G}esIvgo z&kf_d(`$7i$4#^FjHZU3kKlvG;QNQoHRw0YS&Ts@rWeIQvR(=lS zy~=ELyRVYf02y=th`8>!*>0#ao&rDYu-%h{!n#m0)xXwT+4)V>XsFAPrc{^qwdeW- z9_W3*C^XvSTy)eA7y{8wS72RV-*lEsXjWY&Z`>}2Tc)ixsg4^7kt!&=`N|ObQ@Y=u z^{w0Ip_ui4MvJb!k}dzPPly?K5jcppf_%lpy9h*jm-QQ(vGjH{boBAZ!))!^9Sm&) z@EU8+1Kg8a8Kl?#3-nm6p&5u>px7HPa~{&N{)7T1YZbJm=>c!^g?L_kcrxp$(bd0* zxNTUu-OZEQ_xP+(r`|4R5c63)b2^Di)^Ql0c_9Cj5q^LE^Otdb9VYJMXNqH7Ac}SL zSaqwmnq=*49uC??eC#$%8=@8qbE#!-s$3?A5qiswIG!|2y(-5i{Y6K6=!ip24ihO- zf?wL>DJ!q`>~lPw;PkBk@&33J5ZX~j#tJ*6-D22;zG!RD;6kbA@4t3~6}ytw)q#95 zUI>lq@tN3(EoE0GOxGSM0#6t=lHKdOlF78Gn=I1iz{iewIaIO89Un;w>z@NnB2pTE zbw1{y_K%+DQuCe~?AXD@0~ImR=0jXHZ?s#a(;{n2R3fAh6zt)W1E#YY^4;z^4EyU? zdy%N`94{^XKZeTI6e5&#f-u8SPb%31X_y##C__=$nMo@3Bfm4!C)6w=1H1SL^286~ z5L0k$Y_55AnP~c;7pmxhsbpT?N20pnh;X^ms3^wJVjqd61KoR-HF{Z1uq`uC)P}7_ zou5>R{dSnldTpyx!zFnNMGoB}%>4=>hCU6-=rsElRLNN#zqg(A>>%Q+K&KB0UD<&6 zob;6!TjE1@*&omMS@$`U?o=z)vrWgS()ItGU2NJimWG+#F@0W^%l;F*jNn_K{?C&G z5eo_Qm-IpIuuFG16%VuDU9yGCqXS9v1S0@M`@Un(K2oz5!$oqhIQyFzkZFpl>n@4u#L0xK_ zN!n*oL@+h(5pB_XVmJSY4D`}d*f4K{b3%K|7e6@s?IogZ;^9nPow^hc4dA1quHfX( z&&xeC5Z2W4CuRlvar9*?m8N;Ks{EEC`szcdJ%J}vS|EF3{Z)W}r^Czsj65frMip@h zf?+q2YvYep-&#-?{XEehQH}s)#-7NlfND+>vMVCPJX(3;dngZ;B6b)*UGHV^09~*h!2YB!WwoxA;R@f3IACS*I;x)0)D< z0u_{j41WKf5ge2)=t2P-HO>GEmTUIa|CsSxbTkX!X&iDGwiw^bt2WZTO=Rm$$`+-1 zZn)^{k?48d`$d;JJbTB(c_grZTxomRS14li-HB#oGy;y@??MdKBF8pvjyLWJ-9$(u4n^#qFz<@pN)|3cDap8{YfHm%}O7z4+Q?g%b~3k+9s~_R%0yxjk#G}IgH3I z>yG+pKNF;M8G3gH6I|q*{ z&v!Fn7r}mk>7Uyesb?*(axIr%=Ny{5E8AHS4cyf<9fT?1aK$Pi-P{e`$S|kTka5<2 zCj;rvuYo_lv-`?6Wm2b7Tkt#{bm#C#-Y4KU60YlC%OxXr` zPk&gAVo(BV(3XFo+{|UH^ozP?`h`sooNT{kHt6%4j%jZH!wi%`h#WjEw{EwxUjX05 zfB(d8w&hTV1|~vt!P$r+#KmII8OV~V7`#T~Yd6fUXBem+6)%s}wtZv?i59?bS$CtP z@Z9!@BH<^x+E?4^8wKB87T6VZXd_oYzkJgGnbVP!TU|aTUTnVtC|Un;`fj1btEO10 zLk(ml*9{LoS94^?vEf`k-@kTv!7p>g21WsGo z%Mi5aWsNr7CuAbMRWafk%r8%>?a%f2RNq8rBVWwe zi3($D(alChO&eh5IHu%ePr8yCS8)~E{k&!Ea{kWPGVE~3eL~o>P;`R^<4g=wS(_PS zyTppWo@ls6#Pi!PL?Av7a1hOu&#F-=ceTI?En1h#|K-oWHOo%ggWZYHluJJ&xbJ#7 zaK8O}n|h_2_|HQ(cZr~WHZ8X)W#*(XCT4Q1UYp;c+XC|8{~zPAWLHC%di;8lmj(5e zbCoO9QY$pC*68t|hZp(d6yh;INL{nl@_a-jd@s5wh}ooE9cySE$RLXxb-A3pg@}{_ zeVezOLBbf8pi4}QS55)Qa|G@uL4y?J3!x|1am0_g_0}GBE%nl;^vlV$2uxO!9?@9T z>l}NsQiv<&LhCol*=DXI3y{Zpn&D1U*ahxG2Z=}#FjPT!AhZO7QjZU4cV0XbcZd8d z0Ykt4MT8aBjKVgarTQP2I-ElE;s2H)w#O!YGuCd=B)1KS2H~fUj^9Z&KdzGK3w4aJ zhmzOr@NbUppyKZPA;~d7Elm3<7YIle znM~^7|7q{6zoOp4e?Kz}44onkLkZF)-5^~GBHdlmhzK(@N=tWxN=lbB(yeqi(jYxB zcRc5O@4EL-xNF`01DgfR0``3N-cP$%{}!MGP!H4Y4V1Q^C(M#teV+~W24QvK&#^UGjGB525LzB&7D>WBw$i zBxL%Q0**!bk<}a5F!XsYr%bM%;-|qcPlhxagXuYh9$0^R3H!dm|c72^Eo zDW7pDY%V!IVa7i#DubismG%N_b$hjjgguPfXE=LZteaX-U9Z^*@8R9n{a=mQ>ys?Q z#jWMLRmX#tE0M;&y+|5H8x; zjuI+>Q@P(!aIuCrJL8gdiJX5g<|JAEV7lz{VpOZ%w9Q?^?x#Q_%Mf@nU1JJ@PD=O( zFFn=azKe6J`uSkU(UQt|v?TlM8S{tyBs-UnAFh3zp3h7t6Z&;4dnZ?xJ-2QEd2+KS z>u+ZrriVqKrLyxSq32rOvv2}}iNB`N&zxyCA}~;_S10<<9DqG)gRbh>QJU}p*Rt!E z?&;SfQ@zKDRSyZdT9VpUeyqx-DuFU`3RfPu$jN!J1zRk;>+Hbd^EUI3t&AF71{97j zX%ZsbcJY(Ze?B9QyWw;12aUgixU^>-+P{v;M=NNLzHQz(Yt`t;o>&tRns$5ldV6cS zB@^Ye{!b>LfaE${z)_O>GA%v%A7V-+jX>N=3oa7(t=s*<SZ^=RKec1xC{3;g zN}Og$WvW0~LcU+lV)W70y`Hc3JLG*lgV<<5-6QAh(fAV~K% zKH&9gJ^`l6^%hW3>XDAR)BNpAaB1qw#j#=Fvc%vwy8+5!!&>YQZ(lOy+BrLtl$)?p z_hh0wYQ1>Lu`w`@LWYGw@~Uo_vw;jcGSWXoaJh*R56O>Nf~DlF5!MlpbtvFyFYKLd zdka-7UDKW@IXoS=RUU_^+CUm!DPd-h>2|%ReLl|?|3@xJu1W2s0sNyXeNi?IE*SOH zl>h!X(|1?+dl_X>rJxXpP{{CNBMc~FtTzGK^o;*JIE3%KBOPsJ@YQCiK!bnT8|Zm6 zblv8&dFkkhecI|<|0XG{>C(fVoJw%%sEh9}!?ua-KZflbae(j#WR3ApCkx>XC5DzO2cC{nwc+o>uvXlQ0YSwr>bY4-d22B7 z0WZJ1WPHoT6$h%!*07~nMcl*C0RKT=zhxCvX(F>rcrr>*%br;s+$Rhi-_7%vFH?gE zx))z$0w1qW&?neLBorE1;}IQm=dJ_z5ZcqQK~UUI@N zPzm$11+{1z{8w(VtM%wNjNzOJM)fRXG2^C+C@=Q5iciPr#y~h#qgO%j@!xk&3iO{I zmb;&&E3teT&BKk1sqG2(eed|CoQM$I=m*ZG!CFpcGU^)*js8 zTRXOWUb1)-SX9e@AS-VlAMf%EvNf0`=e6z)g)O_Zd(TT0CpltI`D;5AYSlK?0OSk+ zXHpK-avDO9YPF>h(*b9vYx)ip4un0=nnQ!I@C$<>l3k>TrMk!n0GJ>t~O!*!#%#hl5-8n3h?IP{4}4 zR(fjuJOp;ziagJ*(6*m<|4CF)mI*#N1JbYV`vQiunngqt9m zsu9P@Z(|A<-SeAPZ@RE~ia6Rie+8jHm8}6gX^ix_=TYYXp2b<%?-sjQGz%o2_4och z=xVXB0;Sgdy%KrZUfQH$`g;fL-M2YeOhn6rK3Thhxd+-Mg+SQXO#*ae=2tZ?B|9w0je7PGH`+A7RfJj^=hol<1PA}0j( zI1*XcFjTay!*p8f4HBL;P8d+od08EjmKDPsVQ-rfb{vfKBqZ^@o1%g>$*5&c-0;)Vq*=Fe#u_7$&LoIa>sXJbQ@M zlTpZ&5*{CGmz;{8+DUx&%VS(%vVr9Y*1Syu?tVwKxv#Gu)r6yKK0mx=SN=x!D=QsJ z#w+IInQZ)0#v6huo~4bsMZN>La7w)rfH-J_ZAe*)+SR5Jz_a=TNit>#Jau}J!`B(k zY#h=&{V@uREyxdebb>L;$kftO zdETgu2A3@TK3r|@(uE<;F4ujMY%Sag9XsK`QoruVp?j20+<3i~CqT^K`;N~z7;O+U zHNn5i?|8cVF5h&;<=W;q1BX#9B|9;e8cLW_%!NNfnx4KCp7szmm-%vx7{i8786(ev zaaSLJv1uA?4BxYyQq&v&a{ay@{w_JhCmHm%-suBw($8Q^w2#y+3>dkb5H}rn!lCL% ztRedt9NE018_T*|OH0lKKH?1fMmWWWIg?HafyH0eAJCI?{0WGc*J{Nj3MT<{_ZT=Ql;r&ad5P1T5pvOUIKaZ;Glq!%ICpilg zx{BqT*>AnLGc(e?-UExd-{;qdON`wZ9_Wi3?Wgnoo;9lhQqrQIxt z^+og(4d$Hl7LtP$mz^C^N(J z-Ba%E51`KpR(DImP>FR-st0Qk$@0vXar1Q`MrO$jZIef6eO;rGl`JsEk>6t;3+j+J zqty&fSv5xUqW&&zk{!0q`=KpMmvn|rkPNoD2ryZ3s>{3muv%OJycM@`zHe`_pNA41 z)Gg?uG$h2HsHcS=w;4taeq_L<`k5e|S9!WLY>_hih^jZO7)QP6Rc4wEZ(kZblo@LK zKsq3{PKYI@Qzm7|{F_H-onWR&sAwG|8zd>$3r?IWyvR`lStzGsa>wK|_G_A55i zb3yf2s@*v+n$4$nlkFj^dVM7X#=8#)8AHD3|I~Jyhct)uz{+VVE{uG)Fl#D!nkjXyNNj z({f)N-%z%ncnn_l6>I_ABZhoxVcrV9-9k$yL)BT%jOYt~#{Xzr(*L{q+Q0b_9!4Se zTSAx*<`Qc;+V~XKORp7$E6Vp46fsaFLu(Js_1@{rPANuI&mF-7#o;Plk1q*t&U^Tj z$gg@F$#Tz1Cr`6)EPf~UD3Bed=vIv^r>EJ?PB^R{2O@DD&M~8l-+yVmB=s5<9xz&( z8x;BGtP>b7?a7ZOy~gd>4Y`Ldu(0lnKU+czu7=xU?RpG6L&RxFl&h_X_A;`?VNgt^ zoTGJm3fobp=>`?QXK&EHDXK3E9uRk*EJPC{Q7nq)snaHeC!+DqA&QBQSHxt|Me3+C3z4@ zfZ1I`YQtfALf1Q1gx-TUGkb_p*SoqYxcX76$h=DfcbxE1b>MtR|&owPiQBW2Cl~a!p z*u#bWRw9G%3!OwJLM*3kayXEp@23t03|%L9%`fDwF5F%>z+Tro%<5HQ|xOBz1O$nZ+rp&__NA(j%3GJmxUUynKA;LCe_ z!c<17^nAnKap6mRRhy79=!#)dWgg2Kxtj8v9g~&Jv^x=HH_=DX;I<^gFrS-`zajG! z3FfO0-q6`NRP@9jwtQ{c4SY|GZ2-bI;OhzTSj^m8KalAxrG^Nsix(l3EkT zd-g&`vejC?EY9}iJ#X94NsfInL+$3$`*~~6lO-0h?-B2NF|EW2W}*SN%NG7#U;ZLc zsQwF&S+Vk13a7c^g=121<$fa-cQ*;!63aVmej|G3YJg$48pL$uRDvI0Sv6ifE ziBI8epZ_`3+ukA6D~Mmlxlc30%nxlvFb)Ixaxfajff)-VQ(Ds~5*4q^FQs_{C`C`$7FAPazT-U(y1Hta2d#{=^TA3p~V(g`(>L3p#STItms~ ztu=IR%5FyQ_luaERoZ-ZS$uYH>bWA_dga=m{xzhdH6rXP5ZMwhS-I5L>vNC$JLqUNfb|T}X@+`(2lnl#stvj4Q z`6qpMr6t~QKEoUJ=n6VdzmA9=1X2 z^wn?Bb%>nH`qB`G67)71pFFlBxn+Xi)PU*Zf+F)X-UO`MY@NPzV(TcTBqIIT1c9gY zDXMOlmf6`Iw?nch!Zp9nk9i|Mv5dYF0c%g4AMG^!`|p8Ru#ck}aK^1F;!dySCp?x%ZtK z=}9kkSU&2K0-TlhV_i?a)PJB*@)_|;krq>0H}yB@>u<+G);h)YE(QPDGfEa!P*6pi zZ~9uFu(YPhcxZcB)RM0kpp(nfJ=0xF3P^O(aLAuZ^IFXnE~=k#^T-W#zW&`W5&$8- z@urpT`*z1ST4m@zD!8b8iY{<2B|hOW%H7cv!y7^NWo^Yg9fhOF zCum+frK*=VUkmHN(nK0-0#eJq5T}_671Ds`B}AwFIQ7#bF7#Aivxqh*VmmL;MpzYM zddtwpy@t|%5}IGH=<#J5bgQq8XM1)nJM;DX!v|yB=C<;;QPRpr%R4l6mT735VRr=8 z!~NF@*%sY`lO+QM$;=59j)UUS4?A`(<#+Q6-LFN}+xU?qMN@Ia-gczy z{6N9>ta~HI$m_dIYsDh1!fSIwNUwYZCLxpI0ZO*XH7HR99E zzkWqt7OQ(z)-O#xjG|qv0Wl=}#CwGw*E;kYJy~h| z)fZQ(tC|x2Q z;8rGw4&Me1KKkAy1r04&63|Zv0ffCZAr4fM4C)X!&FmvW->AVvjR|8|i`uIoVB@eg93{7N&003=sFQ|FmV!|BB_wS2cs8WI13MhT6! zpx_Hmd0rZ{i*;PAZa=A@d-jya*LBKtW-;Z8hq`NfjGcTIG z_UvAE&As(V$}{^qP?0})DAmDBlm{ni!Jn#m0xcWybx`S9A>Be3dS=y+3u6t$v#TVp zEoyua`%pRas=>4r4W}b!F<=Uc@*1|IM``=^>tpQ=FO!rFrDXIXN9#T`R`qz2tdi5r zor(yl+&K?bG5^4jl>6&<2)6yZlDvB<%$kGMPY*|?-i)N6+Z8s|eXYLDbP+G|ikNC2 znhjKf1XFlzF~xrq0>Sao)#w0D?KxSF9z;}@!8Sh4tX?secy$raehos8?5`A`9Mmjm z`UKpTWedz0K?y63HK0l~RH~CS&C(d$}3^M}EY%`dw&{)hlFJv#Gb}nBg|L)Q?!M40T!`}eI ztYG8*@IbHU-S|Y&M6VA~i}9cB%M1fLkBK^364M1X_8P~u-#zdCShi|tXg$&U$@>qP zTl-1eEJ69Ds| zR;+INmrF&YxwCG-EDzRx5}cd7<*?;d=JA|)uE3^RPKwL7{GtoXWG7xLF?nY#KCH-G z)7&o6K?pR7b zbBsHmqf)Vk}?uAc|Cp)_%vZ3AI3bqZvrZFS8Y+&}Zt zl2I;spEriau%z()Gec{^pYwKSKnW}KRhsVhm~CgaIZkKkC`kok?>Nz8W?#N(+S0ra z0K5^LVk`0Q`JaViD=9Pp<1caK<)qEqge0cfq`G3q6VJ~I+)cD42#aUhx@7g+zTij5 z!-GJ+GWCMkstk{k-u2Ns=2h}GwLv*EOu-F@AD&G~VWojX;6$>%{Ws&-2CikF&^s(s zf0nRB`Qn(9*i}0?mcAKgKqGD+6fCN`w-xNjFDYPoB|+ z0&BAJlCnOTQ{uubFt~DF4t)s`sc5lRndD|P`^?l$U7`(oW{BSLjPH{;wDXFkoN-m3 zIub@Qs4n_QFA|taj}anVz}<%+S=B_4$!l2?;1lkWjE}7|$rQU2=%Udmsa)x#b5C2k z0024_?Zxp&wezH}4c(qycY(cv>6);uM> zW7-`uCMTcqfA?-VrnuYknBjSk{F_GJ`t;8hoE+2_IT?jqPnL~36@E7pJE=4d0yFwc z80>R*L#}bVQ)n7Vu6Md-aIeGFqf-N7=KD|HkSy_g%kj%Dyb0{b^6O${m;U`-S?T+2 zd6ih$zh;xlkW&bGPWn0+mT9_9X`TQ#U%Y1e?Wop zJV`}c4djr6w*uuk(?aKHYRgTLm%aDI44E##!RmJ#%Tt<#2Jz6@#a&u1UL%o4)Hguu} zWbh;>rsP{5p879N9;}M<7n?h2J?BoH({eznJSBj$8x{_u`3JZ8+3w^c(iRa6e&VjG zHHmL#_N4Q_7t9_hU`ryI=(=Gy4`=8~YA{x1SY>XxES5J?@V>!gh1$D^ zSpz|wz|EjcPGKlNFr!vYs}Ya^LZsO!Ng4C5>2Ry6DvOxvajh|XBqEGuH02@; zHp}EM)RKoD-|Ztm0fPnq6!I|VG>qPr;+92eKrmIdld)pcyDlq(m(JjOHShv_^m;lR z#3?b>EG5=3jX$pI<#RC78)H;ljaJE#AJ~M|@l@a*)lfm}4-bw-dvJYJWg+Pb2DUTT zwl&b@eTTx|Zoa|1{H{V6pGzB@V2t%A$rDY|KWJL9%1zw)DlsFlcLj98{smDd?fb^y zLDUShvwhvIJQp}JXPEeeB|po?1d+4+pYX65vqIdIjOh@)m^!1~eWN(b2>@5_S~sR-+#48n-? zw@1q!&xsw_?Jed$Iv}w#YTmj>1U3ZLa*KZoydQ}Ph!<;Xdu~Y810n~0kV3>t{t^=guI5Ugv zmGrr^cxL-+Nsaqq+7t7WL1#xrjepW@PnP}o(OYjX(puK~yX)l!Rm`_OzC+xu_Yqp| zcfsQ?wD1^Wp2+xP_=RznV^NT$|1oCTEWR~!rKkllls0B6oR8%8v$>=@ePP`fNHai^ z!((uGGTTT0#0=UZW~*4z#@s7{@hVWp0;wx40nWI#kM5g~f&dH84>OO@8GPW%z!WC0F`51nf{1-n8HG!G_T*B)%#qFFFJJF@ z7q2Q4a+>!(Cp8g0n3dDb+n1l zVEJ9qfUAw;)2+>3Pv^;J#)h#tW3xDKU;=pG-zBfLPI-49!>(9mj>S=%Fwbw^T`_*_U>cZBsm3p|+nQOXMhvs*DfRe|GT~T}x|+Eror?xi<%0t_jB9K!J_-n6qAF$?_KZVSc^fJzYY= zo1uY@d=cVxH&E;sFejHtRL9yWn4*-4eE5+qxpj)Em}&;J(KoKLnjjmt-x0o849;$14~(Q@&LSLlZ$mY>`$S@6eE%z!XGc8-aqay1J5h zGNv9p4@$khFmjUehK9Y70yL~3#KQ33;M!M0>TfaQ- zi+IeeOb^;Nc-R*3dP8qepqUBQV^u%~R9~1{o@&XXl7t$uFjBcLlp=vHM)|nU&>%B> z0eQt0BYE~1anrmL-#4W`mmR*RCzqG2RHohjHTn8IJD0GRW~wuu@cL^K*PjDoXka7* zGZ9mQa;EDj%~WZfk*d&^UNLi zV7zvJqPHK4qa{GI=L}}^{L&41()s!Uxc(A5)oN}chI~{argU-1dh0JWQ4o*77l{NJ zHChlRHDWX}{UuIR8CW!@1N(VbDxVv@%k@m@+uhNcZ|~G4Y&a`Ae+jk8&bY+S=FE8{ zZ%kFDtVenr6P*GeE>l!EB^5m~cSj!;v+p0J+&|D0@t?!4vZKFWQ3jUKr7IoqgpRdk z29xF7ecUEmWyc(Fngfq*bLt28g0Bn=Im^Df^d6wI?-HV|i*uO1P>+B%7ObgB_oc)eCO@s6;~0Ai8MJfQm3aqb&x0h#gIR70Admtxa=Vvciv3 z%yGao>w*jw$!aLD)XQLAp11P+^8QDLHe_G_?(z>+%3qx$mA{~2&hpIU30{PuWBp3h z3MI(pk#EVw)&DnX}U4m=n_;%`);!9Tj+dZL-YDBcFG{z$j( zxRM9bwkAv^fsu5{zlP}dX^I4zi#MGC0)9^7BD^;!*W8-^RjL^vy}b=u1huQ$*rLR}vz z)vKGcZp3x7XA_-idLr3I&-41*76!0aYc2G0dT@;wl-&+jE^-J?Hit~}8XF05cFKvR z$0ES-O7p^Nh`TS&%(=9Y=Es&U&*3$PlWkJhJ@U1+b-?eKF;j>dF8@rfoe1HIW1EVO zy>=L`jXm!o6xYd1ar``gD2K>5dZP5v%FCvN0M)H7nb|cE;yn9`e$_}h8xS;AT-lQy zz6=MVwOF368!(ATxVq_B5j`CC#^zR=iZFn2Qd|#Yg3`ji9Cb%oKKgdvXihvJCd^p% zbRF=KiT&UbCoHf&0;iuY3{H*Y29r|7l>@)9kgb;Wfq^7}Ow)#8YyJp7dz&}-VliVW z+2u1xL5o`qodXSTOXK1e_BY77?kzEclyVDdo3IO2sc8=M>O_)wqiFTFk#;C|w!Gv& zch>rbgc7rVGaTlU$8kcSAK>Ur{S=l=L0yB;$*~G6nrLj43_$CV108*Iz%B5O&3gi)TR%7)eGn5f zsG~D$hN!1ttN8*f=^x&)>4m~X-DnrM@ni$!>oz>4&^0hGuV-=@qiE#*{Zs{wp@?bU6W&RtFa02D)uV4 zR(-;tk`-7G2eXj^2|?WEp0hhk#rV|T9L6@tDwu_ z!uuk})IhT?73}e8;ti;adDDug#DXrSv$HbMOkDZZHJ{*=xb>DBlXktk$F{nbevm_q zqB9%m34Yp|y$SfKDC~Ua7bethQ*ld)AhKiY_b)JXLRMOQo_9P*+%K85;MI@k)Y2LU z=98c&5X43ZU2QZ#W(+{n@DyH-{sA4OHREH}jBc+U7{?4ZgVsIN4ggIVGzu#Xgmm#D z_lJ(K0Y-h6yXi6}eHd_iv1+19+ji5Mzu6W1e=y>lj$R@V@Jf zMj4s^$^m}LvDNFcL6_lJH zpb*IaoHGjCy=6 zh%IALm2{x4VRl%Ef@ASh%2z+7UwmQR0wHVWX$^wQi|!5G4ptTU)%Ali)d(ZjOP(v? z;$jJzu`8`4@a^s;m=~LCjOdSdo$XkBT;G)tW&2qu{3+?-#sPSiNoa#$SVQ*1{$Bu0< zG7eaYREzm?lY`pbT>stO)Jcaw5JPX>4v7&p{3xeyVD_#1==XS{pNrncAw|7viYJ;E ze2tl3S|C)0)eeHhh^b#SE`#Pc9dMEcd?`x`>S|u}1rP%X>w zngEon)L{u0@x>}Z$0DgQMwTw*H>;(^WiRh}ykAM8dRq(@x*RMDuJS^ION9)TS>MJC zKuETdXgqD-u8E4e(uE<@CfYK+k*JEE!RP-J^b{oxsRR=Sn61%$J;r>v;Y2!(0E9j{ zExa_ZacX$yWJ)c4p^jiDcK&*v!-=0;eqSwA{j=`q{R-hV1a{-~8fh&gizDpL4Sf)D z^5*otJ02gTDbLJ5jy3u;pdUU%T%$2MChJc#C9|S=gKoSILp?)WGBq00j-KYtYmlOP zSaT+eWZQL_f+E$hYnsN2k6hEVqe=y;G{W$|=*&)%&EU@65-;L>_Q#Jv*t`~=O9>+Y zrSR0-?aq^1wX1FYyx84`6#cm^ueE zR^VgoMi${nkF}3h7>;WFok<2~8m%FTa=l=nZauzyWI zx+26;g+WZlvz7$q6h0WWR1RtRc}BSa9#sqt%OaS_MBlkJ@2}(dcU)-yOHM+$8UN?F z#Zpy#^0x--{~o9>|6^XFe$D?e%5$pzJu(0P52IG)f3L5H*>p0YIp!BNYHfO>kb|-xmNZL4<#&|Ce3}szm$j Vlbs$!{`U`8ey$;3A!`=&zW~Abg315@ literal 0 HcmV?d00001 diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index 718e4d0860..4f7e4e2b1f 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -3,9 +3,11 @@ OpenVINO Model Server can perform inference using pre-trained models in either [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets) or [ONNX](https://onnx.ai/) format. You can get them by: -- downloading proper models from [Open Model Zoo](https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/) +- downloading models from [Open Model Zoo](https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/) - converting other formats using [Model Optimizer](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) +This guide uses a [face detection model](https://docs.openvino.ai/latest/omz_models_model_face_detection_retail_0004.html) in IR format. + To quickly start using OpenVINO™ Model Server follow these steps: 1. Prepare Docker 2. Download or build the OpenVINO™ Model server @@ -16,83 +18,74 @@ To quickly start using OpenVINO™ Model Server follow these steps: 7. Run inference 8. Review the results - ### Step 1: Prepare Docker [Install Docker Engine](https://docs.docker.com/engine/install/), including its [post-installation steps](https://docs.docker.com/engine/install/linux-postinstall/), on your development system. -If you are not sure if it has already been installed, test it using the following command. If it displays a test image and a message, it is ready. +To verify installation, test it using the following command. If it displays a test image and a message, it is ready. ``` bash $ docker run hello-world ``` -### Step 2: Download or Build the OpenVINO Model Server +### Step 2: Download the Model Server -Download the Docker image that contains OpenVINO Model Server available through DockerHub: +Download the Docker image that contains OpenVINO Model Server: ```bash docker pull openvino/model_server:latest ``` -or build the openvino/model_server:latest docker image, using: +### Step 3: Provide a Model -``` -make docker_build -``` - -### Step 3: Provide a model - -Store components of the downloaded or converted model in the `model/1` directory. Here is an example command using curl and a face detection model: +Store components of the model in the `model/1` directory. Here is an example command using curl and a face detection model: ```bash curl --create-dirs https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.xml https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/face-detection-retail-0004/FP32/face-detection-retail-0004.bin -o model/1/face-detection-retail-0004.xml -o model/1/face-detection-retail-0004.bin ``` -**Note:** For ONNX models additional steps are required. For a detailed description refer to our [ONNX format example](../demos/using_onnx_model/python/README.md). - - -### Step 4: Start the Model Server Container - -Start the Model Server container: - -```bash -docker run -d -u $(id -u):$(id -g) -v $(pwd)/model:/models/face-detection -p 9000:9000 openvino/model_server:latest \ ---model_path /models/face-detection --model_name face-detection --port 9000 --plugin_config '{"CPU_THROUGHPUT_STREAMS": "1"}' --shape auto -``` - -OpenVINO Model Server expects a particular folder structure for models - in this case `model` directory should have following content: +> **NOTE**: For ONNX models additional steps are required. For a detailed description refer to our [ONNX format example](../demos/using_onnx_model/python/README.md). +OpenVINO Model Server expects a particular folder structure for models - in this case `model` directory has the following content: ```bash model/ └── 1 ├── face-detection-retail-0004.bin └── face-detection-retail-0004.xml ``` - +Sub-folder 1 indicates the version of the model. If you want to upgrade the model, other versions can be added in separate subfolders (2,3...). For more information about the directory structure and how to deploy multiple models at a time, check out the following sections: -- [Prepare models](models_repository.md) -- [Deploy multiple models at once and to start a Docker container with a configuration file](starting_server.md) +- [Preparing models](models_repository.md) +- [Serving models](starting_server.md) +- [Serving multiple model versions](model_version_policy.md) + +### Step 4: Start the Model Server Container +Start the container: + +```bash +docker run -d -u $(id -u):$(id -g) -v $(pwd)/model:/models/face-detection -p 9000:9000 openvino/model_server:latest \ +--model_path /models/face-detection --model_name face-detection --port 9000 --shape auto +``` +During this step, the `model` folder is mounted to the Docker container. This folder will be used as the model storage from which the server will access models. ### Step 5: Prepare the Example Client Components -Model scripts are available to provide an easy way to access Model Server. Here is an example command, using face detection and curl, to download all necessary components: +Client scripts are available for quick access to the Model Server. Run an example command to download all required components: ```bash curl --fail https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/python/client_utils.py -o client_utils.py https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/face_detection/python/face_detection.py -o face_detection.py https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/python/requirements.txt -o client_requirements.txt ``` For more information, check these links: - -- [Information on the face detection script](../demos/face_detection/python/README.md). -- [More Model Server client scripts](../demos/README.md). +- [Information on the face detection script](../demos/face_detection/python/README.md) +- [More Model Server client scripts](../demos/README.md) +- [Clients](./clients.md) ### Step 6: Download Data for Inference -Provide inference data by putting the files in a separate folder, as inference will be performed on all files contained in it. +Put the files in a separate folder to provide inference data, as inference will be performed on all the files it contains. -In this case, you can download [example images for inference](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people). This example uses a file named [people1.jpeg](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people/people1.jpeg) -and use a single people1.jpeg file to run the following script: +You can download [example images for inference](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people). This example uses the file [people1.jpeg](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people/people1.jpeg). Run the following command to download the image:: ```bash curl --fail --create-dirs https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/static/images/people/people1.jpeg -o images/people1.jpeg @@ -100,7 +93,7 @@ curl --fail --create-dirs https://raw.githubusercontent.com/openvinotoolkit/mode ### Step 7: Run Inference -Go to the folder with the client script and install dependencies. Create a folder where inference results will be put and run the client script. For example: +Go to the folder with the client script and install dependencies. Create a folder for inference results and run the client script: ```bash pip install -r client_requirements.txt @@ -112,7 +105,25 @@ python face_detection.py --batch_size 1 --width 600 --height 400 --input_images_ ### Step 8: Review the Results +You will see the inference output: + +```bash +Start processing 1 iterations with batch size 1 +Request shape (1, 3, 400, 600) +Response shape (1, 1, 200, 7) +image in batch item 0 , output shape (3, 400, 600) +detection 0 [[[0. 1. 1. 0.55241716 0.3024692 0.59122956 + 0.39170963]]] +x_min 331 +y_min 120 +x_max 354 +y_max 156... +``` + In the `results` folder, you can find files containing inference results. In our case, it will be a modified input image with bounding boxes indicating detected faces. +![Inference results](quickstart_result.jpg) + +Note: Similar steps can be performed with an ONNX model. Check the inference [use case example](../demos/using_onnx_model/python/README.md) with a public ResNet model in ONNX format. -Note: Similar steps can be repeated also for the model with ONNX model. Check the inference [use case example with a public ResNet model in ONNX format](../demos/using_onnx_model/python/README.md). +Congratulations, you have completed the Quickstart guide. Try Model Server [demos](../demos/README.md) or explore more [features](features.md) to create your application. diff --git a/docs/quickstart_result.jpg b/docs/quickstart_result.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2422220d1e149c72e30dadfdae69052be8647999 GIT binary patch literal 93039 zcmbTdWl$VX)IPcd4H`lqI0Og;2*E8Z5J+%Y++hg>cXxMK+yX&@>$14J2ZAoTxLc6L zojFznt^r=4G&pG`x_p}OlB`YZ-2|z&s08pMUz|#WYE8qpn zf9b#S#eWG6^}iAw4Gk3y104h7e=_DvEDX$-m>3vXI9M;S|4YwzxH#Cj|2_QI$^SNe zfsTrbj*W?d`9D?uKNU}%075L3xEETeD0F}qgea(lC{MirnrAuD|3|!MVgEOwyg)@m z$H06R1LwIy-K%HuQBj|TMt_$0xwg;qdjJ|C`s;U`Vi-iq-!bVN-f#uP=U~x`S9KAq zOq?)q8#xBP#3p%5N=E*kk?8|73lA?JzkuN9uiqpjrKDwKRn^orG_|yKjDMJznweWz zIyt+zy19FJ1_g(NhJ{B&CL|^$r=AFDNW3E-9_9sjaJTXl!ck?&_E;|jofBOaeq)%B8|Tjhk<$Z-Ojgn?)6J@P+j|AXxR z9k9UvU&#J1VE;ER1OOKm<$3T>2?3&j`$Re8m?o-yb^ntV-#Q1I%X7-hA{><-M~AuD zu`{h(hBW1O6b=yT^Yc0^2gpu=Ud()&%)+SWRGBSSCuK}wt*T-lqTiVVl^pp;S;Z$U z0H{7^BbAUdzGFVdONXFp92_X*)+l{n#l~kANNSLw5m4Lz67nE{XWrDm z?C)@()9q?xp0kBNg$39|ju>chU#U6Gso7byFs1K?C|bOPVrMBCCkPzMZN_){3QbSg z@}Z2;CFvFmP$%zw-xo0!WWzQBlqFuPlndHQZKOBPY(R*v$7$z;TQsLunMwsqS?MDD zoj1+WffMFnT>>EMM3MqGyK zKI;s6R+r`sM!zyVFM7xb?U!0*nOulJLs^L5Q{?*xLL)o|^Tz)DAgQA*3O-Pk?rV!I zq|*ds4~4JW+bgN}SpSX@atcx;8K4+Ck?{Ux5moeI*BLHBVNRNZ=0`;BTbQ{S_jABE zx2P@(yLZz_l1g|)I7cLm3^rEfvRH|1*>&yG*uB<7AF;G(|(4R5tOY7FT+@5s_+Gu zl;_!w#=mLFJ~2$a)t!3gK$MIM2hCtX!2wxdPzE8QGWlDWshSo6g~!EUj^Reg-fVh( z{pJofW*xN{T1;|caR4!T3%-=4M%cBzNS%YB>9gi$&{*PCA-FuxuZk}ryv)Cxx+uL= zbxFLy`Aj5HB z6!pb5gfXp#6z|3eTC0eU)sKfq^RX8t>taAfb2>@cyi-uUk9UJ~^teFBS+GxX;zt5t zRWnUPLvgDpN39>Ur4TQ)!~3IGZ1eF=4;e5kxl2X%me{CCb#H7kFpQLnbCks=ZQ#u; zglH(cyNO|hHeJGx-QvdHhZ);>M4^VQ7Ge~pgvmPDv?bD)oG#*!wZtutAi#j~Smtf< z63Pb!__OF$py9=LZS3ib27JfK=kb5oTPRv({beHL+s0aP0{mjm*@U{CAkO)C11no? zT;a2;1@GFM;<3nxY?nKP%9|F1(l-d)nr2ec;uhbMS5eY>FR@D4T$=LX^YrfOhLF23^Ao^ybq&O8Fu_O(O)~N)4O}&*+B>jlzy%2_vMjY# z=VCBtNdvvoe}17Trp2ysv){7N8*|6;{~~-QJ<}-s1jsb!ZI@IdC!gcvM$`rO&V)LC z4gEuz{?($cYC2YagLmTPfwSpE_ND#vzcfHcWrC=m0m#s3dR_9Z zlp(}B?X2T@sw!j>&`t!gm&*vtmzO=hbyFuMWn)QYBR%u0~0>-$O zaiqCFZZR-@JYnI7sU(^5nC$ku>!9yK0zvXkcB~?XD2}c1W59D^p&)Rn{Ua*>fDtpi zOUZ@Z<;Qx(p>x}wRE=4DI4m@eW_d(vYWnnxcX2*n8gOU1YIa=`GT)cnC3q0BaZx}+ z$m8ul<0hK$1c>%600rNhbPqfv(iYFUeeg}NLJFj?fAAxAxqbq~SJ)8!275=63|rtj zf{YYQih31@@^lvi{+0S&TH0R+{dPI6a5^SMy_w-2272}^6VR5p&4W)(O6TdH06#+> zyrVhoV)z}sjFNOi^iXa!w6BBKPMxYZ_Kj#Z6rTVgL2K*MbQ{#|T2BD-n}2q=KD1Rf zuz#Lapjy$@Up>U}QjcKv)~VAk4EQ@iGMle`mGDfNV()>9q>Aln;WP)bK##QoN(KIh z8{Sj*Y`lBd64t3&;a3;RYca}58^Gl~2 z=TE$qsgwD>=USW`^r%4PO?=Xg6}pmaa(7_R6M&ei0%XUd3h$>TN?HScH6NfV#3Vj* z|63LTa|E?O?)aOrkh+O7=3CoehsE9Kk^_`mb9e%9E~sAo#8zHs z#|hND^Bm4@d5r7WFp!XM5!L|;gYI8|H!Ua?iEd_}+pE=%fhP-e4NI{w;}sqeCrGkb z{aTSHfCUIy#8{I9A%bc@cL-vBcmbJpoqP@1up=DW02$VYREC#$Wsv$Ldth zw)FPQiW`nEUWk*TY1i}njCb_&3v01%f)r_*2Ti7v9A>QSXi~??+eZnODKG-;N$-yG zJzB{*=Ve9!nZf%K-y!Z6hkmQKV?No7LyE#fMvFUd)=*FjMX}p9A*(2{1x(4OCmasS zOz+1%`J3eaJ&+a(=Fn}>nx(fI3y_g+0LBfCJOH>3W4vsCY(pojzI2ksXcn-3K|u?2 zFH`A!c_ZX9!t8Ho2Cq{|{J|5+ZcQ4Sl5P`c&wVjsz&F2&dVWzzTAyJT`kjvN%fGQbme0(q{O+iN-ux z-BizebN)bklY>MqdFYK-#O#DQ**|h{1R%dVx>QlS@$L`ZXWe^ zCLnC*Vm+5L08X9kV%0bF=P1J7{P)a`P}QhKw5>yCs~JQiED2M6Gs_>Y^MYX5z{WF# z!HkD;hcbqR(f`yTU{42Q$sw>PJ$l_xP-ivst8|~Ol6g2SL4RD4D683W1!>a77%_FX z007^IHQlLYe7xAW$5E8Uigp1qT@-AL9_pJ+B#8pNDnCl;)Hpqh^Hj4V!+|^|+r-6a z1t~XU(0~liL#HDhi>|&@%yy02jI+&RG7#M)>IJ<}S4TzQ+EBsw;>|Daa06oo+hy(FWVl2Ar2g6UaY!guhut`Sit<(Q?(+uP#$+?SFm8Sw^UuUe8hPbS64Hg|rBprL$2rz9d3bdD zhCisO&viVrp35^S9pB@gQ6#nKzVt4piWfk^&(a+;NpFTn1%Kf3sq>5zaqO0KJe(fO?7(X>DjFh_sAqPUwp&)}W z)d{=8Njf6S6i?rrZ4fpk)nCZ(fS~Lm-n$NThiQ2XUK6rpJ5!hmlhrv~zoEYP<^%JB zRbIs7*fo*y&m!d8NM4NN_Z7eFUeFZYXgwO`hsj*ymGLLqt*;R7eezT&-mB7+Bv*S6)0;gXr8UuNR_JclnOLyCPj231($XF-V@*t)?_vK&igb|!WRQ^#4(mYqU1@#7m5L^iyV@n}Ac-1%3uE}Zu9ASHy zflln%hFCd8BL}*nZe1y_+(TIx^i?HWZ0ei)e=%NbcQh_dJOSnnKT|jI{?#mb98+nC zU<0bJS{#N?RsfkCoZKOO34&_~m5uCXM=vi=%@oa+6r-}KGED-owm1SYtD>Xt>?yRM z*!FF@?yG*9CoWt^=F8}!S{ik2aIDTz2_}<^5S}Pq!QcEqoD&uyjtn;NU>Ma8@I z7H3lKt&l^5j%XRcySLb4AD9{92ETYaz#|`hN-wUn~gCwmuzU4T&U;s}Vd0I9eHtooK+y=f*1HD)%noJ#S`lUw%C0 zn~vcS^uStNkosY)ACtGX5@A3zTL(t-rh(VpmC4B@+q$rV2o&r**ETffndi5A+w9W- z0U4CDd_o*9VAceis|mJ-yGx$!qzmi5sMWWzNgp5I+pzcS_RwAjX@*WCp8$Gktz-AD zMUc!@}df9M9VNk~T&sVRj zM%S;$A?nd$+T;rrtKA~D0vog2sf$KDY^F=5`aMLZAPu-`q9-Xc9=PSR6%vLOnp_gZ z`|!1zM)d33%neCSAR%}C&C086^6olzWc!zYc^0%MDGnqujhbxssa!EI0cGqXfgDaj z1~xkG?x1vgo%Z1)-e}7))5o(hpitS8&b7{kh6TBWiPnbDAK*rTt0!iUX!s+2_?Ypk zzN6M4#rs&rpJfhHltxuHzMNK%6!4vz6ZGh!Fk0OZb)V0#X1QFpj%c`E`$WQWBgc^@ z$9>VFDxcrRy4V;{bVvSBkiC@r@exJEaiAEe4941FmCPW%o9}2bIkko_jQErgAGcH7 zD^0Y@$3oqOdih*U@UPM*oxl2f#RurrH@q}67mHXdru|sqC`w*)QophOFc8C53wz;S zdMn8Gudpdy^pH<_nFrO0tBS=TzKeKy8B0XaT~|YQiF^{?r(=-(P+;R@M69V&4{on6 zJz=19xB9VblPf*2$dzz>EXx zaT+H&nKEEQqVnYD1~SR(Z)?=PyEbga-Vn-BK~yw;`SD56MA}-HZ-gmLQR@yYmUplD zO8_p%^e%Zr_#=TJ2hpVFi=kIVLhlE@YTv4tr#j~U3Kwelk;Niol+3H$B`zZ7FX2Th zLhbr(jEq9@PO7k21a4>@0e1IxXPk8HPl z_(Hyf(XbOO3zm@yim?jTO}p{qs9@OvlC|fS-1?-b`+@uw=CDIqXvfdH%uPvxNlt?l z3Wn8vj%V$)_yipFc~ht-c0ZPG1UqOvGcBS&-$q@JZO3J)v}HZ0zvN?>J#}2wTq%UwH8jiwu<= z-~D2}g&p0BUxZ--BhNXKQc$(Z0X{b?gw$PYPhD;L=yHs+tl*M!phQ+X9MSoj5uciA z09hg+3WnO5j?=j<# z{c63MGJonSw!4CND9`sLB$0%Jg172t1emN-PRcwi1ekk=;87Pu(|Nq61UDf)E&)t7 zV4W(|N8p)uy37GQ3Kpl+sKODg%?T|$O7(pt!7i-Ie*hZ&%S-469Jq-G%B{3cu?5Cs zIWlkLzZh2ATk;3F4F_Bhd#vvGTQYwq{m7!WqFfuv)!Q1c4Ez@8G+@heoYduJd`3(e zx4BVyo}WLiv-K7N*>a~km~h5XNf$1SpG2edLAO`!YGzc76h`y75gyVyTWhP0b>w536oH zzcHh9(#AQ>X{ zh6kPcKwBjA#h68HR?fU6njD$G%@XI_@`gGG3Ust6Y?q4;zINVKyAwNH2TB(kFFJxO z(X@?2L^m`=MwP1?&^*)agBE^Qge#EMrb&1610;nl6l!oY+|J7+=TN${YBz&uL&+-k zx%38jhDvNYSs@qw%o>X)Sp6(JsZ~u>KD*|Ev@Wo^KW%v>)C2FmJ|r>vzp+Q@qPm;S zdK~@CVkJrpmu+8{xEg9RGX%jw_5vKyNLT)w3mPwH$Eo(HdwXYL< zQYR71d+?9rV{g?=1V@EpmE3tBuFmbc)%#3tl9*fMQb;<|lRJpc+CNMd2zfk)m3R|g zcdcb(dh<=SzrBWjVmtZ-lr!-_4UZvKo*j`q!dl3nZsOf*zUu1$tX%(EH-KVPbc+)1 z{;|tTtxMz7!r9u{`n+WIof;=f<# zUx>P8l`mN}jCYuhg?wJlN-%!{)WwE=;wz*WeN{$Ke0IuuPC!_V+g<=Z1e>Tt%lN(i z#GZ1Ta>ON5CSvelX9G%1DrI#{sS^!f!GZz3o&Yba-Lp8_X-@(sjtyHUsny{oN(J&m zl8n(YtKyr6jt=m|hU>o7CjhSd)tAXSFy(G)vto9UB_X%FU(c8odQwdVU;r6P`&!Lj zX2ey=cO)C7*6gfR^$xVBN>?xc+1jBdyKtUC^!iX~_TK4^?4G70VW+blr3I-`8xAa6 zQWUN{qI$Fkh^&kDw=R>cFH#mak{jdCA6aTETr%ksx9?3lz9Vn$NezW~V ztxA5b|5p*SfmfxYq`I7nQuwXPr_+*SzNG~pUj5e-x^iz;xSDV$IE3IL=Uqbdo7p|Y zra!;yFB>XJ3ln%kOtEWqOTs)aKXGf+ypm3!D)%@q)Y0Z&09(QqZ=5cOeVKXwynSH% zg5VNckM|vMK!aM5i3K_;Dhg@tw2Gw-b9btDubU4M)77Smt4k(USCNIiwn2>(EQHNh z)opAS1`s6u7_zthwW~)<{CmmSJhal zZ**zgncz_o?rNtJ-$#Cja%@n5bkc8~a0Wx#-<`n1-`8i>LEq>s?_Ehymc-D?Hd6<` z=K36@XV!BFT662S3q|;Kfg&F1V|x(&k)z&h>&*RzoP%z>wY~8HEpalaqyWhG68FT* zw+}hz5h#L0gFHe_%$ygJ8asR7RHH0HqiLYJgrc&YjLrMM`RTTQcA=ZD z;;Q!Bfz5uNyPnH1lD4wzu>N#V;DssZ^*%MqL_sCT3YKZQ1lJ9OvicMiH{*(O8p^Ho zICe4{2Gn2ez}ml^4TXAB)3*e|dtECn@(4LHKhefJnvAJt>7w4vpG5i1?Odj|YlPB- zzO~duy=q87#*X9@PhERmqFtxj6lH#0cZ=ZxWAFVAa(F+znw+|i@=SSBU2GVg%nP%b zL8X;%s;)MrEC=}otZFtT||#JrI8Z- zBk+=~!eNjhD?_bv*TUgDTi;KiNy*oYP6J93o-xdP+3^F1ubVJ}-jI9_o3~Bk(UfZ@ zDq|nkr$#oWu7TEiT`+Y_*&o-h*bGcfkyfz zcdF)h!6wezr0S6hF$ZpD2Xc}D`I)}I9LrW^c!PWsa<5J|sT()FoYF^Tp}C1#D|U3% z!TRN4^Z@lDBug!;4bO)zsgFHO^lcs&w9^6$Ra}|>JlQ)}!e(mR6VCK!>0*u*PaYE; zN{4Y<%eOhu9O9Scs(-ivB9Fb_t;vC2c{ACO?5E7!CST?u8UMU*$s+b3=jTin?=`bv z*z$CmYy{22*T;fuL1dt{=fRdYoM;s*7%2cRI+GlHAWfXZP~WNibEvZny!Jw#t@g$i z?I8`ua()fM{#~$rqUX(r(Qy8?$C4Z{?e{~F@}3IpyRaJTlPQQB97LZhuua|^#V<+P z@~A;wpwFCb{DikOy!F{h)c(O;+z#d1w|}KOwO&fP8SeaZHH*)JugXmM%+omFPsQB; zIXD<7gAD3Zy-pP$U-D+k`6^tVda0=5mhfThmm?7ymlOnmi4wbZ8oko%7w+a+Up-4r zo0Lr?m+36`+Gw}`>nC3Wj`_XFgL8r?zuK^H;Mbx9YSJm*HYqi@jn3p>#JtDI7T@I1 zn$nH`q3YRA9D%a-Iu}sywPoq^qQ8tU#|P}1z2XFd7EZ6P3{PFGU=oBjhxJK&ORSbb z-ye6Z8CXKgyK)e6=!IJeD3tN}k@*mA%vX=aUI0o7 z@(wpqr?5|;R=oy1*U?V6>Dvv$=bk>*3kaU$aOk=cK%Eqk2QWj;Dq>8*LBA*&%3x)d z_~10ZV!FcOTL0jMU1z4ixM2ouw$*4t%5&s*3zCY!`vvX~LXo`xuC7{k9Q#rQ8SAxr zF4M%~RLwp{JvBKGnV>RLvks3BzCPk6-*^dsV9sK8c1(QRURNc;Oz;Y6?5GnA=mE-IeEN2!w z({*47^nkVuhlaL@ArXO?5t(N1eG3h*id}I`BH#mus zZ9VJZwKl@t`G`_GwE~iH5hd0ip-M|K`&Z97yj4>yOfAEO>Z&Fn%2^&uFgNgKVrau$ z11<4z#FHixNC5TLyng#^g~K1wLXNU~UX2u4&CKlTAIkon>|<6ERH0!wCwtz=-J9Bw z%_^4PSp2)FZij-W3+x(0^rwpN=yQ-)-LSA~_v3ZOjt#B4Hd7j0T?@KJzEbZi?tWBM zdek^B&Mbe8*Z^O`d!ZAWD3@P{B9pfpQq#V~BITEEDmm%I@c}GmUQ`+NG#8!<}3%yqUN{Bcy(g28cF)_>(Q zs{fly)GTx``9Pmta68G#6Clc4?Br;Mf{JS!$@8^TH}jBZh{hv4lZ(_CY7{0aQ4+F&xp5ct2rt*!cuN zRp$9-B7M)T12)Cn>D*KUS>wF=ayxr!%JyM-U{1`9JL`of%uY69m7X?og0FYghJJzg zZG0#~SeSL7gnxW$)ZF6_CAL|dE-iDPd8<$(J-xu5GVoo~81LwoL!RK;P56L8`!r2L zuc7fS%iTiN7*vX_YVgK_kFq&!^j9N2fZDw{zrm2JWRz8AZH&UQsll;nREYddKq2G| zN5>q?kf-~xs37cM~etuN#?dJpIM=2X`4ku+0bW{vqV9}o=31e8GcoI-)0kyDzymsPf#VVPybHBJHFn^`E$ySZ; zc>IIN!v{G>E_oLlXtm6!qo->7-vQj;@#hthcFMYP#CVNrooI%x`EuTsI0MDJ4u-S=)faxGM6zK;BEKPcU$)KA>?OD12yv`%IO3jcFV_VnM=;!GXbx#2J@{+HJMXJ2 z@45T@a%m(AWUXk#b31qpFAl7VZn85SgbO}+KLNxVDF6N?GG}_{gHo1`0b@+|q|ytY zTkCfpfGU5~B@@PAX1yb+$B$vfI=M(*pH4efiA?g)-c7k;b&Q;l<<3e@EZ6V8@?vsm znXGALFKIEIl-l(9n>Ec8UcT(%J-4wugz}<;1lIjiuZ1&(f{;}lw3IT!U*BGoP$IA{ z$T(PG&35sYnHu#4$6XVdK)=w8`a^tWTSDIv;a}^oT#1mN$oX-)mJkN7!8AFp4g1bc z!SepU->RIQT!q`Cm2lG9T>}#-()Z%ROoELu7W8M?^;<#?tXT-DWOMr znvEuVuEA%a39qR*0Lj32)#+#LBZZOU%B5*6HpHMC8=rkB3}eAXw)QrXIYNYAKy+8??(#)pBe;vF~ zq{!(jLzkza&{Ao#>MtOWyW;HE+$c)#GilM|FNPVAn4Z-wV7KnY&mgePojuw7Uf~XH z(6N5w-{E$Qi~a8yUlTz}AC{T)R3*^+zQ7~(xP&*Cyo$Y~pNRD_)K-HYET^M4_Q>Po zDR8pH&MO3!Z6D1?j{Zq~mU@iS;CNGMDBA3#0AGFUS#FLr^!SlHW*}j1MH0nyFV8l` zqgwG&^-3qThaA7EbXi~D!qcP|IX4$!yA{c35jAPK|KJBTsx!E*gtM}g!~j?H&c z*Q!|ewXWW`!QJH$>A}O9MoY(${Ip-TpR1ERvB5)c6^GfaA>`-aP*LhffqU{ga?^ly zZIuavqw|Gn?R8Yx5H!L62~e%kWQLSX%UuZOV;Q^u7=-3aSIu4*1$!kui^l+Yk*p+u z5Ee-u(M6G?qA@c_Px77~4$+Cf%0&p5JO?1;(2&u)i`L##tXYM9oM$)Y5b;OyYR~xD zf@3xLEF6j$BMN$l1K!7EXBHi>Hqs<}KiL1gw8<$0 zMY8I8M9BUSDJX+ux2ndGjj6=P)yKWr5E{ZqhW(gjUSz@JZ(^|+vWk}xAW;|0p$`9Y zVPoy)nn`-NhF{Web%|A!YxNcexZfIWW4#=>?(6H|@C-%*CXv8mtJngA<=UBgR&3Z1 z@ec;n6{zR6_b2K|wY@G$*6J$CeO0|#6QMVOh`|Iu_vO$UY9hwMPQuRB$m#g+$t3iz z-e>&SsVpWRk*wW&^TqpjG1{uH%YrNr`U{>OWpgxW%O)3B(5xnB!5aSt%h7Ya_UL1> z-b(_8bEpFTG6ews>nk&rz@B~*Ybl4y;>he)#Mr=^TvLV+y;%ckS)OD}Hxa{wuMb$-rx6GDY+Cj%~j&S6CUcs$5i3u9U|3uFuwEgdf3TjewA{OKZ>XgSVR&2tc{yLO`eBhCSFQ&W0JA1nL7*TVezNVAef z-)SXxNk+Rc8f+dmY85P^YLDnxa2%7U-fZSQ=(+)kv#uUpFb27o5njbVeuIgF>%Flh zfg+{Y26ECH`TGR0N()MC$S|CaRuny_5HRpY0c>16)}2v;fSdOb941v zT3pT)P<3d(;@Z&6OxE(*1~?d8tPb-uoV4YCcx&1DSu?Dh<;S;>2qAw;w~=S&E2R9{ zZSyfVF+=w^6ZP~x@;tByD54x(B&grik(sx$?ZYT}Q>9|nupf4#h4xC2IvJ-c(O=R^ zV)YLL%3+@bTayO1D$IcO`hbiA{Pmwj8}1%m7ei2IpTwmJ=MlZXQ+oSJV%7zbxRhAe zKI@K^KDTpnBmq_zDQPic2{6d(lV_cskel+6TFP^0+l{JRiDUIOCWD&YD<%snbc54I zgx3^1jmCh;`BVk^+Yr8tta~*_vBVInJM~$AbTZ5iogY;uo-UM5QXIs!a~W4yy|=0qjB+b~k(? zHMTv%^2Y-mTB$)sCi>vnU}GH;K*hZXhyGr;(oEoRhvMie4M8+IBy6&{U2$~n8|f3I z_YjdCc9t&@UNF+!%EUm=DkoAJGm5jAB?;-2g0P|1jVzYkIs<>#^Ah8lcbCN*#P{<1 zLfPd7MiDhmSE?`_wTAbu0k<0Ab?@d~jq>jZmHmr~wscAsP1X2%WoV!IPd344Z?1%h z+}8}(Ta94X4X=Rl%X@TI9qQ6nKbGRZatgS|Rjanb4xN3#`4S-WPH`WY?^>@yQIO}V z#%R8iY?phcE;#dF*Cn?^^A0+}Fg((}0U>#P8`*Jb4Z#QHP%)2K0u~3c&R^84?2k~8o>Q@aYBC%911+j_og>jD15B4tM2xtes2F^k_fl>l zFxb$opLKHtq}cH|-XgBSqq`?Q^z)BUETvQA6F^b~viE&=gM}xvE$de2{?6@8RA4== zSQhW*=Nt70V@aFOr0le*BMpt<1bO3aDz%uoQsRdo@iw4@h`SJM+U|;ml}u80*I7jd zC&JhWYgY`a&)xgFZgQY_xS1P5iBkJuMVBAA>=pQpv&QhpAyru-_b`WcA&D9%VYB73 z)d-iRYVd{yJ*=%Q^3;w_^~@R_Mfi)bWmasC^bDw2I==Sxpk9hKb=}u*?J?WZhjyme zC-sK$v5%iEhnDu!y~|3t--Svc*ith~2`I7ekBSI3PUb9DPimB{@?jPhmtaLJgXf@8 zO{vU(W-tCo6|Is|4u#kF@YQ8f+^a>(9Sx4GwDwx7hl6_tcB`Tb ziUZFU2o3~fQD$lVyC*K&O?7-fh{Kbb=utIsZ1^Dw*6!{%oNqz%=12!Mk};*H(P@dsY+7 z6!>7xpt}~_5^~9Ku~B({u+O2!wYZ)v3nEYpR zq4IY|+MyIN@(9?9wcG)d#BI$@xxtZA(>^$k= zpG+Y7b7rGo6KR~zFUq#jRkztk;>T4u)5nOUelCHAdZ2o$mzu`v`u}3pA7UQ~-HkV% z0KLLryKghFt}uJv#!|kP9sJ&eLc*MCK~V`70D1-)q?UQ3U4vI)VGWO*)9sieF!0>9 zUC-&Fg;j}uA05a67wW=%I+xs~B;AhzA2)5yzO+JKg|y$v9nTF-?TdK6G)dHXU)VautZnVKQz=ucwSJ%kB31b)Bn z*Q?m;pr-14lO=Tn%XqU-0H}-^kz^SS;)iN%Yq_M=?gqvM$E9Q4=A%Re6jpIGV-jvc zz;cAiYrRjR(X7hQ@Ua1RQ(n#cw`;$UpUy}_T5tV!QULNtV)yHxt8UW~-m(717c=E1Y(@`3N+%q1tv%jV`s6fJWn86YDu z+poKnxorIB!u29#k2bo?#dkypd|qxir2bm)((@@mpg|5vXH-p>tq9Y)R>p zEB7!`;eWL%@Jk=8v zaL@U^DZbr~W#7vsUg{p3GK#jDs`am)CyVS}v!gKb*mt|}y<&dB-&-01OVO%< z5Zm^52cr0`G)T!~)tOoazV&ro!E^R^KT??Kz|6WMLJ|0ik`4{a%~7Cvzctr*oAm|V zV@LyGEaoDMWrEaNzjql2grj%y@yv16EacCD!Os*C&dCnzCFAw6Xdod*RIWJ*KdZ5m znbnq`Wkk^{sgFetXQVffz8C?BL!W&X*NJP{$wDU$O|6ZY>RWzMJ9x6<{h+bemyd$n}1Ce$+=NigBgBFJg`1nzaE0u7UW`6 z-gx;2hw$=|sWYC5TIu@F)=e)6Biw|i3cB7c{Kb7aUtj6%cCxbu3l%}%r)HgW)(7hE zix69|qbbYOzx1z9Gm}cDZ$Ge?M-27;a`h|m%4rq<-}Jtol0>IBydQ7!IvVw zn#V*jOREscWKq}4&Htt81wNunO1639>l5R9W$;!*lcavPQ+9|#So)a}kR7Y46KaRE zpgeSav;Vnx{O)Jg^IF2Od*1D@qhAOBc5psBUj+}-DzK?qQ93EruEJ;Cm~o}~^1u6i5-j-bBH zMl!>=Unxl-EUk01s*_!dW#|;=Zs&PxiddAeAVtl^XT53IV+$)GHOw2zBiT2vt4mT( zJZ(DH6NSojyuV4>3U}kBoZvt^+Nk9HeG(`D9^ggouD}r6sPlVc%2V!>%;Y4{DKpm_ z8L?{b85A%#6QvuSlVp6L%-g)`|8j%MYRjsA!AvDedXy6hC#HAhp)2-}s+z|_N;l(e! z8Z+}HcmcQ}h>AliYx35!t4NQm6KIZ+pvbnsvi>3z@)+?{#}0; z>Ts*t8jun1FI-Zx&sT05#mg<-*v}a#L1QYwq9{nUf7&h40}$*+UZ;B^YH(K{Sj`m& z^~%RcI$!hQ;yb0=*Dd-J?>ar7)vTeAHUR|n320xSe5RSHgRJ*Xq<{3+I~*%&W6?P~ z`!MQ-JwcL(X2TGmNb)PuR_Moi17yLCNR#Vv5M^|fK<71_V6bb4QX?0EVa;d%+3@i> z@~UbxkU(s2xedru$DN@MnQFkz+wA{L?l$*F9K+so)!s_(pO=y&_nyl8c!md0&3z#I z@std;B9nv@v-Qm;J_wWeRR3@(I<@yfS8@6Xmn3_EpQhD>LDG9>her199dyyYd zwMp9_$x_r`A%O*YY?uVV*?Jk69mGg-N=T9cRD_yc;)66Q`UQ{KWQi?k88{e(- z!?oZ~fooZ~mGC%n%TX}+2it+aPd2{H_43eJRvxOti&fSQcb;2VQg0uhyN%bhafZ!h>$mEX0@aa3 z-PJYT&7lQmW+e+!yV~DR^9RO>^smd0l0CT~l^021gpyx^_sR|1X!@Ls&aaSv9)$%( z0j1Tw{;Zte2Z)I?|HdnDCRa)?ocVpe07Z4wj6r-U_L|+oSCJU0uxm|yGz(vr@x?zj zoMbn;(c?oLju4bJedb#hv%l3#x>;vW=x)90kad6i|@?xRgL2Q zh;7#H`eh7GLg5o&rhwGBAHvobvfi>oYs?F?D_PNAuxm-61|8GwINV!Cw>s^{r)IsV zO6;lHxAfwvnbsP-LaB6oD}3#I`F8K3QJyy;q5N6#&_tmJ72+g&H9#IiB9V<7Kw+iD0g5`&*$@OB}f*sEB79?li83scGh(;5dtv` zZxuo)`9zu_YQH}u{^*EMm43*1e7iW3o$zkGJ|g1$Z)NE-pVSE1U@e4SmMoM``y6~# zm#u?KBkw*;C5*yCRI~#6nbDJOqe%GQVfcQdj~#6zT4(Kdf{{as{@V@onS0!q3sTCG!$rq%xI;~E^-&2M^75H#1zs`QqY)(W? z`E)f4c#~PxCP*)}Hd$f+08~~@s(r`3WobIy){Ah`loCNG+dL#>GS$6O-oj`;+he** zM>z!j=#QbSVAhckEmN48J-}@ z>x`rNimMgO(z2|QyC`p$b90Kg*FI{YmL1Cx;1j{3Fi_ZhvV*f3XJ(eRNLMQ%JBd?{ zYA^UmgwFQ*glV0`8I^ux?oV^?MyaY#JFTsau@5Q2#~8`_3fItn0{mm~6|ub2eirzW zcy{htE%Y0Q4c~&PikVcy(z~f7y;?VgCn&j3shyzRJ-jmcC5p710p7HtPqr{Os)7j4 z;T(&QPxYv8{3);aHIm209v;*z9RC0X?W4fw@NgepJUOmzu&1quqtdJcp6RkJMdICQxR z$}zS3)=^2P)D}tERbEc3*jG*(gj}1FPRP>_9$7gyZ}?`{#QN8nb*O7sjR=nnk8s2v zOoDm!s-7kBPN5CN_LmyW@);FH(P`Pd-M@En{w#LSO1XF9NGvqwwy>Ctu01P-)Gm$0 zu6OPv<&JyTvykNx%rS0rjiS82;2PoMJVe~5u+2|cn&g<`QWTzBJ!-r=ljMp#vFHG( zVM*tk%T+Adz!j@)r%h=Yl2SJj#yfjg?zwGz7Y^XmXxUVj-{yMq_H<#&inB#Eq)8-z zd6KChXPjpr(xZYldBeHf-F<~y*Ch{g8f*$u1XTd%*Pqs-wu{S;lafwzUu(nhubI=Q zI?wyozKdJ0R>jZzyY*w(k$^Fhd!8w)aQ;DpBtBV_m*<+jr)lwCiDkKzq%kkej-;M~ zp}3YA0@~SNM;IYX9Ax(e6 zl5jKcSK1;uCq9)Fux#oNJoM>b#F@n|N1^sqSwcu#pj-pmqkDX0vQ_4ch2(xU6p`-9 zQOU1|#JygJY+ER_2)OwM2vV@kwfg|6w zMI>iCb{qrwRriULh6fbKiz)}V6&J9YGvtT^v=NimqLvYXfu1_zuS;o#=O-T2*&-?i z(e$GC1Y7uZYSMVE3ZVIIoRj#d{x4qTBw@|Wm*wrcN;V%x&G83(sYzi$q4x1M@|(wPh!PSS8PFi%>&Z5~_( z$Qk3_zWJun7KOl27YEAOjWX$)Py5QCnOLupDx5DNVBMz)_NV)%CYrf=K@WYZUQp zs&b40_B|`m!p_n%lF*?YpdbdwIH^9%xQ1ly)C#*5uy2{p2|l#-v|@#`kQ=Rjxxwx( zyjS(P;$t*{o^4d}6pnw-TKJR3`WzbX#M}J`!-i?~8?O-0Zw0Jrx!zWsfTJ8_vj9G| z_hZ`v*~zb%{{Uv0^qn{M_|fili+JO<9z2piG0*(AGrGii{{W_?KPvwKMf?k^h|999 zelHI7r%g%5-x7+Gx07z#>!tKRHR5gQx844Qj}H7I&@^8Pd{wZ}8+InP{R=CX8Mdi&shhw+c$55&(4U)yT_C9oQ7b1Ew$UR&Ik{pUY* z!x9*0pg1A3UYb8@?}A#_?I+_;ir1H#ji!UHd?tSn!93P@neQ4XP$s~~YGoVd`d8|B zoL>~-@f9lIoT^i4OPiHVRJ17BF5J!3vW=daY<#yqAC*bHR_p#4;Qk~2!EdN)J^+j0 z{{V_~h}Xpb01qwo{c;_1S_t}5tUK;6ZU{_8Xh{mJ3mw6M>6*9jU-mrz0EA1%-Y3yM zDp}}WCev=d5bGM}g*B}j{M^{w%MpoIXk-CNbYj4U0|br;BE1L0pSDlz-Qlf!z+OGm z{8g((a|gpu_&~f*rjHLTz|-yaGbQ|HOvQ1DmORA3 zQSttbC&Qg@M7FxN@C+I>R(H3p8!E*FAzjgoF{@-{S&Jzlh#qhDuOC#yO9?7-R^ z#Ic6+Jhk50H^O zR_ek%LqqWUz}LQ1thl^rS9FpAjHHa)PXG>;>{`E$eja=x{k47>_;*^iw$pq+;awL= zg40uX!~rF;^O|DW#vbN(W^S1zlg4WQ0Q@vwgQ5I1@O8h2J|Ra0`iIBM%Z*wqEm=_g zjqe}kkSk}(Wgz7Iqz?7@UN<_vwn^+#$Jp0$!zZ5<)cw1lam&DC4!oDEW z;)=`T5Y{|Pr&;R>3t#AKX%fy6Lk~G$YXXaq$BYw!TOYJv#~l`534B2DRmX+yHJ=Ii zi^J<{rfQ$?vEZ7;w<;`8ZKtwkIb(b=RUn;*l5t*Cc(*%sBMDGXnp!QKQ&9JMUHi?h z=^M)hOOon)FI&BJ^waNWfcTBIpV-IZ`uI{W69ANwY|!dJWm*R zRVcr8h}=o&2H<>y_LBHl;f*`?i}4S_KZ3fAxVT+bRMYgyqPdanqYOa=!wi5RAcZc# z?Apf!;=Z@|;Ro#F;$PZI^W%q(J}PPPcsIiu7Lj!#Bbg@EoZI=Z+Q5oFZQFi+at9b3 zvhiQCo|&U~$631guc<>DYd$AieM4Bgn;VH}QwzBC8RzQN_~(e+d7E(F8x1TCNkX++ zQueY@O7cz|t5~&ko4ZQt>t2WJUK;SuS({MB*TztdSkkLXF;bM$<(z9dKi$p^yTU15 zwQK3y=BU5moS(JhLfW)f8e;A{w=Syuk8!gbsVFgyaNJkNf49fOe+u~j0K+%P*Lr=! z*~VtHmdfEIXrvt*DoGu|^#;EU;eQl3uPx4FH;Sm^C#v^d@7+o=SM#y@&UeC`B~vYz zu*<0PrKROxTY6Va`6pV^Wzp`5nuVcoTn(OpQ~0qDc16+$_dV(B7*f^czO ztEc=2@n?>F38whpL`bFGp;^3-rK0(NZq+SPSqn6_%6Clw z&js)>r21z(at9Ut<->We_O}n<@l%&H>QkpFY_*gVdMhp8L-~t|@R(e)igEZF3zihC zRfJXBv|~9%E4#O3t#;e)%zz9Hml=$nM{!rQnT>?MYt!Y5WY6zyc#xqo1V`)R#Goy}~7421)hlsy;wb0blG<&UoS&L87ETx`#W)a121Y6zRjsl{N zNhII^E40Qbj zS@c`|XTk7)X@6#yv}>Efu`rTT9@6BjtCl%D3jDv1`0A$(;1uxm)VZYLt8vvrQf}!s zl4{y-uB&tF_(z5)aSkrDaTR4w6riTuAg3pKl^WNZxxO49E8gxIlPsHoZLO8rd@jJxOysd`0Zxc@bTXq~b3-4c5 zd>sD(f?RmZ#d-~_-ZA~5JZ*8QPO4r#D^ij;)sOIv<%#6uBv&WFt z+ty7l`SlC^-&x$F@XO*a?K$EPho|C1J~NNRJ{bPmk4e6;@qNs9meWrr3Pe`ni-1E9 zok#;edb9rkXs?Gq4=sKgX&^MrkyqD>S8MxumqY;}!V40Ub1QZA?V%amB+FUi6H zd9U25@~i7_xcNm&qD@-GpA%i!MCK@FQxcql6B(>~NNn{PnQah)0Ly{$jQZ7?^tRM* zT1(_*wvU3q9DOt1tdA1-e@#WUw9?q5XN4<($JCzn_8C)*aM7iOPIX^ql3u;vk+c`G zr^^J~zXMNB@SdNhHtFwHLIzo*9Zg56-RTx`hS;jAc?=29v8+4K5MN)pba4uk&erK% z_O0UWI@SKqYOi-MaT&-TQ(n$LBg5o42b=LmVGKPzETtry@`_jd6DtpwVzE`VyM2+f zsa!{>-@Vi^F3tjtxXvnVJ4@8HPc&)zeAD?5nP>81XA#CRmuwG}Mn*~CX1O)-PPqat z%=#PsobnhG&~>kN_z&X`1bjh<#kxm^wQG+LJ(t7#m~1VgnPZ<-m*r_hrw7g6K|2O8 zp;UCq`d$_B%fjq#2kO~QIxcWYT6cqKSzU7^<>;^7+pnpK%W;@WY0eJFt8W*<4*}?L zr-)?Kd|jwcH~p7Rm+Uq%pZ2A7GfF*zN9;)EsoVT@gGrR9g}x5>F2XUwHO7o@ok#oE zZy_h&0=@S0_E-1;u6WBq_%~ys_`G<7RqzyA%4;!R#@6x3Vq!sv$C5W8=zD>l^=8lZ zdGN=@uLArS_-&}@+J={`U+A|t6KdAhiRC4tlO*0^mE9zy1ynh}B%t!p5;p@*S>$^yK%m_u2$KBv^5dPk zBZ4!@uT;AIkbEWKj}&-6;tz#1pA+bD*!cd#PPo$b+3nhUs2XG|cN0Oxe>VkAPtzRn z{s-{ht*U<6{w(ntYO+~)f5ILi)in)QQI{_rtZLFG4`R$lG1P@8wR)H=`yk>tyhZ^y zMwDGvlXi1X(S*{{O7^lz-Mt;rtW-S@5cq3hJ=eqyF5Mw=W4%^<4p1rNpHE8qCP{(coj?QR!!QT&JP%{(Uj8A-B#!=^SL2>P zoLoBI>Q493b=!R``xp$RA5enl6Iur6>j=~23jZ7YM$b5~}F7{~`4=An};4pb9g z#rf?nSu^Y?JqiNZLEJ|Kp4A+%f(sFh4(6`3K~D#_T7}{`#s@ts^Nd_yGd(JDy0J_E z4m0%Qr7`Wv=s@R@R?Kb26miy?$+2=*JRX(g$u!LhQo`SLyYW!GTNgV;Uu98{z>s}v zCx_-6dE+LbyC5huUoJr4ie!-GumR(o=Zd;wzzhZlr9|r*xc1PfE&fuxfsVPL+j6R)ao;s~S}O7YKQBB`AQ>Bn9r)t9 zp*yWa-HK^$pq3c|q>j`Bk}y4MhWM}hN&GSRSh87OYgYHVkaCHq+03r${{Zdaou7|B z)y2v2*W#CnbojM<2sHgB##56wm>O-q!M(fVwR$vU>DkE`%1dN>Xt#-900I1}Wq_6A z*V4X!x%k=QUkTn?-)h>lm-Z#`G^Q{?Q~W@WPBZ>ISF3z5_`Rd}_6K_@9BdhAT3uf2Y4k`Vnxh zr?Z4~@2UBJ5^~McPrLk&p}aTyV}8w_w04sQh5KxH7gDhC1Xjy${{RaI!YZOmq)~-8 zP+*2&KZqmevZZog6uumP!7)4^ZguT1_S5kWq9i4v(kwh8_cuo!n7@*#pH|ynDR}S1 zZ>PA@tX!kWGVD$Yucy|%*Wh35dvo#Q;tsLl-wOOR@b#sJpRLbjZF_rls4;MovVzC$ zKxY7U>Us+NYsY^dGT#TY*<}1PI;x~@>MnS?G~S-CO?x_X(JF54R^3|q+5QsvpZ@>^;)_#V8^xN(#o08;CL&v#Z5K{} zZT|KQ(>Ffciua2j+o$#o{hYLRoAx99oLgD(2F9Vy^fJAu)5=)0}Cmx4fh6BN?UHxcZ-4kMMR_3=^+Q z9YrOp{{VP@l?Kl;p5d;pB)O45X&sri@G^P)t7pUh3?3u4BF9sWE?1Vfkap}vQ}Iga zqf&vF7aioY$a4^6|&o9z|Ygxt3%-iwSO+^W{yn!t>|l}gH*p@ zKG!=i>`CUQkX!0%utT7D;$+SUs`Sby^ft|En-k)9-XTxY^1cC=OK44 zTL!;1;v8llIF#xsxJ7D~(RSV1{Z*gH=ak+jUA&9-mO5^vt}bKqW@pP6&QyCEZQqA3 znHjX(Uo0;~2e;!@?4=OMvKGeXIVP`7sZ5GvG7NLLXY{YmX6nK*ohjZg=D)~NbzwUr z=%0t)Cin;YRs28D{uX}F-Y`A}@t=nD^wabY7f&AhO-{{oA+Pp~|vdNIekChbP%#dhk2{0Eyof^=w*ee-nH`cNyTvbF0kYkD+{4 zLxKc>q;g%d%uhf5y7TH~d|QfN?HK4mf77~p{G6oi_ZrV1f``FPxBcm9_au#v+KIjc zYu+x8K>eAlH6f$hX#W6an&(oD{OilT@)n$jC9p$AIScoAQO12g;eYJsB;HO72{f7rZinf()HQeIIdCTj4}C?FeGv6Gx^uK zcss>@BhvLLtUergqB~1l$J!+ydp!3Becx9GT}fr)B#Cd8zVwWSP@$hWEy?z-Q$yB1H28l&z0tHb z)HKU@o9wc#t8wR>=@*YTda>jT<(LeBG19-8p*(aGs~XYflUG(>@ZWLzZE87epR<)2 zD8)NFzw7Tjqr@6@{+qAEr={$%n6&#SP$Q6AZL~n=9k{J83hHxdT71!5>M0$p<&4po z!+C@sdWXe<-^5FI=7)E)wk+WF{{TmhtKG??YLP`_ZKX*a#DD8&GJL1-#ebt52L&zz z%H_+ep+^4z_%0{%?}5edc zDh*pi@P4L-?@s>!MPfc!P-KJHWcIA7bW7_SVI&(&Y(Dbynu^v5FJ~6vj!)kpbRFyJ zl?ywj&xX_Dgtk4$_FmEa3-Q0aVJXqwdew0ggTWxv3+d*qMGk_j6dvm$K+ z+!KO8R?G1fzmI=rkBjx02XxEt+Dli{bnlLT5H%}{O)k>$W7Mwi zOtM>ANA`9Qt4W`iGC#_4Fvp7g2gdBKtqvTObu{R@oYIZGl zbSO0xW2o7f;)hEPlOT-dXV?@oWsd*=IO4nQ590p-?31k9-9uyGn>{1M7ZyHkwZdHq zV|d%EM!gW^V6F$^Fl+eAbfH1>sxq_IEw|)H=7ZF@(@sjGRsNGJofHFoC4t+3vE7yEY@o(YAv#C#WdElX~YB!QJw-Wz-wk^a_UDRLmm%t^sO8J0FHhdG2NXb zLW()fs==mQ+u6kdu|VONkc-O&1O^8_ro64;zm)lRHx9sdW15ldqKd*v7#x(xHT^Rj z1qzKijkohZ6jYMH3B7Y*miTxYP!#~!?reMzLc_>-e9kTly*2w6#R z%uJUuLK%qk_O4lM66Fg?DO6FCRC0B>$R>)gH#{==6(G}0}# zEB^o*X*TUL{N^tQX^IK<{myIWiS@Mb?wO!?i&J-&D}7T_cEHOV#bN*+{F?e(_HO?G zf{J_@(UuPgYd#CVN#lYjDKMthrS2!-}bHd zi>$9*6DAX(X~2{I=!1;?dsoSyv=4{8Met^5{7>M?bY!|;^dqSLwfRm-fy^mlp<29GBev>4mqXCZ__G~X2Rbydm3^Y^Qm0Qo zW84?^@4oPV#yC8EW8f=m-A}_>B;FT^E^b{Qc%gYC5;B5GZs(8>psd(GZx4)?z6bFA zjhBXW3wiu0ek9R##Maz8G5zhnNjW&r z-u!F*U+|+M$+NBx$*NP~{{U-JczRQvS=72|N0&t@Xy)9Vn$cfJq5Sv8w4sLMOv;rd zWgJCaIwkETve|5LcHb0!C;V&iZkek5CjFl@dmT#CT}@7V-3v~XdA9+xiDhTo97LRz z%8cOVxj%^i02{s|TWETp!7JNaJt_s)Zzk~qY2)YAZvN2ajIw9vRX-}I;|MdJD{IBy zw(rMZ5<>PGCaGoN`&UxFMu%o2Ax~iKxtHptYsY1l`tIfCyPc(Xa?(p7EY3;EB%Y+6 z_5CT7@Z2)#!8{c@P)#_-7nA1R^Lj5%&*Xgjc0g#DgH01^qf6Lt|7*i zD7i{fNhGe@dL)w7w)9J>!2C%4ueJR~=Klan_}(d|RhBKLI^;7GJTa?F*`C ztFGDjOExm_w<~=M#N2h>u2lYY{%U4?SBAw_bt~dxud?P=^xXY2j|TCCz6`#*{{XmE zKlu}D;kWH&;(aerHlGnCI-R4bD=aXg%#EJd+8ARz9)0WGJQ3sH3VcM;j;-NML(h#E zEXv4L;$zMj9*6X=lrO#rc=t!SdtFfI(ajryx?dF7p&azu(*ya}NBci`pIGn*#zElC zRwhX&ypvD9w+*;QC~c>eBpfp1Jd9+D;8?saF@EZzmt7iqe2BzlIrdPLvG`h)qTR1J zN-FxhUP)_Y-Mo*ej0;G{=j>Ob*MvAyVJ+La?3CMK7KJ^R6tI!;d zht{MH&*4M^`yS?8ft##!^ilpb*F_6C`9~S{u6yBSxgH!Wi=ET4{D2?Qx`-lPe=}dw z93MxMhp_9&_?*H`;^zFxVo}mUNj{>qZDK3~U>`$OV~7R9@#)7((6fog11a|7uj^l0 z>87KVypNlVU=BbvX(3#UsT_07D?|vu zL7ZUWt3>rBOG^}mv|nsokKk-)rgAFCwHpJGwEqBF5+nAc7~z9t5BLpR`##wL#y}kN z+O$%zH)5(Yg#TI;Nzs0?9nBb`O;9cg5ZFH{(={7O|N1F|jBXs`o8O?lG@yk?;;BSg8wF^Hw{{TmmQ1ci|F?Nke zA)-0RD~?yVBe>$dF5~vy)9(HkkBnY1@D0RTTTkTqqBG?gahfJ=&VYp7H4tEd3^sdew!Q!x4ojU%>H5nxD6xRJrv#u`8u^0(c`t?ON zuNzdFf0_0_!wdUuU&1yrTFS01?baC?7*Py3@4_vh@p z;;YYvmYyK-7QXY_SWDuosjXTV%RHyfZ@NCC9c#kA6#mZs8~BT9W8zIq!-;hUqkOhD zlRejB_$Y_L9aF@|!`I#)@fM4y%z+_ob4h3g(dbGLNF({zg?`eX9zG6h0{2Al z^}eg9-!v!fG1=+wFo>M;>^N2J$*)rQSNjBfF1Gli701M{i*f6c89eJ9GCQluo&It2 zIk>o#s*Hiy5_7y2-qAA%6eDh zcnWzU%v7S}lGgn%f4ufF4>|gaR!&b&m$&46rQ=8v*IyQqAz0(m^z@BL#uQk<0H#`N zvDrl!iqGdx9k+19*BIc|&&ItlX6oVeHb(Ov&0%|TydI4YyM(u{{ZR99leF|y@?*u26E~?P`v*D`n~IS zMYCHZYs=%XsK(dzueH7l_&&vdV-FAfVA5hqFSO5r9w5^-c!0?^^Ig2Q@?*Z%wYeGf zab5oak34hYUx+`qy#D~R@5E0K={lc`!B#$JGA-VISC=MZfr9 zJW=tpQ=M$}NQAc9R9cC*S3&_fLfj9T~v%C2*xy=Ri#R!=5B4bb+Xw~^ibl2x;F^V@1LNa1${ z8h0Dl2kg13=PkR0p|kyUE7{lafi!IXs&Fc<~d)cyQ)z41*)XyjD6*E>(9J zx3YHC$@W&V-JhfImxooeyaUZM3Hux}yHBgPYv`Ao-=b&2-x2gr68Hk;G|v-gvf0SM z6%NY7pZ1iI`qy3HoqxtY61cSSW|5{%a|2r1fdar=AX`F10fUjdaL=eUQHJL1KQi%PS;Qga8x9N zcMLc=ZRZ0x=zG`0_}PWy3VaQO#%I*&#-naBj1;7!8FIaK_hlCpn!LAq=zf9W)qWky z^J@8hDAb%PIL<9jFqbtoQa09{`IMDb_DwX8TGKyiKZIT{&>__PZ}9U}i&BmqlfrDL z2Oxh53xYG~dsolj9R4wQhs2Xb@c#heH^aSZ$4--X+Vu%ExzR%sbF`K0ry1@|d4I;= z0G&4GI~Xjm<;e#b$mzv%+CRXH-w|k+Qq2>`7_^#ppQo4QA#i^%EBU7;%WybMT^vgeExdsZA zw~S!=9x4lKo9hiKO)6gT^S~jw8k*X*6DP{J=?sYC{lT^?zr&`avYltfrTka(u5-2 zxnk5;CDOc_yIL)5y1mck{v!VXg?=E#;rt?yOTUu!kYQhul3DB?%PUZBi%~UA&)A* zIqB_QuMcqzeDbv68ZJDkdEWm3*WP^oBLhZ-p5=0b?84ax$cjrgIvmx_Uh`Jd{0fPt zTuA|*Ti7=vl|M9A`$_i56^E;8_qt`Mic4Vj1AO#Ddjo$rQ|N%1Yts}se0 zERObid~++sB1X77r9uv4Jpt+~&hZ-nu{ks-;;7BTne%c%&Ph5|BA+Z%dP-5b_nZL? zLa&TeYC@L{a4d{6s8>%VOpov)J* z+4VJsNRJr~trIz5Ksi#s(yVyv!Jh|w1>-#nTksdey-x1t>r%)|h1+ihwl?kk;)h~$ z$v>BKy%j?-H}I z7ze`k>N)|&FL>a3SMcT|FRe>3oGQv`CevQddRt3dmrHd&Z-xgHuyKqdLz_o`tkApH zf8$&5J%_{|Ah+>euLp?q`{`cN_ec>%vO*R7tIL%nj)#yjRpFk}$~8?|R`QE~0LxZi zi*$DI=Yo7YH-sd(bkWC=D_lY&GDj?qNh%qJ13I3kp{#3NZhsJ1onB0hpaxuUr`o@- zJPgKQG29%m)K@I2&T3j&zG=!)Z?D5`&&Rx1z~UvvUe;4mi={zbH6*19G~pzgZ8W78 zuA5JE-se+t{usVE((flmCn0W>eBQu{rLLq)Go~}ZtgRl__WXk_s6yQwlgO^W z!JZ57Zo90422rm02L)u{H>{F1wWBz|V8 z#+_vzZB9LvTGz-~Ibqtf@2%4X0qtJ>Yx^zuSH!*+yYc6NzAVFg;mtH^e{a>K(B2~& z+FQpmpDpCcGI^??9ANX!X!u+9V)$v`?N`EHIe&|~XNsWLZLe>1{T^Kcc$VTTduxYh zZ!ncAlEk2^0lUzO^>S__%dqNIF8$W@<#)PQv(raq*2ioV;qNoTJUif6e0kw*FT)qt zf(a(ouY?k@;ZK)m`d6iR5B6~QU+_!Ez8}{7edFs3+tYgt?JBm&?T|1W?_ycK0O^8p zOZ#wqFVVgU-}rKG0cj7V=~mi}<-NtiUbO!JXq(S?MA;yDM7PpzZ*Qb#)U2Y8<`UcT7FFQJ z#BvTsaD5GOUlV>Fd^q?`quuCU5!F{wk?q(`s5S%4a87vWJ-PbVoOrX~l)nz`x_Jv?1ts!doMyj0%ks=cY+X7u zYf5ptNqOmI{{RDFV{;l*>CROsMiXmv`<}1jzX$2Q9`T;5txqFM=H1-f%C6i4B(6e? zbmRHg19xz4u3q4-H!&a%!?3Q$;`fRqzVSArq{jL3@1#jzU|to73-5pjHN{u^KlE$i zT-PA*zb%$#?@`dEqMN@gEzG|DZ@S3*myaqc{Ck?#in4f$wq8nYemjK}QD4x<%ASe) zGQhKZLzUf5)YD`>z?MN>j1Ch{ z#~$uK!n!}&6HC`_Ja6Eu*!I4i4~T4wAczu@na9pgP%uFHkzd7MOHZoDh4IxZUf#(^ zMSETL^?xJuuMH{6rxj)Kj{e@8`==9w_#3Kz&E5{xEEnQdx*ltG9%@5o@*=nUOa=}; zE9_lg_OJb$w4a4`cL&5XMWi-jib(B#^MvRydf|rvcdw4@ehK_4eJA@*Q`hg*tO{Bv z?b|ZG(Vr}%`seQt)~|T8;E%&?GvSAaZdFv@HQn6WvfAAjxbhj7U?X-8z%j@BN1p!x z%5*Q~jN)>Ng=+a!;PfG5-L4=kTS~z7za0vWd3%lIDLk zL5?V`iUnMZ^3pNgjN=0tHPC!B(7Z#a{?T3`*KO~7m|)iKqhi>^u?ZV=V0HV&Nj{)t z*R7Yl9vQ|ooUO|wujb%gb&SR2Scz9rzqX+C@`fPx-ziBLY-gTDHIcEmZbmBCl7JG@ zbp0ukTZK?JX1#W-kD4hulk$Q(@lm>91r=qZ4I$5`tu>+mk`#2w9+i~hlx_t^g?b$G z-!&YuE);XX$4av*Z3LX0HhOlbof+SDKN`)g19Q;+8bYbzTYbrb&gY-^&OeoOi5js1 z54RkDI>_(@44xFTR$OjEK&cX);*+r-8zCgReNZ+WHfl^)A6kvNKVm};~b1v)Ve80Q;S=c>{NsEXP$U8 zRQ9ORk;%v$5OGCt&YNtZe3k$K)Q-N?;}bH1$EWL3EJGoRk_g64ENDQ+N4FLJawggP zOOs+EsbHja=AGrow}XszKGh%{H!AhP;-0Z1^4#%SUrhk2(UEZu6oI&mpQrx-TCQM# zM&XhMPo**9#E`K(8E|;ue5doPA~4I6pdY7d(QZw(D@SM;i30PEa(mS414iH+{xtZN zq6J1za&uE8d0#PyW17`DHeQ4tLb=>Y`r^3%0E~YPYSx7&2-E`HUj!30D^gsEVJ9N@7XGsq3^kFnv5 zimqjw;qaJh&GX4!eAWK|4yVgzxcW4yPFR`>_Gv!H(SNdsgFIK^pY8cI&1xH)D~&=+ zix?8uYsRwyFa?8r*b-x>y?k-|LzN#L{8znqkN8L~vNPBBhQC988)+$~crM2Ccn0dq z30x@u0H!JjKZSmL{?K+>gx?2fXn68)q+eef0aHt^1oV|8V!!R1_7>M*&9U!Zkh!IiPbLF9Gl zG4D6u7-R5$oNe^WkS;=q_XGZ{fsSkCy+gvc{wcrHd^h4fTkNWJJ85qazXIHx65fZQ z_RV*Hv`lweHn26#Ft*Z)ZX?eOKQQt{S-sokOS zd`F>LOF6uZa8N8j{^%9tb?;mTjqxYOUK7->KFrx07G8MsW;2v>X zUOqt=h!G>mQKeV`p1AuQHBwy(&YGH)q{daoWkQ%K`tm*N{mk&&4M!~S=Ox0Yq%fFx zr~Vo+U!nMq9aoI_&oZ34;wyeNBz-OLzxMvqd=K!t-{C|)2e_Kp{4CV`DRzF`piMQ+ z)U(5M0bT@RS53J&+)hVI_3sUS)4v6NIDXMO=f-~l{44PWhvI#6!&g?9*VkIiQd_|9 z6}$;7VGuGfjy7Jr`I+DYU!Rc%G65MN=Zsgrd>>s;;fIGUd{KM9%f7RD&z8KE8Bw3< z?$!8@(h~>6-X8dSm}i(I`I8(*FP(f8dti6@E4NmdnSU0sWz& z({!2n7MB>0>}3A{2-?|Y9e^W_20GWu{u}tk;(aeiyVN{I1d|{#>Xt4+ERPH>*dOo^ zR9B~I{wMg!@l#UMwSNV8;%#2bN|`RLv{lu<$WV+tcLB<>$N|9$M{l67@rT>yc${Rb zPO9eFwyvGrpX+1#k~Hv9;aoi@7p-c|CBMOkN2UJJzwl1qivBp#JUj6F!e1ZeTe~eB zOMLgznBu&YNb!bTx3ZiZ=W*#@IsVt*vww@fXCDTOj~Dpf4SQF%(uJkO)~W)Cp8Cy% zd>nK}&M-2^)SBo%AYOjg9yWWcUx=R>mtM8FMz^)m{{Xbi1}sVD2QHSX0Y^K=&ABQ# z?OZ07zBKs3;;YYv-YD_oYm!5!1d?_qYO+rg0h837jx*mC)oS=yMsuqd7|JqfEq*1| z-+>u_)oNoZ+MJV>970!mX+p02W}---4w$KQ#X zC&Pc))5PBj`~!XVJ1rx?`cyV^K-(lYSNC%)@3?h#So56cK& z<~xo_EKYf?Ju+OV0N2F$JHU8;9QK)pJwmoFq}=_ZT+)v+vr120UEj{f=saJZ)W`DS zu-AncMMX!Wv(oErtckU~Hc5!Tvbk7ih#Whne8h+9~0`O+*-2vo@jPQ1qvO(w__aiugX8$o8s@mEhAplJ_!EFSNAu*7qDiw)cj$1R6el4 zfXe{dyDjoKWAjKk;Z6;GQ~nB}`&Qn5#FoFXMwzR}7J+AZBHe3Y-EOAS_+-;dXDtdo z@<8gPV<&21yyZVno#~=__s6GAv0G>Ek z!pyse^CyYLVU{9lr!5rUEEc*=-uHcyi&1M=bL}zwPmPr&8P;)9vVCOl`gd*I@trEp z{V!y=x3``-8g0?cuA^uj2q5RKJ6C6*_`k%EH1KQpQ(nV7rdN4oCQ~8L%t`yllga0v zwO3ElVbfy~MA-pw0UfG4%ebv%`NlEtU(s|j_+c>VVm14;^tHO5If=wW7~DB-zpuFH zZoFeJhe~RiHOzuV8{?H6p(;ZL0FFWarv|M2Jn?io)~wd^$~G7YP|f^V92(=U{7t0V zTFLhN5p!xbnPDg9&PGa}#0r-|(qq&0B-eZ)bT8~K6^beQ$Z?X42Tq@*a8=BsgT{M2 zD^En)Ulp%w`Kz3@Fcf8kYD&+e>3`9lo#Lx|5A63_+hP!mZVUH*wdMMrim=|vW>hk= z?upd!%bbJt$ga;w@zuPLoflq4NTlK^W&RWW*7Y9skK$cE=RlL~kRYDZJA&^208~$W z^YR~F--Rmp$l@t`M(&pV*UPcRmnyhwj+@cFyFX7eBfuB-_K@EAV^EEAaKi=11;Xt= znuR0iX?!{1_-#B^wvgNGQu$X0BC(n&THawyS{x~WSmodTwAxR zPTgJ4)iS*5oG|c)BG!*pugTl+vFpAhwzszUZK`VenuN?r1(a-m);>Wb4`3MnRpve% zxAL@63+>vkxq;wg)YZ=wd_>efFzEN%cZOz`&rH#_y4`svbeo$VTBjo|*Cc{=3dqwu zH+>WfH;i>zlF)tKJh0y}{{U}j8UFxuekQ*~@WU)zE0xs4Q~m7fOPW(p_i63Qo3m+k zcC&5T=gx61XZ^8VPF~`Tl8gACU6)sVucuRIP1mQoc`s~Z&9rVKt5AD1>@;lFO2>tX|}Pxo#APo;g}U`f}}6t zI-30>6`fPcY5QzVTYhiRZhXEU13YbY?4h$-_Vs&Q?R*jOKgHS?hV@Sb+xVvT-%i$| zM8DGDf8AXLb!@QcF}JT?-3b-welPu_V7Agc1!3U@j>hxBIyK&%EY}K?M?aqO7s{2| zL5@a7QahUQ&xF1K@ov8w>-yfI_Io$~096c;=1ElWag1yj=zmJ?uKW|>+nZmp&!xo? zgN(+zNcZ>m&3kzkFZ*K%v2v$qrrL5=ZtIrqC#q|#t*1fb>R~Xrmrc(7`F2GA01*Bs z>wgyh84U;G55_ljH>8%;C7NBrBwr&1BYf;HHyj_{u{qtE^4|uxfqXT4<@mS9o*LCH zw*uDEMXDxw^;M%2@?lXm8mp0-az_MRWkV^)+MOcffF zj48!AI_b4e5vLZ^YpCCrD)x#^tJv7_&ZqGE#4OQxXT}~A(XH}v6cFmIkEv@yO=IZa z4)o1Uq_`gy^cj&*fwJ3CA2ApmZ3lBkpYWdcd$ANSnG_$Gh3G4-8a|uhn@Fu=hkHi+ zqAucGasdOcY>vaOEAl!jaFV5kt0w;d?cF@v$b zi?g;j6&S(is-K{*?!OBBZSg;XygQ{vhFr567<X4_UL4+V0YJhU(r`!$%^VGM{ioeX-zw+JoVji~b-< z;qL={N4UOSa4>BvQ-=*_EO8g!hXCO1am9J3?Yr=sO4V&3`1Rq)mKn7OAeULU zkM79NzCSfaI6I;_^$ZVVpRw?B>3f@TkBAWA?km8<2C*7YaZ^v8_E4Lf*PB$NlJ~5w z+tSZ?D+z+iez}0H?RkDJt#0byEiZF`{gfjoU)7}tm9~t28Dsq`*F0(ATW=G`Wi-H~ z+I{>tP{_T0M^~w$TpjLulhapwe0Mx5Pp&ja7GR}#w$Nr_&a^zX)Qcc zZL8^be2HPTg87G(*(U>V=cli=Zg|tgKMTGI_^Nwv32BewI~&+H-CMz~Mqq%KZ`}nD zu^<2l%aem#tp5NU{t(M>WsBfRjzDr5HG(jHh9U>$U*q<3lIL7KoY%nD!!+XT)#>Zi zUeztL=+D;a(a7;QXi8a~EB^q85pS3KI}=*gd`V?uCh-)$Ot&m$iq&M?BaHOtj;B2< z>2D2q148h=r+=(jM$dDo-Ag>uOM|z3h03oTGJX12%hPz@!2U7UqttZ|25U$}i?tWT zFqBX_41JS0A6oi%~Xp;D82s^s90VWE|u5sLBJ*+rVzez_Ve>N*JC5}YylzPwNgesmO1%hB+^%qXb|d_2=p83* zjulb$Cbd$4AYH>5#xeR)Qrm#d<)P|$%Krd`6~$7FT(!AMqDxUxV^TRe$2k=BRUH%# zP6#w2FAyw$*72YBTl{Kt!1;>x z7&xs~td*d##xs>(G1D~lP^2#0ZuAt3DyahiWcTe+>l((JplQ!)HXgnvtZ7nPmx{7mzBIh?{-IK74DLFs=dhjpX7xt^S_$}diA@Kf^ zZuK@)RWeJqIMw$6+;VwBJ90YKuf*@#=fm1nl-9lo)7th>FfQi?IEg$BtM{9pKZd?< z_@i@i;*C*!PpP=PX_LykGL7J_a)%wZo}R?wzoj?_z+NE6!dU$7kFtV`kPtnG5a4bCl%{o3H~a0 zH^9CY({#@cc$VFlScFHaX_EyH73!}n7bM`QP#b`F9c#>fKG|v-o}n|_y|{*TJ#&zM z3jCWt;jC{A@m?nbTI!pVanU6xw(QbzeuwILUl?Gs{Fa6#;b~5+AqY0*SxWGVg1;?j ztJy75SKh}9fAO>955&D{4-Q^@anf%tudY(wG!{34M|WVMNBS@vk_R}ir1bv)jGqWT z09jagL&F*wn(oF{mr|DI-~)9lMn{;_LbC^KH%@~X7{)8(-Cs>V4lKXnE{ff5F0Jk( zc*2wN;lXD726^p@si=HeyVNIVnNG(TSqaZRhtj`&@P~(TnwV;ti^NV6gpx`TrP}vx z(w3Lgq46A5#ki_9qeeMx3Ef?}OA_tn{_uSVny@4i0%I&h2&CEN#j|40A>g3O?}XCm{XruUhl{GsN>;MVXH|;QETw zwbee;hPJrc zIb(R?$NUT*n0xj0rs@{gS72PhF3X6{Vw4X4htwYRPV((_n?q-+cvj)=Q-V~17(Su> z57RaI?;G&`PsaFvVVKkR<97*i-p|RT+IlOkq`wb)g}9eJ;d!}bxQWw@tfZqi8>O}E z*D_ajz4=>q+`-j+dGU7EMwR?=;zV7$_WuBB&OTgZAHCBY*7w4XkAD%gTWk9bV&nU1 zZomrK#r4}gsR!>y(M*gEIKd@ONyczSY|HUReIfiGWv-aCsF=C4d0FF+I6FppPV~{d_CxSmJ`9BT#CkMir#IYF45t{f{H6-HI*OS)k%GN%^ zGVu?IadbIpW9r|^wD11_r5=s(O8)@hkBs!#ykqeDU-1=|x8%aQ)`Q|i6Qq*t3h!xk z86~4q2gy341;!pi2pGN;wP*PwzeyHF=HGYBt?7!$NO}d22R#RRyQpjWUV&u`=NgH? zj%7G#7uy|M-1N^J^IzP&IpSsmD#}(56BinAjiBAF6qmajC3O{RCa%)cQ}bM1O4*HR zN~D{V+j5h9&9|#%{*6XL!$zHTiBO)umFF4K2;b znoO-4^+jXcoxFP*_MJ;b(8q`Lp9@_@Z2_K3mbp;|*S3qxi*s%{F2RbZJ-}|6!2bX? zzLefBaaR~-@X2x2)1yW6s8f$DS2EL9+mlO6yK8+B`sV;;xL9F6wFtD|yLmgkKLv81 z#XlT;6_dfycxyz{t^WYEZ5~Cpj6xJNPV6?E9G%g*9eS;J55ce6cj5=Y{Z?CF26%f+ zg4a)Xf=gJ;k0U@El~%cVly>T*SG#Dxvel1_B2S2VmZ;V@5^7G-*i9J*I3rhOB=ih+ zF(ce#rF@_9g5Kl7zZLbpE+hcQc<9K#%k{73UK*i;#!96Kw>>WPYp;`6-2D!|dzoca zVNM*1Dow6xbmwOzlWEQpYU(k%R@ol2@l*Z^KQ67O%)TA?av{Aa!DhWnATmY?V{dH} zH~sWJqO$)0V~>dbAGX!*ym8{48pB6hseqRH(o2ROY2g`!(#gOi91K^^{vdcZ%>;#3 z;!I*b{G9Z!p+9D?g>h@TKA++3e{?oCQ^eECqdWqnkH>C4roM|j$?$n@Rd$?I+WS&^ zFDw3M4GuBRarmi4x_=C!+-OmYw3CdIlBT4u7Sm~`W?%S?dwZySO4htds7aHkOpcDH zhT9scBOQ8muMYU%<485VMRYq^+`(qY%{y=!PKT4|IQ;AA`^ofQhgz1qF1z5g@hzT@ z1<1Fux|&F%GMJ=_7A+ndYDiV^2;JrFBmzkIr&hft>Etm5k(ieGdC%it(;OG#RuVoM z(92-|0C`K?miotd+vRKDL-IU#52`##QyDE(wU4h;q40IwYb#ss87?@_TJJRVK!6Z( z2c>x*hHj=h)KJGDLnGsJHv{XM^}7>wZExk=;~;bL;=gq8qdksul+<#Ky_~Io#PKug z9?GI=?p@qM?j1dAvi+X^CC%`&;_i{+uQ2%{e9Ava-Vx9+Yh(8=(!5$Kid|IZqR>n59*zv<7 zd4oKR6+_N?5No~tueIG0&&JmO00}%HsmpVucuo(p#_uU*Gq%$v&cPIbsxUAC&m@|w z@TuA|uPD0LXVtX{wM|BLl6#q2 zNu+bjvl2=CNw1a4X?uKhB~$)f#@D{`d%ItvUx}ONuTuRXrN6z`TWf8!C$+YFx#P8u z8|9E9bKIN?!|^JtO6ev+LO|n?dxKh1-T7ci6vaq9GM+*8u2053Ao3!Sq?Mi7vgN+^ zeNWfcy7-GQh6@Wim1L9@es{m!YknuCmr$Kbs>s8?)g&?)#Gi2PJonB=>0MvL%Zcsh zU2DT*dXU2I$Q*fc3<6Yl2d}kx_nYREAP47ct#USZvMan|TUnO`nyYc(y<e_z!di5C4Y{?r z)^l?$vY}xVNX&@X{__&uM?@IvYtTLq{4V%WCxxZ)H;g<~iKv;`BogWt2|Nl0K@nZd zn}N?zt~vpN4tXCM+}y`|oqt@kwo3?7&(DVnvY3X`uYSibRrdT6P@Jly0`v)epf1dV6yPU z-Z;Ixjla?4n{JAijZ)V`lENgox^T@5LzrS1 z>U~d4`_=72=EuTu?zXiP>X)2B5pB=U@j2`H)ISZh%dhxMq`SJ4?H5c+?<4Or^#jmk zpUS-KRX9epC`B%2(rbWcUclsKZ%Fsb?h9HC+rGM`?dURKDv-wvFKs_t;3Kpdf zMYXJa8>uBznLST3y7)h@+gnMfMdPD=cwCPr>Qm)1pH$}ur%JQnT`N!UUZHaz#1u;n zK4{F1a*n1yFexA@Byvy60VAmFGhTswblYRHJ3lsZxl}>UJxT3Y_I7$UyQnqBr)veo zk#PiV9_5GJXZVk=rYrNFA@I9}^PDrK8eEcnVCg_Ko_Qy)szYmR9BqXF9B`ta_B}p%l4)$Ca)&XqASF+z^skwv;C5Swk1ig+Fa5ju z-QAt_`128ppEYW2{{Y+HrLFiJ--fi!M(7=TSb)SCb8l?J=0xXcZ}z#X*E%Fu7ER`! ztGSQ8Ye+<@R4B)DRCU{*vuVr$0JA90du|o`R#k$bk!7^-v_2YY-o4+?bK@}>y7=5v zt0%0P&S_EH{66u0w~VywUGiHN-3XANm0b!l?i{ZSc@=-gUmlmoT3&>tvlfDnqoqX|Kr z6F17T+{ZP-=G7y7Mix+)`4}qzH=JzI=`|bWmV;7Mf zx0R;>5~~fqUl_+vrE`8i*F06?NJfpQYj+b{!&lI}yH2djo?qdjyF`q+JZmsOr zQzTO%2h**0emL-7?{kx-vt)~uu*_3>^K|4TptWe>s3AKty=F}E~#`!&S$($QymXu9U_hGC*mK9 zv~*bEzca?a!En3(01Wm0Yo*bCG3l3Y_I+2seC_%l9>`qmvrZ|%d~mHX+V`KEa}e-dXn7ix2Dc6zVG_PZTc!&igt z8fD~DFRX2nKBKqeRv8-vfs>45HR3)W@k?l$)OS`lXLL?H_;qKN0*tTo;O`Qx8$qp;Gp3YZkuDt)=|*^gmVMz99bEr6|;oElBRW zr`24EknIO=CXz%#p-At@1FcW6=Y=@;H7=KF_Ki;730d;!&f1)g zNfxaXvt)tT6Wc$BN-A4&jzLmLE7SmS?bq6h;hhww`H;DUD}@;yc?UEwVU+AEl!xPY z!kxMQ0EGKa{{VqbDn0P$45r;O&a zr7!#i{{WGG+LQZB?>c#yGel1XUy={jvwTnEOK%C=i+?Nl;`i+%q z&}x1R)@@%=lJCnJMyq8gmRPosU&|n11CCpd#}(lk-;R7?F8j!y_P>E6cTs{c^TlGla8IMm+@bVrPJ;1 z1*7u}dzFCeU+9m6oC%fi2Mj|MkE01=Dqc=@PEm`uEK_RhOo8pKr^}C-BT|NuBXved1KMsF9 z*Ds^^!fkTlF76};?GTX4Iu-G4&Lfd7NMqJ8+)h*5<%Q`8OOeVI{hmXo7Jkl zy%_T=>bkY%>7zb27#eh>=~i}j-?KU|5ovPivd61KxqPAojHK{0!+r<+@m{xk;SE#8 zI!>E!H<*|5+n~6LSWIM`1H2C4{{TKK=1&*+uj6z&RInXlA1Wjwe=fy0GIu#3{*~z8 zvuEu+t52@zx@U>yj`BSsMFQUSP8|yZK`2#94l~0zVJXTOnhDMv z=_fU7$`P}7+UYxLv_D4Rj8#mF5~W^hbY}TwX5GJ*PU%|TlRAGMd@i5Emn&_qz$K3E z(tB|OW2BJCfsel8E9Uj{G<`@|Pi_h_g_1X34h{%4_2TtV!h{oX|1uD_*@X7L(Ks*e0{{RY+Vus`&Us~ql)%6#ZqS4~>CSs$O#~J7Jt-T%Y zH5>HO?ICGyq;RPWYFXWnKmg>L?7`!yI+EtSy!QEird6=>gcZNw{K&N}BKFGJwE!-R zO5u0Hyaqi6dIMZD%{}G)&DFtn#KZvaxPUmXps%O6)4mPfX&S$Vt)sq3Bv|CT+Z@U~ zz{P-I7?KK&oR-)KKPg)9T_4Jt!|l`U0!LdM~t;(FK> zDs&rr=zCx6qvCxd!urZ*0sBj6GyN3nf&7b zyN*JuB?%y?4QX3;q<_>GoP-xYh0~RycgIkpd8f<;POp zQGMTc@bmK5_E0`lkVLE_Y~&|C)%O1Y!Y=^$82mc_0EExtEc;}+n%dt=-k_Fw?OFOB zp-5x(TKyZxJT)8+GYbgD(Mx5e-k+iJyboX7!mOn3{Lh{>ABz`O{vN)QS+b10mhP7z zC?uq^J1OV7?j#?=zDMykof~R%BLJ+jFv|2i*U(=bye+GEcH&PI+Y-8c#xAml2m zfU12k2Omo0ycO`P;U9{8Js*fXQSsK(PSoUgX0npVm4R}0sb^)&cjF*uk6QD%2PUbH zyQeEc>h8_k?mZ0a6HXIX-&2I}_k{czZQ$88d+lc7b^ieE6kB@(YnZ;xwC;W2@Y}m~ z1Ki_6(&to$A1iT4RN+ZE&(g1W%R=xchII&sg}ih%-Lm3Vj@!#!q#Tj+HhIT21Irb{ z??Hq?gYe207?O zShBRdznEKCTFE<}c10Ou`3n0uPmx~#0Ey>GJJ}cYi!T$}={kIxoai+9%#&GLha1(- z12ZpCkO2hs9E!+$>zi+xTj|ApG5bjT2>5s4-`Gp`e)!Abj{#i8<6j<6;(_CRP3~@G zl4R5GYz57UDhLUhWPC-=|_em85q(AzPAEca@y4`O)Dd{-XlIOUW$PFI7ss(1X4TQ0-Wg=@uXIWPbrx>s@VM&O-OQMBN@TcfBa zxF`A=$?z}3yRQ_;O!|ay#J_lgS-yY_e=%Dg2(z~E=fsQMHp!UB96?ELr;(i3!SKHk z;Og-n6AeyRl9s+nroU6KIKbCgMM|<(O=x;5C9UGE;wvPn(b2Gel{oR7jEw^ut?0ip z$A>uo0C4m8ilX+8*NtOuaw2CQbPUAfJ;%7F!K7)I_huRH#fG9A_hv~5Mn18~4+q%S z^l!GTqwZ+`0I$US)l#D;dYY~MKj3~u)}ASXX<2l=cS@L^B(>Bt5kAB)BQ%;0siom*I^#u4KIO!%oZJ1oZmWV%zEZ zh2!{V;gzdQbB#-0xaE(iRy{to*5!+*NhoQ}wwk~2tsC`fDQ|OV{aSPP_2u7P7pvFq zyKVL;=qToEoqF(Zojz4*IVNT}1MutZieCil@1}T$+TIC3=1tGsk2CxUrM|hbjyBVD zHs57n#t7#h^7C~604qPZQJj7?$Z0x*r^G2>M=YZ1K!YE2bI0djF`H&j0g1z?>v^Qw z?%uEOR{39J(xrx2nl*3RZ89wZX4CF?fM4rNCyC^{KWUjV2O)FTzgD4)r8+8C-2D1jI?$-oS{id*ERMFN z3M_+i#>DS%KaKI*JqP$zhSecEF$`Bl`(SX(SlzQ2=ot6+8ADsGBllYRxNQ^QJ5;M1^Ym+d; z(Z|wIbtMZs-Y&*;<5HCFqjPuSM~yrqsrbJ8Q}9Nf_Bu?3KFVUuSAz9ulzO=jykroOD8wVNx0PEHckE*t{1aM6@*N$ab z81jBo*W8c?<6d4bGxj#aLQUCOHt%~L&KE9HkG7KQsoD)fRPv%o%SkisjeQO|C+I6O z>r^vdUro5cXjk8>i z(fYcaT$)c+XvPXn$?CU%frKy=u~Kcy&2RVHL-GFrT=4Fzqu)uTc$Y@Gu(*f;Exq-- zM(dH0m0$?S=CdQw7p=GeeFc8P_{T)F)IJ|;@_46Ik5utrhkQK3+V<@E&wF#GXo+Pc zj9WXIZ?&nHoczg-)y4c|(LM^;cw=7ph2kFod_9Iw2xxlcj*W5RZ2~s51ZuYWeTvSo zMqXC`071Oi#1_oNZdJ)1ZWF~yRjNi6aE|TUNp*I$uCHsqk@^ zLow=eRtX^F0n)!rF8=^wkJ+PLvG5Ovyj}2XO|#aoEn864S3uP#w|n^E(H`#hIX`-1 zw^hBkvxxx+%A_2XIjBYeFNiI4e-LJ?ax28LlJ8$lHr6pC zLzxss9m|vaOca6uAp3NDPRJ`$=A360wd|VR_IK&q@zCU{lw&!!Px}7=f_yKnOR3qw z*VfXZEBBSNl05};mwNl!z|*D-lQHs@js|+?=m#~~{B5z)JQML3#C{FYZ{@nvw5@XP z&d%cCFPS7UOz4i=Kp83lIO4o=`&tirZQfEcc0=f@PbcyprFG`H#TpP(gq@_7ujl=K zM}Av^b*`kg{7#ofyO3S6^9f@RV%^EkbNSc5{15R1>pl@Sk~uGFAW#oKDdc{+ubccY zcO~T48k;5rjH?poq738wYuZ0&tNAVU>GXd!6kS>w6-xngBrDG@Ct#{vGT4JxEIb0AO7M+>6-`KU!n|0KqwbXL<4e01|7_ z$5DFc{{Yus=Lutgr>{;r{S8kEW*f3e>O<|%*1yibtzwtkXXrLa`ThR@;F_Paw3)?~ z@CT?jgk$qx=TXW300hGQo@N9u_JSPty9e~y{cGsRgtCy3f?GKXIsGa{d5~>6RtFqp z5&728#^RH_W=l%v%@6+o1hM!K92hUPD5M;#dGV4DVly5;I`|j%jglV${7>;8gkboI zKZxwK-Cjs^$?h#MA)}HeEa_FvJMt08Qobqe;uLbftIi6utH^#@E(X?sbTS@BL z^ICo9$njQXi=l>CDl>76^_{wM5IE|r3HKa>`WpRzHR1^=PD^c9%Wuf> zFga~CPBCj;{%1qs3*sQRkzb_p0r`*Xn)FRF>24!f+X1nTeQV8pDxOxIW|<`A$;WbB z*K=*Y*-}L$am9MM4kmw8gr(VjDI8h7I6Cg$W~J*~c@jq?D%*O4>FHcezKLO^_=elW znk};b0HIwhw>Bu-Br${g#Z%~4ILAZz*Pv;W!K=VlVgm)i_pDzK_*%v7=F~6QVuA(x z_{%Zu2e~!q=Gk;Hw3S4z)~W2*TioEw@qbsU;i|t*{zr516UTZti7qsYKMv{*AU9An zQggd$A(R}BNCO7AjVHl2zAN$0scUsVl@eW&^T~1LMy$<&zCw->axs!RkUqNq0JHX; z4xQs&7hmyin%5eBfpXH`!srj~9yt~wbH;ZtQSK@^Z*O#uh&r~t;Z0KN#lN&C7W3K6 z(8v(E<0^6nF~)1>a*U3i3ZK<5FyxB7liyiBX)jdY-DB0oW;H6}T}t!jX(jSjOYlhY zua7<-weXLQhlVw)xd+)3ARrm}ut01Q17m_Y!31`!oflKm0tL31kM`wL{VK=CS8MMZ zHSOfDE4mWWNZX4Y$0U=~=e=U-@uPxW#sKIBE8uZBoJAfb#9b!&tlhPCNgqi@6zXO0 z-(#iwOl)>RCz_!KG@fw9qQGZNqH~YBp3{M`ewe%4^h{G z?*9NI&YV;m?2GZ+wY(7+WeX&Lmh}T87^xmfUNo1?VIsH7AY-^6T+?n?iB}od!1ftW} zj&a+*eJcL|jV8^fmAn0J0sIYS_+!JTRJq*l-dg?R>MP32@{HSvxSBa$UrzFzY}(Sv z_5T2Zdstd@Y2kgXPWSnnzY4rt;y(`|7JeW$w#wf~iXu=)y1 zEk9Devu#59LmXSNYYAi5+PpqD;Vc&rUNHoky|fWMh5WG16J$JYK@}y8GpxU8xSi#l-!Chi zu{p2Hat{cePsFrwbm#lHH6?Ye@1pr0jz5W|jLYXrYu=WR{LfAB#m$0C6U`vVkxouM zee3A^pNJj@)qFdlX%~JWx7(#zLvN?r!Eg>_4H`Yek^Bg;;hUiu1HF8WJflp~z_}#L zImBo)s;J)EzqYqGen(s@#Pn9iz}^pRIjwirf`0D8taJN!nEZ0ABw9*Hgpc zOvUB3=ANowq1XP?Ul%O3FA(Y;0kKw=(j7)wtzBf==r;g}*vMhD79jU-2;^6g-^lZ* zxk#i?1|M93PMnhX0|6y~IGei|ztsHSx+hvh$ruBLRh zjw2nsyTRnQ@)fJ`C;k!ZpBRlZL9&Vntt}ULiEs>=$v7UwezlQv8YCg(7~S`Wq3>U} z@J&kiju)eeoH>1>ak_Akeji5KU9^xPZ}^zP1WhX} zX^td9!z>h>f=D?0M%JVYHO6`U;p_DxzcS#AMkZA8POO`=YyRF>=IOe6GtivlR#CMj z2GV_XPi@=jsZhvVMyb%KtozL>`&HLqu(OD~kM{zOr!{T3b!2Xf*1MmD8cbK((ZBN4 z$pU3_&Otx^s-8TqfQNW9iaQ;?V5;0q~nr@Nb4rrL07tl-x*c z^NRA_0{;N*9z4}`OJT!F@)A2_kMOTx@m{-sCY&_=OIK@4c@&2zyO@rcQQeP0&w9!5 zr-QVOErzWhhpcQY#AT+q)UQ0rB*t-&G5{F|0B7l%{(IpZ@#MLu_Ld=YJLr-2yg`75 zFAW?W4HDkuNH4W31=K99B$7xDZzcg1i?gJmF z9^FNBnk+ibqp3kRhcp>HTV*P3n%Qi8nB!BA^->HlJalEp>r?*#!WpZ0ZtDGfb)jA8 z`fc#@w6$3CEKDD32FM@6NiF`!>0hqV@%FSaww4w?$=d14@_v0a^3&#gT)aDS!*fP* z=3P>%Z(xk8EG3SSjb#}(?0O)^I zmxvioZH2;B!PJZ*pCSy?@Q?c~4U`JO6#GvogN zi3~sCD7hM5kf)0qc{y-Dyxso*l~VCOv1uD8!taLA2$ja$>$uk&6SxP|)$bqp$6XfV zOIXFcld(&C7)Ds&<1Bj*YR1$fgW+$5HwhL^Yf=ZwEICp~E`5OM?_XPpuZQ+-ROH&V z=TYy<){&2AEs{qK2vwA#rjmEGAhu8WrQY6VdPQaZzMdb`0Q(+|5%X9c9%I*@`04e{ zdH#`n{{ZlreXl%ysgK}EuJ^*$LrUPF5 zepKo8tUUoYS|R~}@^wCgirJ0h@{}Pbbez}d8C}*HT=^_zomOf6uMo$TE09lLQ;NdW zHOpIDxgeHL-Xr60KEL5wT9wF2eol%M41N`b<9$6IWz=F{^^xI@JJ((=y-3n-8>0** zI&!L`Etw>~DSJ!C8}g$W1XHyi6Tx#2$spXyO8T678sl}rcP5VpyKg%bjQMv_?%)yM z@~9+B{Z~bKbm>o@WJIVJt0%AFUn_~vaWHd&jr6+nUSEms!z-ZqWq!6b^<8Jrg>}6( zm8Dhv^dtUS1wS@NbI{~f4J!Tx@e!KZT*GvtNPuqVJbnhTzqOj`BXerPCuR(AKh?@O z&(XRXr(=6G8rW1_r{=_X`>cC?GhY*l&)?Rm&!cPpAM|VWx$Q$QQKKmN9+vvdcPSp1 z5hVjKB!3Y+yM9<-p*3PHK_!<3wxC|!hf3zOM3whkV6z|HTQ7D$#NVGaz2Vyfsp>%# z=HA`^0C@Yjk;h@{iu@b)%--#VBk9kvjSwKdp2S>iXuD zbr!LrYO~tPggh4((-e5y)mU;3ay~eYPqN=$qA5#h6L>BG>N*amq3P1PFALtqJ-X>P z@mq$V&Ub2Y~IV$2TCgFJjS;x9i)!yO{(D42Y%vWXCnsH}FAQoOeNMwN72 zDfyElBEAza!Q_oRJHoB+Xx=?7wwKfK=z8?9wPk7hPrp;B_^0t-SiA9NrK;N5q_%gL zl1&=jpfe53TjpfI&PnJoTp#wLYkIQImi}1Vf|&K|{OZKAT->3G`rVkG31+Owt>m!Q zqFsem&jkHz=QC_(Hg)OqNxEFoR(jj!dw5(u8dTbhQkAz|Pe<^roNX4?;w6*k!0Z68 zW%w)NT_;FImr~L6=8`L!BzudQqKZ^WBUBq1m@{ovkOje9$&=2~Jb%HE8LWe{k{Uw9 ze=OGfy4xQ<^6n^?w)p_h;%oIRB^)+CDE1W{qb0ASy8Mrm#MPW}SBB~SN7BhPgszth zIuo}6w~mMLuAAYEFmV6uWf8kb@<7qWJs3f_Jtnjs} z{q$m_u6n7)LG&WK-wXcA-VyjW;)d}?k2IGTcXq-yc@zzcyO03&<3B-Pg<(4>>)-~irIXel*sp((RIUWf(#UCqDca_-Ga0vm&``mDU&(f**!^C=rhIEwFbhL@3 z^Fpthg_i^pc+a=3NHVRqR^z$GanH46d_^Ps1S&-GNRCd$t2=$4 zGAO&&NZ$B!<2%nBXmefa7Ewa*&9M0-;PL2liYq_hvNgYk=bd6GCcI}@++O&9;^8Zl_grm0K?(@7SH&Aqu~)RDPHhJTi35ydIrRQMZuH z0Z<dN755kX7MH|R-+Wl`hl7h|J3kLeb{BU>n&Fj? z;viqjz7Fx#>s#7+I5@`u*Y*d7SXgCz2~Mk4bLN-c?7sw$%`>Wx6`il(4{^Y#CP2FGdGQ;GRZlyi@p0D6d zUtRI9#b`V&;hkpQ&RH$www_zJ{oKkTXK)8|kaqOOI*O#8Fl#$mF5Xc0H!`yY>!@#oL7~C;%r?kU0GDVy5FX|+s?}0 ziP>8zr8viy)>hNdcaQHOy4P*@9EZ2Yf7;2<^Q_x=u4kET;DC&rb5<>$&tIMNyQbRs z4IBRebk>ULa_Cn6ZO!62#s&?1)alg5<0p$>44v++r$(Ps(^8HejA=iKExVG-7(|Op zfPxeQD`TmxJ8vpD_5wH+jcYJ+Wj#-Ny=^NmB>r{wSPF_d+jGpX7S*5h%Z(wDLP?`I z=AnjlWF(xN)N@>a32(x^Pi++ooXQt=DQcQ5`pxWcN5UpqiS3%))3n=tHq=`{xb;4! zs(5nf+{Z9;%FB%VR*Z~RGqhy%1Kz)~d?DdpAIW%9o_vyZYQf4=e`PD(F3BaYRy<8= zuGC)0$MST@7*Bs}7!Co&I!nQOGP;eipu}tGT@=0=iuwp{Eu#e{h;JVK>lXLH+U9{T z+Xbu3ZM!9c6SlvRvq;vdj;UHPyd@^|=&yD1vmHE53XaZR_h!OoNXS9M5y`HD!hRO> zqkVf&aUHx1xHAq7QShIKE<9P{2<=rqMfs#-(LnE7KM{OU9;tI5!`}xD&_=;$mNEB_ zrxa6-DimHOs=f4z?#?>WsZye$PkrS3FY{)8ogwiyxi+^b#jU6!Abo3@@c=nN#=e!; z+G4nT^g)Bpy(7XN8yXe8!o1{SQ6D`16~OAcjlIR3up>8^ae&K=@+;+YT<<8O zIALXogPrd$d1(7@(DiccE+Tb(s;1O7j-5OF%69srSs+;xd``bQALm+{SH&y61}28p zM7dleNt~$ktQqz+3Z&-)n$LK`z6h-NuM+cI4Ll|yFs({9alWoMTG^+w?W#VL4CO{{ zPh?YqwjajHd>rT?OBNq7bY{u zKH{|7PY}cQ?JoH)ZSol%4=|rnJw0pavdrHDi_2k81tsnzrv&}zG_9iD#rIdYQ^~|( z=Nhfl>$B+oV)=^;;hVkSZb1# zRF}ma8otdjM+~b%%_rS6^&{i2{1g}d3C-~v_Hpq?!p#O0T5ZGudd7$0L9iNkd zZZai*k}w7a4l%&w*X9PfEH5m^2}>hR+gXMQ88u1uyO~DDAangiM;y;H?rAgB{x$JA zWh}EOg-TdTl9xNX=I_<**ROVVDpS8Sx+K0!Z@7_NTII6GfxsV1?fwty-(=M0iR3EY zLG7O4kI)+7tz<-f@;c;>0c>ag0Igkjh4o8)4hZkGtF@F%3ncTZkM*RVlxLBFgI;eG zV(`^?YZFGKmFE_$+TTW!Tivv+q3C7&^z#aHOJ1^J#)b4AoTTr*q`9{kG?njSA=-q#9E?g5D?Rd@1`olIoqAw=lOA8GI%pc zyR+AY{nV}Iwu`G7cW0IsHvkV#m^Jsez>kb`{0#U_27#pLdJ1Y*R!KhXTUA?AH_8bh zk~!x9VD{<7e<)_u6l%CZDK{9Urpofz{qG{p@z`D-qbhahPCnYRYBQ8me3*M(Ca!(K z;ZJ~nviFAc;b%9(tsdxzf{Cx}qnWZhlHlgN6ZVz(SMbW}`twZiZ-nf0@}oB9!r5Wx z(3AqX?Jxcc2QB{q!fE~|>K+-mSna14*G+Hx#7T(AP}m#2N8w(7sD9kGTK@os?sfkF z5qMKZ`%UKLx3+`CB1^R%tsHCeA7l8}%|j`hcTm~>C%rB)%^ zpND#?>l$^*xYRs1;(I%^gHgOLpdh$nyJaOvBOtC0`Ve%F7pzr|mLx+jXJ z_!rC--DTz;m1h#yPhfgccfvvoZ3niW#DH!TDsuO%Nk) ze8HTQTKyx3Dbl4MXFDY1mG-sc9ez(%eqG`Xd|o~nJWQu4R8*wzCgkO6_0mnZccMvl zN0x{*tseJO_?6m&X)Q$x=g?E^FHmTsa5IzHO$*wX_|(vpcxi%Htm}@BWN9S^{--22&iHy&803# zsQ1;j?e`<2E^=6TOS^3DG==hPQ#b)4#(txku-CS_d&TydLV=Dc{uZ%=RhU^`UA4SP zZ8Y-996Q)7Q9OqVS#nOybscLm>%tlj#BEJ2d^O@<5Xmq&5W(TdoVWf?3i)0vFOC-adm8ZA6_agbHnmqX|n1*9=S5x{pswaIWc`R*Xvp4*>)bi z2Z_U`r}0^A`rR+`)jpOu@c5dPr|jyxt+o7{{{TH1tD;(Jw{}Smk2$!WNcKh%hwjw< z=O_4m0P9_bm8Z?&tvPhvJs~r);!C*@1&`bl%OBFX{{Ro$Sm@UA%^A6J=ntI*)POL- zUNMu6LFGA8*upx09nQ}#y;;pwdr6u+lO#ibvo4>ovmo5?f0aU z^SS3@^4xYJrz{fX^XvJE;{9jDKk$!_4bS2mtBW%r0tS?n`@<&#5F~L;A;Z!8kmim8Pj-tx;J$k{T zlx?_&i4gS!RXJ_%;6`mfQ7Ff(av&&ujXw6?NbVP3(J)+(-oj7b1E~7b=e@POwp;ih zEE|SEa8bL0EA<+cl}K_;`^jkguItTpK57wge9+nS?E4sJP=@s-(e8BLCRF+Sp~%N} zUs3H`#-$7zb@%qBn59!czC(Z;@Et3p)}*kzxJ4dirP>_Z%_cUVZ>4A7XwmA>J;;HQ z;W(NkZim%-VAsaxG-Zi=iQY?8_J8o}@khOdlxgbpPj&rt{ESU^QnZFOn#Rl>!#;a1 zLi&N~YTv>C0E9j({h|IU+4v8_O^Ke~Nt$_vRw&~_#hV>6aHHLncdYAWo;H(1)R2hW zpDEb%HTIwU61U;5kD>fK*S;c406LxauG)W@C?6ydB0km6OtgfbU5n{oKUNPHQ97#e zv%0mdCH{Nd_h`x#Anm_L0i$o{&|~2H=Z9EU9|}`mF%pZnk_#A?6T~;4Di0vjWw#h`u*qZ z#i&F3#Oc_I$nls4<};u9W!LdE?*!{DtawvQ)#L=5ojMs#PI+Vp`qe)fUL)RZ50J5p zeQ{r|aK$HAH-wk+{{WHlSe+oz%xcb&qYU*m%iV)4(m2iwdjA0R>t|QFU@{5Ew`#?b zP^`o*Tl?Kbe^7A#3iOj_i8*RTB4NXB!Sy4r<|`Yhscb(x!No2t7Y^tI9Eojd7{><{u!oc8j_6ew@e!lyM6adtbzKGwG+jk2?CJ<|vC^sB2n{{X8lkLUvApM3h(9+9nE-b9j;p|PAPJbRB?(~jeL z46-o;p$aklxcYRj@{XL7*!?b0vMog-XP8GlasVgar!_E~9tR|kTz2*U06)^HEb4$w>-_7wih45#WAhvS3YU4}8z0)6QoHe-Fu5nH23- zzyOcVs@rQ?b)+rjDQR<%0N|fmt*lyJNvb}curLrgM&tOq{{V$)=~^zAq(}DcZcqT_ zU{Ac&>fv(?c58_C)Z(EVd{1P%oK-1N!$Xoz@3QJUK=7ug;yIsDy_H~verYal4iN{} z*VuDb?saQecld9>us$VSw|a|=NhtIm+)r-({VA_Lz2ivyxPY_MEzf_I6nmcLs$E~( z#s2^jUk%1cW42j<`J_MLUfcFKTs^4X(%x_5o9zYBd7O#VsXu>p(QLhYzaj}@g3ZR8 zr?;PQ+({_tzQg=#%2`Kb7RgRzDjHHhm0PsZAn@Ihy1OB+TyQ-zQP{&Ixw)T~Zdlg1 zd00hbSnepD^N7wvOu4y`FhVLC^xhs5L>7p_Dfk0I%g@y)L@tos|!2P+uWv>c+ z2DtwKgx|r}k_{_Z)}>nu6~jkq0R_@9#o5@X<1N;_=bUBp&1Hv-^=U68_IK%XNJ3uD zE%!clmerz~-%z)32Eo0HU}HSglgVc+zG_}O=kBX>ihh|jmEsE<%W*6~5E**owM!?4 z-ap>yVCFa<`_yB+!O%|g$)^Z8VeDUpU0kZWpHCpp+T zQZ)QA0fDYC|Mwd?oO8hl;e15QMmyZ>%f@-GXQ2kt5HX zr#&n+W{7nk=YQJW#?f(D(>vUcAkxQk@cFPNvgc0|X+t(F#Jt|#w8y+FNLKNi2LL_7T ztnpZ}{8#Xe$VQt)x>-wMJnLhO{Y6Z(L-7JXCT|{C!Dgp2Sy?{QkM^^Q{&>RX*+wmU zJS}L|mt`gR>vVXvFu1yN=c@+}?W_ID%+1vNM{A;6OW?nT&n@M}%i~bhG0Fb`WpU6{ z-X79CFX0;pO-}t~)Hh;SkmQ1TjQaZ4+*h6m@J^XC>6(SKvO^0Pqm9>s2Yhuk=zp`9 z!%qWzUeu%T#;lf>vqz@s^G$JYq1ilyOJxL%OUnzFCE1vk$@z~xG5Cj#d_{`LGTJn0 z)V%5`H_Ggx2=cB~WZmTDtKQ2~^)C*xEFO02;;|BxDbtlWuW?ROokck&)Td1)6;1Ve zw5_{dd4GA}%gr$&vePZ5VxKB8052VQ7UruyldW?J5?!FiCBh2d7z}D6(JkhhdPVGI?w#USA zUk_2rFw}80DoRRnlwULyB%D+v^i!H?sjY6E)6-`Dy!Z|K5cpf-Pwbhcj~K^2rm3nt zx}Kk?+d-LamQD~5Nb?bq5sI8}cLFd?ct6Cy*z@CW!aKP&3oRe}S4{^aR?&AzqBHzb zMn8EF$97%5gpO(_*yvy)c3wz^sFv^Xi@ zsm~6R?0#i@X3~5=@cZHw{{VyhC$3y-7BT#SL-iv0%hZ|vppTgO(iL#ucPPP?7SCjNcK;kdxe zsOtOx1~R|_(!4+7kNgwg#la-fz83kAY0?L~nhWdq@}qd;W4m;)#&}S}zB9`j`fR_% z+{#o{8r9pk^ZVcMPFdyDQ@USvhtBi-CDT&UZAO2J{v7h+8>EgoW@%fe3Zgt7NEu?I z-@SC768;ZA4I*jv>F|;q)*WwR} z^+dOpB)5g-+AgLHL0)-P#^xCG7_Vs4{{Un!2>6#mv0oc_%I-VM6iYiXZ)jv!hyqk> z1TlP$p@1DQYixfQVtt}=#92MmSL(Iwt6_IvQ>z@=6t{8RAetq!AqYsJ2ieWqH& z94;3NxGX{4KqGEgf&k=#KN#GB^Nbg5y072GV}$ zuRHjQ@OR<`y!W0q@%N1F#51y@ww|Uga(E{{H_98Z>C(K~SPE_O`)0Yf^ZkBim9q+M zb4v`Z9l99*0Jf#=r;BWFV$iLjRf}4(ks8@BrI{}H}zEF*B?yy9c4m4CB6Q>2k=*mL*WVl9J!IEO$P8iK<4n3kO!(x-$O& z)Qaby_=@5SwTo1-w=ns~Y>kV-zv%E|ZVMjg=qqnm)^zU%>91#LGBwK#1dQ}G{e!`{7V`LGF%tJ}rTZ`IPN(3x z#wp6H!%<%K_3}Bd9NtH(c)~3@JQ%H)Xan`EC9qpx9tBwNtsdo7KG^=1XGUk2QJ+<~ zb;-`=X6f9YVeeHU)nL8x%FSs${3gKQa7gt1YtgM%6)%XN+m@c44RG|QMw_P{lp6V+ z-iLVBZ+Jn&u7|2H{{Vp4&THSlVb6uuHhvk?^}SlrM7N9fnP!u?I4TBv0gCzSLV`_C z#B%76FU>xWsl&&qu($wzwcU8bz~2^p3#maCi4WLhj#&QyvtA3Eg_3v(&Ufw#!?w_; z82rDP{AILl%`xRlzpKxpQ%l3@;F{*qf@x=N`61}N%+HK^#nIj*-#BjuP z&(PPq6mTEQi}n%;1a7EUr-_<0AK@!uIi@oVA*w4M*uwEqAQ z>Md;}$!&8b;k}L72QEH9$fwj}AoE|7aP=%k4SPjZI6c+-bzj!UyTo-c*vIVMH|;HS z?f4#Jt1&+dG*(l9{vp)!{>kzzG5o7T;nukf(MjRmRD&FxC~nmUi0qyZhB{rmu(WPG zMX8BVh6*@_$2@lFT-KL!GESFKwjEB^$EdH?xDz~{Pnbfo_+=?6d8amfZysPiw_QQ* zsNtIRSb9eOt)$-gA6_#=3tCLs&umqT<=bv`Z3ae-ijwFL1b3~QJ!4JrCYAl4tSHG` z0>=az=dU5S(B~RmwBk=O@eosz0X=K?{Kgb;_&L?4?@g=u#s2`oKQ5sDnKj3 z*k;U3eqrcp8}mJt!)kh1^8=6HL64SE{SUoh>e{N>u(s78lGXnJ<&>?0t@w=p06Md$ zYO~#2v+36%HTX58pnn;1!$f}iB1$hBo9MYu5OIe{l8CEx4T~Po*U~|VAkw>Ne|ey zWER(8U`SUA$KTSpW$bY?T+f!LNiCn3lkz%Z>QarZcy(P@`TCW0NHp&dMsA}UWbQg( z*V=#ZP2E=7=is&WmLn^1X|GCW@4IwSZEwhamGJJjr0W_|HN;k~&@F(v;1v0{bT_c4C&KltbCTRt(nCsUXQ#sC-}O60#{m|EBNW$=tZk-EW{ zXVA*$`qx3?>C0;u0r>srRs)*;ir@-Pt}>f@@-NW*&oOqKmzywkJCLikKo4Nt=Cb5` zj0MLT+yz_JCETS@jt&kv&1GCWZW#U^cY1$1{0y!JPe}!gJuB%{5>~N{UbkmMrpm1#C;|1x zarqi5mW^|n08z^I1Yt9FOYF}qWKmi!e2#)g0FK1fsPz($wPeX1Mles{ zYqEqKmWL~7`A_zfzDrM!9wdTxRsrpZ0E}~M7fSi_;@+hN#8ND1px}Z>diDPR+Hb`A zKZw3Qc!uXoz7q%@8+qM&d9lKZ%kQ*}r`!=cNY4#SuHa@mT&Kkgn5?lFFh{;XFnSSKdMAf%Wswh-(@e*{Nzc;1sQ6A8oWm}o%DJB^-{D^` zw9m%#bx#Y7sJ2&oZ};>?S?;y(7CKr*3ftN*2?T4kRDW9Obgv26_)|_grTznL$Su{6 zuyOuUokzxZ-6U(4s& zBz){~6)}&Jn&qaqeg6Q@)TcGX-XPS>sxcCiuy(5&oPxs8#}+qjs}}LwY0#3GK9y3& z(ti{9io-y+BYpMkP$o}zaHILxlZ%_d;U_t})Rye*r~B%PF;v)2n+)MTctPp_x8- z@NziF>5+`*#Xcc^!CnuV%Hzg=3BDlQTiRI8tE_l`UbbO*r)2Ux$drZ{h^7SaDw?EyQBEP8>6Kc^il4KbiaBJ1R zKX{ws_J!j~^&bj&M)denJ6IsnP2QH1Mqxx8u~j4Q7RK(_$gX!s_=K(H-)jB`(e6to zcBYYM84cej$ZYfU`uDH2!gzs?tw~CB)SbFBf>~uG^w{9rSCEjl(3bW9o@&*kUMah= z)F-rxIR5~%?G44s%_5_O-ieTma&TO5ImSpV&EgU9zNJ4(^Z4a9+Mw8$f9t>Py* z+Y}`b;CE#{`L3hkhrx|k!B!Cwcp4-?Az=A&y3wWN;6&)%DubjIE4z;TjCT>Wd@ziKTfOYsN7y+6g) zo>MiQ%8N&Z09JxUPnAIEzjT4v3<~lz_+zN+_8GX=3SewRcvJ!a>Bu?y``7d*i5@HA z_?N*PrBZG)TX~`{+9SN3 z0*onJfPqIRgu~s*4MsE;l5dt7ZsUJ8K8~eXyQw3Pdi1ZU{{Uu> ziW(KJwS8}GrGchQVv^DZhiFJ17}-;9PB0EJlZyDhE}k{0%UMFty|<0m&ri7hPl6s5 z@b)G*wMANw<4(Q4?M`n?{{VuJ{?ncd@pL){xA1P#9X@$fc`@6cF|srAFg)YvYqI^P zzu=)7zrojo!X7aAdvw~R{e*Xh)meXa3#mM}ZuRPVji12Z14kdkomb)ZiQ&j@?;(O+ zc6~ZH$~3Kv?#4!W=ia;{_Kbgp{{R`jJ8D|T_Wi!AVI7sauA`>h{MKZz5r3imIGnrFlB3;nOc zUJ0=`mlwK5qHZp59(fd5Pk>5+#&7`jBED(3@n48#)odE><7?PPEv@%LKX)0(#xcp{ z_2$2!d?Ucqi^bz9MaBvdO8#zb{zt)ay-ZD9lLrd3PHL@t+x!@F?_Y#Eg}Ug%#iNdU zL{K(5h6o4nl_2!>?lJF<+6%ouP%8zEwWL<*k`^0qfc6MW)1i~glHMlda9cm2 z{{R|?#`5?+`%Aadbt^}P2b+AbTS~~Y;{Yq3>64PV=L6_@xqMfp_+H7iO?LTE1>N%< z>HzJ}IjXO) z*Tq{2{9R!;hc!w3`&=xcNo=ci9RC3I>#%Q#Fi&VCwzIssW*<8_5yq#}p#F8*^6Ak^ z{Y-tLZphp5F0XxmCB?<`z$~k^P0Bp!8ukl^1aaMiSAl$N*L3C8qq(~cbp@&=ulNHVIQY~*hxT6?S z&&ErRrHRH*Qb#`C#$QX>NpmwwN>h_-p~iS)T+{Tw5Y2V3rNkDGV|!~MwTMXQZY`vT zAsLV{1;Hb5=tg>1U+@Rx4wLXh$9C7g6ZnU(%OBbXY;HU?2!+mle%eUS)9|k_y1Kfx z)r9lI<`!nc$@2`HkN&+>xSB}y6858r*K(fgj91YA0A{>Y+*F$8idOes?RK|Mx$0qf zgj9`kXE!Kx_0@WqKEzpi{K9q*vY0{d>LbK$U(T8(iSRD zag+UPz&<_x(-z+!rHANGgRB?)5 z4e0XS_<4q#XvRy4QSj4eEUfw3F`STg_2(mkMr*2+XGSvSmE5I!zn5gXF2{kGWB&lO zr%5Vtry6kOjAPAYxuq@TDM_~}N$V%QmX=4g_AC_P0j*b@s2;akKGb zE#c}onlp33sK(9_i*Z)!t<0>qm*S7kqb?n-m`)Gft9NwX?*9PRhtK~29RC1i`wxfT z1Y5>>E$!vZ)|#Sys%wMtv@P;*ssI>rrza<`HR0X_wOMty)FdNqv}J;y;w0dIJpL8- zXYFs}O%F);O>yBnjZ)mjabYsVg@UxMMgc|6cx-cyeMUL>`@k)y+U2gvGF;2F1JP7> z{PX!&(D;9yWAjXtmmN1IsyBH)mcQma)|@C)lBT*_L$~l1=+pdC_xB8O4w0xz?T&b| zoS)@iPgs82H`<|t!P*CfMA~MVX}?jpjsWvxqO4$$0mn)!1@fuz=NL}Mav3+isG z*Mns^6}$U9KZd+%rLMKAo1t@~T*v*Z1-;CX`Ot%cTgxGek=%4S^uhc~jcCx!_=^)> zs=qHPUwoJ7e!Igs%tix*Fm!2ss0&dS!i^^?kXm)qPr0^Wd|wN|#en$d?S-TgX`@UGLv+Q9K&hrB&2aI$!pQi?pDgUQpt{&kP3czx}) zQGC!ijqUe8)p%fjgdfaT*yWa?PB40FFT!m3o-CKP%+#0s`osP>dPZdT4SwHbw7D_K z)p^H0l(;HG40v}^wTj{&5(`^`K6IT9(0z!lKMZJB z8eQuqsj98aGAQzhmwI~kJa(_y@rxP#l&jUgu~u=CY2UiN*W7+(3Y5LGg7eX{*IVsgj)1_#uRYx)O-upY}Qh3&QV z{{Vt~u4P#~L;VD^w78PVtnVWGWvB48IDMnsec}05Ro{i=)2@c8;ax{#q0f^i2kBe( zkihMiyNIy+lUYbN56_yh{j9+Kp6LbZ1Awza8NV<%udbE{4PKg6I&Au{N5AHIixEnD<}IiFSwA7KsDI#_Hn3XU__t8BR7j^wACNviU}w)yt$fj` zT;FT*podOtwN4S^pJ_jaIj?j5g*;Iof_@_1cslz|nmKjLK+z8|SzOd7@jpaB-0GC;rMS#XMUlm-5EP zk}x}0y?)wqJl`8Mtf_7OA4T~`0D00J*M@4g5$fJ8vC?f9BJTPqa0lKj!F?>nQ#yAF$;$Ie2;!eUH+A*~3V04}1~uo{c)Ncyudm>OZ_=41b+7#L}?Tt_!Ho zns~G+{!h?$G z7^CxCbkj}uY|PXoCqE(N^TlRdF=BA}$GFJ+X)kWw$s*yi+c`B-M<kP9F z>EV&!)Rv>9b=|f|@CQ(*y*F|XxL5))&rJSx1hSP}0g?xEj8uMbWZYB(>JJ9JXu!`_ zja?q$Qs64ELVGi3Kj-{ts=8dWHNv=V{0+a?{F*C{wLjh)Me@Yh_$%>KU)H>3Xu2nd zEHupu5Z-0ApH}%!+_Mr%&N=`NE9osCTU*ODx!e;xGUQ`F=e2xxzY}%6BU_T=?JWhQ zkras9B7X1XJ5@*BP)X+lJbMb8MEKd_b9eSTd;5inU!2UuFR<^Q^NRlfBqxylUHdgH zkJ7Oi_lTcAExix6E%gERaKXVC?nwjG44hKyI_=(t;p^Q`POy^pYg;*Gg3SosJ1Q0t zXB-wF4u=HS$n*SM*G;^SABZ(g!9j$j=D<2+5EOg&HO=ck8-5^YwzF#M)QUQ;_FD@zq;1vyOJp7 zE~+JB_{rqqw{_;b9}{UaT=NSCvPfkZE~FAM)RGGy z!oHVZCjS6JD}cw#>5OBKdht9{%VU-Cm2YOP8FI;A&MNEV?vJ_PTvNqlbkw(v)}Fe4 z2g3gV8hk%u-xF@`H7n(cJ&}O9$>3Hz-ZattJ#KA0HD)}_VBEha@U8FK^G3R~_^K`} z+C8$_u2xBQl0&fX?TYiuNOc(F-uep2KfN4*kM*zH_$xQ541$Dc!++Mjmh?lZT=TL)lW-u>FR<@o9mKHia;IBQoDBDoyTxIW3^tF zOx2!0v}(7@Zn*hk&I?vtk@-!5HshYnoc@BmtkSh)lw}sL)$jK$QPD;*e+yqDXTts{ z)ig<>wo94*&Ta$Dw$2hpQ-Hx-f>$78)1HRDp8b|DTHgL?rMs4Py1HhK<~d~>Q~*iI z&lx%En(@zoKL*pq*F#+KAu(DY%rTMkPTjtWKVPMKFYLvsO=IJe;rqmG@!UxfmuC#h zPZ=Ld{O5>q_)3%>7LA;ivT1$F{cY6sF!bsr&m-w=A4-<@$6sl&xNCccOu;(;01E-o z`h&>DbUNpRv~L++#@dv>*ghqJHd^YI=mJ~}=VJ!>WKw#gaZBJmD9z$UmtZ-MAdc15 zc)HTc;?1<(M@|x>@Xiw?d-{*4=D#|euURB_!{Io&d{bo`KY)HT{6EvJuO*1Z2CZqP z#<5%}7z_>w1dhkQL(;s1UHy>$59petTD{D764{H$%#g`Bi5@cJaQO!VZhBX@&pqr~ zVet2jG$XBPeIdC3nIj#u-;Z9ki>g^@8m^YU6o*!6tmKpk*6?NF9Q>ywWDi{9*EQdV zsVXnrcE0x8{sD~D&kMEiN5MNU7EP>ZFx=>|2{B~4Yyw8W;1M*;+X?mBrnt`%c-!J0 zf$`!kU*Rs4ZluusyPKG9o=5vu)GTDUc3|!RdbbD zGNpGVwa$8wT&+)I$NoC~vLArjeb%QOX)V^RZ1YD30*8?9V52-PSm1U54%N!|cF)4{ zqW!3+h*YM(e zGjXA6QLHJb!pMN``y_ce^xQZf&*@*MKN8P2s$y&{{XYz`DY&ccpsf$Yab1~5qg8{k=^fT@<6?I zc+aLYkI(h3O?E37p~jsrmdG+0Pbbqpzl~+b;+Ja|fNhR+9F-Uh4t{UUe?MCN{{STL zkA!jzI;o2P0BEYZ%i+J2R*(MPZ1@~s6K0s_`159wvBOe&SBKFdnw>xep4dGi0Sp(s(;{K^}%b`ziibl zUQWmB{utNA<1iS^G;2rOxlx={=LlUvS5j%t zQCd01>i5?6vUc@Pj6bvw#9xfQ0J={R>d@HuZsGPu6uT?!9l#xZw*l>5H{joikMMrR z=G(#xZFQyGus6>rZ{9HlbG4KZaqo^QEidBcm+<$)mp%d2d_}F_*vR{(j!z8RlqVPm zo3rid(zxFgc-O!lBh#YM$BK39YjGhl!xx3^`kue?`(Mp@eq9;At14~5y)7B*@=?9d z(s491MHN@uQRhppBM)`ue`{+RJ|6gksp@|aZncdPX(174nxmlyJc}9KK?fk=l}-*u zMRJ<2#lI42tfy7Avbb<^Bx`M_KY;`EuDjvgkHhZ|Lveqn>sr0ktz&ZxX>+2*2rVtU z#0eth@CN*z4@#k^cuPstQ)ICzW84l$zH9okg0eSH0O5vC2~(#V$}*2MxvIHoB%aca zwri#IKQ`h_#+Dl{p@+oE5~`}!o8+lZrK2cNQEk(dqORdB8@`un&0pPq7Rd&w9k!(u zyP8x+eDXmc99O1zlf~Bl7t+JCjH+~yWD}0{;9m}WQ~v-6u9Y3MlHzN4RaRv^amEkj zns1Ced2{0Hli^!ULuAsac+Am;R&PPLbv698o@CYV`PFI_6}e>7`7@&riG@r&Wf!_3 z__y&-Qt@5OS`8}QB4@z+E4r8Z07yUg&3TGkNo(T8x4N}hN0bi6B0{}KK8K%9D7%*P zP1I%6^_gBbM=K<3MI=FAn_^$nH&hl(;5|AXP^h4&b=%y9u%qaLu(khNxQSr zJR3EY&x)4jf3+gYFpX&%HZh!KSd5Ry@UKdO<}+)C)r*n965C1&kNf8pc9BfnbnKPfyrFl5zYL+S+T*lXL2+WZylBX(m1sNY-@UJ-W z?~L{BV_Hp7)lb=Ak|1MDE9OUS{_U0{J$PSA>^xh$58SZP5*V5lW z(jwFEt{O?CnS-c^1&Kp2Zgba01_nFg`IQq?b?2v8vPG}uv5n8ki9-+lWloz{UgQCF%hGKca^WP^q=h6 z@bg3XmGHeZxNaBiwzH&Hk+)R@WncZ$0q507?Oq-6f5x5$)pe__Bf{2_$z`Wqyi0W< zAL&I~0I2>IC!TpD(AQV+hvLtOJ{oIxS2`uQwYr+hTVz*1xtubPy#D|TD`1TMJJ$l* zK9;(bmxi?H5>+=N&5_1S7AiL%P%)os=Hh+`mO+ZdW8&{ZO>%3hZ8tR6m9%}G&djrj zKEH;gg=*Cm`Lx|>%G2}f<}q}gbHvwB>H2n_%N>*^7-Ehix<`@1mOtKN`96RyE41+C zyjIt@UI%z%`$f&%8F+s5Ml#@!_HxXGj<}CKYR8Q{U#ECJ?&C%9-J~sZa=_{rQOs9{ z%B(0?46C;Rwvf%!cHR#?;kjb)PNf#Te6k}56=l!AwPk(yzp95X#TD7uhcfiLqZ zTFc~p-73Fa!9mn_mW$5&FZd?qmF2&K_3K-|5t)U%Y1eiaEpHOLr<((+VOf+MZ(a!< z4Ptyc@!x^I417QyEzrDcud}XF=Gpu$0giajRX^#!j&WK(KO!f`lXu4WZwDLC`!(z8~(X*;K?y|h0~z-PbN z`CTkD@bv1ca+Koa>ZY8OBP)A4kXC9D-9Ns{{U{bd)SU3rHjLRY*q%3Dp%hSm*HI; z_l}oBvhc={r1)Cy{&8cf-QCGO^@>~n0If@80YXbAapV$7BdFrL?*Z9q-w`y{*{8hi zG)~hkvKNvzWy82p$sNu;iLO_~Iz`WkkCSO0%UBm+V7M#?LH$qv09vT{Zch<-3ixUk zmsb}4OoB+~wsu(}IpNffK?nTyuhBD~4|8lTQwh*cQ&*FIZSy^?_% zj9{9CRAYNaFoaA?m^^!HD5!uhRaNA zh>>lbDwZVrNe-h;Y(l@dXEWQi@E zHnvELA`tV`1Jl!v#MdF>Z9>@HG;z+)5}l#js~=PJ9jjMc@f;B-x3SaYwR0E@@?A85 zob_A|K8CA&Tliq9uk^TVQYF+PQrwd6IIf5O5>kjEw!O5J z0gMThH|lH3t?giy0yVMDIw7cTtbWejCx{O0roO`g;$9;GMN#^)(#uI(^gPL8b$&OB%=guxrP3{X%<5Kug(Fc4+16 zT2Sly@p6gc8<0D0n;`ym?P2^<%G9r04>jp8_;1MNoG(=;waZY#Ns^uWLI_YLtoP`z9Cr6 z@sj%QQ_*6$U$@!mQ$~|X69!+AxB-Kh&s-c4Uc5Y2$YzvbPOmf3EjRV%{SGR*1}2(v zZZ=-O?h~h`ee0Lg+TS72uXTJD;`P z&2uXSXp@%MxdfaX4|ALWn)|!ppN79>nf?>at$aB6IdeVtinM)iOSOvecekAlt2u^g zo=F)bmL|#+8+kb#3jF;2n(pm<1^Y&6O{qIW3=>?!b^=c@$f`~{L|~eO6#k?M6G+hbn12G_zE(k&B=L5UOH~Jzpc-re`|}xlj1g!eG-82XrKZz zOJ-RPJ-Szmd@X|GS@_xEyAc`r{6%e*WA}3^o}#^X;{D8?D!rdsj#UyEL{P*}3Z9uA z@=5%0Sbwr-!>G02kFaXGg~3$OZQk%kjsCI02&3>XpTfTy#AEQ>OX3|`*f-4yR7x+p zic0>D*FQ?FgsYYC^dVP9qpzzz^wX~xC?l}v73Vs|_?A{aXU&t%b++1vlWgv-(lUDc zSDWg0<8c6vdW!zQ@Y@FEkF4$dtMflC#k8W3%bTYX6))eXU#>sTYK!g66pf#9O7V{JX&QHBBoJ(W3rVdSQ%V15Tf5WiunPgHk=m+>wN#VC4#$+cTWqbYQ3GXV-Br}Z=M!-(l3!o+^TcNTO1Rb z^vzEI`#=Q0_RshWqD?KHmEsLcSHQvzKTdQF$E@2i#(P((X__Cyzm4As*3#}kJ9$Y_#{3Mp~ z+QafK+|2G&w^QY#ckDkpp6m8B_^W57onFJkGtR2Jw(0UtU+`HWP=7l5pH=t~;SYmQ zM=yf3m9&hg6TDHC-_KG{CcQJ^{C<9g6p@>W7^vbaouAJ&!H47g!!E;9qk@Eelypi_ zyZOEU021?Tf>6Y|n3*>z>vZ({wm&j;tq;Y19KCA|M^L=AnRqa27VMAz0FPF5_kJsH zm-|ui{{T$n3!lorN^+XKNL$v?|V7m3QEE;do!xy1#__e1pe2W%R@6n?#68 z*y_T*c}}EaHI!NN-|K_?O9gvTw`wY)_7rmbw(`cAEf3xt{(5k*ZvIgYs{Lfn^|PT#zy4^xbX3HCgis=zbZ(RR`G|#KMu^o(@Qsb=pr3{ zqPUM5{Als+nRuhbUKxtVV)=-X-2l$r6l4%bTJ;@wR9DkaigevV0E(OB0Yd)(I{A+G z$1(WV$NI(H@mtHNuNDg;2gw23mtZ;)17jz?ewFUvu(GVBE=#EI`58&s?s^Y|JZJHc z+YMptBD6`L%rQox=3sG*pP=dQTb~g=IBNbB@b;If>mCV}r`7LUO=#tqL`Wtl6JV=u z=teP)di1U)*TnbMXk9GzZ#*^{TQ^1X2dUkRpL*1?_`BnME;q2#{v7?aql$KWy?att zNRieqK#XJ&<(`ZW&tA4vp&ZJbrADpGAd2@BBMhM?G?_`NSUckO05{lYjupHNg1S#j(e9G;1dLBV5RH)YtZp3gwh| zKZhw(tNccnGw&y>{{VyLkIXUEYUdc%qOAnomqfe#{{SOjQ1OIT@335Fa0KM{BDu{| z#xD%MV!t^nocibdE11-^rkR`>GmH_>`y#1oH^$+jK-{#aJo;CanDISImr;FxQ`N%c z@Pk)6Jx9dr1?{!WhY^{YH$H*D{-9O+PZTePJWck`F!?udD#+Q!2p9zZ09PNY#C1&~ zSgpq9B}#MOaQ^@kSoewI2`wU$=To?EEw(bdXCF~t1;zM@XE{x5y1h3j%a(DHx4PDj z_xt{A`V66cOBD=7Ia$W(uIZhpjdfin;yLBCSoh1-vCvl!r)f7g*S~6Lm?E$dgXm9M zx1`)yTifsP1k!>K-4>*j9QYK?Rn z73a*j7{_6pf1jl@#9CyRld@V!R_YetZ6ITuj@%l}@Z{c1(Mz0yNdwdh*VgV=?E-k5 zo^lRr@Q)O7xnc8qd4v{~maqHE{MEk`>M}aX(UlhdAItt6hW*pV`daGkZjowsKt-s; z&9`vqvIEZ@j~#2zHJgyWE&wVGqhS#y&^rGBpIY<_%bzZE*7V@9T%zH}MV>$6Q|n%5 zs9YKRLoMb24RFL`w>Ynb$>>Uc)!RiE_@2dS+Sl%Sx5CKOnQoa0Ab?gKOLgmiSnOb%r9>8Rp^UsGfAG0luxErQBrC>hl*JrJ5O&z0+juAQLzH=8F;x#pn z$DHezGqvh7T!e46E+mwbu#?pF;8fbZwxzFYi)(XiFtGW9KgYP>_2-|mH%~Sg-H-9%|bDGAHzfyP7@v)XAAohF#kA8Xg73uyL@WeV+ zmj#q-ov1k-&wtjq?*r*_J4t1KlPtFpJOBdQcJ3VeV?X!MuU)i?-%GU@vH(~vB0@Xk zsju0*8OkFpEqq+ow^2{abK*GP8sl1>2zI%dap7GoM9|*GzH6^dr1^e?1C3+CT(ZNA4dh(O(w71^>00he5igjIC){}Z! zFJ8ZLInN72;tgWz=TVjcGu&+xTu5`}fKTLd03S?uuA@k`yz$nxEv@>xd5W?mYs!Sj z&4bf~3O=CLev_@)SNkGw^q2?M?_JG*Yp+rNt9#jW>a!%-u128-{gL4h{Mvs z(^lxd{{Z-M*14m!z8scTjIt%vv9&~74+_XW73x<{ z8QCC|z`%9bqbH}WYZSXi@v8-wOMeLr*n`U?CnCBxLs zGtaBiSA6!+`%KQXFjxrAdR-Pg1FQIJ!QKkAyVsc#>S%*oS_T41I}?M@an$}5qvKfH zR+i65*KJ|b9pn2{_Jokppg2(Ej0}^XpcRAQy#m`x)?x8BqQ7cOc|)`k9EnvVBrmAl zjB#Cc%HH^=#TT9=)#FdL>Js_4rFm2UVIl|76a^nl_pjf$Gs3JcO^c2@5lv2{n|#uL z3ck@^nqBL;@ti}(c&vUMVWBQsaY%mULVq8UF&xX#G@pM0RVS7u7kt= z8`1A9mgZX+-EJ5#4xr{Rc>Ma)R?fj?5JzPTvBq~qI(}U%Ao!YNs2i&pPG*e$J@TqP_I2es<+jz4;rtxv6+R!`Jt6y{?yRkLDqHe38IW!tv`|e~)|#ZKcHb zmXbT#Z~f|Ln8E%f$8T@)HfQmacBSoY^p?1#x;J-(n?A9zU_BiQRwBlmSvXJQc`XmQ`w`-ych7lT=<#sZ$S7>pxVaw zz980ZtSqEv!iJtzMaUfnSY-Wc*1zDgJ`mD=AAZ~Vr^Af~CQU!Xej3npy9Hs4Lk^W{ zi|yODdg%WEW1V}!Uj{#F-vxYQ@!|4xpAKqn_m>ZBWeAN7ijEh#S7kZs3YP6%NBkE~ z&%?ev_|@UB+7H1uvE6BYDDd*Z_PDLxJjpGa8EwNE^5leYPu;Hz55oAIHw4w9sar|& zHMcw^A9nsxT*-Eo$BWMKFU1T61sC;dn0majwvMEw7t4LFp$Db*W^Gk`BQY;+k&20z{lRiS^ViFakp$RkbmCJe_Dnu>w3XUAtQ`&UVN8B zRk6@pT@}QHM8tH=I6tLqcynL=&%PHDxkNFtxmfdqkCgZREPhpkWjx3Mk)x0w?a>OIIvjSl91t)t20dv&nADv~H)TvVkQ=Uz^ zZrg3YSNM7xa8IG)z2o*Z@a@H)hAuuJ>d`7(YcmmQ zsr)wTa5(<}X@NgLE3?APn!Ro!r#P#q-cQMNK7)&M3U37I-(45asq~4|L|SFU$2br_ zQIa^XIMnY+e1^sjMr*CUzf{m}2sp$I+}E9I7i6&{l1cAh-ToEe<2tpc{pI-|l49ks z=D${zgC`jN6aITt_V+3qDUJpTfHV4v$2Xt6+0O@@b^I|_t>(xaaz|YCujtuPN;W*} zSk{hR=a4#agOgH;2qP!!lhfZOs#-fb0s%NYgVXV;j^lumWE1blwQ|Y}n#4U0m%|1| zD#(}$c^z^y`q5DMT!fo{Ah&GeC;4WI@bOwzE-Ot>oGm^t{84uEFZ@5EN&Vv$ibTip z6?Wgo);gKaz2O;=yS=}b`1%4XHEg3(jk))$5yFVMC=VTL{FHOH$otjU)wk3B8)_~j z^JR^g@-H;x82pDdpYadE9uEHigmP=ULt@YOl$|v|IQJRFJ|KXO0Oqs&c0A1rTsCm0 z2k@@k1|oDZl;<7OUzrkx)vkIk?0e!1TmJxrFlckKu(8zShk{1ayh+>r{{W?ZO`t&( zR$gl~YKTH^VaLUR-pgI^OpZs}pflR-eWXaFsloN-5BvtV?>;qrX0TuQ zM(qA1%&?X7#B-6pIS25$BRxpsy$_@Z3}Wy;C*tQQs4POKUOGE9TAXI>{{U|`pPD|~ zg0mbwEVag3wC(q3d(Z8i@m>!C=r+1_wYi$+{&j`~+1#H_y)*e$Pk^5gKeRLkvTY(+ zZlff5Px7krKrG$A44V0~$G#`m{6}Y`c;m(1E0Xf&EBGb2Y1{o;6)G{GvL`ai!Tjjj3Jfc1Gp? z^s5d^kM4$UgXjf#_w6CzT{p&l6uGdT(Mw1B*or91vYhet!5@u#2CaG)0?@W|?Ou7~ zS*Do^x(svAt!IOrqftpcQS(!@v}coiJG#>JZ3%RpG8K(u9IAnmeNU}k)b(Mo?QG}N z*EK7ynXS(V3xO*y>;C}it#$Gwgi2Q=4C1||P4n}xqL&VmM+Xugk{+ z+PshB1(eajEx@#rNZ6e1d~NryPSJczVdA|iJDYisIT>i%Irr^cZ^R#q-wHk$OK~5> zy+_TCLVTS%(%8#1hty{Zzfx<$%i~c^#YMN+Z6>!o2VMBT7l(W!sc9ONM*2vtWoroM zlMChrBxL(__r^Qd4e(G)Zk}z`)3wB`WtL9t>~hDaT$*R?1@T|PpAmIK{{RRl!)SCX z8D!dLo?|W4LQ+q9S*R_s3-v^pv@`|qoP0rTlw(34hugg=* z_3c{IN7Lkj&BU^{P0i1#>w#WJtZIzc?{?glEO1ZdUB8dD8%v+vY4Uk$!zX7ekEd$$ zUlH7F20%tR861IM(wt?RDDtRDa>ZR+Z;|o&9ut*bOIN>ggu1(@F4-(G`EmTI))HJ^ zF_m+`*&iRoPbRYMBYi^bHsTR;&!uiym|~6PMcT0pAoo2#t$t6JRK#Pd`$@~(TY2~U zkF2X0(v6bT+SB$d8xxQnK>7-f_XQ_I&PtJ=UZ2vLrOX6C7T{ER!^p9Mq-B^_+EkoY z8Tl{k^E`;g_1Y`x+EjNJZ!E~W`EA>?$NvDTy)WUHfV>^5_+|C)5cq@1jLN{G*vUFa z!TF>p>$4vz`Hl!~{FU?lDjhvGEbBZr#BG&=29OTq0zfD6720V2Kky%ew1KDT9v;2A zkj`X|dzfs}Rp4QK#)eiP5DqvUM{{0HT2iNmjA+KvdbYgZb9uKpyR$RKi{VSZ7RbI2 zv?*t9n+$;QGbtm?05buRj)aWljt(o9xwv`tDXue-4iY3C+syn58~ypme)U~;$wfl+DkNfw)T zba{E@5vlbg^HSa<&0^8P!{CGjbtBi(jxotrcf*sT1@FOg$JKQCqHq|$t7DX&&qw`w znHSeGG)~Q(iZXIDis5b?nLN-j^Jcp(e$!IXv{@5Wh{ZL?{m^sOax;%gGvO7h@bBwk52e?skWPB_t_>_K#(eon(doU@g28?=GFCBK!MkWXB)Q@ zzyrNcVTCZdtV5AS7Pmp^+4{{Yop$BsN5ZQ+>HShCcW%o4_Yrv&C**-$#<4Wphj zo}EQ;v8-C>i1f&<4(TV<5-osoGRgq`Yv>P-zYe_b2-@kk(Pmp*yiN}%1BGvYn>p@O z6JFkPj-C$zf}S@ow^>0;tg&o^tm94d=?oFI)mD{y?(+i zXI5z}FJoV{#0-!zDnaXyOasPq$R72deXUYGow?L=)LaEU0Hgkc6M;~9#~8?;GXYKBc{{5Jstj=Jjf(QOMjIeFc>HV z;Gdz-x6-NKT|Jk9EH0*rl^R(iw`0>FF5&Nz%zd%$Xt%vf4+TT3rmN)2(#aXNVo8@E zfaB42voF^^q>7#*)vPYGtv|!Iw_BFpCV!c5H#y`FTw@vUiu|XCVOm^EPm$`S+eDIg z`QJtAeRg5&-dNl1_qNrH*$t$(mb1Yg8c52>!2W0QC-bhjhL1uTR8FE>=@-or43QY< zqPS~YN#9kq)1T#Ia*CyoX5;)T-@GOJJl$Sc$Esd*~bUJ@BXD{iWZG zqPCJhl-(D(E`DAKeZJrVx%VVj!*L&rxP0#{sY?w*mYP~QYcIKIzZ2HOa7}!#6$w_U zdAHML{%5gxf8pD=z}rigwkY$PnQ-+zyV6JX22XCIj+l7{hK}_crhch@d$iQ3Xue>+mF9n+k?~z22omxw+IdR9!Yj(3R;J3-<`F!$(3|ICZ zK3TRO+5K6mMJqe$*YsVJKN_bRScyqa`n?Y%@!qLFg>>7b^2%Pw#@SK91rBie9YGyG zI^eah5}gj`%C?nWOIt0l>^wBGagq7w@T0|kI$sc9`JN=am`F=B*Jm%8Sdv+fUP;bt zhq$?x9d0O|;bj7FRgtn74B(H>ygctbjV&P6?w6;3Klo$la=fiZo$4;{*ZNZEm5H&GP0)lG;eT#FPRK(wOHsDt(yO-Twga zSYI3b8u;!100idnSC0G}Z54&o8l9c~mtz%!L@nW#2_u@^9ie_#+mbR5mB$>{!~Xyc zpT$29_01RJ7OUZVyK5M=7@@qIP}7`S%LKn9h{eX%R*jeu+%BQ$|=Q^ElL-n^Mx!r^@cE z?)})MuU4*)htIMsZa+TCF|;D@g{<3r)=CnK=DM@KsU^1ip9wO6-3AY+t43)RMZMB5 zBjt0*r%PojK=@HRT!*Vx9e(CNj_*aOix;cePZCS;%*Q2rf#TnfdZezGc z$Y_;&us}0V+uO@M!xS5eBNBauSh-2zZGeemKX)TI{A%}uA%^QyxU?zr%Vjd4><-%J z;e40!&9-+Qt&sZ-fO$gN{=B3+Dft;+5h!S*$)1tzYS(&mr5^C0gsOZ`uZB~3qsD?jg zNeAxW93A-nCeK08V?BCPnLM$dQB}1F!v&E?12vp~OUT zKsf=A_*NR<#>5pMk&K+ywx2q(d5fNyI2is_`~2y(4NfO>dd5XkBsj(~gWsC8@#p3Y zJ%(^aXX&%A_qfQ(IqCXVnSV0e{`EQZ9FOz$uOhFzBQ~sU_(4C?0uzyu&VMe{Qg}qD zmuZxeNAZmA9q6y0#w*rkMXMh?1Yb^Rb_{d+)0Kf7bI&}|o@@M*XWyin8m9w4ooD#S zk)lb4&Oya#DB}XNJYrk>7EJWof0cA$Tl&Nong_Av9uU`Wd@-!SuULPqwz+27INUc7 zSMdao#=T?3z7)_tBUr=ZJqKDAcZLASCYe0`q%mu^px;&w-@+hm+>}P3*zupAR}tdnxVYQ_BoETJ4zK1zwND4B&sxR5hWAsK?3S=Jj>qLx zOq2Lmn}_#VM}_$2WU}#X#B;~7Bt&2pGus*Dj&coK(;LmVSMywP@`8#1A5h2nRKFT+ zEv@zI3uq8cGNiw0p4{(Nsa3izP@GSEzGYW2j6Ipto#m-?ONl=`~m<2JbKsc9DjuRoUr&M{{Sv1Qj7lp7b=v$ z@@CH)4Egz`-c6rMcq?7-_LbqAt#Ori*}_cGX8_=EezoCVAivSRDQefcpNXV}W|Bs4 zJ%=%<=m)SL%DdlxCHJ;%UTP6->~MZLwbN5rdwm*qlHFJs<#_N;22MxUHMih< z*sX!{+sjje8>*l1slGb+YTr)yhp1{g6TG@@{LQFoHvxW0ZKiGO)6kNK(4hweRV@!q z7m;Q=g|{ag;zTDW*ChV{g@0asBr*tO(95v&oK_y43UH>DvRtat=2p?UU7jXeQ=Q56 zyKMFU0D;yr`0L^|mWYvuw1K?)$Bls-7Y7`Aoul!uZvCQsL1Uxn2Jsk&NQ_aS#xdz! zN9?2E;ozSP>K-+`iJn`P+a4p!6dZB*l7A{six&FdihRqf{n1-4O2YkpYyA1*3{){W z-?LKpY3b_vpSAGr4dC)lXCHc|zb=Q%-Y?S<_WFCfNzJ>+fuqdv-pNPRMSc4 z-4`4T@GJG~H7L@ily=#Er^wWkZ}BhN%3_EG9GW#5ciqEj$i!uv>qV|G8MpvdD6W3f zsmU>H43MPy3iN8#e6fY~lV5Vw<);+p-L^Fr>M;~{!>~CgxvM@D(~iHa38WwB@bRBq zb*w22+MV*)+j3d4kzJJY+xS1jCJ2ZUI&^u8j#lS{Ee(Arx(j;h$&s=agd#b@|l((_M}2(N(5(D{&j!2E}yJQpKdDVlj6-MPVq*StQi(q?Cjx=Q1KEs4xZ8fuIiCpu>?yLQJ*ajW0T8c2a*R~mGp1y{i^?yU}2IVW83!? zaog9o=U9KWK80g@rOCG9DZ-!LMURz;_pj$(Bk_}ov+T#}FiM3DoMC&lx4kX?O33|B z0paFTg39HHZPcl2ru0eqCHc0`j&<2B{AYJ^H7toDwUMpt1;#1Z0t(J!|f`W(nlmTULAyFHB93 zSolrh%j>^}@py+wxmBLZ<=5p0c5)6g+;dkvLjEA}>e|}r_Oi`zjFa!yvHW%8So|fbeXGJ{;k&pW=?V|Yfzyv%*XGr#POfE9 z#eJt{{{U0fgl$P&?!E^64!rRP#@`C){{Y#HmiltZeRf%Kyr$i){{VfJ`~mc@O8EU9 z-jlChSn0w<5sZHCc5>Zuj-4xi{t3mP>K-H0b#ID(BpJH6Z?i`!9QkqjdVIw2Ju745 z$sd;?1UxGdj_=#q) zYr8ok4V?Al)^?S2Q9x$pPAga9P5fGyimv5|VHL`#A6n z1G@(Yp%wPImJWt%MO2!-zXo`eD$fr-R5wH_Xq94hQ4l=~U~UQqG6(nyO>A3g zpJuhPA#N>gf(1Nzha~?1-&sG%cB1c(FoGjVw&v z)O5e(pPBaD1y`3DIuYux_YwR_xOnXtb=-nG*Fc+|ed;t`=oA;gTKV;uD*a(Mf8Wjz+;hbg>v8m42_ld1T&s zhsM&pUowx%-u2JUv;126rA$29dSB;#4$5B{sCL}@h&VYtYNh`GiXym=0f*u%oOYI1 zmL+T#Cuzre%oAMABUrI3H}HmT8`{68aXIyBapY+H`i@yeyRL z7hH^-`tUi%LE*duEu&gTeLkUWAC$WbNW*srj=t5ed3&cUhT>gb&n$lTo}c4iTa;JA zSHLMwqSY-gYb9>ax#MEtQ|lb6F3SBL?C(55@#6QzI%{}-e}}pvTbP7#Jl7ZZFKz}g zv(LG3IskGH8LQqayYTmlwapVk@eAC?r&;NCkE&m2)~htXJSyzr9gs6+&T_*(wdRoc zm%^5%ck9aRP6|g8wtj>P(9kcu^D(!X*$7asz=Fy;WFJgd^#1^WHL{xZF?dWpROXVE zqa`a|>ifTy^tqKR9a@!=sTUQkovp36*RkgQEz`fVyh(Qg=VMAp%HWWB&N;Od1po6aMYLg0_AfBy#wnUDuHVllf!%)(o7!^C7|PPbRm18E#(`ss8|` zEuYf7{{RK4!hS1c@shfA>HcPz#^Z&ZyA&^Dmcvq>)W`Cq%&ZS=9<_4IgepOA#F>G!?1Z{pXtfk3-L6nhG1WH= z{+07MNV(OD?(O%EPey%}@H0%CMfg{s>GvRq8e~rE+!h;%{4-tUq=lVWzkciYK~pmXU3O4Uxo@XBXr0( zJ!{S{wH>dtzd`F=U&OsoS$JwF_v3ZKjt{Y~EWXs`xIIW6f$d+u_-B$s3y~=`tfeRV zeuPQ$>~t17%1Cj^tt#djU zW86q>x#^C5b6+)H7j`mg=S|@@{{Tqc3C~m4AA!bwe_AP?4v?95d7V0pdeL7yi%nKz zr+Xhit8FLK)|Y=cBk`yfRy-Pba8v>ZBEQJyed^qLI0G5)n$7WkKe60nvG46#3Ukk; zW_ZQF;!fv00BfTM{__j>Uzt`sjQ{}g-2VVGN{-REc-*Tg1zAoAKBAEbDo8!g^E8$I z@np1V*5Uj$IV9rgK{e5*1)=%%R@GMg+3jBs{6zjI@Z5TW0A_`Rcd>;87Xv%6M;RD7 z9=OjH-1vW5fT#~34W2@S{cFPh40wn8AI351)6NnNM0qc}7~lLIeg0qg6?+hlTM5kU z50+mA*zgab>x1>L@ssH<;e?{{RH8y)E4R3ns?>Zw93P zC8Ik_eMw~lY-ID-e_zZ00IgHp3(Ly~f=ke_F!|ttb|Vi)?&I0Z3Z>!8CB2XBF~-ua z23kHvZ^vtCau(TTRAJ7?;Qk(`=qvfD+7mtK?09#_O$IiO%T^Ia;?m@+23+A^jEwt* z{Og{!lXbXMjdB+YgdA6#-Zbz;(!ZCsDZhjX6qY0SdU4S8t{cD}9`O%| zwM!(ANfFyL@wD=6!1DJ3eq)}a(!T43Fcoub%MC^`ZZfG9^wQT?zmi{p$%>q5LALrC z9~vXm^$&|WuAd|gDAVk0;()8;YOT6|k4n?{DPyshZO37eR6a4oYiIFGU%r{a zG8lr4_Yp=v#-{i~tI0L^SP4THDp!IHe#PRPY1HDJeM)Zeb9T1YTUhb1Z}$=OB>kZL zeQ&S$_FWTBpL`lx`7Bib02uqgdgG;VH~O{owlYAmg@o-=Ny!A9 z-pc;~-S~cPZCm~U&zV(m!^w4G>6(FbOM9!?b`*4mBE*GL$-u$D&p%ULpTnOAYL_~k z5bAc4#RD)U#tA9LNXK3)BjN9W^#1@D-G1v-cy6L?q7w*pWhaBw5zp4W!@xcP{=i7= zZ!gWvs@PT^bw{;*uhNGV{iDY9XnZQ1V;6O9jq@(|zUL-d{gfj2JAF%Cw(y;`gqmH{ zQEAN>SfGiw7C0MM9)Nf6#d7}u68u=&o|`PuD=7(#ZO3k&^_i>Q-q`q(Z6Zev9l0U` z>(LJ&55tP1uWNSGY0YnNKp-3*!oSW8Ze2VkK2WKwl%LOC&)=xyp@_l8P|-BMBjs<5 zQr_Kox@W)2#T;RmBN@p*;aq%9S@=JNci*%jS#@ZxE&38wxau><^{zL=x@fp_B*zM) zEZFEP`Y(g|F?n_xofz87UtXu<`QmhO)m+{6)6kX$7WU>=1Q#583Xb3Bml)s?nv+wR zUSHj=7ov|!WB{%l0h7u1ufCTm#B#@IqrzIo{mWN39#x^*ImQ;K#yqv%{70oe?oTnq zVAyHlrZ$1q-=Nt!_iv8+TNnP0hP(^SUx%N z3+o!2MGxEupy&l}`0K=wXxeA?MWf)A!)-k)gRqW!`S+IEuEQRtpA%;o462+m$lsM~ zK3(6J&##e&TZgHNs_|7mUWZF(;tvgKnl-+S;;ks7TCbOHe`$c>I^lpu8_mnv$L0;(hS~>1{&=om_N>&c ztu#xPkv7S<1xetXa6#`~Ux9pSZx?{l`bk_Xz7{yhEgFC@NcZbr1^Z$6_w5#GqgiS` zW3!0CBb<}h--`ZF!_uvpaU7~z&Qotrn%w>46IL01My~Wx{SOfMiLcG9_$oN%vWOyG zfRiF%vFER^ZhdRd;I=n6QCkHpWsy`U>T+?4_0I(SJ<(;-&EBA*;E~FbF`vilSUR_a zbm8KwDYRr*S}XW`$n$_#c<0-W_5BT$dmOhYogTWh{$KD9!s}O6@prMOH;?RmJD^&_ zVG5*%*aHHLl_MjQ+mLGuMDZ4hscR3Zc%w$TyJj6AVT2yPTJ^09Q1I`BJWBo-@!p3G z=833ENo$A{n^_6QV7Oxyv5P8vw4p;~Ixjwwtk^a^C-L98u zsmN{s2#j2V&Uvq({0(bw;2(mPapXtj>EIl2Iu0w~&)WCmd+OSZ+9kciGhEyjlji%V z4SatI<@K|!B5{w}mcEz&03-D-B*fFovRJ3~Vwd4#!Ea@InWUZDmtx2Bs!wttkV*h7 zoP}4++ssb~+coLB z5>i!ZtykgX{eI&WQ8mjxzmJ;qBh<8LrvM~mmp{8H*FK*0_K(4z4C%iEeg|7YBw*c} zPcr56kY!=Hfc6CA@UNHt0Qh!q75q!E(Df(@OF4eiaHMrqAdmceZGBba>$q-oRMXE9 zLl!ZdcCY788gnW&^Bg{3Nm}0Y*Ph8g#h>KM7g8l<&--v0pCk<@D1rH+?+*7kG1*`DG^Z5a1bNL>41{-&+?drO;0(PtMC zku0*|!*!}i_RVhEGuX!O<|~_qT$KPEDC3OellWG&vm{z<7TfaMGDdTcO8%qa%%|#@ zhgG!F^49iAyY+6`pOEA64y+VytAB>(VU(oByMxxG)ih+)ZN=rIVquO4b5K78hqieX z`#oG~$Ip?*0pt^k`dXgBw+i12>)Xu1DvHvE=ysdq1hvMO;=L_yRen(@#nJy99I|MKM~7k;_)5LtdU1?9>5*9bDWHt_Ny%_C$tv~xXAC{y?-q6n^-LV ztmx?p>X-HAe$&G+jx#ERC#<8RJmXZ*e^2x88>H+I#*xP&m*INvy)C-Ex9NTSw$$4&la3C4hS$XSB3mmf z*B^F9Mn1%6`d2EEY*93M+J5i{I5pgO{?uG}Mph#x7fCqwI6tAUKC!HeN{UpmNns`LJ8jwuNnPc>-_jySKsz_xy81<=QV~v5h z1mM(KkUPTz_qgNMznbup_mcBpncX#G?(YuI_Wdf_?8*MR1Yyris5P}}1AqV-@5XBr z!xqnHqFGvzmX6jmUjN*RMQ=$>x`AoF0daSEGDSB0*+^0FB1FA8Ji9E0A!p*%Lvu(<9R zqLi06z5f8EGbyJ{v`1rY<)8*KPqD3?5^0UYTwWo>f z46!ZUq;eo7M2ZT6RFZKZ8OO}LS2;}wa8QC9s5y5;vIx`oti zK4hBqS0DKERvx+HU01*sEggo2ZNsVaWs?ds?O(%Aq;U;KJD+_|D0JWNC8v>_`)KQ0 zeYb(GG>s}k_Y<^hJhC{>!s9Bc4;l3*JoFXV_!mwQ+CYhaZ-E+s4a)JyPECBh@l(Y* zuZ%T~ml}ggra_cmp2EI~(VW=m&vDbrftZ8FaJl|MziHt5c#OY?)U?)n^}Uzizf4EO+jcy-wN5vC9WJSPYaQH^vWM9* z2&_N_iN_@I*S|s6wQ!#m;kME~AnLkamw9V-1+~t|?riQBNaVtSxmigJGoC@Oq(5Q* z0NKV*+ZXn-@Zap*v)x-stc$t-0JLrFmgDRu-qvZBX(Mbbkq1N$q^{*9Ln$DSY5k|a z;NJfL+FQf%-~37VWAQt~J|(x8Mw?js73G`g^UD>zs!MibMlwW!U6>$g0Ch%f!2GBC zE5JNo4tP_FQHn$d1bEi1Jrt6C)XJ~uedu&~8L4LS14TBQ2wmXB-Q zrETulbMxa*)wO>NNo%iI-8_D9D&<=l!wwJVe+u?rguXV_uWpw1_fQ5Bo1L8YJ^8OJ zw$NtQ{2gT0XBd@(90S;q*0p>Er)!=byI-}JSY8PR?3{GZ_?r4}9x_~)iE@lqCamKe zW*RkH{{1+)I9>NiS#@_up@fv_)sDN^(A0h~YF;b&sjFO@GY*=JAk+7z((0MZ5PD}h z9@O6uc!o)aWV!iABq0L0JDqt2-s@B)#H{6qWgWjNzdoz>n{fCCe?mQL{M6#?vXx4B zY;`DEIPy#UQMH}>xV85mr({^!dkIjJ)oQ&N%KT4te&YkrHOGt8hTH76r>V;} zI@e|5n>nv+Zer@&NKyXRU+}L#v)$E)PBC9sz&PmA!^($L_wr|j#8hvIRUXX~xGdp7 z-aqa!*`#uh8QrsktrW&z;e-eJ09XM&u+*lE<3urk=~e z`WYpZNJrvMDzAt1yI&3ICse;7TW&xnn&SLj;+eIH+D*j9IE_K;UhXd$hT&|h+D}^2 zS})Y|y`1w*Pb7W_y?n~LmGs*7q-2s$l$^eM3fIwXZ4o@_BnWyhAE2&&$z7ky1cRKp z^{d}$V1Xm_5w|~>E5~cOz*;@AF0Og@D=(;`vv~Z{u;LSTqxw+P%PcqIBn0=BN2GPJ7PJ@d0+9;z!_YzIQo!dGR zYt6`MW!RYgdk;&J`djrqEb6vpL;Z^pSDxD6)bt%c$G0~6LPZS5HD~?MCj|R}=~T77 zUPdc(=MwE6SRTCBC24gmlKrmlAxGSZ95a1A2fb)nUNrigQd|Jrt|Q#N0QrXl*b4f| zR({Q+|JJq@kCSxN7X=iJxYcyWTBU4eM3d-K`#OUOJz z%@!vxSZKd?8?Q5*kHT?V$G8$)_a~ay_%WqvHva$`ygLtxb(?)RPP^43w7b==?S#=- zET_vCPKN=OBH~Z@2Y!`u_Q{iW!U_SAp4H6wf_=!=z;b%Iuf44mD7Mef6LU^4W2EsX zhxKg%xFtv02jNBUgM49WCB5&2e0_6gs#r?cdwYwB1afie8%-deTvwk-ueF8PTlm`I zBll*FwzwY6{9i-*S9#)Z7HVEG@Wj_4ZzQ_Z?gielad#YSbsMW&bLLECI62!S?T)N+ zNp>R~MgE-9+oqjt;z;8rS#qQ0&Oe|7@UC1QKNW+UsfU*Qo73lS z*5^~8>-rA8XZBm2MjNQW!H(wBwgKp*$e`;S~#Pw_|g zW%yy?9{_*Cai#0}o{uE@a!K|%C64(F?;iDxwjg+cU{ny30o1n!yh~BgJVBsefp4fR zkLAZ9n#yc}y%n*M+-C>SgI}QFyhd=R7-F#8&3&lK&fbkX`hC_s$z?FAoaofD?AQ4< z_#aLF%03-b)iq6WP_nErq!8Q{!-0~_tUD3%*M0FLP!MUw10Dg%&r0V%WiJok=)VP_ z)iqnWB)Qb*lI5C5B$YoZ@_m#a#OrX!xX8>>uzWByFPdL%c?fB;?_v!%EaS$Kj*OhYd=r0GD0MY3ml99GtE%c z?;BLTxYQX)X{E~KbYJtueHH%z1nu~1;Qs&~d{7#Xjc>e|G>c(#WQHkmCDC7^qawbxeAqfPlKVFO4}n!9h^tm|+1&EYCf@G(-R|L)9z3GP>OoZmor5F98A*V?_e!oLE(H~di7buS-lejb7dwM)4g)f!?JIKvf0L4%0ZRaHOP z1B_NTivIv(AB`4Tz-TP=`6G5Yoi4~#9Q6eOAAmLcmKTorLkpEsg-mrdReVwASHE37 zPm^3P#8vSUoN%*nwvUzA@>`-}h|Ir@W86VKk5z{<@K85=x{1))Kt3R=P%_^Yti0TR%g)VJO!TPW6rrF9_~s03hzVZuMPMIS@8wP zveI-ZZzk@sCPIBrLHO6NcmwuKzP#CU<2%nPEc0h%-#5{p7$1k{Ue`VMfIJgqEp&Z0 z-YC#~sTu_zkgik2za6aXkc%Cuv=9Q<urd@mGf}7wozTaN%d%|C9bHtfDNawjZJvfdJInCY{gxgv zid~XM?zgH*c*TQl-JFiNs`^;WF-jEt_Xib)4xy;(k%I;!iO(`S0CDUqZr<%;vX0+! z{_XOoKG?7H`{;S$=6%BJ8=Gb{T^Lq`BMxmGm&D~~t+unar{la+n_;nYj3Xs4@ib2&ZEg=W=KAf5Ti!^?4%bFL zVvdYGMR$EF_z~l=G&B4>`K8IpDDUv8}mdSM|IAh4CL2+p!i0?sGU%JP?*1wW) z^Y(RAk7h}@*!#cX&xucq{28YBlgmUm)=|pwuOV20kUg`S>MgKF+yTJg3i1B{*(=35 zPP6d3(?Yh^QhUuk38{$(TDnRO21Yy4p)$A6ruwcGX9CgKhnOBK%HL$U6T@NAQ z;-=Bf{6-9J?NhLfkOAqF2V<(ZpAK~xTvUI;MX7ZTr#dJC} zO0n)aVB{$IMl(h)iV|!UE34S)d@pPfTOn`&6~r^CUCQRM zrqa;)RV8LX8o3NyjTOQE z>8sI8mixU$W_aFl{f8kVA1Tf%sV3^6*!kyKas8`5Jo$#H7h}k)8p8vrPIrIHL&a1s zGk`N+*4PT|Wwc1T;1F;K;=Bjqb<8)H^Iu)RIVDHEe>~+hW9_dfpI?rt zIMH@zp02SLn=0$Nt(cJ`xLIVjM%%emgRnLZ3F)5MuS?UvYi|c?*3aSSz8T#?E#waj za~*UrjIxoqgBKum#zlDl0E?zsbn|XkK5T#=r&^`p%P3>lt-yswS%Q!;)kv?{vS`)% zr!(63`;uRw`Tjmh)v@X~GJUeo?RM7Rx<4y$2kGDIS|19Pqqw-Yh;p)iJAWFI*Fm&=_DQU1ZW4Inl{QF<#1f>gM&W`wf(=3O1O5sR`)mAqg?=4) z&-SA6#+j zrwTvDx!>92QPYRTYduR@)g!fz$|sp;l;KhFwMqP2KU`P!eIFR|MU&y1!cv|h5R84K zsm4=DYLabVPMwdRc|0Xo+D&SYsF&e~hI|J*L1(YYC7hcJ*~!_|V32d)(2g-&{{Y3W z9qG{cVe}mm#wea)wWbO;D)cx#vNAs!#nt?48{NvMUV`+gA!de9!GY*TE0gh6^W9rX zcX0brMoY2jUx4_7;$C;f7^!J4JnTX^pD+lZo?F(tb?W6cMq2h;K7amcN$ zH%r%am5)o-w8?ERts;%u-r)+djnp!dI`X7qzHbGEdGN@_k z^88dQ{hQiekNyeMM+`E*l&T7R$0w;gcds?^E|hdH65LoHx?6f5V}t!`*Ss;Jooh%M ztYrd)%I#b#`yb2mHA~0e0lZ#+5=C^z!p+Jzoi1nuO?a*Wv_O8=F{gJ*q>lcX9 z{5ZT@bx$#XmLOyUoGx5s*B|4ZBgEcpx^A^<+FTYgnBKxchmDtl7=M2}V!xoWyxS?Q zPF3*rBU)6|)MF&={pnfh75MZ&JI7$L)GZ3Il{GEfO?;Pk`5gZMjr?%h<js?O#PHT8{YhD`1PRpErFKQ-2JCeQaCV2^AK^_j@17EfmiD#xVG@u zh3&MhKHNZzpS+AO0QLPiA3O*J z>+LTQ{BY4F)=ri*n~4vRB<&`^02u6h=Ogp?ugdtswDDP@q_vC};`M)_`mYFJV?!zS z&qp@j$!Nb;e2@PC1srQ#Pr`*-VrR0}B6a5ghFKI2di=cS+P*crFhd{^LIVNIu_Tl0 z&3$G600kHEO`fr(_-QP3{{XY5rz75yWXni_aKBJDpU%Ebxlp9Zk%7Pmy?vJpMp4P( zey#rik@1{V?VQRhqF<1a$WVEVNpd=oQV+wgyad`a;q#ae%Y zd?RHg)&8k+uv)~K!6foY9TW_bdjL&+601&)=RJ~V&c>BW(@=udw0%8bem4A7)Aaop z;q8XYS#6M}rFSawm>)M%a5`rltLk5ZU$akzJ{f#4-Q!COJ$qA@CX-OV)NImLnnpYU zyfzU=03O-vPyLtw0A}w8{2B0Nj)iAyAKHAF*HO7L7JI3bVcg{Knav$X*~9q_EUpT*eQL>km;4^={M-Gc zY+!$n+Qq%35+qh)C~e_+-cKoZ2P&nBA#(f-;B*)m70O)Qt(;2t zsj0b*k}MVOMRT|JQCyJ|L_x>{1Ftmetwct5Dv~+e2Wq7Hghq6^Mn5YCP{SbNyQ)pL z(l{XmkF`H<4G-b%=j@qhC&SN#Ho6w8;Qd!iO>t_aC&{KtdE zd?g*s_SXe&Ady2T*ykYa##H_wkF9+@`$2!fQGPG}(BA+i{hxeg@Xx|ldUUC5iK%I^ zT&2p084CURSxGwvc>JrvybIwQzlxqH@ZW{49TI!1m9U-5V60$*C2kJ^8*f9zRpV2H z=;^Zi&bmrbrSEhvYrY`y9RC0gbWJJ`5nbL{-HGJ7wz|1}i84sas=JQR!ZN3f96uwa zWcY7Fz40%JEp!_@N!IUBx{er-61Zm|40hTx_yJ#Ae%dpB$)B@7?D=W?82k;;Vexji zf2gLb;$Ml{m7x=?vC7CJu?>&2+meo)b1MAqPGiTJ{>9%8EU&&JOX9h;CAg1EwavUv zxWvzp2_DKD@glLtV_pj^sfuf4?dQ=QxmFYVM=+;`iqcQ;-Tnu-YxXvN3Gl6_hV+mG zc6O-{9?GCs%RjUyi{+N-G-#NImQ1>n{2>1TUX}G%#M@(Opk8Yi5{;JdG>**N0fKAc z-D_RC@mGrOH9*M1X*Nb30mA2uXQBT98u(5U!OIn1Ri&-9eysfyi}9-shm8p7b!F&` z{V_Dx)-9J^c_Xs((QcY99b?bTM+cBI+P=p9m$eUumKX79KMA}ACZ%M^r%<`Lk&itQ zn7$Vro(4MfubqAXd=-o0?}{&B)UVMjbW0t7Kx4+}tT-o_<;yF$hR#~r`@Rpx#^_>Xa@+shT4E{NTC za7gsWVOG8o{>Xo{myh(jZA-&e3%{;;aO!^O7K-_sOr7%^E_vo z*2KFvsD8%(0JB7YwHL;XE8vOvui~;`Q~M>9A}C%`h44vt*ijcBGcXCqce+yyk}s9Fa)PextfV1h z`6a8(cVQae+iqVew#Y{tdH(gJ# z5zFBi7Dh><%qZjSU%Pa4lsV+K01>8lAAo=0t)K8ry(9h!@8drScz;6p)fbIm@h*im z*wkXue#)xw+%ytw+bpRhOjjy*haWP!E&*?uB>1JTTrhi`ZtdIc27JBCljfXePEX!A zA2(hqj=$p@tNk_`S?-PUXc za6oc8M(2RvL(}VzTH?QG_nto*SzS85*)-Gv&qg8NBkD3gI^FPh#7_W4qg&c*UL{nJ z+p63|_Hr->Xar;_2D#6To+Hq;n_V|d&~^CrC@tDnNkne17(0GO+y-%u52s#5dK^uT ztD0aXTD?oo(fdiaYbAR<>|sVKHEm;(AznY3bLmh>gE=5`>M7cE!btqLJdf68)E{jsUVUW2I%HP^05B^A8I}#e`a6a zop(aipuVt|)B$Ln%^#!$pO=Oy`m2>CXKLO8FZ|*q$6Z(2okp;*Z z8OZI^>0i5GG3rp<@ns&S-G=ZLbAo#D*T3W5x;-F;0B+%OUvcBu`+rL1?VdmobAy3{ zn&>o!ISQa;atkOPzx{f6%yin(ml!)8cAGy~IzqcahTycSp;J2mrE?^s5mS&H<;bs{@=qEYE~~P z!Di0h?Ot2rZS&qV&OeI^EAxE1oK)_6nA(m|Ie!o^mf|=``B6b9-2VXjs_(=7DBpOl z&KGU5$03#5)le43Jx($B)_wVD?qrb{IcyB_Jv&#R{2cgQu4|eV&y1poEJEPh#_f^9 zfLw#n4pe8P2 zdnL4CxUj$_$-n@ReuBPPR+Ob^rtWFWq2ph)9Txgn!TJ2_E}z#~wU@@7YQ;3zyhxgLtb8x_ zQTScq+kYEd-uQyv;$dX3s@zR9fbUpFNjd&l00E9G&2;|&9r!oKGI+=0XT_aL;yqeT z7#|HP^qAJ-TPIP1j>jO6O#c8gUuyUt;g^S})1kOs7TkqF zAdn#1Ml+B-a%=OC;19*?kAl82p8n+|HW${0*H92)-(kT7DZn`AjPyRW`bpsLiyjZt zd@JFd9$S-b2EC*f)1`LZ_P^bRI3xX$pQdZ`zCWReubfL27+F-;O|Nfz?{{|dKS;4zjNS4A@VkRMXm|UD-JZomeJ=dIG{zu6= zl(1P$T1bn>8+_^;1^MZkt~FTirD-9ApeH+XoKw6^zFJAf51IfO`qoW^{zTizuhu5g zsM|j!gK8EwBP`!|W74Hn0eCqAqC&)O2RR&6i6$~f;aR!GvXZg&AA#f4z8L<>78(zS z@1jjt#@eJf3g0)GJE}$xpb>>X@FQNW@mu25-W$}sLwp`6=d;j-wThQeRUMA`4o@U` zN;vhepZ+6wvrjhP40PKxnh0!u!t!rIp@N8jV1J9hHN^aA_>br7I;Ew(=biSLqD9M> z!UD$`KU(+~md_uFQ+vtDzx)X$^0Gfc{?n&`!n$i{#xY;`3wQiYZ`!BDcHSek@MsG% zPi5g-SZ7IGU=@lmNCT!yl5<`qBem4I##D3_sp9_t7V2Iv)SFVi44cp-aR($K0gMd( zHCo!#M+lMlAI`ql1Bd#y7Bu@`@pnEKABpJDPtAbRb?4;6dc0HaLbZKS| ztdedqZ71CNbK_6#J3qr;gf{;G5pFc+x79pK~Ha( zTCWuRN7wV-@$ zEL0X@js|i+3Wwpp#hp9AHl8QcKPrxQaCs&dk%?D}5ITl&2b%Y10@oVw4&pAqX5$>uz|q^d|5 z7(=jNeHOSaM_q(52wR@{{Wz#~4NC6zd2ep#W}0_0Oy`}hCB31ZH0GuHrn)oJw@qfN1Z#Gkywf%NtN z06bMkx@+k?sGK5>i_bq_Q(fbAuN1rrRvdM^^){V#Cz@93W4=p``0cb66vj* z^e&whEshu&?prAvGdBvyzpxeT{{XT#!e1EpgT`9cp`%*4y3zEq_c0TOE4g=~aDREU z@$bcB4;OqO@PT`;0O)gGUSNoB@5yqW;E)wh&}XRVYuo<-U|$=L2KaBro;lQDG88QjAI{k)=j$9UG%@+6P&Qtly$Q??~2|CzWB%SOTxN$fi(3xz4n%6 z)SUrnZSDNR6{I6OLj2u*LXO$@zXf~`@TbEI33PuBI>B=~t6km5FlLYpZouJkbA=r~ zyq_-9d|P4gEi~^FrmiowpAc%6I(EIew=`ZsSf)|#X7k+pFdeJt&2%oM2=n3zCPU;5 z90AbR=G;@7V{6VhbnQuLr`l@QZ_%HsaF$b-;i*dyqx;5_*Iro5WW85Y!M|x69~4LM z^I6k8Ju!1M59jS)`DOtv&(LG>uaf*H;k|EM)@<}I4rv#c*Y{H_cJ@}0w31B3{nbST zlk^q$n|wCbzA(Jr0`QX&aj1EbO>sFpAWl`JZ%{MSzXQE}E&DBg#~%g%0A?);7(64T zeXCiPEw}z5yg=7?B#uBo^2$8)MdKLGdBx%tJ`@~Xg1qsyOsma*;|x{@Z8*-?L9uh_+9V|;OD^I zIc&Zfcn-%)0ZPeuvG2H(x!o&e$R{Tm#deckTiV3@a-)y&u3KF3C@knX9f;|Vt$l?Y zw+jlgtkvf&+dn+S=D5s77}A`*$n3p$HSe_>WQ!KksLxP8opV~Rh{|L%!Ekev*BPu` zQ^ru*N*B&OI0SGz@m$uuudK+^G}tV}eQ{odX~$-HuB$pdQ^kTj#AMD#4f)m%r{Ym> znIpK)?nW1(_4?N-sB7}bPmn+>&elD@9-sYc#J=%VYYR;^;=3TrfcOi}K4N%ZsjK9S zOu5|7nfy^b{EnA_Rz)}@uMN)xRarbh_Da!6n2t_%0h5~JZoF4&^7)WV&mi55IRRT8 z*p5eRs9{)o$B(|%13Su~k%ou(dSHT2Ez`dq)mI49l6so|02DlXrJWtL)rxC}V}i(l zE(av%Jrv=%&rEvb%zSfuqw1PBlG|Ja!b)+-$6BZ2e;cuKv(1njV zN9IL%WD~l+(>=&#ZhSv{eTWnYV->~+2`Yn_(6Pc zXW)DJG~sS%k(L6+IEVbSWgLaaKEsoMK|Q#>Da20-<4#q{vhC}EWct}6Duq|c*kH#$w>R+idkiHdSEryNxmQV@`G2SHxuBLjR5 z-^Fjx&J8J{p^OZ-F%gXOk4m`LOk)zFWii316h|TufQ|g)9+Z~TEwlM@jH4V7PfGa6 zwR5PqY{cc5j@YX^vB^CuIAK&dh~yt$^za-WPi|}2qeV`VGUjQWkHTM!8pp#uNtQ?r z{*!lw)HKOo<&k=BU+%c|IzFi$rKQeco-@G* zgX*W$9^lvSSzcjJ6G}BH%SN~Ecoiw~%FP{BvvMR>VUgTtJ?m#om@JCJf(K2+WFFs- z<;_siXVZLRZf-P)MZTpPX6H@^i2xZd7pU%6sOK3Ri7cDSa~WgQ0Dq_QuBqaA^nC$A vy^f Date: Thu, 15 Dec 2022 15:31:08 +0100 Subject: [PATCH 111/130] Fix klocwork detected issue with InferenceResponse default constructor (used in tests) (#1604) --- src/inferenceresponse.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp index 268281ea64..434b777e89 100644 --- a/src/inferenceresponse.cpp +++ b/src/inferenceresponse.cpp @@ -26,9 +26,12 @@ #include "status.hpp" namespace ovms { + +const std::string RESPONSE_SERVABLE_NAME_USED_ONLY_IN_PREDICTION_TESTS = "CONSTRUCTOR_USED_ONLY_IN_PREDICTION_TESTS"; + // this constructor can be removed with prediction tests overhaul InferenceResponse::InferenceResponse() : - InferenceResponse("CONSTRUCTOR_USED_ONLY_IN_PREDICTION_TESTS", 42) {} + InferenceResponse(RESPONSE_SERVABLE_NAME_USED_ONLY_IN_PREDICTION_TESTS, 42) {} InferenceResponse::InferenceResponse(const std::string& servableName, model_version_t servableVersion) : servableName(servableName), servableVersion(servableVersion) {} From 520dd7187fdd1e9f7c8141946ff652bb0cb93c07 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Thu, 15 Dec 2022 15:34:28 +0100 Subject: [PATCH 112/130] Move TODO's into task tracking tool (#1602) JIRA:98964 --- src/buffer.cpp | 2 -- src/capi_frontend/capi.cpp | 18 ++++++++---------- src/config.cpp | 1 - src/deserialization.hpp | 2 +- src/exit_node.cpp | 1 - src/inferenceparameter.cpp | 3 +-- src/inferenceparameter.hpp | 1 - src/inferencerequest.cpp | 4 ++-- src/inferencerequest.hpp | 2 +- src/inferenceresponse.cpp | 3 +-- src/inferenceresponse.hpp | 8 ++++---- src/inferencetensor.cpp | 2 +- src/main_benchmark.cpp | 11 +---------- src/ovms.h | 6 ------ src/precision.hpp | 2 +- src/predict_request_validation_utils.cpp | 4 ++-- src/prediction_service_utils.cpp | 2 +- src/serialization.hpp | 2 +- src/server.cpp | 2 +- src/statefulmodelinstance.cpp | 1 - src/test/c_api_tests.cpp | 15 +++------------ src/test/capi_predict_validation_test.cpp | 4 +--- src/test/inferencerequest_test.cpp | 3 +-- src/test/prediction_service_test.cpp | 9 +++------ src/test/stateful_modelinstance_test.cpp | 2 +- 25 files changed, 35 insertions(+), 75 deletions(-) diff --git a/src/buffer.cpp b/src/buffer.cpp index c05dbbf070..7ad7e0ed32 100644 --- a/src/buffer.cpp +++ b/src/buffer.cpp @@ -23,8 +23,6 @@ Buffer::Buffer(const void* pptr, size_t byteSize, OVMS_BufferType bufferType, st byteSize(byteSize), bufferType(bufferType), bufferDeviceId(bufferDeviceId) { - // TODO in later stages - // it can be advantageous to have custom made release fn especially with buffers passed by external users if (!createCopy) return; ownedCopy = std::make_unique(byteSize); diff --git a/src/capi_frontend/capi.cpp b/src/capi_frontend/capi.cpp index fda2790b12..e53dfdc7bc 100644 --- a/src/capi_frontend/capi.cpp +++ b/src/capi_frontend/capi.cpp @@ -182,7 +182,7 @@ OVMS_Status* OVMS_ServerSettingsSetGrpcBindAddress(OVMS_ServerSettings* settings return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (grpc_bind_address == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->grpcBindAddress.assign(grpc_bind_address); @@ -205,7 +205,7 @@ OVMS_Status* OVMS_ServerSettingsSetRestBindAddress(OVMS_ServerSettings* settings return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (rest_bind_address == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->restBindAddress.assign(rest_bind_address); @@ -218,7 +218,7 @@ OVMS_Status* OVMS_ServerSettingsSetGrpcChannelArguments(OVMS_ServerSettings* set return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (grpc_channel_arguments == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->grpcChannelArguments.assign(grpc_channel_arguments); @@ -261,7 +261,7 @@ OVMS_Status* OVMS_ServerSettingsSetCpuExtensionPath(OVMS_ServerSettings* setting return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (cpu_extension_path == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->cpuExtensionLibraryPath.assign(cpu_extension_path); @@ -274,7 +274,7 @@ OVMS_Status* OVMS_ServerSettingsSetCacheDir(OVMS_ServerSettings* settings, return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (cache_dir == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->cacheDir.assign(cache_dir); @@ -315,7 +315,7 @@ OVMS_Status* OVMS_ServerSettingsSetLogPath(OVMS_ServerSettings* settings, return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (log_path == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ServerSettingsImpl* serverSettings = reinterpret_cast(settings); serverSettings->logPath.assign(log_path); @@ -328,7 +328,7 @@ OVMS_Status* OVMS_ModelsSettingsSetConfigPath(OVMS_ModelsSettings* settings, return reinterpret_cast(new Status(StatusCode::NONEXISTENT_SETTINGS)); } if (config_path == nullptr) { - return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); // TODO name->string + return reinterpret_cast(new Status(StatusCode::NONEXISTENT_STRING)); } ovms::ModelsSettingsImpl* modelsSettings = reinterpret_cast(settings); modelsSettings->configPath.assign(config_path); @@ -336,7 +336,6 @@ OVMS_Status* OVMS_ModelsSettingsSetConfigPath(OVMS_ModelsSettings* settings, } // inference API OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, OVMS_Server* server, const char* servableName, uint32_t servableVersion) { - // TODO should we allow to create requests to not yet loaded models? if (request == nullptr) { return reinterpret_cast(new Status(StatusCode::NONEXISTENT_REQUEST)); } @@ -503,7 +502,7 @@ OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* res, uint32 *shape = tensor->getShape().data(); *dimCount = tensor->getShape().size(); *bufferType = buffer->getBufferType(); - *deviceId = buffer->getDeviceId().value_or(0); // TODO how discriminate betwen undefined & actual device 0 + *deviceId = buffer->getDeviceId().value_or(0); // possibly it is not neccessary to discriminate *data = buffer->data(); *bytesize = buffer->getByteSize(); @@ -578,7 +577,6 @@ static Status getModelInstance(ovms::Server& server, const InferenceRequest* req } const ovms::Module* servableModule = server.getModule(ovms::SERVABLE_MANAGER_MODULE_NAME); if (!servableModule) { - // TODO: Should we wait for servable manager to appear? And for model to get loaded? return ovms::Status(ovms::StatusCode::INTERNAL_ERROR, "missing servable manager"); } auto& modelManager = dynamic_cast(servableModule)->getServableManager(); diff --git a/src/config.cpp b/src/config.cpp index ddaaf3b3e5..5c25edfbf6 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -34,7 +34,6 @@ const uint MAX_PORT_NUMBER = std::numeric_limits::max(); const uint64_t DEFAULT_REST_WORKERS = AVAILABLE_CORES * 4.0; const uint64_t MAX_REST_WORKERS = 10'000; -// TODO: Not used in OVMS - get rid of. Used only in tests. Config& Config::parse(int argc, char** argv) { ovms::CLIParser p; ovms::ServerSettingsImpl serverSettings; diff --git a/src/deserialization.hpp b/src/deserialization.hpp index 136f1fa733..0b9f80760e 100644 --- a/src/deserialization.hpp +++ b/src/deserialization.hpp @@ -462,7 +462,7 @@ Status deserializePredictRequest( return Status(StatusCode::INTERNAL_ERROR, "Failed to deserialize request"); } ov::Tensor tensor; - // TODO binary input handling + // binary input handling /* if (requestInputPtr->getDataType() == "BYTES") { SPDLOG_DEBUG("Request contains binary input: {}", name); return StatusCode::NOT_IMPLEMENTED; diff --git a/src/exit_node.cpp b/src/exit_node.cpp index 7de469b7a0..7f21bdf1d3 100644 --- a/src/exit_node.cpp +++ b/src/exit_node.cpp @@ -62,7 +62,6 @@ Status OutputGetter::get(const std::string& name, ov::Tensor& template Status ExitNode::fetchResults(const TensorMap& inputTensors) { OutputGetter outputGetter(inputTensors); - // TODO fix name version (it does not work properly anyway right now) static const std::string name{""}; static const model_version_t version{1}; return serializePredictResponse(outputGetter, name, version, this->outputsInfo, this->response, getOutputMapKeyName); diff --git a/src/inferenceparameter.cpp b/src/inferenceparameter.cpp index 4f63384bea..f10e2ef92b 100644 --- a/src/inferenceparameter.cpp +++ b/src/inferenceparameter.cpp @@ -20,7 +20,6 @@ #include "ovms.h" // NOLINT namespace ovms { -// TODO should we own our own copy of value? // size_t DataTypeToByteSize(OVMS_DataType datatype) { static std::unordered_map datatypeSizeMap{ @@ -40,7 +39,7 @@ size_t DataTypeToByteSize(OVMS_DataType datatype) { {OVMS_DATATYPE_FP32, 4}, {OVMS_DATATYPE_FP64, 8}, {OVMS_DATATYPE_BF16, 2}, - // {"BYTES", }, TODO + // {"BYTES", }, }; auto it = datatypeSizeMap.find(datatype); if (it == datatypeSizeMap.end()) { diff --git a/src/inferenceparameter.hpp b/src/inferenceparameter.hpp index 72a5821cc3..9cd7b3545e 100644 --- a/src/inferenceparameter.hpp +++ b/src/inferenceparameter.hpp @@ -22,7 +22,6 @@ namespace ovms { size_t DataTypeToByteSize(OVMS_DataType datatype); -// TODO should we own our own copy of value? class InferenceParameter { const std::string name; OVMS_DataType datatype; diff --git a/src/inferencerequest.cpp b/src/inferencerequest.cpp index e7a116a138..56322fee04 100644 --- a/src/inferencerequest.cpp +++ b/src/inferencerequest.cpp @@ -91,13 +91,13 @@ const InferenceParameter* InferenceRequest::getParameter(const char* name) const } Status InferenceRequest::getBatchSize(size_t& batchSize, size_t batchSizeIndex) const { if (inputs.size() == 0) { - return StatusCode::INTERNAL_ERROR; // TODO test + return StatusCode::INTERNAL_ERROR; } // we make here the same assumption as with bs=auto in TFS/KFS API const InferenceTensor& tensor = inputs.begin()->second; const auto& shape = tensor.getShape(); if (batchSizeIndex >= shape.size()) { - return StatusCode::INTERNAL_ERROR; // TODO test + return StatusCode::INTERNAL_ERROR; } batchSize = shape[batchSizeIndex]; return StatusCode::OK; diff --git a/src/inferencerequest.hpp b/src/inferencerequest.hpp index 8439323204..bba5c359fb 100644 --- a/src/inferencerequest.hpp +++ b/src/inferencerequest.hpp @@ -61,7 +61,7 @@ class InferenceRequest { Status setTimeoutMicorseconds(uint64_t microseconds); InferenceParameter* getInferenceParameter(const char* name); InferenceTensor* getTensor(const char* name); - // TODO add tests & seek if those can be removed by potentialy exposing inputs map? + Status getBatchSize(size_t& batchSize, size_t batchSizeIndex) const; std::map getRequestShapes() const; }; diff --git a/src/inferenceresponse.cpp b/src/inferenceresponse.cpp index 434b777e89..fc533cec72 100644 --- a/src/inferenceresponse.cpp +++ b/src/inferenceresponse.cpp @@ -44,7 +44,6 @@ model_version_t InferenceResponse::getServableVersion() const { } Status InferenceResponse::addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount) { - // TODO insert tensor with wrong shape/datatype/name/dimcount auto it = std::find_if(outputs.begin(), outputs.end(), [&name](const std::pair& pair) { @@ -104,5 +103,5 @@ uint32_t InferenceResponse::getParameterCount() const { void InferenceResponse::Clear() { outputs.clear(); parameters.clear(); -} // TODO remove +} } // namespace ovms diff --git a/src/inferenceresponse.hpp b/src/inferenceresponse.hpp index 5fb8b6474e..8d9ab4abf3 100644 --- a/src/inferenceresponse.hpp +++ b/src/inferenceresponse.hpp @@ -30,15 +30,15 @@ class Status; class InferenceResponse { const std::string& servableName; const model_version_t servableVersion; - std::vector parameters; // TODO after benchmark app verify if additional map wouldn't be better - std::vector> outputs; // TODO after benchmark app verify if additional map wouldn't be better + std::vector parameters; + std::vector> outputs; public: // this constructor can be removed with prediction tests overhaul InferenceResponse(); InferenceResponse(const std::string& servableName, model_version_t servableVersion); Status addOutput(const std::string& name, OVMS_DataType datatype, const size_t* shape, size_t dimCount); - Status getOutput(uint32_t id, const std::string** name, const InferenceTensor** tensor) const; // TODO consider in the future if we need getOutput by name + Status getOutput(uint32_t id, const std::string** name, const InferenceTensor** tensor) const; Status getOutput(uint32_t id, const std::string** name, InferenceTensor** tensor); Status addParameter(const char* parameterName, OVMS_DataType datatype, const void* data); @@ -52,7 +52,7 @@ class InferenceResponse { Status setId(); Status getId(); InferenceParameter* getInferenceParameter(const char* name); - // TODO this can be removed with prediction tests overhaul + void Clear(); }; } // namespace ovms diff --git a/src/inferencetensor.cpp b/src/inferencetensor.cpp index 13efc6606d..5eaaa9797a 100644 --- a/src/inferencetensor.cpp +++ b/src/inferencetensor.cpp @@ -33,7 +33,7 @@ InferenceTensor::InferenceTensor(OVMS_DataType datatype, const size_t* shape, si Status InferenceTensor::setBuffer(const void* addr, size_t byteSize, OVMS_BufferType bufferType, std::optional deviceId, bool createCopy) { if (nullptr != buffer) { return StatusCode::DOUBLE_BUFFER_SET; - } // TODO validate against byteSize == 0 + } buffer = std::make_unique(addr, byteSize, bufferType, deviceId, createCopy); return StatusCode::OK; } diff --git a/src/main_benchmark.cpp b/src/main_benchmark.cpp index b9d9d12c66..456aca42cb 100644 --- a/src/main_benchmark.cpp +++ b/src/main_benchmark.cpp @@ -182,7 +182,7 @@ OVMS_InferenceRequest* prepareRequest(OVMS_Server* server, const std::string& se OVMS_InferenceRequestNew(&request, server, servableName.c_str(), servableVersion); OVMS_InferenceRequestAddInput(request, inputName.c_str(), datatype, shape.data(), shape.size()); size_t elementsCount = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); - OVMS_InferenceRequestInputSetData(request, inputName.c_str(), data, sizeof(float) * elementsCount, OVMS_BUFFERTYPE_CPU, 0); // TODO sizeof + OVMS_InferenceRequestInputSetData(request, inputName.c_str(), data, sizeof(float) * elementsCount, OVMS_BUFFERTYPE_CPU, 0); return request; } @@ -220,13 +220,6 @@ void triggerInferenceInALoop( averagePureLatency = std::accumulate(latenciesPure.begin(), latenciesPure.end(), 0) / (double(niterPerThread) * 1'000); } } // namespace -// TODO support more than 1 input -// TODO support 3 modes: -// * fair play -> don't recreate request, but set buffer inputs -// * full request prepare -> recreate request -// * inference only -> dont reset buffers -// TODO change plugin_config -// TODO change nireq in model int main(int argc, char** argv) { installSignalHandlers(); @@ -295,7 +288,6 @@ int main(int argc, char** argv) { return EX_USAGE; } std::string inputName = inputsNames[0]; - // TODO add support for many shapes/inputs // datatype handling OVMS_DataType datatype = OVMS_DATATYPE_FP32; // shape handling @@ -402,7 +394,6 @@ int main(int argc, char** argv) { })); } // sleep to allow all threads to initialize - // TODO rework to use signals std::this_thread::sleep_for(std::chrono::milliseconds(100)); /////////////////////// // start workload diff --git a/src/ovms.h b/src/ovms.h index d458c16228..554846fe7d 100644 --- a/src/ovms.h +++ b/src/ovms.h @@ -27,7 +27,6 @@ typedef struct OVMS_Status_ OVMS_Status; typedef struct OVMS_ServerSettings_ OVMS_ServerSettings; typedef struct OVMS_ModelsSettings_ OVMS_ModelsSettings; -// TODO reuse this in precision.hpp typedef enum OVMS_DataType_enum { OVMS_DATATYPE_BF16, OVMS_DATATYPE_FP64, @@ -182,22 +181,17 @@ OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_ServerSettings* server_settings, OVMS_ModelsSettings* models_settings); // in fact only --config_path // Unload all and cleanup -// TODO: Should not be possible to re-start? -// OVMS_Status* OVMS_ServerStop(OVMS_Server* server); // OVMS_InferenceRequest OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, OVMS_Server* server, const char* servableName, uint32_t servableVersion); void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); -// OVMS_Status* OVMS_InferenceRequestAddInputRaw(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype); // TODO consider no datatype & handle the parameters NOT IMPLEMENTED // ownership of data needs to be maintained during inference OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, const void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId); OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, const char* inputName); OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); // this will allow for reuse of request but with different input data -// OVMS_Status* OVMS_InferenceRequestRemoveAllInputs(OVMS_InferenceRequest* request); -// OVMS_Status* OVMS_InferenceRequestAddRequestedOutput(OVMS_InferenceRequest* request, char* inputName); // TODO consider the other way around - add not usefull outputs OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize); OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* request, const char* parameterName); diff --git a/src/precision.hpp b/src/precision.hpp index f5e5272dec..3ce2f4c44b 100644 --- a/src/precision.hpp +++ b/src/precision.hpp @@ -43,7 +43,7 @@ enum class Precision { DYNAMIC, MIXED, Q78, - BIN, // TODO remove BIN,Q78, DYNAMIC, CUSTOM, MIXED + BIN, PRECISION_END }; diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index d4b4d84ed0..473f7f2e35 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -657,7 +657,7 @@ Status RequestValidator bool RequestValidator::checkIfBinaryInputUsed(const InferenceTensor& tensor, const std::string inputName) const { - // TODO no strig no bytes currently, will implement one of those types with binary input. + // no string no bytes currently, will implement one of those types with binary input. return false; } diff --git a/src/prediction_service_utils.cpp b/src/prediction_service_utils.cpp index 506e334ac8..570c9056d8 100644 --- a/src/prediction_service_utils.cpp +++ b/src/prediction_service_utils.cpp @@ -102,7 +102,7 @@ std::optional getRequestBatchSize(const InferenceRequest* request, co size_t bs = 0; auto status = request->getBatchSize(bs, batchSizeIndex); if (!status.ok()) { - return std::nullopt; // TODO sth different? + return std::nullopt; } return bs; } diff --git a/src/serialization.hpp b/src/serialization.hpp index b925aa6081..2dd5db2298 100644 --- a/src/serialization.hpp +++ b/src/serialization.hpp @@ -213,7 +213,7 @@ Status serializePredictResponse( } const std::string* outputNameFromCapiTensor = nullptr; status = response->getOutput(outputId, &outputNameFromCapiTensor, &outputTensor); - ++outputId; // TODO C-API test serialization 2 outputs + ++outputId; if (!status.ok()) { SPDLOG_ERROR("Cannot serialize output with name:{} for servable name:{}; version:{}; error: cannot find inserted input", outputName, response->getServableName(), response->getServableVersion()); diff --git a/src/server.cpp b/src/server.cpp index 67300e7fd6..1b48acfab2 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -365,7 +365,7 @@ Status Server::start(ServerSettingsImpl* serverSettings, ModelsSettingsImpl* mod return StatusCode::SERVER_ALREADY_STARTED; auto& config = ovms::Config::instance(); if (!config.parse(serverSettings, modelsSettings)) - return StatusCode::OPTIONS_USAGE_ERROR; // TODO: Have separate code for each option validation failure + return StatusCode::OPTIONS_USAGE_ERROR; configure_logger(config.logLevel(), config.logPath()); logConfig(config); return this->startModules(config); diff --git a/src/statefulmodelinstance.cpp b/src/statefulmodelinstance.cpp index 368e17db5d..79307ff281 100644 --- a/src/statefulmodelinstance.cpp +++ b/src/statefulmodelinstance.cpp @@ -202,7 +202,6 @@ Status StatefulRequestProcessorsequenceId = sequenceProcessingSpec.getSequenceId(); if (!sequenceManager.sequenceExists(this->sequenceId.value())) return StatusCode::INTERNAL_ERROR; - // TODO should be able to search & get in one go sequence = &sequenceManager.getSequence(this->sequenceId.value()); sequenceLock = std::make_unique>(sequence->getMutex()); diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 23e8ec28f1..b581309cd8 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -19,8 +19,6 @@ #include -// TODO we should not include classes from OVMS here -// consider how to workaround test_utils #include "../inferenceresponse.hpp" #include "../ovms.h" #include "test_utils.hpp" @@ -172,7 +170,7 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(serverSettings->grpcBindAddress, "2.2.2.2"); EXPECT_EQ(serverSettings->restWorkers, 31); EXPECT_EQ(serverSettings->restBindAddress, "3.3.3.3"); - // EXPECT_EQ(serverSettings->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(serverSettings->metricsEnabled, false); // EXPECT_EQ(serverSettings->metricsList, ""); EXPECT_EQ(serverSettings->cpuExtensionLibraryPath, "/ovms/src/test"); EXPECT_EQ(serverSettings->logLevel, "TRACE"); @@ -196,7 +194,7 @@ TEST(CApiConfigTest, MultiModelConfiguration) { EXPECT_EQ(cfg.grpcBindAddress(), "2.2.2.2"); EXPECT_EQ(cfg.restWorkers(), 31); EXPECT_EQ(cfg.restBindAddress(), "3.3.3.3"); - // EXPECT_EQ(serverSettings->metricsEnabled, false); // TODO: enable testing once metrics will be configurable via api + // EXPECT_EQ(serverSettings->metricsEnabled, false); // EXPECT_EQ(serverSettings->metricsList, ""); EXPECT_EQ(cfg.cpuExtensionLibraryPath(), "/ovms/src/test"); EXPECT_EQ(cfg.logLevel(), "TRACE"); @@ -277,8 +275,6 @@ TEST(CApiStartTest, StartFlow) { ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings), StatusCode::SERVER_ALREADY_STARTED); - // TODO: Is infer ok? - OVMS_ModelsSettingsDelete(modelsSettings); OVMS_ServerSettingsDelete(serverSettings); OVMS_ServerDelete(srv); @@ -463,7 +459,6 @@ TEST_F(CapiInference, NegativeInference) { ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, nullptr, modelsSettings), StatusCode::NONEXISTENT_SETTINGS); ASSERT_CAPI_STATUS_NOT_NULL_EXPECT_CODE(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, nullptr), StatusCode::NONEXISTENT_SETTINGS); ASSERT_CAPI_STATUS_NULL(OVMS_ServerStartFromConfigurationFile(cserver, serverSettings, modelsSettings)); - // TODO add check for server ready strict in 2022.3 not available in C-API OVMS_InferenceRequest* request{nullptr}; OVMS_InferenceResponse* response = nullptr; @@ -524,12 +519,10 @@ TEST_F(CapiInference, NegativeInference) { OVMS_ServerDelete(nullptr); } -// TODO negative test -> validate at the infer stage -// TODO reuse request after inference namespace { const std::string MODEL_NAME{"SomeModelName"}; const uint64_t MODEL_VERSION{42}; -const std::string PARAMETER_NAME{"sequence_id"}; // TODO check if in ovms there is such constant +const std::string PARAMETER_NAME{"sequence_id"}; const OVMS_DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; const uint32_t PARAMETER_VALUE{13}; @@ -639,5 +632,3 @@ TEST_F(CapiInference, CallInferenceServerNotStarted) { OVMS_InferenceRequestDelete(request); OVMS_ServerDelete(cserver); } -// TODO make cleaner error codes reporting -// todo decide either use remove or delete for consistency diff --git a/src/test/capi_predict_validation_test.cpp b/src/test/capi_predict_validation_test.cpp index c15ac55959..8518f06d0c 100644 --- a/src/test/capi_predict_validation_test.cpp +++ b/src/test/capi_predict_validation_test.cpp @@ -412,7 +412,7 @@ TEST_F(CAPIPredictValidation, RequestIncorrectInputWithNoBuffer) { std::array shape{1, 1, 1, 1}; request.addInput("Input_FP32_1_1_1_1_NHWC", OVMS_DATATYPE_FP32, shape.data(), shape.size()); auto status = instance->mockValidate(&request); - EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); // TODO change retcode? + EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); } TEST_F(CAPIPredictValidation, RequestIncorrectContentSizeZero) { @@ -792,8 +792,6 @@ TEST_F(CAPIPredictValidationDynamicModel, RequestDimensionInRangeWrongTensorCont EXPECT_EQ(status, ovms::StatusCode::INVALID_CONTENT_SIZE) << status.string(); } -// TODO: Add request parameters name validation tests - class CAPIPredictValidationPrecision : public ::testing::TestWithParam { protected: std::vector requestData{10000000}; diff --git a/src/test/inferencerequest_test.cpp b/src/test/inferencerequest_test.cpp index 7ce2c7e940..435d82be50 100644 --- a/src/test/inferencerequest_test.cpp +++ b/src/test/inferencerequest_test.cpp @@ -41,7 +41,7 @@ using ovms::StatusCode; namespace { const std::string MODEL_NAME{"SomeModelName"}; const uint64_t MODEL_VERSION{42}; -const std::string PARAMETER_NAME{"SEQUENCE_ID"}; // TODO check if in ovms there is such constant +const std::string PARAMETER_NAME{"SEQUENCE_ID"}; const OVMS_DataType PARAMETER_DATATYPE{OVMS_DATATYPE_I32}; const uint32_t PARAMETER_VALUE{13}; @@ -196,4 +196,3 @@ TEST(InferenceResponse, CreateAndReadData) { status = response.addParameter(PARAMETER_NAME.c_str(), PARAMETER_DATATYPE, reinterpret_cast(&PARAMETER_VALUE)); ASSERT_EQ(status, StatusCode::DOUBLE_PARAMETER_INSERT) << status.string(); } -// TODO logging diff --git a/src/test/prediction_service_test.cpp b/src/test/prediction_service_test.cpp index f18aa98981..6cd3f796e0 100644 --- a/src/test/prediction_service_test.cpp +++ b/src/test/prediction_service_test.cpp @@ -29,7 +29,7 @@ #include "../buffer.hpp" #include "../deserialization.hpp" #include "../executingstreamidguard.hpp" -#include "../inferenceparameter.hpp" // TODO move bytesize util +#include "../inferenceparameter.hpp" #include "../inferencerequest.hpp" #include "../inferenceresponse.hpp" #include "../inferencetensor.hpp" @@ -52,9 +52,6 @@ using ovms::InferenceResponse; using ovms::InferenceTensor; using ovms::StatusCode; -// TODO: These tests test both TFS and KFS for prediction, -// but output is always serialized to TFS, therefore we only test TFS serialization here. -// TODO: Add C-API tests when predict/infer becomes ready. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wnarrowing" void serializeAndCheck(int outputSize, ov::InferRequest& inferRequest, const std::string& outputName, const ovms::tensor_map_t& outputsInfo) { @@ -235,7 +232,7 @@ class TestPredict : public ::testing::Test { << readableError(expectedValues.data(), actualValues.data(), expectedValues.size() * sizeof(float)); } static void checkOutputValues(const ovms::InferenceResponse& res, const std::vector& expectedValues, const std::string& outputName = INCREMENT_1x3x4x5_MODEL_OUTPUT_NAME) { - InferenceResponse& response = const_cast(res); // TODO decide if output should be const + InferenceResponse& response = const_cast(res); size_t outputCount = response.getOutputCount(); ASSERT_GE(1, outputCount); size_t outputId = 0; @@ -354,7 +351,7 @@ void TestPredict::checkOutputShape(const ovms::InferenceResponse& while (outputId < outputCount) { const std::string* cppName; InferenceTensor* tensor; - InferenceResponse& response = const_cast(cresponse); // TODO decide if output should be const + InferenceResponse& response = const_cast(cresponse); auto status = response.getOutput(outputId, &cppName, &tensor); EXPECT_EQ(status, StatusCode::OK) << status.string(); EXPECT_NE(nullptr, tensor); diff --git a/src/test/stateful_modelinstance_test.cpp b/src/test/stateful_modelinstance_test.cpp index 121d56f76c..2d8c2d1b2b 100644 --- a/src/test/stateful_modelinstance_test.cpp +++ b/src/test/stateful_modelinstance_test.cpp @@ -226,7 +226,7 @@ class MockedStatefulModelInstance : public ovms::StatefulModelInstance { if (!status.ok()) return status; - if (waitBeforeSequenceLock) { // TODO remove since right now it is in singel step + if (waitBeforeSequenceLock) { std::cout << "Waiting before waitBeforeSequenceLock" << std::endl; waitBeforeSequenceLock->get(); } From 1a757361b6181211c5ea373d881451ffe554e21b Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Thu, 15 Dec 2022 16:50:07 +0100 Subject: [PATCH 113/130] Mkulakow/kfs raw input contents (#1592) Add support for binary inputs placed in raw inputs contents in KFS Co-authored-by: bstrzele --- check_coverage.bat | 4 +- src/binaryutils.cpp | 51 ++++++----- src/binaryutils.hpp | 3 +- src/deserialization.hpp | 10 +-- src/predict_request_validation_utils.cpp | 35 ++++++-- src/test/binaryutils_test.cpp | 107 ++++++++++++++++------- src/test/deserialization_tests.cpp | 2 +- src/test/predict_validation_test.cpp | 52 +++++++++++ 8 files changed, 192 insertions(+), 72 deletions(-) diff --git a/check_coverage.bat b/check_coverage.bat index f70c442075..ece4d722b6 100755 --- a/check_coverage.bat +++ b/check_coverage.bat @@ -5,8 +5,8 @@ #MIN_FUNCTION_COV=88.3 #Rhel -MIN_LINES_COV=74.8 -MIN_FUNCTION_COV=75.7 +MIN_LINES_COV=75.0 +MIN_FUNCTION_COV=75.6 LINES_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | head -n 1` FUNC_COV=`cat genhtml/index.html | grep "headerCovTableEntry.*%" | grep -oP ">\K(\d*.\d*) " | tail -n 1` diff --git a/src/binaryutils.cpp b/src/binaryutils.cpp index f60aed82f1..527f3a1ec6 100644 --- a/src/binaryutils.cpp +++ b/src/binaryutils.cpp @@ -165,7 +165,7 @@ static Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv:: static bool checkBatchSizeMismatch(const std::shared_ptr& tensorInfo, const int batchSize) { OVMS_PROFILE_FUNCTION(); - if (!tensorInfo->getBatchSize().has_value()) { + if (!tensorInfo->getBatchSize().has_value() || batchSize == 0) { return true; } return !tensorInfo->getBatchSize().value().match(batchSize); @@ -188,7 +188,8 @@ static Status validateInput(const std::shared_ptr& tensorInfo, const } static Status validateTensor(const std::shared_ptr& tensorInfo, - const tensorflow::TensorProto& src) { + const tensorflow::TensorProto& src, + const std::string* buffer) { OVMS_PROFILE_FUNCTION(); auto status = validateLayout(tensorInfo); if (!status.ok()) { @@ -214,13 +215,14 @@ static Status validateTensor(const std::shared_ptr& tensorInfo, return StatusCode::STRING_VAL_EMPTY; } } - return StatusCode::OK; } static Status validateTensor(const std::shared_ptr& tensorInfo, - const ::KFSRequest::InferInputTensor& src) { + const ::KFSRequest::InferInputTensor& src, + const std::string* buffer) { OVMS_PROFILE_FUNCTION(); + bool rawInputsContentsUsed = (buffer != nullptr); auto status = validateLayout(tensorInfo); if (!status.ok()) { return status; @@ -232,7 +234,8 @@ static Status validateTensor(const std::shared_ptr& tensorInfo, return StatusCode::INVALID_SHAPE; } - if (checkBatchSizeMismatch(tensorInfo, src.contents().bytes_contents_size())) { + size_t batchSize = !rawInputsContentsUsed ? src.contents().bytes_contents_size() : 1; + if (checkBatchSizeMismatch(tensorInfo, batchSize)) { SPDLOG_DEBUG("Input: {} request batch size is incorrect. Expected: {} Actual: {}", tensorInfo->getMappedName(), tensorInfo->getBatchSize().has_value() ? tensorInfo->getBatchSize().value().toString() : std::string{"none"}, @@ -240,16 +243,20 @@ static Status validateTensor(const std::shared_ptr& tensorInfo, return StatusCode::INVALID_BATCH_SIZE; } - for (int i = 0; i < src.contents().bytes_contents_size(); i++) { - if (src.contents().bytes_contents(i).size() <= 0) { + if (!rawInputsContentsUsed) { + for (size_t i = 0; i < batchSize; i++) { + if (src.contents().bytes_contents(i).size() <= 0) { + SPDLOG_DEBUG("Tensor: {} {}th image of the batch is empty.", src.name(), i); + return StatusCode::BYTES_CONTENTS_EMPTY; + } + } + } else { + if (buffer->size() <= 0) { + SPDLOG_DEBUG("Tensor: {} raw_inputs_contents is empty", src.name()); return StatusCode::BYTES_CONTENTS_EMPTY; } } - if (src.contents().bytes_contents_size() <= 0) { - return StatusCode::BYTES_CONTENTS_EMPTY; - } - return StatusCode::OK; } @@ -331,7 +338,7 @@ inline static int getBinaryInputsSize(const ::KFSRequest::InferInputTensor& tens } template -static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo) { +static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std::vector& images, const std::shared_ptr& tensorInfo, const std::string* buffer) { OVMS_PROFILE_FUNCTION(); Dimension targetHeight = getTensorInfoHeightDim(tensorInfo); Dimension targetWidth = getTensorInfoWidthDim(tensorInfo); @@ -340,8 +347,10 @@ static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std:: bool resizeSupported = isResizeSupported(tensorInfo); bool enforceResolutionAlignment = !resizeSupported; - for (int i = 0; i < getBinaryInputsSize(src); i++) { - cv::Mat image = convertStringToMat(getBinaryInput(src, i)); + bool rawInputsContentsUsed = (buffer != nullptr); + int numberOfInputs = (!rawInputsContentsUsed ? getBinaryInputsSize(src) : 1); + for (int i = 0; i < numberOfInputs; i++) { + cv::Mat image = convertStringToMat(!rawInputsContentsUsed ? getBinaryInput(src, i) : *buffer); if (image.data == nullptr) return StatusCode::IMAGE_PARSING_FAILED; cv::Mat* firstImage = images.size() == 0 ? nullptr : &images.at(0); @@ -349,7 +358,6 @@ static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std:: if (status != StatusCode::OK) { return status; } - if (i == 0) { updateTargetResolution(targetHeight, targetWidth, image); } @@ -381,10 +389,8 @@ static Status convertTensorToMatsMatchingTensorInfo(const TensorType& src, std:: // if (i == 0 && src.contents().bytes_contents_size() > 1) { // // Multiply src.string_val_size() * image resolution * precision size // } - images.push_back(image); } - return StatusCode::OK; } @@ -437,15 +443,14 @@ static ov::Tensor convertMatsToTensor(std::vector& images, const std::s } template -static Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo) { +static Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo, const std::string* buffer) { OVMS_PROFILE_FUNCTION(); - auto status = validateTensor(tensorInfo, src); + auto status = validateTensor(tensorInfo, src, buffer); if (status != StatusCode::OK) { return status; } - std::vector images; - status = convertTensorToMatsMatchingTensorInfo(src, images, tensorInfo); + status = convertTensorToMatsMatchingTensorInfo(src, images, tensorInfo, buffer); if (!status.ok()) { return status; } @@ -456,6 +461,6 @@ static Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Te return StatusCode::OK; } -template Status convertBinaryRequestTensorToOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); -template Status convertBinaryRequestTensorToOVTensor<::KFSRequest::InferInputTensor>(const ::KFSRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +template Status convertBinaryRequestTensorToOVTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo, const std::string* buffer); +template Status convertBinaryRequestTensorToOVTensor<::KFSRequest::InferInputTensor>(const ::KFSRequest::InferInputTensor& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo, const std::string* buffer); } // namespace ovms diff --git a/src/binaryutils.hpp b/src/binaryutils.hpp index 02b40a6b2e..12f67d0891 100644 --- a/src/binaryutils.hpp +++ b/src/binaryutils.hpp @@ -16,11 +16,12 @@ #pragma once #include +#include #include "tensorinfo.hpp" namespace ovms { class Status; template -Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo); +Status convertBinaryRequestTensorToOVTensor(const TensorType& src, ov::Tensor& tensor, const std::shared_ptr& tensorInfo, const std::string* buffer); } // namespace ovms diff --git a/src/deserialization.hpp b/src/deserialization.hpp index 0b9f80760e..f3df1e42d1 100644 --- a/src/deserialization.hpp +++ b/src/deserialization.hpp @@ -350,7 +350,7 @@ Status deserializePredictRequest( if (requestInput.dtype() == tensorflow::DataType::DT_STRING) { SPDLOG_DEBUG("Request contains binary input: {}", name); - status = convertBinaryRequestTensorToOVTensor(requestInput, tensor, tensorInfo); + status = convertBinaryRequestTensorToOVTensor(requestInput, tensor, tensorInfo, nullptr); if (!status.ok()) { SPDLOG_DEBUG("Binary inputs conversion failed."); return status; @@ -406,17 +406,17 @@ Status deserializePredictRequest( } ov::Tensor tensor; + auto inputIndex = requestInputItr - request.inputs().begin(); + auto bufferLocation = deserializeFromSharedInputContents ? &request.raw_input_contents()[inputIndex] : nullptr; + if (requestInputItr->datatype() == "BYTES") { SPDLOG_DEBUG("Request contains binary input: {}", name); - status = convertBinaryRequestTensorToOVTensor(*requestInputItr, tensor, tensorInfo); + status = convertBinaryRequestTensorToOVTensor(*requestInputItr, tensor, tensorInfo, bufferLocation); if (!status.ok()) { SPDLOG_DEBUG("Binary inputs conversion failed."); return status; } } else { - auto inputIndex = requestInputItr - request.inputs().begin(); - auto bufferLocation = deserializeFromSharedInputContents ? &request.raw_input_contents()[inputIndex] : nullptr; - tensor = deserializeTensorProto(*requestInputItr, tensorInfo, bufferLocation); if (!tensor) { status = StatusCode::OV_UNSUPPORTED_DESERIALIZATION_PRECISION; diff --git a/src/predict_request_validation_utils.cpp b/src/predict_request_validation_utils.cpp index 473f7f2e35..cd43e529b6 100644 --- a/src/predict_request_validation_utils.cpp +++ b/src/predict_request_validation_utils.cpp @@ -130,7 +130,7 @@ class RequestValidator { Status validateTensorContent(const InputTensorType& proto, ovms::Precision expectedPrecision, size_t bufferId) const; Status validateNumberOfShapeDimensions(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; Status validatePrecision(const ovms::TensorInfo& inputInfo, const InputTensorType& proto) const; - bool checkIfBinaryInputUsed(const InputTensorType& proto, const std::string inputName) const; + bool checkIfNativeFileFormatUsed(const InputTensorType& proto, const std::string inputName) const; Status validateRequestCoherency() const; Status validate(); }; @@ -780,7 +780,7 @@ static Mode getShapeMode(const shapes_info_map_t& shapeInfo, const std::string& } template <> -bool RequestValidator::checkIfBinaryInputUsed(const TFSInputTensorType& proto, const std::string inputName) const { +bool RequestValidator::checkIfNativeFileFormatUsed(const TFSInputTensorType& proto, const std::string inputName) const { if (proto.dtype() == tensorflow::DataType::DT_STRING) { SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.string_val_size()); return true; @@ -788,7 +788,7 @@ bool RequestValidator -bool RequestValidator::checkIfBinaryInputUsed(const KFSTensorInputProto& proto, const std::string inputName) const { +bool RequestValidator::checkIfNativeFileFormatUsed(const KFSTensorInputProto& proto, const std::string inputName) const { if (proto.datatype() == "BYTES") { SPDLOG_DEBUG("[servable name: {} version: {}] Received request containing binary input: name: {}; batch size: {}", servableName, servableVersion, inputName, proto.contents().bytes_contents_size()); return true; @@ -796,11 +796,23 @@ bool RequestValidator -bool RequestValidator::checkIfBinaryInputUsed(const InferenceTensor& tensor, const std::string inputName) const { - // no string no bytes currently, will implement one of those types with binary input. +bool RequestValidator::checkIfNativeFileFormatUsed(const InferenceTensor& tensor, const std::string inputName) const { + // TODO no strig no bytes currently, will implement one of those types with binary input. return false; } +static bool shouldValidateBinaryBatchSizeMismatch(const ovms::InferenceRequest& request) { + return true; +} + +static bool shouldValidateBinaryBatchSizeMismatch(const TFSRequestType& request) { + return true; +} + +static bool shouldValidateBinaryBatchSizeMismatch(const KFSRequest& request) { + return request.raw_input_contents().size() <= 0; +} + template Status RequestValidator::validate() { Status finalStatus = StatusCode::OK; @@ -836,14 +848,19 @@ Status RequestValidator:: } const Dimension& batchSize = inputInfo->getShape()[batchIndex.value()]; Mode shapeMode = getShapeMode(shapeInfo, name); - if (checkIfBinaryInputUsed(proto, name)) { + if (checkIfNativeFileFormatUsed(proto, name)) { status = validateNumberOfBinaryInputShapeDimensions(proto); if (!status.ok()) return status; - status = checkBinaryBatchSizeMismatch(proto, batchSize, finalStatus, batchingMode, shapeMode); - if (!status.ok()) - return status; + if (shouldValidateBinaryBatchSizeMismatch(request)) { + status = checkBinaryBatchSizeMismatch(proto, batchSize, finalStatus, batchingMode, shapeMode); + if (!status.ok()) + return status; + } else if (batchSize != 1) { + SPDLOG_DEBUG("When the image is placed in raw_inputs_contents batch size cannot be bigger than 1."); + return StatusCode::INVALID_BATCH_SIZE; + } continue; } diff --git a/src/test/binaryutils_test.cpp b/src/test/binaryutils_test.cpp index aaeaa4e13c..e14710a55b 100644 --- a/src/test/binaryutils_test.cpp +++ b/src/test/binaryutils_test.cpp @@ -48,7 +48,6 @@ class BinaryUtilsTest : public ::testing::Test { tensor.mutable_shape()->Add(batchSize); tensor.set_datatype("BYTES"); } - void prepareBinaryTensor(tensorflow::TensorProto& tensor) { size_t filesize; std::unique_ptr image_bytes; @@ -82,7 +81,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingBatchsize) { auto tensorInfo = std::make_shared(); tensorInfo->setShape({5, 1, 1, 1}); tensorInfo->setLayout(Layout{"NHWC"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::INVALID_BATCH_SIZE); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_BATCH_SIZE); } TYPED_TEST(BinaryUtilsTest, tensorWithInvalidImage) { @@ -94,7 +93,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithInvalidImage) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorInvalidImage, tensor, tensorInfo), ovms::StatusCode::IMAGE_PARSING_FAILED); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorInvalidImage, tensor, tensorInfo, nullptr), ovms::StatusCode::IMAGE_PARSING_FAILED); } TYPED_TEST(BinaryUtilsTest, tensorWithEmptyTensor) { @@ -106,9 +105,9 @@ TYPED_TEST(BinaryUtilsTest, tensorWithEmptyTensor) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); if (std::is_same::value) - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorEmptyInput, tensor, tensorInfo), ovms::StatusCode::STRING_VAL_EMPTY); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorEmptyInput, tensor, tensorInfo, nullptr), ovms::StatusCode::STRING_VAL_EMPTY); else - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorEmptyInput, tensor, tensorInfo), StatusCode::BYTES_CONTENTS_EMPTY); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(requestTensorEmptyInput, tensor, tensorInfo, nullptr), StatusCode::BYTES_CONTENTS_EMPTY); } TYPED_TEST(BinaryUtilsTest, tensorWithNonSupportedLayout) { @@ -116,7 +115,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithNonSupportedLayout) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NCHW"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::UNSUPPORTED_LAYOUT); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::UNSUPPORTED_LAYOUT); } TYPED_TEST(BinaryUtilsTest, tensorWithNonSupportedPrecision) { @@ -124,7 +123,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithNonSupportedPrecision) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::MIXED, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::INVALID_PRECISION); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_PRECISION); } TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingShapeSize) { @@ -132,7 +131,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingShapeSize) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1}, Layout{"NC"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::UNSUPPORTED_LAYOUT); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::UNSUPPORTED_LAYOUT); } TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingNumberOfChannelsNHWC) { @@ -140,7 +139,7 @@ TYPED_TEST(BinaryUtilsTest, tensorWithNonMatchingNumberOfChannelsNHWC) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 1}, Layout{"NHWC"}); - EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::INVALID_NO_OF_CHANNELS); + EXPECT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_NO_OF_CHANNELS); } TYPED_TEST(BinaryUtilsTest, positive_rgb) { @@ -150,7 +149,7 @@ TYPED_TEST(BinaryUtilsTest, positive_rgb) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 3); uint8_t* ptr = static_cast(tensor.data()); EXPECT_EQ(std::equal(ptr, ptr + tensor.get_size(), rgb_expected_tensor), true); @@ -173,7 +172,7 @@ TYPED_TEST(BinaryUtilsTest, positive_grayscale) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 1}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(grayscaleRequestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(grayscaleRequestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 1); uint8_t* ptr = static_cast(tensor.data()); EXPECT_EQ(std::equal(ptr, ptr + tensor.get_size(), grayscale_expected_tensor), true); @@ -193,7 +192,7 @@ TYPED_TEST(BinaryUtilsTest, positive_batch_size_2) { for (const auto layout : std::vector{Layout("NHWC"), Layout::getDefaultLayout()}) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{2, 1, 1, 3}, layout); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(batchSize2RequestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(batchSize2RequestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 6); uint8_t* ptr = static_cast(tensor.data()); EXPECT_EQ(std::equal(ptr, ptr + tensor.get_size(), rgb_batchsize_2_tensor), true); @@ -207,7 +206,7 @@ TYPED_TEST(BinaryUtilsTest, positive_precision_changed) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::I32, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 3); uint8_t* ptr = static_cast(tensor.data()); int I32_size = 4; @@ -221,7 +220,7 @@ TYPED_TEST(BinaryUtilsTest, positive_nhwc_layout) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 3); uint8_t* ptr = static_cast(tensor.data()); @@ -231,7 +230,7 @@ TYPED_TEST(BinaryUtilsTest, positive_nhwc_layout) { TYPED_TEST(BinaryUtilsTest, layout_default_resolution_mismatch) { ov::Tensor tensor; std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 3, 1, 3}, Layout::getDefaultLayout()); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::INVALID_SHAPE); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_SHAPE); } TYPED_TEST(BinaryUtilsTest, positive_resizing) { @@ -241,7 +240,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 2, 2, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_size(), 12); uint8_t* ptr = static_cast(tensor.data()); EXPECT_EQ(std::equal(ptr, ptr + tensor.get_size(), rgb_expected_tensor), true); @@ -254,7 +253,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_cols_smaller) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, {2, 5}, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedColsNumber = 2; EXPECT_EQ(tensorDims[2], expectedColsNumber); @@ -277,7 +276,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_cols_bigger) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, {1, 3}, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedColsNumber = 3; EXPECT_EQ(tensorDims[2], expectedColsNumber); @@ -293,7 +292,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_cols_in_range) std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, {1, 3}, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedColsNumber = 1; EXPECT_EQ(tensorDims[2], expectedColsNumber); @@ -309,7 +308,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_rows_smaller) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, {2, 5}, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedRowsNumber = 2; EXPECT_EQ(tensorDims[1], expectedRowsNumber); @@ -332,7 +331,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_rows_bigger) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, {1, 3}, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedRowsNumber = 3; EXPECT_EQ(tensorDims[1], expectedRowsNumber); @@ -348,7 +347,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_dynamic_shape_rows_in_range) std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, {1, 3}, 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedRowsNumber = 1; EXPECT_EQ(tensorDims[1], expectedRowsNumber); @@ -364,7 +363,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_any_shape) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, ovms::Dimension::any(), ovms::Dimension::any(), 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedRowsNumber = 1; EXPECT_EQ(tensorDims[1], expectedRowsNumber); @@ -380,7 +379,7 @@ TYPED_TEST(BinaryUtilsTest, negative_resizing_with_one_any_one_static_shape) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, ovms::Dimension::any(), 4, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::INVALID_SHAPE); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_SHAPE); } TYPED_TEST(BinaryUtilsTest, positive_resizing_with_one_any_one_static_shape) { @@ -388,7 +387,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_one_any_one_static_shape) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, ovms::Dimension::any(), 1, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); size_t expectedRowsNumber = 1; EXPECT_EQ(tensorDims[1], expectedRowsNumber); @@ -411,7 +410,7 @@ TYPED_TEST(BinaryUtilsTest, positive_resizing_with_demultiplexer_and_range_resol TypeParam requestTensor4x4; this->prepareBinaryTensor(requestTensor4x4, image_bytes, filesize, batchSize); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); ASSERT_EQ(tensorDims[0], batchSize); EXPECT_EQ(tensorDims[1], 1); @@ -435,7 +434,7 @@ TYPED_TEST(BinaryUtilsTest, positive_range_resolution_matching_in_between) { for (const auto& batchDim : std::vector{Dimension::any(), Dimension(batchSize)}) { std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{batchDim, {1, 5}, {1, 5}, 3}, Layout{"NHWC"}); - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(requestTensor4x4, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); shape_t tensorDims = tensor.get_shape(); ASSERT_EQ(tensorDims[0], batchSize); EXPECT_EQ(tensorDims[1], 4); @@ -470,7 +469,7 @@ TEST_P(BinaryUtilsTFSValidPrecisionTest, Valid) { Layout{"NHWC"}); ov::runtime::Tensor tensor; - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(stringVal, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(stringVal, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_shape(), (ov::Shape{1, 1, 1, 3})); ASSERT_EQ(tensor.get_size(), 3); ASSERT_EQ(tensor.get_element_type(), ovmsPrecisionToIE2Precision(testedPrecision)); @@ -485,7 +484,7 @@ TEST_P(BinaryUtilsTFSInvalidPrecisionTest, Invalid) { Layout{"NHWC"}); ov::runtime::Tensor tensor; - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(stringVal, tensor, tensorInfo), ovms::StatusCode::INVALID_PRECISION); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(stringVal, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_PRECISION); } const std::vector BINARY_SUPPORTED_INPUT_PRECISIONS{ @@ -564,7 +563,7 @@ TEST_P(BinaryUtilsKFSValidPrecisionTest, Valid) { Layout{"NHWC"}); ov::runtime::Tensor tensor; - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(inferTensorContent, tensor, tensorInfo), ovms::StatusCode::OK); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(inferTensorContent, tensor, tensorInfo, nullptr), ovms::StatusCode::OK); ASSERT_EQ(tensor.get_shape(), (ov::Shape{1, 1, 1, 3})); ASSERT_EQ(tensor.get_size(), 3); ASSERT_EQ(tensor.get_element_type(), ovmsPrecisionToIE2Precision(testedPrecision)); @@ -579,7 +578,7 @@ TEST_P(BinaryUtilsKFSInvalidPrecisionTest, Invalid) { Layout{"NHWC"}); ov::runtime::Tensor tensor; - ASSERT_EQ(convertBinaryRequestTensorToOVTensor(inferTensorContent, tensor, tensorInfo), ovms::StatusCode::INVALID_PRECISION); + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(inferTensorContent, tensor, tensorInfo, nullptr), ovms::StatusCode::INVALID_PRECISION); } INSTANTIATE_TEST_SUITE_P( @@ -597,4 +596,50 @@ INSTANTIATE_TEST_SUITE_P( [](const ::testing::TestParamInfo& info) { return toString(info.param); }); + +class BinaryUtilsTestKFSRawInputsContents : public ::testing::Test { +public: + ::KFSRequest::InferInputTensor requestTensor; + std::string buffer; + void SetUp() override { + requestTensor.mutable_shape()->Add(1); + requestTensor.set_datatype("BYTES"); + + size_t filesize; + std::unique_ptr image_bytes; + + readRgbJpg(filesize, image_bytes); + buffer.append(image_bytes.get(), filesize); + } +}; + +TEST_F(BinaryUtilsTestKFSRawInputsContents, Positive) { + uint8_t rgb_expected_tensor[] = {0x24, 0x1b, 0xed}; + + ov::Tensor tensor; + + std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); + + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, &this->buffer), ovms::StatusCode::OK); + ASSERT_EQ(tensor.get_size(), 3); + uint8_t* ptr = static_cast(tensor.data()); + EXPECT_EQ(std::equal(ptr, ptr + tensor.get_size(), rgb_expected_tensor), true); +} + +TEST_F(BinaryUtilsTestKFSRawInputsContents, Negative_batchSizeBiggerThan1) { + ov::Tensor tensor; + + std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{2, 1, 1, 3}, Layout{"NHWC"}); + + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, &this->buffer), ovms::StatusCode::INVALID_BATCH_SIZE); +} + +TEST_F(BinaryUtilsTestKFSRawInputsContents, Negative_emptyString) { + ov::Tensor tensor; + + std::shared_ptr tensorInfo = std::make_shared("", ovms::Precision::U8, ovms::Shape{1, 1, 1, 3}, Layout{"NHWC"}); + + std::string empty; + ASSERT_EQ(convertBinaryRequestTensorToOVTensor(this->requestTensor, tensor, tensorInfo, &empty), ovms::StatusCode::BYTES_CONTENTS_EMPTY); +} } // namespace diff --git a/src/test/deserialization_tests.cpp b/src/test/deserialization_tests.cpp index 548b904ee2..a24430c692 100644 --- a/src/test/deserialization_tests.cpp +++ b/src/test/deserialization_tests.cpp @@ -430,7 +430,7 @@ TEST_P(DeserializeKFSTensorProtoNegative, ShouldReturnNullptrForPrecision) { TEST_P(DeserializeKFSTensorProto, ShouldReturnValidTensor) { auto [testedPrecision, getInputFromRawInputContents] = GetParam(); std::string* bufferPtr = (getInputFromRawInputContents ? &buffer : nullptr); - if (!getInputFromRawInputContents && (ovms::Precision::FP16 == testedPrecision)) { + if ((!getInputFromRawInputContents && (ovms::Precision::FP16 == testedPrecision))) { GTEST_SKIP() << "Not supported"; } SetUpTensorProto(TensorInfo::getPrecisionAsString(testedPrecision), getInputFromRawInputContents); diff --git a/src/test/predict_validation_test.cpp b/src/test/predict_validation_test.cpp index 5f49391e78..5964a4784c 100644 --- a/src/test/predict_validation_test.cpp +++ b/src/test/predict_validation_test.cpp @@ -785,6 +785,58 @@ TEST_F(KFSPredictValidation, ValidRequestBinaryInputs) { EXPECT_TRUE(status.ok()); } +TEST_F(KFSPredictValidation, ValidRequestBinaryInputs_RawInputsContents) { + modelConfig.setBatchingParams("auto"); + std::string inputName = "Binary_Input"; + ::KFSRequest binaryInputRequest; + + auto input = binaryInputRequest.add_inputs(); + input->set_name(inputName); + input->set_datatype("BYTES"); + const int requestBatchSize = 1; + std::string bytes_contents = "BYTES_CONTENTS"; + auto content = binaryInputRequest.add_raw_input_contents(); + *content = bytes_contents; + input->mutable_shape()->Add(requestBatchSize); + + servableInputs.clear(); + ovms::shape_t shape = {1, 3, 224, 224}; + servableInputs[inputName] = std::make_shared( + inputName, + ovms::Precision::FP32, + shape, + ovms::Layout{"NHWC"}); + + auto status = instance->mockValidate(&binaryInputRequest); + EXPECT_TRUE(status.ok()); +} + +TEST_F(KFSPredictValidation, RawInputsContentsBatchSizeBiggerThan1) { + modelConfig.setBatchingParams("auto"); + std::string inputName = "Binary_Input"; + ::KFSRequest binaryInputRequest; + + auto input = binaryInputRequest.add_inputs(); + input->set_name(inputName); + input->set_datatype("BYTES"); + const int requestBatchSize = 2; + std::string bytes_contents = "BYTES_CONTENTS"; + auto content = binaryInputRequest.add_raw_input_contents(); + *content = bytes_contents; + input->mutable_shape()->Add(requestBatchSize); + + servableInputs.clear(); + ovms::shape_t shape = {2, 3, 224, 224}; + servableInputs[inputName] = std::make_shared( + inputName, + ovms::Precision::FP32, + shape, + ovms::Layout{"NHWC"}); + + auto status = instance->mockValidate(&binaryInputRequest); + EXPECT_EQ(status, ovms::StatusCode::INVALID_BATCH_SIZE); +} + TEST_F(KFSPredictValidation, RequestWrongBatchSizeBinaryInputs) { std::string inputName = "Binary_Input"; ::KFSRequest binaryInputRequest; From ee71e02e418f50d5e7567bb3489b4acaae3bc100 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Fri, 16 Dec 2022 11:16:16 +0100 Subject: [PATCH 114/130] Add descriptions to ovms.h (#1603) JIRA:CVS-98964 --- src/ovms.h | 294 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 260 insertions(+), 34 deletions(-) diff --git a/src/ovms.h b/src/ovms.h index 554846fe7d..2fb7427efa 100644 --- a/src/ovms.h +++ b/src/ovms.h @@ -27,6 +27,10 @@ typedef struct OVMS_Status_ OVMS_Status; typedef struct OVMS_ServerSettings_ OVMS_ServerSettings; typedef struct OVMS_ModelsSettings_ OVMS_ModelsSettings; +// OVMS_DataType +// +// Tensor and parameter data types recognized by OVMS. +// typedef enum OVMS_DataType_enum { OVMS_DATATYPE_BF16, OVMS_DATATYPE_FP64, @@ -53,6 +57,10 @@ typedef enum OVMS_DataType_enum { OVMS_DATATYPE_END } OVMS_DataType; +// OVMS_BufferType +// +// Types of memory used by OVMS. +// typedef enum OVMS_BufferType_enum { OVMS_BUFFERTYPE_CPU, OVMS_BUFFERTYPE_CPU_PINNED, @@ -63,6 +71,10 @@ typedef enum OVMS_BufferType_enum { typedef struct OVMS_InferenceRequest_ OVMS_InferenceRequest; typedef struct OVMS_InferenceResponse_ OVMS_InferenceResponse; +// OVMS_LogLevel +// +// Levels of OVMS logging. +// typedef enum OVMS_LogLevel_enum { OVMS_LOG_TRACE, OVMS_LOG_DEBUG, @@ -74,80 +86,177 @@ typedef enum OVMS_LogLevel_enum { //// //// OVMS_Status //// Structure for status management. -//// Whenever C-API call returns non null pointer it should be treated as error with code and detailed message. +//// Whenever C-API call returns non null pointer it should be treated as error with code and string message. //// The status should be deallocated with OVMS_StatusDelete afterwards. //// -// Deallocates status memory for given ptr +// Deallocates a status object. +// +// \param status The status object void OVMS_StatusDelete(OVMS_Status* status); +// Get the status code from a status. +// +// \param status The status object +// \param code Value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_StatusGetCode(OVMS_Status* status, uint32_t* code); +// Get the status details from a status. +// +// \param status The status object +// \param details The status details +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_StatusGetDetails(OVMS_Status* status, const char** details); //// //// OVMS_ServerSettings -//// Structure for server settings for both: single and multi (with config.json) management +//// Structure for server settings for both: single and multi (with config.json) management. //// -// Allocates memory for server settings and returns ptr +// Allocates memory for server settings and returns ptr. +// +// \param settings The server settings object to be created +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsNew(OVMS_ServerSettings** settings); -// Deallocates server settings memory for given ptr + +// Deallocates server settings object for given ptr. +// +// \param settings The settings object to be removed void OVMS_ServerSettingsDelete(OVMS_ServerSettings* settings); -// --port +// Set the gRPC port of starting OVMS. Equivalent of using --port parameter from OVMS CLI. +// If not set server will start with gRPC port set to 9178. +// +// \param settings The server settings object to be set +// \param grpc_port The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetGrpcPort(OVMS_ServerSettings* settings, uint32_t grpc_port); -// --rest_port +// Set rest port for starting server. If not set the http server will not start +// Equivalent of starting server with +// --rest_port. +// +// \param settings The server settings object to be set +// \param rest_port The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetRestPort(OVMS_ServerSettings* settings, uint32_t rest_port); -// --grpc_workers +// Set gRPC workers server setting. +// Equivalent of starting server with +// --grpc_workers. +// +// \param settings The server settings object to be set +// \param grpc_workers The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetGrpcWorkers(OVMS_ServerSettings* settings, uint32_t grpc_workers); -// --grpc_bind_address +// Set gRPC bind address for starting server +// Equivalent of starting server with +// --grpc_bind_address. +// +// \param settings The server settings object to be set +// \param grpc_bind_address The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetGrpcBindAddress(OVMS_ServerSettings* settings, const char* grpc_bind_address); -// --rest_workers +// Set REST workers server setting. +// Equivalent of starting server with +// --rest_workers. +// +// \param settings The server settings object to be set +// \param rest_workers The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetRestWorkers(OVMS_ServerSettings* settings, uint32_t rest_workers); -// --rest_bind_address +// Set REST bind address server setting. +// Equivalent of starting server with +// --rest_bind_address. +// +// \param settings The server settings object to be set +// \param rest_bind_address The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetRestBindAddress(OVMS_ServerSettings* settings, const char* rest_bind_address); -// --grpc_channel_arguments +// Set the gRPC channel arguments server setting. +// Equivalent of starting server with +// --grpc_channel_arguments. +// +// \param settings The server settings object to be set +// \param grpc_channel_arguments The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetGrpcChannelArguments(OVMS_ServerSettings* settings, const char* grpc_channel_arguments); -// --file_system_poll_wait_seconds +// Set config check interval server setting. +// Equivalent of starting server with +// --file_system_poll_wait_seconds. +// +// \param settings The server settings object to be set +// \param seconds The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetFileSystemPollWaitSeconds(OVMS_ServerSettings* settings, uint32_t seconds); -// --sequence_cleaner_poll_wait_minutes +// Set sequence cleaner interval server setting. +// Equivalent of starting server with +// --sequence_cleaner_poll_wait_minutes. +// +// \param settings The server settings object to be set +// \param minutes The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetSequenceCleanerPollWaitMinutes(OVMS_ServerSettings* settings, uint32_t minutes); -// --custom_node_resources_cleaner_interval_seconds +// Set custom node resource cleaner interval server setting. +// Equivalent of starting server with +// --custom_node_resources_cleaner_interval_seconds. +// +// \param settings The server settings object to be set +// \param seconds The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetCustomNodeResourcesCleanerIntervalSeconds(OVMS_ServerSettings* settings, uint32_t seconds); -// --cpu_extension +// Set cpu extension path server setting. Equivalent of starting server with +// --cpu_extension. +// +// \param settings The server settings object to be set +// \param cpu_extension_path The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetCpuExtensionPath(OVMS_ServerSettings* settings, const char* cpu_extension_path); -// --cache_dir +// Set cache dir server setting. Equivalent of starting server with +// --cache_dir. +// +// \param settings The server settings object to be set +// \param cache_dir The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetCacheDir(OVMS_ServerSettings* settings, const char* cache_dir); -// --log_level +// Set log level server setting. Equivalent of starting server with +// --log_level. +// +// \param settings The server settings object to be set +// \param log_level The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetLogLevel(OVMS_ServerSettings* settings, OVMS_LogLevel log_level); -// --log_path +// Set the server log_path setting. Equivalent of starting server with +// --log_path. +// +// \param settings The server settings object to be set +// \param log_path The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerSettingsSetLogPath(OVMS_ServerSettings* settings, const char* log_path); @@ -155,53 +264,170 @@ OVMS_Status* OVMS_ServerSettingsSetLogPath(OVMS_ServerSettings* settings, //// OVMS_ModelsSettings //// Options for starting multi model server controlled by config.json file //// Models management settings for starting OVMS. Right now only using config.json file -//// is supported +//// is supported. //// -// Allocates memory for models settings and returns ptr +// Allocates memory for models settings and returns ptr. +// +// \param settings The models settings object to be created +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ModelsSettingsNew(OVMS_ModelsSettings** settings); -// Deallocates models settings memory for given ptr + +// Deallocates models settings memory for given ptr. +// +// \param settings The models settings object to be removed void OVMS_ModelsSettingsDelete(OVMS_ModelsSettings* settings); -// --config_path +// Set the server configuration file path. Equivalent of starting server with +// --config_path. +// +// \param settings The models settings object to be set +// \param config_path The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ModelsSettingsSetConfigPath(OVMS_ModelsSettings* settings, const char* config_path); //// //// OVMS_Server -//// Handler for all management activities +//// Handler for all management activities. //// // Allocates memory for server and returns ptr +// +// \param server The server object to be created and set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerNew(OVMS_Server** server); -// Deallocates server memory for given ptr + +// Deallocates server memory for given ptr. +// +// \param server The server object to be removed void OVMS_ServerDelete(OVMS_Server* server); -// Start with configuration file config.json -// Return error if already started +// Start server with configuration file config.json. +// Return error if already started or any other loading, configuration error occured. +// In preview only using config file is supported, providing model name and model path is not. +// +// \param server The server object to be started +// \param server_settings The server settings to be used +// \param models_settings The models settings to be used +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_ServerStartFromConfigurationFile(OVMS_Server* server, OVMS_ServerSettings* server_settings, - OVMS_ModelsSettings* models_settings); // in fact only --config_path -// Unload all and cleanup + OVMS_ModelsSettings* models_settings); // OVMS_InferenceRequest +// +// Create new inference request object. In case of servable version set to 0 server will choose +// the default servable version. +// +// \param request The request object to be created +// \param server The server object +// \param servableName The name of the servable to be used +// \param servableVersion The version of the servable to be used +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceRequestNew(OVMS_InferenceRequest** request, OVMS_Server* server, const char* servableName, uint32_t servableVersion); void OVMS_InferenceRequestDelete(OVMS_InferenceRequest* response); +// Set the data of the input buffer. Ownership of data needs to be maintained during inference. +// +// \param request The request object +// \param inputName The name of the input +// \param datatype The data type of the input +// \param shape The shape of the input +// \param dimCount The number of dimensions of the shape +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceRequestAddInput(OVMS_InferenceRequest* request, const char* inputName, OVMS_DataType datatype, const uint64_t* shape, uint32_t dimCount); -// ownership of data needs to be maintained during inference -OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, const void* data, size_t bufferSize, OVMS_BufferType bufferType, uint32_t deviceId); +// Set the data of the input buffer. Ownership of data needs to be maintained during inference. +// +// \param request The request object +// \param inputName The name of the input with data to be set +// \param data The data of the input +// \param byteSize The byte size of the data +// \param bufferType The buffer type of the data +// \param deviceId The device id of the data memory buffer +// \return OVMS_Status object in case of failure +OVMS_Status* OVMS_InferenceRequestInputSetData(OVMS_InferenceRequest* request, const char* inputName, const void* data, size_t byteSize, OVMS_BufferType bufferType, uint32_t deviceId); + +// Remove the data of the input. +// +// \param request The request object +// \param inputName The name of the input with data to be removed +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceRequestInputRemoveData(OVMS_InferenceRequest* request, const char* inputName); -OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); // this will allow for reuse of request but with different input data + +// Remove input from the request. +// +// \param request The request object +// \param inputName The name of the input to be removed +// \return OVMS_Status object in case of failure +OVMS_Status* OVMS_InferenceRequestRemoveInput(OVMS_InferenceRequest* request, const char* inputName); + +// Add parameter to the request. +// +// \param request The request object +// \param parameterName The name of the parameter to be added +// \param datatype The request object +// \param data The data representing parameter value +// \param byteSize The byte size of the added parameter value +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceRequestAddParameter(OVMS_InferenceRequest* request, const char* parameterName, OVMS_DataType datatype, const void* data, size_t byteSize); + +// Remove parameter from the inference request. +// +// \param request The request object +// \param parameterName The name of the parameter to be removed +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceRequestRemoveParameter(OVMS_InferenceRequest* request, const char* parameterName); -// OVMS_Inference Response +// OVMS_InferenceResponse +// +// Get the number of outputs in the response. +// +// \param response The response object +// \param count The value to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceResponseGetOutputCount(OVMS_InferenceResponse* response, uint32_t* count); -OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* bytesize, OVMS_BufferType* bufferType, uint32_t* deviceId); + +// Get all information about an output from the response by providing output id. +// +// \param response The response object +// \param id The id of the output +// \param name The name of the output +// \param datatype The data type of the output +// \param shape The shape of the output +// \param dimCount The number of dimensions of the shape +// \param data The data of the output +// \param byteSize The buffer size of the data +// \param bufferType The buffer type of the data +// \param deviceId The device id of the data memory buffer +// \return OVMS_Status object in case of failure +OVMS_Status* OVMS_InferenceResponseGetOutput(OVMS_InferenceResponse* response, uint32_t id, const char** name, OVMS_DataType* datatype, const uint64_t** shape, uint32_t* dimCount, const void** data, size_t* byteSize, OVMS_BufferType* bufferType, uint32_t* deviceId); + +// Get the number of parameters in response. +// +// \param response The response object +// \param count The parameter count to be set +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceResponseGetParameterCount(OVMS_InferenceResponse* response, uint32_t* count); + +// Extract information about parameter by providing its id. +// +// \param response The response object +// \param id The id of the parameter +// \param datatype The data type of the parameter +// \param data The parameter content +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_InferenceResponseGetParameter(OVMS_InferenceResponse* response, uint32_t id, OVMS_DataType* datatype, const void** data); + +// Delete OVMS_InferenceResponse object. +// +// \param response The response object to be removed void OVMS_InferenceResponseDelete(OVMS_InferenceResponse* response); +// Execute synchronous inference. +// +// \param request The request object +// \param response The respons object. In case of success, caller takes the ownership of the response +// \return OVMS_Status object in case of failure OVMS_Status* OVMS_Inference(OVMS_Server* server, OVMS_InferenceRequest* request, OVMS_InferenceResponse** response); #ifdef __cplusplus From 217449b8032d3f6624c407d9409c0c71e0efa079 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Fri, 16 Dec 2022 12:23:18 +0100 Subject: [PATCH 115/130] Hadolint for build and release images (#1593) - remove boost for HDDL on ubuntu due to HDDL being not part of the 2022.3 scope Co-authored-by: Dariusz Trawinski --- Dockerfile.redhat | 85 +++++++++++++------------- Dockerfile.ubuntu | 104 ++++++++++++++++---------------- Makefile | 2 +- release_files/Dockerfile.redhat | 19 +++--- release_files/Dockerfile.ubuntu | 50 ++++++--------- tests/hadolint.sh | 58 ++++++++++++++++++ 6 files changed, 183 insertions(+), 135 deletions(-) create mode 100755 tests/hadolint.sh diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 66e8ac0e7c..f94607e7b2 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -19,15 +19,11 @@ FROM $BASE_IMAGE as base_build LABEL version="1.0.0" +SHELL ["/bin/bash", "-xo", "pipefail", "-c"] + ARG ov_source_branch=releases/2021/2 -ARG ovms_metadata_file -ARG INSTALL_DIR=/opt/intel/openvino -ARG TEMP_DIR=/tmp/openvino_installer -ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools -ARG DL_DIR=/tmp ARG JOBS -ARG CHECK_COVERAGE=0 # build_type=[ opt, dbg ] ARG build_type=dbg @@ -36,7 +32,7 @@ ARG minitrace_flags ENV TF_SYSTEM_LIBS="curl" ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" -RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && yum update -d6 -y && yum install -d6 -y \ +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && dnf clean all && yum update -d6 -y && yum install -d6 -y \ boost169-atomic \ boost169-chrono \ boost169-filesystem \ @@ -77,6 +73,7 @@ RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.n yum clean all # Build and install pugixml +# hadolint ignore=DL3003 RUN git clone -b v1.13 https://github.com/zeux/pugixml && \ cd pugixml && \ cmake -DBUILD_SHARED_LIBS=ON && \ @@ -90,12 +87,12 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ chmod +x bazel-*.sh && \ ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - cd / && \ rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh ####### Azure SDK needs new boost: WORKDIR /boost -RUN wget https://sourceforge.net/projects/boost/files/boost/1.68.0/boost_1_68_0.tar.gz && \ +# hadolint ignore=DL3003 +RUN wget -nv https://sourceforge.net/projects/boost/files/boost/1.68.0/boost_1_68_0.tar.gz && \ tar xvf boost_1_68_0.tar.gz && cd boost_1_68_0 && ./bootstrap.sh && \ ./b2 cxxstd=17 link=static cxxflags='-fPIC' cflags='-fPIC' \ --with-chrono --with-date_time --with-filesystem --with-program_options --with-system \ @@ -107,27 +104,23 @@ COPY third_party /ovms/third_party/ ####### Azure SDK WORKDIR /azure -RUN git clone https://github.com/Microsoft/cpprestsdk.git && cd cpprestsdk && git checkout tags/v2.10.16 -b v2.10.16 && git submodule update --init - -RUN git clone https://github.com/Azure/azure-storage-cpp.git && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage && git checkout tags/v7.5.0 && mkdir build.release +RUN git clone --recurse-submodules --depth 1 --branch v2.10.16 https://github.com/Microsoft/cpprestsdk.git && \ + git clone --depth 1 --branch v7.5.0 https://github.com/Azure/azure-storage-cpp.git && \ + patch -d /azure/cpprestsdk/ -p1 < /ovms/third_party/cpprest/rest_sdk_v2.10.16.patch && \ + patch -d /azure/azure-storage-cpp/ -p1 ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ +RUN if [ "$CHECK_COVERAGE" = "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" ; fi ; \ bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} -RUN cd /ovms/src/example/SampleCpuExtension/ && make - # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel +# hadolint ignore=DL3059 RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:capi_cpp_example + # C-API benchmark app RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" -# Copy binary for //src:ovms +# OVMS RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms -RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so -RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make +# hadolint ignore=DL3059 +RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so # C-api C/C++ app with gcc -RUN cd /ovms && make -f MakefileCapi cpp && make -f MakefileCapi c +RUN make -f MakefileCapi cpp && make -f MakefileCapi c +ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json -RUN cd bazel-bin/src/ && ./ovms --version && ./ovms && cd /ovms +RUN /ovms/bazel-bin/src/ovms --version && /ovms/bazel-bin/src/ovms COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE COPY client /client COPY demos /demos - diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 6563674e47..52e7a3a4ac 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -19,16 +19,10 @@ FROM $BASE_IMAGE as base_build LABEL version="1.0.0" -ARG INSTALL_DIR=/opt/intel/openvino -ARG INSTALL_DIR=/opt/intel/openvino -ARG TEMP_DIR=/tmp/openvino_installer -ARG DL_INSTALL_DIR=/opt/intel/openvino/deployment_tools -ARG DL_DIR=/tmp +SHELL ["/bin/bash", "-xo", "pipefail", "-c"] + ARG JOBS ARG APT_OV_PACKAGE=openvino-2022.1.0 -ARG CHECK_COVERAGE=0 -ARG NVIDIA=0 -ARG http_proxy ARG CMAKE_BUILD_TYPE=Release # build_type=[ opt, dbg ] @@ -39,8 +33,7 @@ ENV HDDL_INSTALL_DIR=/opt/intel/openvino/deployment_tools/inference_engine/exter ENV DEBIAN_FRONTEND=noninteractive ENV TF_SYSTEM_LIBS="curl" ENV TEST_LOG="/root/.cache/bazel/_bazel_root/bc57d4817a53cab8c785464da57d1983/execroot/ovms/bazel-out/test.log" -SHELL ["/bin/bash", "-c"] -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install --no-install-recommends -y \ libboost-atomic1.71.0 \ libboost-chrono1.71.0 \ libboost-filesystem1.71.0 \ @@ -90,20 +83,22 @@ RUN apt-get update && apt-get install -y \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +ARG NVIDIA=0 # Add Nvidia dev tool if needed +# hadolint ignore=DL3003 RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ - wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin -O /etc/apt/preferences.d/cuda-repository-pin-600 ; \ + wget -nv https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin -O /etc/apt/preferences.d/cuda-repository-pin-600 ; \ set -exuo pipefail ; \ rm -f /etc/apt/apt.conf.d/docker-clean ; \ apt-get update && \ - apt-get install -y \ + apt-get install --no-install-recommends -y \ gnupg2 \ software-properties-common; \ if [[ ${enable_tensorrt-} == "1" ]] ; then dpkg -i /nv-tensorrt-repo-*.deb ; fi; \ apt-key adv --keyserver-options http-proxy=$http_proxy --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub ; \ add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"; \ - apt-get update && apt install -y \ + apt-get update && apt-get install --no-install-recommends -y \ libzstd-dev \ cuda-11-8 \ libcudnn8 \ @@ -118,9 +113,7 @@ RUN if [ "$NVIDIA" == "1" ] ; then true ; else exit 0 ; fi ; \ chmod a+x /usr/local/bin/sccache ; \ curl https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-x86_64.tar.gz -L | tar xzvC /usr/local --exclude={doc,man} --strip-components=1 ; \ curl -L https://github.com/ccache/ccache/releases/download/v4.3/ccache-4.3.tar.xz | tar xJv ; \ - cd ccache-4.3 ; \ - mkdir build ; \ - cd build ; \ + mkdir -p ccache-4.3/build ; cd ccache-4.3/build ; \ cmake -DCMAKE_BUILD_TYPE=Release -G Ninja .. ; \ ninja -v install @@ -131,14 +124,14 @@ RUN curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHT curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ chmod +x bazel-*.sh && \ ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - cd / && \ rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh ####### Azure SDK needs new boost: WORKDIR /boost -RUN wget https://sourceforge.net/projects/boost/files/boost/1.69.0/boost_1_69_0.tar.gz && \ +# hadolint ignore=DL3003 +RUN wget -nv https://sourceforge.net/projects/boost/files/boost/1.69.0/boost_1_69_0.tar.gz && \ tar xvf boost_1_69_0.tar.gz && cd boost_1_69_0 && ./bootstrap.sh && \ ./b2 cxxstd=17 link=static cxxflags='-fPIC' cflags='-fPIC' \ --with-chrono --with-date_time --with-filesystem --with-program_options --with-system \ @@ -150,28 +143,23 @@ COPY third_party /ovms/third_party/ ####### Azure SDK WORKDIR /azure -RUN apt-get update && apt-get install -y uuid uuid-dev && rm -rf /var/lib/apt/lists/* -RUN git clone https://github.com/Microsoft/cpprestsdk.git && cd cpprestsdk && git checkout tags/v2.10.18 -b v2.10.18 && git submodule update --init - -RUN git clone https://github.com/Azure/azure-storage-cpp.git && cd azure-storage-cpp/Microsoft.WindowsAzure.Storage && git checkout tags/v7.5.0 && mkdir build.release - -WORKDIR / -RUN cp -rf /ovms/third_party/cpprest/rest_sdk_v2.10.16.patch /azure/cpprestsdk/ -RUN cd /azure/cpprestsdk/ && patch -p1 < rest_sdk_v2.10.16.patch -RUN cp -rf /ovms/third_party/azure/azure_sdk.patch /azure/azure-storage-cpp/ -RUN cd /azure/azure-storage-cpp/ && patch -p1 < azure_sdk.patch -WORKDIR /azure +RUN apt-get update && apt-get install --no-install-recommends -y uuid uuid-dev && rm -rf /var/lib/apt/lists/* +RUN git clone --recurse-submodules --depth 1 --branch v2.10.16 https://github.com/Microsoft/cpprestsdk.git && \ + git clone --depth 1 --branch v7.5.0 https://github.com/Azure/azure-storage-cpp.git && \ + patch -d /azure/cpprestsdk/ -p1 < /ovms/third_party/cpprest/rest_sdk_v2.10.16.patch && \ + patch -d /azure/azure-storage-cpp/ -p1 ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml $(bazel info output_path)/_coverage/_coverage_report.dat ; fi ; \ +RUN if [ "$CHECK_COVERAGE" == "1" ] ; then bazel coverage --combined_report=lcov --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || { cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; } && genhtml --output genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" ; fi ; \ bazel test ${debug_bazel_flags} --jobs=$JOBS --test_summary=detailed --test_output=streamed //src:ovms_test > ${TEST_LOG} 2>&1 || (cat ${TEST_LOG} && rm -rf ${TEST_LOG} && exit 1 ; ) && tail -n 100 ${TEST_LOG} && rm -rf ${TEST_LOG} # C api shared library RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:ovms_shared # C api app with bazel +# hadolint ignore=DL3059 RUN bazel build ${debug_bazel_flags} --jobs $JOBS //src:capi_cpp_example + # C-API benchmark app RUN bazel build //src:capi_benchmark && ./bazel-bin/src/capi_benchmark --niter 2 --threads_per_ireq 2 --nireq 1 --servable_name "dummy" --inputs_names "b" --shape "b[1,10]" -# Copy binary for //src:ovms +# OVMS RUN bazel build ${debug_bazel_flags} ${minitrace_flags} --jobs=$JOBS //src:ovms -RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so -RUN cd /ovms/src/example/SampleCpuExtension/ && OPENVINO_PATH=/opt/intel/openvino make +# hadolint ignore=DL3059 +RUN bazel build ${debug_bazel_flags} --jobs=$JOBS //src:libsampleloader.so # C-api C/C++ app with gcc -RUN cd /ovms && make -f MakefileCapi cpp && make -f MakefileCapi c +RUN make -f MakefileCapi cpp && make -f MakefileCapi c ARG ovms_metadata_file COPY ${ovms_metadata_file} metadata.json -RUN cd bazel-bin/src/ && ./ovms --version && ./ovms && cd /ovms +RUN /ovms/bazel-bin/src/ovms --version && /ovms/bazel-bin/src/ovms COPY release_files/thirdparty-licenses/ /ovms/release_files/thirdparty-licenses/ COPY release_files/LICENSE /ovms/release_files/LICENSE diff --git a/Makefile b/Makefile index 3631e434c9..09abb504b2 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,7 @@ style: venv clang-format sdl-check: venv @echo "Checking SDL requirements..." @echo "Checking docker files..." - @bash -c "if [ $$(find . -type f -name 'Dockerfile.*' -exec grep ADD {} \; | wc -l | xargs ) -eq 0 ]; then echo 'ok'; else echo 'replace ADD with COPY in dockerfiles'; exit 1 ; fi" + @./tests/hadolint.sh @echo "Checking python files..." @. $(ACTIVATE); bash -c "bandit -x demos/benchmark/python -r demos/*/python > bandit.txt" diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index f0aaceba25..22280a45d5 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2021 Intel Corporation +# Copyright (c) 2020-2022 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,12 +16,11 @@ ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.7 FROM $BASE_IMAGE as base_build -RUN yum install -y xz +RUN yum install -y xz && yum clean all WORKDIR / COPY ovms.tar.xz / RUN env -RUN tar -xf ovms.tar.xz -RUN groupadd --gid 5000 ovms && \ +RUN tar -xf ovms.tar.xz && groupadd --gid 5000 ovms && \ useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --shell /bin/bash --skel /dev/null ovms && \ chown -R ovms:ovms /ovms RUN mkdir /licenses && ln -s /ovms/LICENSE /licenses && ln -s /ovms/thirdparty-licenses /licenses/thirdparty-licenses @@ -43,9 +42,9 @@ ENV INSTALL_DRIVER_VERSION=$INSTALL_DRIVER_VERSION ARG GPU=1 ENV GPU=$GPU WORKDIR / -RUN set -e ; \ - set -x ; \ - microdnf upgrade -y ; \ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=DL3003,DL3041,SC2164 +RUN microdnf upgrade -y ; \ microdnf install -y pkg-config && rpm -ivh https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm && \ if [ "$GPU" == "1" ] ; then \ case $INSTALL_DRIVER_VERSION in \ @@ -101,7 +100,7 @@ RUN set -e ; \ rpm -ivh http://mirror.centos.org/centos/8-stream/BaseOS/x86_64/os/Packages/numactl-2.0.12-11.el8.x86_64.rpm; \ rpm -ivh http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/ocl-icd-2.2.12-1.el8.x86_64.rpm; \ else \ - microdnf install tar gzip; \ + microdnf install -y tar gzip; \ mkdir /tmp_ovms ; \ cd /tmp_ovms ; \ curl -L --fail -o deps.tar.xz "$INSTALL_RPMS_FROM_URL" ; \ @@ -111,7 +110,7 @@ RUN set -e ; \ cd / ; \ rm -rf /tmp_ovms ; \ fi ; \ - microdnf install shadow-utils; \ + microdnf install -y shadow-utils; \ cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt ; \ groupadd --gid 5000 ovms && groupadd --gid 44 video1 && \ useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --groups 39,44 --shell /bin/bash --skel /dev/null ovms @@ -119,4 +118,4 @@ RUN set -e ; \ COPY --from=base_build /ovms /ovms COPY --from=base_build /licenses /licenses USER ovms -ENTRYPOINT ["/ovms/bin/ovms"] +ENTRYPOINT ["/ovms/bin/ovms"] \ No newline at end of file diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index baf49328b1..ad24f39ec1 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2020-2022 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -49,6 +49,7 @@ RUN ./bootstrap.sh && \ make -j4 WORKDIR /opt/libusb-1.0.22/libusb +# hadolint ignore=DL3003 RUN /bin/mkdir -p '/usr/local/lib' && \ /bin/bash ../libtool --mode=install /usr/bin/install -c libusb-1.0.la '/usr/local/lib' && \ /bin/mkdir -p '/usr/local/include/libusb-1.0' && \ @@ -80,13 +81,12 @@ SHELL ["/bin/bash", "-c"] WORKDIR / COPY drivers /drivers - -RUN set -e ; \ - set -x ; \ - apt update -y ; \ - apt install -y curl libpugixml1v5 libtbb2 && \ +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=DL3003,SC2164 +RUN apt-get update -y ; \ + apt-get install -y curl libpugixml1v5 libtbb2 --no-install-recommends && \ if [ "$GPU" == "1" ] ; then \ - apt-get update && apt-get install -y libnuma1 ocl-icd-libopencl1 && rm -rf /var/lib/apt/lists/* && \ + apt-get update && apt-get install -y ca-certificates libnuma1 ocl-icd-libopencl1 --no-install-recommends && rm -rf /var/lib/apt/lists/* && \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \ @@ -117,27 +117,27 @@ RUN set -e ; \ dpkg -i intel*.deb && rm -Rf /tmp/gpu_deps ; \ ;; \ "22.35.24055") \ - apt-get update && apt-get install -y gpg-agent && \ - curl https://repositories.intel.com/graphics/intel-graphics.key |gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \ + apt-get update && apt-get install -y --no-install-recommends gpg gpg-agent && \ + curl https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \ echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu focal-legacy main' | tee /etc/apt/sources.list.d/intel.gpu.focal.list && \ apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ intel-opencl-icd=22.35.24055+i815~u20.04 \ intel-level-zero-gpu=1.3.24055+i815~u20.04 \ level-zero=1.8.5+i815~u20.04 && \ - apt-get purge gpg-agent --yes && apt-get --yes autoremove && \ - apt clean ; \ + apt-get purge gpg gpg-agent --yes && apt-get --yes autoremove && \ + apt-get clean ; \ rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* ; \ ;; \ *) \ dpkg -P intel-gmmlib intel-igc-core intel-igc-opencl intel-level-zero-gpu intel-ocloc intel-opencl intel-opencl-icd && \ - apt-get update && apt-get -y install dpkg-dev && rm -rf /var/lib/apt/lists/* && \ + apt-get update && apt-get -y --no-install-recommends install dpkg-dev && rm -rf /var/lib/apt/lists/* && \ cd /drivers/${INSTALL_DRIVER_VERSION} && \ dpkg-scanpackages . > Packages && \ cd - ; \ echo "deb [trusted=yes arch=amd64] file:/drivers/${INSTALL_DRIVER_VERSION} ./" > /etc/apt/sources.list.d/intel-graphics-${INSTALL_DRIVER_VERSION}.list ; \ apt-get update && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ intel-opencl-icd \ intel-level-zero-gpu level-zero \ intel-media-va-driver-non-free libmfx1 && \ @@ -154,26 +154,16 @@ COPY --from=base_build /usr/local/lib/ /ovms/lib/ COPY --from=base_build /ovms /ovms # for HDDL -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - libboost-filesystem1.71.0 \ - libboost-thread1.71.0 \ - libboost-thread1.71.0 \ - libboost-program-options1.71.0 \ - libboost-chrono1.71.0 \ - libboost-date-time1.71.0 \ - libboost-atomic1.71.0 \ - libjson-c4 && \ - rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + # for NCS RUN if [ -f /ovms/lib/hddl/etc/rules.d/97-myriad-usbboot.rules ]; then mkdir -p /etc/udev/rules.d/ && cp /ovms/lib/hddl/etc/rules.d/97-myriad-usbboot.rules /etc/udev/rules.d/ && ldconfig ; fi # for NVIDIA RUN if [ "$NVIDIA" == "1" ]; then true ; else exit 0 ; fi ; echo "installing cuda apt package"; \ - apt update -y && \ - apt install -y curl && \ - cd etc/apt/preferences.d && curl -o cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin ; \ - apt update && apt install -y --no-install-recommends libcudnn8 libcutensor1 libpng16-16 && \ - apt clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + apt-get update -y && \ + apt-get install -y --no-install-recommends curl && \ + curl -o /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin ; \ + apt-get update && apt-get install -y --no-install-recommends libcudnn8 libcutensor1 libpng16-16 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/*qg RUN echo "The source code of added GPL components is stored in https://storage.openvinotoolkit.org/repositories/openvino/ci_dependencies/container_gpl_sources/ubuntu20/" > /ovms/thirdparty-licenses/GPL.txt USER ovms diff --git a/tests/hadolint.sh b/tests/hadolint.sh new file mode 100755 index 0000000000..bb1c9a554a --- /dev/null +++ b/tests/hadolint.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -o pipefail + +files_to_scan=( + "./release_files/Dockerfile.ubuntu" + "./release_files/Dockerfile.redhat" + "./Dockerfile.ubuntu" + "./Dockerfile.redhat" +) + +files_no_proxy_setting=( + "./release_files/Dockerfile.ubuntu" + "./release_files/Dockerfile.redhat" +) + +docker run --rm -i hadolint/hadolint:latest hadolint -v -V + +has_issues=0 +while IFS= read -r -d '' dockerfile + do + if printf '%s\0' "${files_to_scan[@]}" | grep -Fxqz -- $dockerfile; then + echo "Scanning $dockerfile with sha256: $(sha256sum $dockerfile | head -n1 | cut -d " " -f1)" + docker run --rm -i hadolint/hadolint:latest hadolint \ + --ignore DL3006 \ + --ignore DL3008 \ + --ignore DL3013 \ + --ignore DL3016 \ + --ignore DL3018 \ + --ignore DL3028 \ + --ignore DL3033 \ + --ignore DL4001 \ + - < "$dockerfile" || has_issues=1 + else + echo "Skipping $dockerfile" + fi + if printf '%s\0' "${files_no_proxy_setting[@]}" | grep -Fxqz -- $dockerfile; then + echo "Searching for proxy in $dockerfile" + grep -in proxy "$dockerfile" && has_issues=1 || true + fi + done < <(find ./ \( -name 'Dockerfile*' \) -print0) + +exit "$has_issues" \ No newline at end of file From 3b132c869c4341c0d3d8a09f5345d95777e6c253 Mon Sep 17 00:00:00 2001 From: Bartosz Strzelecki Date: Fri, 16 Dec 2022 13:28:32 +0100 Subject: [PATCH 116/130] KSereve c++ samples with resnet (#1439) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added cmake lists * added infer example with dummy * args fix * removed redundant defaults * added documentation for grpc infer dummy * Apply suggestions from code review Co-authored-by: Miłosz Żeglarski * converted ns to ms * removed redundant output * added server live to readme * added mentions of dummy model * changed file extensions from .cc to .cpp * image classification * converted ns to ms * minor fixes * minor fix * added resnet download * changed file extensions from .cc to .cpp * sending native request * cleanup * doc fix * param fix * removed opencv * rebase fix * added rest infer * minor doc fix * minor fix * rest doc fix * fixed stats * doc fix * rps fix * updates for documentation test * fixed help msg * review fix * added error handling * Update client/cpp/kserve-api/README.md Co-authored-by: Miłosz Żeglarski * fix * fix * help fix * rest fix Co-authored-by: Miłosz Żeglarski Co-authored-by: ngrozae --- client/cpp/kserve-api/README.md | 207 +++++++++++++----- client/cpp/kserve-api/samples/CMakeLists.txt | 3 +- .../kserve-api/samples/grpc_infer_resnet.cpp | 185 ++++++++++++++++ .../kserve-api/samples/http_infer_dummy.cpp | 8 +- .../kserve-api/samples/http_infer_resnet.cpp | 197 +++++++++++++++++ .../samples/http_model_metadata.cpp | 6 +- .../kserve-api/samples/http_model_ready.cpp | 6 +- .../kserve-api/samples/http_server_live.cpp | 6 +- .../samples/http_server_metadata.cpp | 6 +- .../kserve-api/samples/http_server_ready.cpp | 8 +- 10 files changed, 560 insertions(+), 72 deletions(-) create mode 100644 client/cpp/kserve-api/samples/grpc_infer_resnet.cpp create mode 100644 client/cpp/kserve-api/samples/http_infer_resnet.cpp diff --git a/client/cpp/kserve-api/README.md b/client/cpp/kserve-api/README.md index d85a2d3e02..e103f0ad27 100644 --- a/client/cpp/kserve-api/README.md +++ b/client/cpp/kserve-api/README.md @@ -4,21 +4,19 @@ OpenVINO Model Server introduced support for [KServe API](https://github.com/kse This guide shows how to interact with KServe API endpoints on both gRPC and HTTP interfaces using [Triton](https://github.com/triton-inference-server)'s client library. It covers following topics: - GRPC API Examples - - grpc_server_live.py - - grpc_server_ready.py - - grpc_server_metadata.py - - grpc_model_ready.py - - grpc_model_metadata.py - - grpc_infer_resnet.py - - grpc_infer_binary_resnet.py + - grpc_server_live + - grpc_server_ready + - grpc_server_metadata + - grpc_model_ready + - grpc_model_metadata + - grpc_infer_dummy - HTTP API Example - - http_server_live.py - - http_server_ready.py - - http_server_metadata.py - - http_model_ready.py - - http_model_metadata.py - - http_infer_resnet.py - - http_infer_binary_resnet.py + - http_server_live + - http_server_ready + - http_server_metadata + - http_model_ready + - http_model_metadata + - http_infer_dummy ## Before you run the samples @@ -30,7 +28,7 @@ cd model_server ### Start the Model Server Container with Dummy Model ```Bash -docker run --rm -d -v $(pwd)/src/test/dummy:/models -p 9000:9000 openvino/model_server:latest --model_name dummy --model_path /models --port 9000 +docker run --rm -d -v $(pwd)/src/test/dummy:/models -p 9000:9000 -p 8000:8000 openvino/model_server:latest --model_name dummy --model_path /models --port 9000 --rest_port 8000 ``` ### Build client library and samples @@ -47,9 +45,6 @@ cd samples This section demonstrates inference on a simple model, which increments each provided value. - -Once you finish above steps, you are ready to run the samples. - ### Run the Client to get server liveness - Command @@ -125,8 +120,8 @@ Usage: ```Bash ./grpc_server_metadata --grpc_port 9000 --grpc_address localhost -Name: "OpenVINO Model Server" -Version: "2022.2.c290da85" +Name: OpenVINO Model Server +Version: 2022.2.c290da85 ``` ### Run the Client to get model readiness @@ -155,7 +150,7 @@ Usage: - Usage Example ```Bash -./grpc_model_ready --grpc_port 9000 --grpc_address localhost --model_name resnet +./grpc_model_ready --grpc_port 9000 --grpc_address localhost --model_name dummy Model Ready: True ``` @@ -185,24 +180,21 @@ Usage: - Usage Example ```Bash -./grpc_model_metadata --grpc_port 9000 --grpc_address localhost --model_name resnet -model metadata: -name: "resnet" +./grpc_model_metadata --grpc_port 9000 --grpc_address localhost --model_name dummy +name: "dummy" versions: "1" platform: "OpenVINO" inputs { - name: "0" + name: "b" datatype: "FP32" shape: 1 - shape: 224 - shape: 224 - shape: 3 + shape: 10 } outputs { - name: "1463" + name: "a" datatype: "FP32" shape: 1 - shape: 1000 + shape: 10 } ``` ### Run the Client to perform inference @@ -258,15 +250,15 @@ Requests per second: 189.041 ```Bash ./http_server_live --help -Sends requests via KServe rest API to check if server is alive. +Sends requests via KServe REST API to check if server is alive. Usage: http_server_live [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --timeout TIMEOUT Request timeout. (default: 0) ``` @@ -284,15 +276,15 @@ Server Live: True ```Bash ./http_server_ready --help -Sends requests via KServe rest API to check if server is ready. +Sends requests via KServe REST API to check if server is ready. Usage: http_server_ready [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --timeout TIMEOUT Request timeout. (default: 0) ``` @@ -310,15 +302,15 @@ Server Ready: True ```Bash ./http_server_metadata --help -Sends requests via KServe rest API to get server metadata. +Sends requests via KServe REST API to get server metadata. Usage: http_server_metadata [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --timeout TIMEOUT Request timeout. (default: 0) ``` @@ -327,7 +319,7 @@ Usage: ```Bash ./http_server_metadata --http_port 8000 --http_address localhost -{'name': 'OpenVINO Model Server', 'version': '2022.2.c290da85'} +{"name":"OpenVINO Model Server","version":"2022.2.c290da85"} ``` ### Run the Client to get model readiness @@ -336,15 +328,15 @@ Usage: ```Bash ./http_model_ready --help -Sends requests via KServe rest API to check if model is ready for inference. +Sends requests via KServe REST API to check if model is ready for inference. Usage: http_model_ready [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --model_name MODEL_NAME Define model name, must be same as is in service. (default: dummy) @@ -366,15 +358,15 @@ Model Ready: True ```Bash ./http_model_metadata --help -Sends requests via KServe rest API to get model metadata. +Sends requests via KServe REST API to get model metadata. Usage: http_ready [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --model_name MODEL_NAME Define model name, must be same as is in service. (default: dummy) @@ -394,16 +386,16 @@ Usage: - Command ```Bash -./http_infer_resnet --help -Sends requests via KServe rest API. +./http_infer_dummy --help +Sends requests via KServe REST API. Usage: http_infer_dummy [OPTION...] -h, --help Show this help message and exit --http_address HTTP_ADDRESS - Specify url to grpc service. (default: + Specify url to REST service. (default: localhost) - --http_port PORT Specify port to grpc service. (default: + --http_port PORT Specify port to REST service. (default: 8000) --input_name INPUT_NAME Specify input tensor name. (default: b) --output_name OUTPUT_NAME @@ -418,7 +410,7 @@ Usage: - Usage Example ```Bash -./http_infer_resnety --http_port 8000 +./http_infer_dummy --http_port 8000 0 => 1 1 => 2 2 => 3 @@ -434,4 +426,117 @@ Number of requests: 1 Total processing time: 2.18683 ms Latency: 2.18683 ms Requests per second: 457.283 +``` + +## Examples with Resnet Model + +### Download the Pretrained Model +Download the model files and store them in the `models` directory +```Bash +mkdir -p models/resnet/1 +curl https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.bin https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin/2/resnet50-binary-0001/FP32-INT1/resnet50-binary-0001.xml -o models/resnet/1/resnet50-binary-0001.bin -o models/resnet/1/resnet50-binary-0001.xml +``` + +### Start the Model Server Container with Resnet Model +```Bash +docker run --rm -d -v $(pwd)/models:/models -p 9000:9000 -p 8000:8000 openvino/model_server:latest --model_name resnet --model_path /models/resnet --port 9000 --rest_port 8000 --layout NHWC:NCHW --plugin_config '{"PERFORMANCE_HINT":"LATENCY"}' +``` + +Once you finish above steps, you are ready to run the samples. + +### Run the Client to perform inference using gRPC API +```Bash +./grpc_infer_resnet --help +Sends requests via KServe gRPC API. +Usage: + grpc_infer_resnet [OPTION...] + + -h, --help Show this help message and exit + --images_list IMAGES Path to a file with a list of labeled + images. + --labels_list LABELS Path to a file with a list of labels. + --grpc_address GRPC_ADDRESS + Specify url to grpc service. (default: + localhost) + --grpc_port PORT Specify port to grpc service. (default: + 9000) + --input_name INPUT_NAME Specify input tensor name. (default: 0) + --output_name OUTPUT_NAME + Specify input tensor name. (default: 1463) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: resnet) + --model_version MODEL_VERSION + Define model version. + --timeout TIMEOUT Request timeout. (default: 0) +``` + +- Usage Example + +```Bash +./grpc_infer_resnet --images_list resnet_input_images.txt --labels_list resnet_labels.txt --grpc_port 9000 +../../../../demos/common/static/images/airliner.jpeg classified as 404 airliner +../../../../demos/common/static/images/arctic-fox.jpeg classified as 279 Arctic fox, white fox, Alopex lagopus +../../../../demos/common/static/images/bee.jpeg classified as 309 bee +../../../../demos/common/static/images/golden_retriever.jpeg classified as 207 golden retriever +../../../../demos/common/static/images/gorilla.jpeg classified as 366 gorilla, Gorilla gorilla +../../../../demos/common/static/images/magnetic_compass.jpeg classified as 635 magnetic compass +../../../../demos/common/static/images/peacock.jpeg classified as 84 peacock +../../../../demos/common/static/images/pelican.jpeg classified as 144 pelican +../../../../demos/common/static/images/snail.jpeg classified as 113 snail +../../../../demos/common/static/images/zebra.jpeg classified as 340 zebra +Accuracy 100% +======Client Statistics====== +Number of requests: 10 +Total processing time: 96.651 ms +Latency: 9.6651 ms +Requests per second: 103.465 +``` + +### Run the Client to perform inference using REST API +```Bash +./http_infer_resnet --help +Sends requests via KServe REST API. +Usage: + http_infer_resnet [OPTION...] + + -h, --help Show this help message and exit + --images_list IMAGES Path to a file with a list of labeled + images. + --labels_list LABELS Path to a file with a list of labels. + --http_address GRPC_ADDRESS + Specify url to REST service. (default: + localhost) + --http_port PORT Specify port to REST service. (default: + 9000) + --input_name INPUT_NAME Specify input tensor name. (default: 0) + --output_name OUTPUT_NAME + Specify input tensor name. (default: 1463) + --model_name MODEL_NAME Define model name, must be same as is in + service. (default: resnet) + --model_version MODEL_VERSION + Define model version. + --timeout TIMEOUT Request timeout. (default: 0) + +``` + +- Usage Example + +```Bash + ./http_infer_resnet --images_list resnet_input_images.txt --labels_list resnet_labels.txt --http_port 8000 +../../../../demos/common/static/images/airliner.jpeg classified as 404 airliner +../../../../demos/common/static/images/zebra.jpeg classified as 340 zebra +../../../../demos/common/static/images/arctic-fox.jpeg classified as 279 Arctic fox, white fox, Alopex lagopus +../../../../demos/common/static/images/bee.jpeg classified as 309 bee +../../../../demos/common/static/images/golden_retriever.jpeg classified as 207 golden retriever +../../../../demos/common/static/images/gorilla.jpeg classified as 366 gorilla, Gorilla gorilla +../../../../demos/common/static/images/magnetic_compass.jpeg classified as 635 magnetic compass +../../../../demos/common/static/images/peacock.jpeg classified as 84 peacock +../../../../demos/common/static/images/pelican.jpeg classified as 144 pelican +../../../../demos/common/static/images/snail.jpeg classified as 113 snail +Accuracy 100% +======Client Statistics====== +Number of requests: 10 +Total processing time: 115.804 ms +Latency: 11.5804 ms +Requests per second: 86.3526 ``` \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/CMakeLists.txt b/client/cpp/kserve-api/samples/CMakeLists.txt index bc7f15e3c8..c867dc3bb5 100644 --- a/client/cpp/kserve-api/samples/CMakeLists.txt +++ b/client/cpp/kserve-api/samples/CMakeLists.txt @@ -27,8 +27,9 @@ function(sample sample_name) endfunction() sample(infer_dummy) +sample(infer_resnet) sample(server_live) sample(server_ready) sample(server_metadata) sample(model_ready) -sample(model_metadata) \ No newline at end of file +sample(model_metadata) diff --git a/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp b/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp new file mode 100644 index 0000000000..52638dc09f --- /dev/null +++ b/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp @@ -0,0 +1,185 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include + +#include "grpc_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + return 1; \ + } \ + } + +std::vector Load(const std::string& fileName) { + std::ifstream fileImg(fileName, std::ios::binary); + fileImg.seekg(0, std::ios::end); + int bufferLength = fileImg.tellg(); + fileImg.seekg(0, std::ios::beg); + + char* buffer = new char[bufferLength]; + fileImg.read(buffer, bufferLength); + + return std::vector(buffer, buffer + bufferLength); +} + +int main(int argc, char** argv) { + cxxopts::Options opt("grpc_infer_resnet", "Sends requests via KServe gRPC API."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("images_list", "Path to a file with a list of labeled images. ", cxxopts::value(), "IMAGES") + ("labels_list", "Path to a file with a list of labels. ", cxxopts::value(), "LABELS") + ("grpc_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("grpc_port", "Specify port to grpc service. ", cxxopts::value()->default_value("9000"), "PORT") + ("input_name", "Specify input tensor name. ", cxxopts::value()->default_value("0"), "INPUT_NAME") + ("output_name", "Specify input tensor name. ", cxxopts::value()->default_value("1463"), "OUTPUT_NAME") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("resnet"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value(), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + if (!args.count("images_list")) { + std::cout << "error: option \"images_list\" has no value\n"; + return 1; + } + if (!args.count("labels_list")) { + std::cout << "error: option \"labels_list\" has no value\n"; + return 1; + } + if (args.count("help")) { + std::cout << opt.help() << std::endl; + return 0; + } + + std::string input_name(args["input_name"].as()); + std::string output_name(args["output_name"].as()); + + std::string url(args["grpc_address"].as() + ":" + args["grpc_port"].as()); + std::string model_name = args["model_name"].as(); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerGrpcClient::Create(&client, url), + err); + + std::string img; + int label = -1; + std::vector imgs; + std::vector labels; + std::ifstream images(args["images_list"].as()); + while (images >> img >> label) { + imgs.push_back(img); + labels.push_back(label); + } + + std::vector shape{1}; + + // Initialize the inputs with the data. + tc::InferInput* input; + + FAIL_IF_ERR( + tc::InferInput::Create(&input, input_name, shape, "BYTES"), + "unable to get input"); + std::shared_ptr input_ptr; + input_ptr.reset(input); + + tc::InferOptions options(model_name); + if (args.count("model_version")) + options.model_version_ = args["model_version"].as(); + try { + options.client_timeout_ = args["timeout"].as(); + } catch (cxxopts::argument_incorrect_type e) { + std::cout << "The provided argument is of a wrong type" << std::endl; + return 1; + } + std::vector inputs = {input_ptr.get()}; + + std::vector results; + results.resize(imgs.size()); + for (int i = 0; i < imgs.size(); i++) { + std::vector input_data = Load(imgs[i]); + FAIL_IF_ERR( + input_ptr->AppendRaw(input_data), + "unable to set data for input"); + FAIL_IF_ERR( + client->Infer(&(results[i]), options, inputs), + "unable to run model"); + input->Reset(); + } + + std::vector classes; + std::ifstream lb_f(args["labels_list"].as()); + std::string tmp; + while (std::getline(lb_f, tmp)) { + classes.push_back(tmp); + } + + int acc = 0; + for (int i = 0; i < imgs.size(); i++) { + std::shared_ptr results_ptr; + results_ptr.reset(results[i]); + // Get pointers to the result returned... + float* output_data; + size_t output_byte_size; + FAIL_IF_ERR( + results_ptr->RawData( + output_name, (const uint8_t**)&output_data, &output_byte_size), + "unable to get result data for output"); + + int lb = std::distance(output_data, std::max_element(output_data, output_data + 1000)); + std::cout << imgs[i] << " classified as " + << lb << " " << classes[lb] << " "; + if (lb != labels[i]) { + std::cout << "should be " << labels[i] << " " << classes[labels[i]]; + } else { + acc++; + } + std::cout << std::endl; + } + + std::cout << "Accuracy " << float(acc) / imgs.size() * 100 << "%\n"; + + tc::InferStat infer_stat; + client->ClientInferStat(&infer_stat); + std::cout << "======Client Statistics======" << std::endl; + std::cout << "Number of requests: " + << infer_stat.completed_request_count << std::endl; + std::cout << "Total processing time: " + << double(infer_stat.cumulative_total_request_time_ns) / 1.0e+6 << " ms" << std::endl; + std::cout << "Latency: " + << double(infer_stat.cumulative_total_request_time_ns / infer_stat.completed_request_count) / 1.0e+6 << " ms" << std::endl; + std::cout << "Requests per second: " + << double(1.0e+9 / (infer_stat.cumulative_total_request_time_ns / infer_stat.completed_request_count)) << std::endl; + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_infer_dummy.cpp b/client/cpp/kserve-api/samples/http_infer_dummy.cpp index 7f4d1b310a..9b5d588df3 100644 --- a/client/cpp/kserve-api/samples/http_infer_dummy.cpp +++ b/client/cpp/kserve-api/samples/http_infer_dummy.cpp @@ -59,13 +59,13 @@ void ValidateShapeAndDatatype( } // namespace int main(int argc, char** argv) { - cxxopts::Options opt("http_infer_dummy", "Sends requests via KServe rest API."); + cxxopts::Options opt("http_infer_dummy", "Sends requests via KServe REST API."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") ("input_name", "Specify input tensor name. ", cxxopts::value()->default_value("b"), "INPUT_NAME") ("output_name", "Specify input tensor name. ", cxxopts::value()->default_value("a"), "OUTPUT_NAME") ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") @@ -120,7 +120,7 @@ int main(int argc, char** argv) { tc::InferRequestedOutput* output; FAIL_IF_ERR( - tc::InferRequestedOutput::Create(&output, "a"), + tc::InferRequestedOutput::Create(&output, output_name), "unable to get output"); std::shared_ptr output_ptr; output_ptr.reset(output); diff --git a/client/cpp/kserve-api/samples/http_infer_resnet.cpp b/client/cpp/kserve-api/samples/http_infer_resnet.cpp new file mode 100644 index 0000000000..75579ddd8e --- /dev/null +++ b/client/cpp/kserve-api/samples/http_infer_resnet.cpp @@ -0,0 +1,197 @@ +//***************************************************************************** +// Copyright 2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include + +#include "http_client.h" + +namespace tc = triton::client; + +#define FAIL_IF_ERR(X, MSG) \ + { \ + tc::Error err = (X); \ + if (!err.IsOk()) { \ + std::cerr << "error: " << (MSG) << ": " << err << std::endl; \ + return 1; \ + } \ + } + +std::vector Load(const std::string& fileName) { + std::ifstream fileImg(fileName, std::ios::binary); + fileImg.seekg(0, std::ios::end); + int bufferLength = fileImg.tellg(); + fileImg.seekg(0, std::ios::beg); + + char* buffer = new char[bufferLength]; + fileImg.read(buffer, bufferLength); + + return std::vector(buffer, buffer + bufferLength); +} + +int main(int argc, char** argv) { + cxxopts::Options opt("http_infer_resnet", "Sends requests via KServe REST API."); + + // clang-format off + opt.add_options() + ("h,help", "Show this help message and exit") + ("images_list", "Path to a file with a list of labeled images. ", cxxopts::value(), "IMAGES") + ("labels_list", "Path to a file with a list of labels. ", cxxopts::value(), "LABELS") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "GRPC_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("9000"), "PORT") + ("input_name", "Specify input tensor name. ", cxxopts::value()->default_value("0"), "INPUT_NAME") + ("output_name", "Specify input tensor name. ", cxxopts::value()->default_value("1463"), "OUTPUT_NAME") + ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("resnet"), "MODEL_NAME") + ("model_version", "Define model version.", cxxopts::value(), "MODEL_VERSION") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") + ; + // clang-format on + + auto args = opt.parse(argc, argv); + + if (args.count("help")) { + std::cout << opt.help() << std::endl; + return 0; + } + + if (!args.count("images_list")) { + std::cout << "error: option \"images_list\" has no value\n"; + return 1; + } + if (!args.count("labels_list")) { + std::cout << "error: option \"labels_list\" has no value\n"; + return 1; + } + std::string input_name(args["input_name"].as()); + std::string output_name(args["output_name"].as()); + + std::string url(args["http_address"].as() + ":" + args["http_port"].as()); + std::string model_name = args["model_name"].as(); + + // Create a InferenceServerGrpcClient instance to communicate with the + // server using gRPC protocol. + std::unique_ptr client; + + FAIL_IF_ERR( + tc::InferenceServerHttpClient::Create(&client, url), + err); + + std::string img; + int label = -1; + std::vector imgs; + std::vector labels; + std::ifstream images(args["images_list"].as()); + while (images >> img >> label) { + imgs.push_back(img); + labels.push_back(label); + } + + std::vector shape{1}; + + // Initialize the inputs with the data. + tc::InferInput* input; + + FAIL_IF_ERR( + tc::InferInput::Create(&input, input_name, shape, "BYTES"), + "unable to get input"); + std::shared_ptr input_ptr; + input_ptr.reset(input); + + tc::InferOptions options(model_name); + if (args.count("model_version")) + options.model_version_ = args["model_version"].as(); + try { + options.client_timeout_ = args["timeout"].as(); + } catch (cxxopts::argument_incorrect_type e) { + std::cout << "The provided argument is of a wrong type" << std::endl; + return 0; + } + + tc::InferRequestedOutput* output; + + FAIL_IF_ERR( + tc::InferRequestedOutput::Create(&output, output_name), + "unable to get output"); + std::shared_ptr output_ptr; + output_ptr.reset(output); + + std::vector inputs = {input_ptr.get()}; + std::vector outputs = {output_ptr.get()}; + + std::vector results; + results.resize(imgs.size()); + for (int i = 0; i < imgs.size(); i++) { + std::vector input_data = Load(imgs[i]); + FAIL_IF_ERR( + input_ptr->AppendRaw(input_data), + "unable to set data for input"); + FAIL_IF_ERR( + client->Infer(&(results[i]), options, inputs, outputs), + "unable to run model"); + input->Reset(); + } + + std::vector classes; + std::ifstream lb_f(args["labels_list"].as()); + std::string tmp; + while (std::getline(lb_f, tmp)) { + classes.push_back(tmp); + } + + int acc = 0; + for (int i = 0; i < imgs.size(); i++) { + std::shared_ptr results_ptr; + results_ptr.reset(results[i]); + // Get pointers to the result returned... + float* output_data; + size_t output_byte_size; + + FAIL_IF_ERR( + results_ptr->RawData( + output_name, (const uint8_t**)&output_data, &output_byte_size), + "unable to get result data for output"); + + int lb = std::distance(output_data, std::max_element(output_data, output_data + 1000)); + std::cout << imgs[i] << " classified as " + << lb << " " << classes[lb] << " "; + if (lb != labels[i]) { + std::cout << "should be " << labels[i] << " " << classes[labels[i]]; + } else { + acc++; + } + std::cout << std::endl; + } + + std::cout << "Accuracy " << float(acc) / imgs.size() * 100 << "%\n"; + + tc::InferStat infer_stat; + client->ClientInferStat(&infer_stat); + std::cout << "======Client Statistics======" << std::endl; + std::cout << "Number of requests: " + << infer_stat.completed_request_count << std::endl; + std::cout << "Total processing time: " + << double(infer_stat.cumulative_total_request_time_ns) / 1.0e+6 << " ms" << std::endl; + std::cout << "Latency: " + << double(infer_stat.cumulative_total_request_time_ns / infer_stat.completed_request_count) / 1.0e+6 << " ms" << std::endl; + std::cout << "Requests per second: " + << double(1.0e+9 / (infer_stat.cumulative_total_request_time_ns / infer_stat.completed_request_count)) << std::endl; + + return 0; +} \ No newline at end of file diff --git a/client/cpp/kserve-api/samples/http_model_metadata.cpp b/client/cpp/kserve-api/samples/http_model_metadata.cpp index 13fee6e2d7..4abed6d356 100644 --- a/client/cpp/kserve-api/samples/http_model_metadata.cpp +++ b/client/cpp/kserve-api/samples/http_model_metadata.cpp @@ -33,13 +33,13 @@ namespace tc = triton::client; int main(int argc, char** argv) { - cxxopts::Options opt("http_ready", "Sends requests via KServe rest API to get model metadata."); + cxxopts::Options opt("http_ready", "Sends requests via KServe REST API to get model metadata."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") diff --git a/client/cpp/kserve-api/samples/http_model_ready.cpp b/client/cpp/kserve-api/samples/http_model_ready.cpp index db7fc17918..20afe7bd5f 100644 --- a/client/cpp/kserve-api/samples/http_model_ready.cpp +++ b/client/cpp/kserve-api/samples/http_model_ready.cpp @@ -33,13 +33,13 @@ namespace tc = triton::client; int main(int argc, char** argv) { - cxxopts::Options opt("http_model_ready", "Sends requests via KServe rest API to check if model is ready for inference."); + cxxopts::Options opt("http_model_ready", "Sends requests via KServe REST API to check if model is ready for inference."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") ("model_name", "Define model name, must be same as is in service. ", cxxopts::value()->default_value("dummy"), "MODEL_NAME") ("model_version", "Define model version.", cxxopts::value()->default_value(""), "MODEL_VERSION") ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") diff --git a/client/cpp/kserve-api/samples/http_server_live.cpp b/client/cpp/kserve-api/samples/http_server_live.cpp index dd514ef9c3..a903c00fe0 100644 --- a/client/cpp/kserve-api/samples/http_server_live.cpp +++ b/client/cpp/kserve-api/samples/http_server_live.cpp @@ -33,13 +33,13 @@ namespace tc = triton::client; int main(int argc, char** argv) { - cxxopts::Options opt("http_server_live", "Sends requests via KServe rest API to check if server is alive."); + cxxopts::Options opt("http_server_live", "Sends requests via KServe REST API to check if server is alive."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") ; // clang-format on diff --git a/client/cpp/kserve-api/samples/http_server_metadata.cpp b/client/cpp/kserve-api/samples/http_server_metadata.cpp index c909842d86..5e8980d358 100644 --- a/client/cpp/kserve-api/samples/http_server_metadata.cpp +++ b/client/cpp/kserve-api/samples/http_server_metadata.cpp @@ -33,13 +33,13 @@ namespace tc = triton::client; int main(int argc, char** argv) { - cxxopts::Options opt("http_server_metadata", "Sends requests via KServe rest API to get server metadata."); + cxxopts::Options opt("http_server_metadata", "Sends requests via KServe REST API to get server metadata."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000"), "PORT") + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") ; // clang-format on diff --git a/client/cpp/kserve-api/samples/http_server_ready.cpp b/client/cpp/kserve-api/samples/http_server_ready.cpp index d1828bc5d0..596ecc5ef0 100644 --- a/client/cpp/kserve-api/samples/http_server_ready.cpp +++ b/client/cpp/kserve-api/samples/http_server_ready.cpp @@ -33,14 +33,14 @@ namespace tc = triton::client; int main(int argc, char** argv) { - cxxopts::Options opt("http_server_ready", "Sends requests via KServe rest API to check if server is ready."); + cxxopts::Options opt("http_server_ready", "Sends requests via KServe REST API to check if server is ready."); // clang-format off opt.add_options() ("h,help", "Show this help message and exit") - ("http_address", "Specify url to grpc service. ", cxxopts::value()->default_value("localhost")) - ("http_port", "Specify port to grpc service. ", cxxopts::value()->default_value("8000")) - ("timeout", "Request timeout.", cxxopts::value()->default_value("0")) + ("http_address", "Specify url to REST service. ", cxxopts::value()->default_value("localhost"), "HTTP_ADDRESS") + ("http_port", "Specify port to REST service. ", cxxopts::value()->default_value("8000"), "PORT") + ("timeout", "Request timeout.", cxxopts::value()->default_value("0"), "TIMEOUT") ; // clang-format on From 4a7f29d205928edb35c358865d66c065b9acf83f Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Fri, 16 Dec 2022 15:43:08 +0100 Subject: [PATCH 117/130] Remove unnecessary libs (#1605) Co-authored-by: Trawinski, Dariusz --- DockerfileMakePackage | 4 +++- release_files/Dockerfile.redhat | 3 +++ release_files/Dockerfile.ubuntu | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/DockerfileMakePackage b/DockerfileMakePackage index 347fe7a7c7..37164dd535 100644 --- a/DockerfileMakePackage +++ b/DockerfileMakePackage @@ -61,7 +61,9 @@ RUN if [ -f /ovms_release/lib/libjava.so ] ; then true ; else exit 0 ; fi ;cd /o # Remove capi temp libraries RUN if [ -f /ovms_release/lib/libsrc_Slibovms_Ushared.so ] ; then true ; else exit 0 ; fi ;\ - rm -rf /ovms_release/lib/libsrc_Slibovms_Ushared.so + rm -rf /ovms_release/lib/libsrc_Slibovms_Ushared.so \ + /ovms_release/lib/libprediction_service_proto.so-2.params \ + /ovms_release/lib/libovms_shared.so-2.params RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.so*' -exec cp -vP {} /ovms_release/lib/ \; RUN find /opt/intel/openvino/runtime/lib/intel64/ -iname '*.mvcmd*' -exec cp -v {} /ovms_release/lib/ \; diff --git a/release_files/Dockerfile.redhat b/release_files/Dockerfile.redhat index 22280a45d5..2130922850 100644 --- a/release_files/Dockerfile.redhat +++ b/release_files/Dockerfile.redhat @@ -24,6 +24,8 @@ RUN tar -xf ovms.tar.xz && groupadd --gid 5000 ovms && \ useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --shell /bin/bash --skel /dev/null ovms && \ chown -R ovms:ovms /ovms RUN mkdir /licenses && ln -s /ovms/LICENSE /licenses && ln -s /ovms/thirdparty-licenses /licenses/thirdparty-licenses +# Remove capi library +RUN if [ -f /ovms/lib/libovms_shared.so ] ; then rm -rf /ovms/lib/libovms_shared.so ; else exit 0 ; fi ; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 as release @@ -116,6 +118,7 @@ RUN microdnf upgrade -y ; \ useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --groups 39,44 --shell /bin/bash --skel /dev/null ovms COPY --from=base_build /ovms /ovms + COPY --from=base_build /licenses /licenses USER ovms ENTRYPOINT ["/ovms/bin/ovms"] \ No newline at end of file diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index ad24f39ec1..6ab43658e5 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -65,6 +65,8 @@ RUN groupadd --gid 5000 ovms && useradd --home-dir /home/ovms --create-home --ui --gid 5000 --shell /bin/bash --skel /dev/null ovms && \ chown -R ovms:ovms /ovms +# Remove capi library +RUN if [ -f /ovms/lib/libovms_shared.so ] ; then rm -rf /ovms/lib/libovms_shared.so ; fi # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # FROM $BASE_IMAGE as release From 91f0f523846d0c27f3813f9a7bb5ca884fff5e48 Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Sun, 18 Dec 2022 23:50:56 +0100 Subject: [PATCH 118/130] fix ca-certificates installation (#1607) --- release_files/Dockerfile.ubuntu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release_files/Dockerfile.ubuntu b/release_files/Dockerfile.ubuntu index 6ab43658e5..8efee94a79 100644 --- a/release_files/Dockerfile.ubuntu +++ b/release_files/Dockerfile.ubuntu @@ -86,9 +86,9 @@ COPY drivers /drivers SHELL ["/bin/bash", "-o", "pipefail", "-c"] # hadolint ignore=DL3003,SC2164 RUN apt-get update -y ; \ - apt-get install -y curl libpugixml1v5 libtbb2 --no-install-recommends && \ + apt-get install -y curl ca-certificates libpugixml1v5 libtbb2 --no-install-recommends && \ if [ "$GPU" == "1" ] ; then \ - apt-get update && apt-get install -y ca-certificates libnuma1 ocl-icd-libopencl1 --no-install-recommends && rm -rf /var/lib/apt/lists/* && \ + apt-get update && apt-get install -y libnuma1 ocl-icd-libopencl1 --no-install-recommends && rm -rf /var/lib/apt/lists/* && \ case $INSTALL_DRIVER_VERSION in \ "20.35.17767") \ mkdir /tmp/gpu_deps && cd /tmp/gpu_deps ; \ From 0a60c649a47338fc898ec46582a1d8ef47625f8b Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Mon, 19 Dec 2022 13:59:08 +0100 Subject: [PATCH 119/130] Cvs 98446 capi demo (#1601) * Capi Docker --- demos/README.md | 2 + demos/c_api_minimal_app/Makefile | 67 ++++++++++++ demos/c_api_minimal_app/README.md | 100 ++++++++++++++++++ .../capi_files/Dockerfile.redhat | 52 +++++++++ .../capi_files/Dockerfile.ubuntu | 53 ++++++++++ .../capi_files/demos/MakefileCapi | 25 +++++ .../capi_files/demos/config.json | 8 ++ .../demos/config_standard_dummy.json | 10 ++ 8 files changed, 317 insertions(+) create mode 100644 demos/c_api_minimal_app/Makefile create mode 100644 demos/c_api_minimal_app/README.md create mode 100644 demos/c_api_minimal_app/capi_files/Dockerfile.redhat create mode 100644 demos/c_api_minimal_app/capi_files/Dockerfile.ubuntu create mode 100644 demos/c_api_minimal_app/capi_files/demos/MakefileCapi create mode 100644 demos/c_api_minimal_app/capi_files/demos/config.json create mode 100644 demos/c_api_minimal_app/capi_files/demos/config_standard_dummy.json diff --git a/demos/README.md b/demos/README.md index dec904378b..5a3ef52d45 100644 --- a/demos/README.md +++ b/demos/README.md @@ -11,6 +11,7 @@ ovms_demo_optical_character_recognition ovms_demo_face_detection ovms_demo_face_blur_pipeline + ovms_demo_capi_inference_demo ovms_demo_single_face_analysis_pipeline ovms_demo_multi_faces_analysis_pipeline ovms_docs_demo_ensemble @@ -50,6 +51,7 @@ OpenVINO Model Server demos have been created to showcase the usage of the model ## C++ | Demo | Description | |---|---| +|[C API applications](c_api_minimal_app/README.md)|How to use C API from the OpenVINO Model Server to create C and C++ application.| |[Image Classification](image_classification/cpp/README.md)|Run prediction on a JPEG image using image classification model via gRPC API.| |[Benchmark App](benchmark/cpp/README.md)|Generate traffic and measure performance of the model served in OpenVINO Model Server.| diff --git a/demos/c_api_minimal_app/Makefile b/demos/c_api_minimal_app/Makefile new file mode 100644 index 0000000000..8d17604498 --- /dev/null +++ b/demos/c_api_minimal_app/Makefile @@ -0,0 +1,67 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +.DEFAULT_GOAL := all_web + +HTTP_PROXY := "$(http_proxy)" +HTTPS_PROXY := "$(https_proxy)" +NO_PROXY := "$(no_proxy)" +BASE_OS_TAG_UBUNTU ?= 20.04 +BASE_OS_TAG_REDHAT ?= 8.7 +REDHAT_PACKAGE_URL ?="http://repository.toolbox.iotg.sclab.intel.com/projects/ovms-c/2022.3-RC1/RC1/dist/redhat/ovms.tar.gz" +UBUNTU_PACKAGE_URL ?="http://repository.toolbox.iotg.sclab.intel.com/projects/ovms-c/2022.3-RC2/RC2/dist/ubuntu/ovms.tar.gz" + +OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server +OVMS_CPP_IMAGE_TAG ?= latest +BASE_OS ?= ubuntu +DIST_OS ?= $(BASE_OS) + +ifeq ($(BASE_OS),ubuntu) + PACKAGE_URL = ${UBUNTU_PACKAGE_URL} + BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) +endif +ifeq ($(BASE_OS),redhat) + PACKAGE_URL = ${REDHAT_PACKAGE_URL} + BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) + DIST_OS=redhat +endif + +clean: + rm -vrf capi/$(DIST_OS) + mkdir -vp capi/$(DIST_OS)/demos && cd capi/$(DIST_OS) + +from_docker: + docker run $(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG) bash -c \ + "tar -c -C / ovms.tar.gz ; sleep 2" | tar -x -C capi/$(DIST_OS)/ && cd ../../ + -docker rm -v $$(docker ps -a -q -f status=exited -f ancestor=$(OVMS_CPP_DOCKER_IMAGE)-pkg:$(OVMS_CPP_IMAGE_TAG)) + +from_web: + wget -P capi/$(DIST_OS) ${PACKAGE_URL} + +build_image: + cp -vR capi_files/* capi/$(DIST_OS)/ + cp -r ../../src/test/dummy capi/$(DIST_OS)/demos + cp -r ../../src/main_capi.c* capi/$(DIST_OS)/demos + sed -i s+/ovms/src/test/c_api/config.json+/ovms/demos/config.json+ capi/$(DIST_OS)/demos/main_capi.c + sed -i s+/ovms/src/test/c_api/config_standard_dummy.json+/ovms/demos/config_standard_dummy.json+ capi/$(DIST_OS)/demos/main_capi.cpp + cd capi/$(DIST_OS)/ && docker build $(NO_CACHE_OPTION) -f Dockerfile.$(BASE_OS) . \ + --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" \ + --build-arg no_proxy=$(NO_PROXY) \ + --build-arg BASE_IMAGE=$(BASE_IMAGE) \ + -t $(OVMS_CPP_DOCKER_IMAGE)-capi:$(OVMS_CPP_IMAGE_TAG) + +all_web: clean from_web build_image + +all_docker: clean from_docker build_image diff --git a/demos/c_api_minimal_app/README.md b/demos/c_api_minimal_app/README.md new file mode 100644 index 0000000000..75ba1155a5 --- /dev/null +++ b/demos/c_api_minimal_app/README.md @@ -0,0 +1,100 @@ +# C API inference demo (C/C++) {#ovms_demo_capi_inference_demo} + +This demo demonstrate how to use C API from the OpenVINO Model Server to create C and C++ application. +Building the application is executed inside the docker container to illustrate end to end usage flow. + +## Prepare demo image +Enter the directory with the example and build the demo docker with all dependencies and examples that will be named `openvino/model_server-capi`. +The example image also contains dummy model and config.json required for the applications. +```bash +git clone https://github.com/openvinotoolkit/model_server.git +cd demos/c_api_minimal_app +make +``` + +The make command downloads the `ovms.tar.gz` package from web and executes the c_api_minimal_app/capi_files/demos/MakefileCapi to build the applications. +And executes the /ovms/bin/demo1 application and /ovms/bin/demo2_c application in the image environment. + +You can find the source code for the example applications in the ovms repository path src/main_capi.c and src/main_capi.cpp. +or make modifications in the built image: +```bash +docker run -it openvino/model_server-capi:latest +cat main_capi.c +cat main_capi.cpp +``` + +Afterwars rebuild and run the modified examples using the MakefileCapi rules: +```bash +make -f MakefileCapi c +make -f MakefileCapi cpp +``` + +It will link the main_capi.cpp binary with the `libovms_shared.so` library from /ovms/lib and use the headers from /ovms/include directory: +``` +g++ main_capi.cpp -I/ovms/include -L/ovms/lib -lovms_shared +``` + +The example output is: +``` +[2022-12-19 11:39:41.428][14][serving][info][modelinstance.cpp:797] Loaded model dummy; version: 1; batch size: 30; No of InferRequests: 12 +[2022-12-19 11:39:41.428][14][serving][debug][modelversionstatus.cpp:88] setAvailable: dummy - 1 (previous state: LOADING) -> error: OK +[2022-12-19 11:39:41.428][14][serving][info][modelversionstatus.cpp:113] STATUS CHANGE: Version 1 of model dummy status change. New status: ( "state": "AVAILABLE", "error_code": "OK" ) +[2022-12-19 11:39:41.428][14][serving][info][model.cpp:88] Updating default version for model: dummy, from: 0 +[2022-12-19 11:39:41.428][14][serving][info][model.cpp:98] Updated default version for model: dummy, to: 1 +[2022-12-19 11:39:41.428][14][modelmanager][info][modelmanager.cpp:478] Configuration file doesn't have custom node libraries property. +[2022-12-19 11:39:41.428][14][modelmanager][info][modelmanager.cpp:495] Configuration file doesn't have pipelines property. +[2022-12-19 11:39:41.428][14][serving][info][servablemanagermodule.cpp:44] ServableManagerModule started +Server ready for inference +[2022-12-19 11:39:41.428][14][serving][debug][capi.cpp:606] Processing C-API request for model: dummy; version: 1 +[2022-12-19 11:39:41.428][14][serving][debug][modelmanager.cpp:1350] Requesting model: dummy; version: 1. +[2022-12-19 11:39:41.428][14][serving][debug][modelinstance.cpp:1013] Model: dummy, version: 1 already loaded +[2022-12-19 11:39:41.428][14][serving][debug][modelinstance.cpp:1187] Getting infer req duration in model dummy, version 1, nireq 0: 0.002 ms +[2022-12-19 11:39:41.428][478][modelmanager][info][modelmanager.cpp:900] Started model manager thread +[2022-12-19 11:39:41.428][14][serving][debug][modelinstance.cpp:1195] Preprocessing duration in model dummy, version 1, nireq 0: 0.000 ms +[2022-12-19 11:39:41.428][14][serving][debug][modelinstance.cpp:1205] Deserialization duration in model dummy, version 1, nireq 0: 0.019 ms +[2022-12-19 11:39:41.428][479][modelmanager][info][modelmanager.cpp:920] Started cleaner thread +[2022-12-19 11:39:41.429][14][serving][debug][modelinstance.cpp:1213] Prediction duration in model dummy, version 1, nireq 0: 0.369 ms +[2022-12-19 11:39:41.429][14][serving][debug][modelinstance.cpp:1222] Serialization duration in model dummy, version 1, nireq 0: 0.011 ms +[2022-12-19 11:39:41.429][14][serving][debug][modelinstance.cpp:1230] Postprocessing duration in model dummy, version 1, nireq 0: 0.000 ms +[2022-12-19 11:39:41.429][14][serving][debug][capi.cpp:650] Total C-API req processing time: 0.474 ms +output is correct +No more job to be done, will shut down +[2022-12-19 11:39:41.429][14][serving][info][grpcservermodule.cpp:177] GRPCServerModule shutting down +[2022-12-19 11:39:41.429][14][serving][info][grpcservermodule.cpp:180] Shutdown gRPC server +[2022-12-19 11:39:41.429][14][serving][info][grpcservermodule.cpp:184] GRPCServerModule shutdown +[2022-12-19 11:39:41.429][14][serving][info][httpservermodule.cpp:54] HTTPServerModule shutting down +[evhttp_server.cc : 320] NET_LOG: event_base_loopexit() exits with value 0 +[evhttp_server.cc : 253] NET_LOG: event_base_dispatch() exits with value 1 +[2022-12-19 11:39:41.604][14][serving][info][httpservermodule.cpp:59] Shutdown HTTP server +[2022-12-19 11:39:41.604][14][serving][info][servablemanagermodule.cpp:54] ServableManagerModule shutting down +[2022-12-19 11:39:41.604][478][modelmanager][info][modelmanager.cpp:916] Stopped model manager thread +[2022-12-19 11:39:41.604][479][modelmanager][info][modelmanager.cpp:930] Stopped cleaner thread +[2022-12-19 11:39:41.604][14][serving][info][modelmanager.cpp:985] Shutdown model manager +[2022-12-19 11:39:41.604][14][serving][info][modelmanager.cpp:993] Shutdown cleaner thread +[2022-12-19 11:39:41.604][14][serving][info][servablemanagermodule.cpp:57] ServableManagerModule shutdown +``` + +It is also possible to use custom model but that requires copying it to the built image and adjust the configs and example applications accordingly. +To make the changes permanent in the resulting demo image you can modify the specific dockerfiles in c_api_minimal_app/capi_files/ directory +Dockerfile.ubuntu +Dockerfile.redhat + +And run the demo make with os specific arguments: +```bash +cd demos/c_api_minimal_app +make BASE_OS=redhat +``` + +## Build libovms_shared.so +Alternative to getting `ovms.tar.gz` package from web you can build it yourself from sources. To build the capi docker image, you must first build the `ovms.tar.gz` package with the `libovms_shared.so` library and `ovms.h` header. +run `make` command in ovms git main directory. +```bash +git clone https://github.com/openvinotoolkit/model_server.git +make +``` + +And then execute the alternative make target: +```bash +cd demos/c_api_minimal_app +make all_docker +``` \ No newline at end of file diff --git a/demos/c_api_minimal_app/capi_files/Dockerfile.redhat b/demos/c_api_minimal_app/capi_files/Dockerfile.redhat new file mode 100644 index 0000000000..8e371ff94b --- /dev/null +++ b/demos/c_api_minimal_app/capi_files/Dockerfile.redhat @@ -0,0 +1,52 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi:8.7 +FROM $BASE_IMAGE as base_build +SHELL ["/bin/bash", "-c"] +RUN yum install -y xz git shadow-utils make gcc gcc-c++ +WORKDIR / +COPY ovms.tar.gz / +RUN env +RUN tar -xf ovms.tar.gz +RUN groupadd --gid 5000 ovms && \ + useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --shell /bin/bash --skel /dev/null ovms && \ + chown -R ovms:ovms /ovms +RUN mkdir /licenses && ln -s /ovms/LICENSE /licenses && ln -s /ovms/thirdparty-licenses /licenses/thirdparty-licenses + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 as release +SHELL ["/bin/bash", "-c"] + +RUN microdnf upgrade -y +RUN microdnf install shadow-utils make gcc gcc-c++; \ + cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt ; \ + groupadd --gid 5000 ovms && groupadd --gid 44 video1 && \ + useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --groups 39,44 --shell /bin/bash --skel /dev/null ovms + +RUN rpm -ivh https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm + +COPY --from=base_build /ovms /ovms +COPY --from=base_build /licenses /licenses +COPY demos /ovms/demos + +WORKDIR /ovms/demos +RUN make -f MakefileCapi cpp +RUN make -f MakefileCapi c + +USER ovms +ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/demos/c_api_minimal_app/capi_files/Dockerfile.ubuntu b/demos/c_api_minimal_app/capi_files/Dockerfile.ubuntu new file mode 100644 index 0000000000..a612f53f30 --- /dev/null +++ b/demos/c_api_minimal_app/capi_files/Dockerfile.ubuntu @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=ubuntu:20.04 +FROM $BASE_IMAGE as base_build + +SHELL ["/bin/bash", "-xo", "pipefail", "-c"] + +RUN groupadd --gid 5000 ovms && useradd --home-dir /home/ovms --create-home --uid 5000 \ + --gid 5000 --shell /bin/bash --skel /dev/null ovms + +COPY ovms.tar.gz / +RUN tar -xf ovms.tar.gz +RUN chown -R ovms:ovms /ovms +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + +ARG BASE_IMAGE=ubuntu:20.04 +FROM $BASE_IMAGE as release +ENV DEBIAN_FRONTEND=noninteractive +SHELL ["/bin/bash", "-c"] +WORKDIR / +RUN set -e ; \ + set -x ; \ + groupadd --gid 5000 ovms ; \ + useradd --home-dir /home/ovms --create-home --uid 5000 --gid 5000 --groups 39,44 --shell /bin/bash --skel /dev/null ovms + +ARG BUILD_DEPENDENCIES="build-essential libxml2 curl libpugixml1v5 libtbb2" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ${BUILD_DEPENDENCIES} && \ + rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +COPY --from=base_build /ovms /ovms +COPY demos /ovms/demos +WORKDIR /ovms/demos +RUN make -f MakefileCapi cpp +RUN make -f MakefileCapi c + +USER ovms +ENTRYPOINT ["/bin/bash"] \ No newline at end of file diff --git a/demos/c_api_minimal_app/capi_files/demos/MakefileCapi b/demos/c_api_minimal_app/capi_files/demos/MakefileCapi new file mode 100644 index 0000000000..9baeb1e661 --- /dev/null +++ b/demos/c_api_minimal_app/capi_files/demos/MakefileCapi @@ -0,0 +1,25 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +cpp: + mkdir -vp /ovms/bin + g++ main_capi.cpp -I/ovms/include -L/ovms/lib -lovms_shared -fPIC --std=c++17 -o /ovms/bin/capi_cpp_example + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/lib /ovms/bin/capi_cpp_example + +c: + mkdir -vp /ovms/bin + gcc -c main_capi.c -I/ovms/include -o /ovms/bin/main_capi.o -std=c99 + gcc -o /ovms/bin/capi_c_example /ovms/bin/main_capi.o -L/ovms/lib -lovms_shared + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/lib /ovms/bin/capi_c_example \ No newline at end of file diff --git a/demos/c_api_minimal_app/capi_files/demos/config.json b/demos/c_api_minimal_app/capi_files/demos/config.json new file mode 100644 index 0000000000..4646c93fbe --- /dev/null +++ b/demos/c_api_minimal_app/capi_files/demos/config.json @@ -0,0 +1,8 @@ +{ + "model_config_list": [ + {"config": { + "name": "dummy", + "base_path": "/ovms/demos/dummy", + "shape": "(30, 20)"}} + ] +} \ No newline at end of file diff --git a/demos/c_api_minimal_app/capi_files/demos/config_standard_dummy.json b/demos/c_api_minimal_app/capi_files/demos/config_standard_dummy.json new file mode 100644 index 0000000000..732abbf73d --- /dev/null +++ b/demos/c_api_minimal_app/capi_files/demos/config_standard_dummy.json @@ -0,0 +1,10 @@ +{ + "model_config_list": [ + {"config": { + "name": "dummy", + "base_path": "/ovms/demos/dummy", + "shape": "(1, 10)" + } + } + ] +} From 4f4007d8eaf8bfc298901bf3eb1275697a0d380f Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Mon, 19 Dec 2022 23:08:46 +0100 Subject: [PATCH 120/130] update docs about TF importer and accelerators (#1609) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Miłosz Żeglarski Co-authored-by: Damian Kalinowski --- demos/README.md | 2 ++ .../python/README.md | 2 +- docs/accelerators.md | 36 +++++++++++++------ docs/models_repository.md | 17 ++++++--- docs/ovms_quickstart.md | 5 +-- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/demos/README.md b/demos/README.md index 5a3ef52d45..d3aba4788d 100644 --- a/demos/README.md +++ b/demos/README.md @@ -17,6 +17,7 @@ ovms_docs_demo_ensemble ovms_docs_image_classification ovms_demo_using_onnx_model + ovms_demo_tf_classification ovms_demo_person_vehicle_bike_detection ovms_demo_vehicle_analysis_pipeline ovms_demo_real_time_stream_analysis @@ -40,6 +41,7 @@ OpenVINO Model Server demos have been created to showcase the usage of the model |[Model Ensemble Pipeline](model_ensemble/python/README.md)|Combine multiple image classification models into one [pipeline](../docs/dag_scheduler.md) and aggregate results to improve classification accuracy. | |[Image Classification](image_classification/python/README.md)|Run prediction on a JPEG image using image classification model via gRPC API.| |[Using ONNX Model](using_onnx_model/python/README.md)|Run prediction on a JPEG image using image classification ONNX model via gRPC API in two preprocessing variants. This demo uses [pipeline](../docs/dag_scheduler.md) with [image_transformation custom node](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/src/custom_nodes/image_transformation). | +|[Using TensorFlow Model](image_classification_using_tf_model/python/README.md)|Run image classification using directly imported TensorFlow model. | |[Person, Vehicle, Bike Detection](person_vehicle_bike_detection/python/README.md)|Run prediction on a video file or camera stream using person, vehicle, bike detection model via gRPC API.| |[Vehicle Analysis Pipeline](vehicle_analysis_pipeline/python/README.md)|Detect vehicles and recognize their attributes using a pipeline of vehicle detection and vehicle attributes recognition models with a custom node for intermediate results processing via gRPC API. This demo uses [pipeline](../docs/dag_scheduler.md) with [model_zoo_intel_object_detection custom node](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/src/custom_nodes/model_zoo_intel_object_detection). | |[Real Time Stream Analysis](real_time_stream_analysis/python/README.md)| Analyze RTSP video stream in real time with generic application template for custom pre and post processing routines as well as simple results visualizer for displaying predictions in the browser. | diff --git a/demos/image_classification_using_tf_model/python/README.md b/demos/image_classification_using_tf_model/python/README.md index 076c042723..fd37641277 100644 --- a/demos/image_classification_using_tf_model/python/README.md +++ b/demos/image_classification_using_tf_model/python/README.md @@ -1,4 +1,4 @@ -## Overview +# Model Server demo with a direct import of TensorFlow model {#ovms_demo_tf_classification} This guide demonstrates how to run inference requests for TensorFlow model with OpenVINO Model Server. As an example, we will use [InceptionResNetV2](https://storage.googleapis.com/download.tensorflow.org/models/tflite/model_zoo/upload_20180427/inception_resnet_v2_2018_04_27.tgz) to perform classification of an image. diff --git a/docs/accelerators.md b/docs/accelerators.md index 18a534c6ea..0adde47da0 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -81,7 +81,7 @@ Example: make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 ``` -## Starting a Docker Container with Intel GPU +## Starting a Docker Container with Intel integrated GPU, Intel® Data Center GPU Flex Series and Intel® Arc™ GPU The [GPU plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html) uses the Intel Compute Library for Deep Neural Networks ([clDNN](https://01.org/cldnn)) to infer deep neural networks. For inference execution, it employs Intel® Processor Graphics including @@ -121,15 +121,24 @@ docker run --rm -it --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/rende ``` -> **NOTE**: -> The public docker image includes the OpenCL drivers for GPU in version 21.38.21026 (RedHat) and 21.48.21782 (Ubuntu). - -Support for [Intel Arc](https://www.intel.com/content/www/us/en/architecture-and-technology/visual-technology/arc-discrete-graphics.html), which is in preview now, requires newer driver version `22.10.22597`. You can build OpenVINO Model server with ubuntu base image and that driver using the command below: +GPU device can be used also on Windows hosts with Windows Subsystem for Linux 2 (WSL2). In such scenario, there are needed extra docker parameters. See the command below. +Use device `/dev/dxg` instead of `/dev/dri` and mount the volume `/usr/lib/wsl`: ```bash -git clone https://github.com/openvinotoolkit/model_server.git -cd model_server -make docker_build INSTALL_DRIVER_VERSION=22.10.22597 + +docker run --rm -it --device=/dev/dxg --volume /usr/lib/wsl:/usr/lib/wsl --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ + +-v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest-gpu \ + +--model_path /opt/model --model_name resnet --port 9001 --target_device GPU + ``` + + +> **NOTE**: +> The public docker image includes the OpenCL drivers for GPU in version 22.28 (RedHat) and 22.35 (Ubuntu). + +If you need to build the OpenVINO Model Server with different driver version, refer to the [building from sources](https://github.com/openvinotoolkit/model_server/blob/develop/docs/build_from_source.md) + ## Using Multi-Device Plugin If you have multiple inference devices available (e.g. Myriad VPUs and CPU) you can increase inference throughput by enabling the Multi-Device Plugin. @@ -240,15 +249,22 @@ THROUGHPUT ## Using NVIDIA Plugin -*Note:* To build container with NVIDIA plugin use commands: +OpenVINO Model Server can be used also with NVIDIA GPU cards by using NVIDIA plugin from the [github repo openvino_contrib](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/nvidia_plugin). +The docker image of OpenVINO Model Server including support for NVIDIA can be built from sources + ```bash git clone https://github.com/openvinotoolkit/model_server.git cd model_server - make docker_build NVIDIA=1 OV_USE_BINARY=0 + make docker_build NVIDIA=1 OV_USE_BINARY=0 OV_SOURCE_BRANCH=releases/2022/3 OV_CONTRIB_BRANCH=releases/2022/3 ``` +Check also [building from sources](https://github.com/openvinotoolkit/model_server/blob/develop/docs/build_from_source.md). Example command to run container with NVIDIA support: ```bash docker run -it --gpus all -p 9178:9178 -v ${PWD}/models/public/resnet-50-tf:/opt/model openvino/model_server:latest-cuda --model_path /opt/model --model_name resnet --target_device NVIDIA ``` + +Check the supported [configuration parameters](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/nvidia_plugin#supported-configuration-parameters) and [supported layers](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/nvidia_plugin#supported-layers-and-limitations) + +Currently the AUTO, MULTI and HETERO virual plugins do not support NVIDIA plugin as an alternative device. \ No newline at end of file diff --git a/docs/models_repository.md b/docs/models_repository.md index 20c0564a17..26ee01c7a7 100644 --- a/docs/models_repository.md +++ b/docs/models_repository.md @@ -1,14 +1,18 @@ # Preparing a Model Repository {#ovms_docs_models_repository} -The AI models served by OpenVINO™ Model Server must be in either of the three formats: +The AI models served by OpenVINO™ Model Server must be in either of the four formats: - [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets), where the graph is represented in .bin and .xml files - [ONNX](https://onnx.ai/), using the .onnx file - [PaddlePaddle](https://www.paddlepaddle.org.cn/en), using .pdiparams and .pdmodel files +- [TensorFlow](https://www.tensorflow.org/), using frozen graph format with .pb extension (preview feature) To use models trained in other formats you need to convert them first. To do so, use OpenVINO’s [Model Optimizer](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) for IR, or different [converters](https://onnx.ai/supported-tools.html) for ONNX. +The feature of direct import of Tensorflow models is currently a preview feature. Currently it supports only the frozen graph and not all topologies can be used that way. +For unsupported models you can use the Model Optimizer to convert the model to IR format. + The models need to be placed and mounted in a particular directory structure and according to the following rules: ``` @@ -29,10 +33,13 @@ models/ ├── model3 │ └── 1 │ └── model.onnx -└── model4 - └── 1 - ├── model.pdiparams - └── model.pdmodel +├── model4 +│ └── 1 +│ ├── model.pdiparams +│ └── model.pdmodel +└── model5 + └── 1 + └── TF_fronzen_model.pb ``` - Each model should be stored in a dedicated directory, e.g. model1 and model2. diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index 4f7e4e2b1f..13283d28c2 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -1,7 +1,7 @@ # Quickstart Guide {#ovms_docs_quick_start_guide} OpenVINO Model Server can perform inference using pre-trained models in either [OpenVINO IR](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_IR_and_opsets.html#doxid-openvino-docs-m-o-d-g-i-r-and-opsets) -or [ONNX](https://onnx.ai/) format. You can get them by: +, [ONNX](https://onnx.ai/), PaddlePaddle[https://github.com/PaddlePaddle/Paddle] or TensorFlow format [https://www.tensorflow.org/]. You can get them by: - downloading models from [Open Model Zoo](https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/) - converting other formats using [Model Optimizer](https://docs.openvino.ai/2022.2/openvino_docs_MO_DG_Deep_Learning_Model_Optimizer_DevGuide.html) @@ -124,6 +124,7 @@ In the `results` folder, you can find files containing inference results. In our case, it will be a modified input image with bounding boxes indicating detected faces. ![Inference results](quickstart_result.jpg) -Note: Similar steps can be performed with an ONNX model. Check the inference [use case example](../demos/using_onnx_model/python/README.md) with a public ResNet model in ONNX format. +Note: Similar steps can be performed with an ONNX model. Check the inference [use case example](../demos/using_onnx_model/python/README.md) with a public ResNet model in ONNX format +or [TensorFlow model demo](../demos/image_classification_using_tf_model/python/README.md ). Congratulations, you have completed the Quickstart guide. Try Model Server [demos](../demos/README.md) or explore more [features](features.md) to create your application. From 2e7e17cfa79d3e7d5b1c3c4f362c94407fda0a6f Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Tue, 20 Dec 2022 10:56:12 +0100 Subject: [PATCH 121/130] Update baremetal documentation (#1610) Co-authored-by: Trawinski, Dariusz Co-authored-by: Rafal Sapala --- docs/deploying_server.md | 39 +++++++++++++++++++++++++++++++++++---- docs/ovms_quickstart.md | 2 +- docs/parameters.md | 2 +- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/docs/deploying_server.md b/docs/deploying_server.md index c9a7b52de6..78a2aa6930 100644 --- a/docs/deploying_server.md +++ b/docs/deploying_server.md @@ -71,13 +71,44 @@ zebra ``` If everything is set up correctly, you will see 'zebra' prediction in the output. -## Deploying Model Server on Baremetal +## Deploying Model Server on Baremetal (without container) +It is possible to deploy Model Server outside of container. +To deploy Model Server on baremetal, use pre-compiled binaries for Ubuntu20 or RHEL8. +Find latest binary package in [release](https://github.com/openvinotoolkit/model_server/releases) page. +Alternatively it is possible to build package from source: -There are two ways to start the server: +```bash +git clone https://github.com/openvinotoolkit/model_server +cd model_server +make docker_build +``` + +The `ovms.tar.gz` package will appear in `dist/ubuntu` or `dist/redhat` directory. +Unpack the package: + +```bash +tar -xzvf dist/ubuntu/ovms.tar.gz +``` + +Install required libraries depending on the OS. +For Ubuntu 20.04: +```bash +apt update -y && apt install -y libpugixml1v5 libtbb2 +``` +For RedHat 8.7: +```bash +microdnf install -y pkg-config && rpm -ivh https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm +``` + +Start the server: + +```bash +./ovms/bin/ovms --model_name resnet --model_path gs://ovms-public-eu/resnet50-binary +``` -- using the ```./ovms/bin/ovms --help``` command from inside the directory where OVMS is installed -- in interactive mode as a background process or a daemon initiated by ```systemctl/initd``` depending on the Linux distribution and specific hosting requirements +or start as a background process or a daemon initiated by ```systemctl/initd``` depending on the Linux distribution and specific hosting requirements. +Most of the Model Server documentation demonstrate containers usage, but the same can be achieved with just the binary package. Learn more about model server [starting parameters](parameters.md). > **NOTE**: diff --git a/docs/ovms_quickstart.md b/docs/ovms_quickstart.md index 13283d28c2..84acb54303 100644 --- a/docs/ovms_quickstart.md +++ b/docs/ovms_quickstart.md @@ -85,7 +85,7 @@ For more information, check these links: Put the files in a separate folder to provide inference data, as inference will be performed on all the files it contains. -You can download [example images for inference](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people). This example uses the file [people1.jpeg](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people/people1.jpeg). Run the following command to download the image:: +You can download [example images for inference](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people). This example uses the file [people1.jpeg](https://github.com/openvinotoolkit/model_server/tree/releases/2022/1/demos/common/static/images/people/people1.jpeg). Run the following command to download the image: ```bash curl --fail --create-dirs https://raw.githubusercontent.com/openvinotoolkit/model_server/releases/2022/1/demos/common/static/images/people/people1.jpeg -o images/people1.jpeg diff --git a/docs/parameters.md b/docs/parameters.md index 9d37ab3ab2..38b8fcd5a9 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -10,7 +10,7 @@ | `"shape"` | `tuple/json/"auto"` | `shape` is optional and takes precedence over `batch_size`. The `shape` argument changes the model that is enabled in the model server to fit the parameters. `shape` accepts three forms of the values: * `auto` - The model server reloads the model with the shape that matches the input data matrix. * a tuple, such as `(1,3,224,224)` - The tuple defines the shape to use for all incoming requests for models with a single input. * A dictionary of shapes, such as `{"input1":"(1,3,224,224)","input2":"(1,3,50,50)", "input3":"auto"}` - This option defines the shape of every included input in the model.Some models don't support the reshape operation.If the model can't be reshaped, it remains in the original parameters and all requests with incompatible input format result in an error. See the logs for more information about specific errors.Learn more about supported model graph layers including all limitations at [Shape Inference Document](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_ShapeInference.html). | | `"batch_size"` | `integer/"auto"` | Optional. By default, the batch size is derived from the model, defined through the OpenVINO Model Optimizer. `batch_size` is useful for sequential inference requests of the same batch size.Some models, such as object detection, don't work correctly with the `batch_size` parameter. With these models, the output's first dimension doesn't represent the batch size. You can set the batch size for these models by using network reshaping and setting the `shape` parameter appropriately.The default option of using the Model Optimizer to determine the batch size uses the size of the first dimension in the first input for the size. For example, if the input shape is `(1, 3, 225, 225)`, the batch size is set to `1`. If you set `batch_size` to a numerical value, the model batch size is changed when the service starts.`batch_size` also accepts a value of `auto`. If you use `auto`, then the served model batch size is set according to the incoming data at run time. The model is reloaded each time the input data changes the batch size. You might see a delayed response upon the first request. | | `"layout" `| `json/string` | `layout` is optional argument which allows to define or change the layout of model input and output tensors. To change the layout (add the transposition step), specify `:`. Example: `NHWC:NCHW` means that user will send input data in `NHWC` layout while the model is in `NCHW` layout.

When specified without colon separator, it doesn't add a transposition but can determine the batch dimension. E.g. `--layout CN` makes prediction service treat second dimension as batch size.

When the model has multiple inputs or the output layout has to be changed, use a json format. Set the mapping, such as: `{"input1":"NHWC:NCHW","input2":"HWN:NHW","output1":"CN:NC"}`.

If not specified, layout is inherited from model.

[Read more](shape_batch_size_and_layout.md#changing-model-inputoutput-layout) | -| `"model_version_policy"` | `json/string` | Optional.The model version policy lets you decide which versions of a model that the OpenVINO Model Server is to serve. By default, the server serves the latest version. One reason to use this argument is to control the server memory consumption.The accepted format is in json or string. Examples:
`{"latest": { "num_versions":2 }`
`{"specific": { "versions":[1, 3] } }`
`{"all": {} }` | +| `"model_version_policy"` | `json/string` | Optional. The model version policy lets you decide which versions of a model that the OpenVINO Model Server is to serve. By default, the server serves the latest version. One reason to use this argument is to control the server memory consumption.The accepted format is in json or string. Examples:
`{"latest": { "num_versions":2 }`
`{"specific": { "versions":[1, 3] } }`
`{"all": {} }` | | `"plugin_config"` | `json/string` | List of device plugin parameters. For full list refer to [OpenVINO documentation](https://docs.openvino.ai/2022.2/openvino_docs_IE_DG_supported_plugins_Supported_Devices.html) and [performance tuning guide](./performance_tuning.md). Example:
`{"CPU_THROUGHPUT_STREAMS": "CPU_THROUGHPUT_AUTO"}` | | `"nireq"` | `integer` | The size of internal request queue. When set to 0 or no value is set value is calculated automatically based on available resources.| | `"target_device"` | `string` | Device name to be used to execute inference operations. Accepted values are: `"CPU"/"HDDL"/"GPU"/"MYRIAD"/"MULTI"/"HETERO"` | From 387113453847f2dd6ad4f7ba652c9c7a210c2e47 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Tue, 20 Dec 2022 13:28:58 +0100 Subject: [PATCH 122/130] Mkulakow/fix documentation (#1611) --- docs/clients_kfs.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/clients_kfs.md b/docs/clients_kfs.md index 48b75cbb87..36b01b9e40 100644 --- a/docs/clients_kfs.md +++ b/docs/clients_kfs.md @@ -85,3 +85,14 @@ When creating a Python-based client application, you can use Triton client libra @endsphinxdirective For complete usage examples see [Kserve samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/python/kserve-api/samples). + +## C++ Client + +@sphinxdirective +.. raw:: html +

C++
+@endsphinxdirective + +Creating a client application in C++ follows the same principles as Python. When creating a C++-based client application, you can use Triton client library - [tritonclient](https://github.com/triton-inference-server/client). + +See our [C++ samples](https://github.com/openvinotoolkit/model_server/tree/develop/client/cpp/kserve-api/README.md) to learn how to build a sample C++ client application. From a1ff749a55e9e5fc636e12a24f3e1fa2e8e8b642 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Tue, 20 Dec 2022 14:47:30 +0100 Subject: [PATCH 123/130] C API documentation (#1608) JIRA:CVS-95629 --- demos/c_api_minimal_app/README.md | 3 +- docs/api_reference_guide.md | 8 +++-- docs/features.md | 6 ++++ docs/model_server_c_api.md | 49 +++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 docs/model_server_c_api.md diff --git a/demos/c_api_minimal_app/README.md b/demos/c_api_minimal_app/README.md index 75ba1155a5..3eb36d32d8 100644 --- a/demos/c_api_minimal_app/README.md +++ b/demos/c_api_minimal_app/README.md @@ -2,6 +2,7 @@ This demo demonstrate how to use C API from the OpenVINO Model Server to create C and C++ application. Building the application is executed inside the docker container to illustrate end to end usage flow. +Check C API full documentation [here](../../docs/model_server_c_api.md). ## Prepare demo image Enter the directory with the example and build the demo docker with all dependencies and examples that will be named `openvino/model_server-capi`. @@ -97,4 +98,4 @@ And then execute the alternative make target: ```bash cd demos/c_api_minimal_app make all_docker -``` \ No newline at end of file +``` diff --git a/docs/api_reference_guide.md b/docs/api_reference_guide.md index 2ac9646a0c..7c9b95c7af 100644 --- a/docs/api_reference_guide.md +++ b/docs/api_reference_guide.md @@ -10,16 +10,20 @@ ovms_docs_grpc_api_kfs ovms_docs_rest_api_tfs ovms_docs_rest_api_kfs + ovms_docs_c_api @endsphinxdirective ## Introduction -OpenVINO™ Model Server exposes two sets of APIs: one compatible with TensorFlow Serving and another one, with KServe API, for inference. Both APIs work on both gRPC and REST interfaces. Supporting two sets of APIs makes OpenVINO Model Server easier to plug into existing systems the already leverage one of those APIs for inference. Learn more about supported APIs: +OpenVINO™ Model Server exposes two sets of network APIs: one compatible with TensorFlow Serving and another one, with KServe API, for inference. Both APIs work on both gRPC and REST interfaces. Supporting two sets of APIs makes OpenVINO Model Server easier to plug into existing systems the already leverage one of those APIs for inference. Learn more about supported APIs: - [TensorFlow Serving gRPC API](./model_server_grpc_api_tfs.md) - [KServe gRPC API](./model_server_grpc_api_kfs.md) - [TensorFlow Serving REST API](./model_server_rest_api_tfs.md) - [KServe REST API](./model_server_rest_api_kfs.md) -If you already use one of these APIs, integration of OpenVINO Model Server should be smooth and transparent. \ No newline at end of file +If you already use one of these APIs, integration of OpenVINO Model Server should be smooth and transparent. + +Additionally OVMS provides preview of in process inference with its C API: +- [OVMS C API](./model_server_c_api.md) diff --git a/docs/features.md b/docs/features.md index bab8d50904..22e392ae0e 100644 --- a/docs/features.md +++ b/docs/features.md @@ -14,6 +14,7 @@ ovms_docs_stateful_models ovms_docs_metrics ovms_docs_dynamic_input + ovms_docs_c_api ovms_docs_advanced @endsphinxdirective @@ -61,6 +62,11 @@ Configure served models to accept data with variable batch sizes and input shape [Learn more](dynamic_input.md) +## Model Server C API +Use in process inference via model server to leverage the model management and model pipelines functionality of OpenVINO Model Server within an application. This allows to reuse existing OVMS functionality to execute inference locally without network overhead. + +[Learn more](model_server_c_api.md) + ## Advanced Features Use CPU Extensions, model cache feature or a custom model loader. diff --git a/docs/model_server_c_api.md b/docs/model_server_c_api.md new file mode 100644 index 0000000000..d0cccd8419 --- /dev/null +++ b/docs/model_server_c_api.md @@ -0,0 +1,49 @@ +# OpenVINO Model Server C-API (preview feature) {#ovms_docs_c_api} + +## Introduction + +This document describes OpenVINO Model Server (OVMS) C API that allows OVMS to be linked into C/C++ applications. With exceptions listed at the end of this document, all capabilities of OVMS are included in the shared library. + +## API Description + +Server functionalities are encapsulated in shared library built from OVMS source. To include OVMS you need to link this library with your application and use C API defined in [header file](https://github.com/openvinotoolkit/model_server/blob/develop/src/ovms.h). + + +To start model serving you need to spawn process that will keep OVMS alive. Then you can schedule inference both directly from app using C API and gRPC/HTTP endpoints. + +### Server configuration and start + +To start OVMS you need to create `OVMS_Server` object using `OVMS_ServerNew`, with set of `OVMS_ServerSettings` and `OVMS_ModelsSettings` that describe how the server should be configured. Once the server is started using `OVMS_ServerStartFromConfigurationFile` you can schedule the inferences using `OVMS_Inference`. To stop server, you must call `OVMS_ServerDelete`. While the server is alive you can schedule both in process inferences as well as use gRPC API to schedule inferences from remote machine. Optionally you can also enable HTTP service. Example how to use OVMS with C/C++ application is [here](../demos/c_api_minimal_app/README.md). + +### Error handling +Most of OVMS C API functions return `OVMS_Status` object pointer indicating the success or failure. Success is indicated by nullptr (NULL). Failure is indicated by returning `OVMS_Status` object. The status code can be extracted using `OVMS_StatusGetCode` function and the details of error can be retrieved using `OVMS_StatusGetDetails` function. + +The ownership of `OVMS_Status` is passed to the caller of the function. You must delete the object using `OVMS_StatusDelete`. + +### Inference + +To execute inference using C API you must follow steps described below. + +#### Prepare inference request +Create an inference request using `OVMS_InferenceRequestNew` specifying which servable name and optionally version to use. Then specify input tensors with `OVMS_InferenceRequestAddInput` and set the tensor data using `OVMS_InferenceRequestSetData`. + +#### Invoke inference +Execute inference with OVMS using `OVMS_Inference` synchronous call. During inference execution you must not modify `OVMS_InferenceRequest` and bound memory buffers. + +#### Process inference response +If the inference was successful, you receive `OVMS_InferenceRequest` object. After processing the response, you must free the response memory by calling `OVMS_InferenceResponseDelete`. + +To process response, first you must check for inference error. If no error occurred, you must iterate over response outputs and parameters using `OVMS_InferenceResponseGetOutputCount` and `OVMS_InferenceResponseGetParameterCount`. Then you must extract details describing each output and parameter using `OVMS_InferenceResponseGetOutput` and `OVMS_InferenceResponseGetParameter`. Example how to use OVMS with C/C++ application is [here](../demos/c_api_minimal_app/README.md). While in example app you have only single thread scheduling inference request you can execute multiple inferences simultaneously using different threads. + +**Note**: After inference execution is finished you can reuse the same `OVMS_InferenceRequest` by using `OVMS_InferenceRequestInputRemoveData` and then setting different tensor data with `OVMS_InferenceRequestSetData`. + +## Preview limitations +* Launching server in single model mode is not supported. You must use configuration file. +* DAG pipelines cannot be used directly through C API. DAG inferences can be scheduled only by gRPC/HTTP endpoints. +* There is no support for native file format (jpg/png) through C API. +* There are no server live, server ready, model ready, model metadata, metrics endpoints exposed through C API. +* Inference scheduled through C API does not have inference success/failure, request time metrics counted. +* You cannot turn gRPC service off, HTTP service is off by default but can be enabled. +* There is no API for asynchronous inference. +* There is no support for stateful models. + From aa64f0d18d81ee296b1c7da28ca4b06de23cb1dc Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 21 Dec 2022 09:31:53 +0100 Subject: [PATCH 124/130] Fix links (#1615) * Fix links --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 97e5272e07..eaa6a98ce2 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,6 @@ Google Cloud Storage (GCS), Amazon S3, or Azure Blob Storage. Read [release notes](https://github.com/openvinotoolkit/model_server/releases) to find out what’s new. -Review the [Architecture concept](https://docs.openvino.ai/2022.2/ovms_docs_architecture.html) document for more details. - Key features: - support for multiple frameworks, such as Caffe, TensorFlow, MXNet, PaddlePaddle and ONNX - online deployment of new [model versions](https://docs.openvino.ai/2022.2/ovms_docs_model_version_policy.html) @@ -22,7 +20,7 @@ Key features: [Intel Movidius Myriad VPUs](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_MYRIAD.html), [GPU](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html), and [HDDL](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_HDDL.html) -- works with [Bare Metal Hosts](docs/host.md) as well as [Docker containers](https://docs.openvino.ai/2022.2/ovms_docs_docker_container.html) +- works with Bare Metal Hosts as well as [Docker containers](https://docs.openvino.ai/2022.3/ovms_docs_deploying_server.html) - [model reshaping](https://docs.openvino.ai/2022.2/ovms_docs_shape_batch_layout.html) in runtime - [directed Acyclic Graph Scheduler](https://docs.openvino.ai/2022.2/ovms_docs_dag.html) - connecting multiple models to deploy complex processing solutions and reducing data transfer overhead - [custom nodes in DAG pipelines](https://docs.openvino.ai/2022.2/ovms_docs_custom_node_development.html) - allowing model inference and data transformations to be implemented with a custom node C/C++ dynamic library @@ -45,9 +43,7 @@ For more information on using Model Server in various scenarios you can check th * [Model repository configuration](https://docs.openvino.ai/2022.2/ovms_docs_models_repository.html) -* [Using a docker container](https://docs.openvino.ai/2022.2/ovms_docs_docker_container.html) - -* [Landing on bare metal or virtual machine](https://docs.openvino.ai/2022.2/ovms_docs_baremetal.html) +* [Deployment options](https://docs.openvino.ai/2022.3/ovms_docs_deploying_server.html) * [Performance tuning](https://docs.openvino.ai/2022.2/ovms_docs_performance_tuning.html) From d0fba37592c347115bf86a57d4dbff1c4f13601f Mon Sep 17 00:00:00 2001 From: Rafal Sapala Date: Wed, 21 Dec 2022 09:32:17 +0100 Subject: [PATCH 125/130] help fix (#1613) --- client/cpp/kserve-api/samples/grpc_infer_resnet.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp b/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp index 52638dc09f..a80dc677fc 100644 --- a/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp +++ b/client/cpp/kserve-api/samples/grpc_infer_resnet.cpp @@ -65,6 +65,10 @@ int main(int argc, char** argv) { // clang-format on auto args = opt.parse(argc, argv); + if (args.count("help")) { + std::cout << opt.help() << std::endl; + return 0; + } if (!args.count("images_list")) { std::cout << "error: option \"images_list\" has no value\n"; return 1; @@ -73,11 +77,7 @@ int main(int argc, char** argv) { std::cout << "error: option \"labels_list\" has no value\n"; return 1; } - if (args.count("help")) { - std::cout << opt.help() << std::endl; - return 0; - } - + std::string input_name(args["input_name"].as()); std::string output_name(args["output_name"].as()); From a80736c683a4da3adc47464ef74510c7dd44f2e0 Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Wed, 21 Dec 2022 10:59:53 +0100 Subject: [PATCH 126/130] Add info about new kfs binary inputs features to the docs (#1614) --- docs/binary_input_kfs.md | 48 ++++++++++++++++++++++++++++++- docs/model_server_grpc_api_kfs.md | 4 ++- docs/model_server_rest_api_kfs.md | 22 ++++++++++++++ docs/model_server_rest_api_tfs.md | 13 +++++---- 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/docs/binary_input_kfs.md b/docs/binary_input_kfs.md index 482cc28072..20889d8aca 100644 --- a/docs/binary_input_kfs.md +++ b/docs/binary_input_kfs.md @@ -68,13 +68,16 @@ KServe API also allows sending binary encoded data via HTTP interface. The tenso ``` For binary inputs, the `parameters` map in the JSON part contains `binary_data_size` field for each binary input that indicates the size of the data on the input. Since there's no strict limitations on image resolution and format (as long as it can be loaded by OpenCV), images might be of different sizes. Therefore, to send a batch of different images, specify their sizes in `binary_data_size` field as a list with sizes of all images in the batch. -The list must be formed as a string, so for example, for 3 images in the batch, you may pass - `"9821,12302,7889"` +The list must be formed as a string, so for example, for 3 images in the batch, you may pass - `"9821,12302,7889"`. +If the request contains only one input `binary_data_size` parameter can be omitted - in this case whole buffer is treated as a input image. For HTTP request headers, `Inference-Header-Content-Length` header must be provided to give the length of the JSON object, and `Content-Length` continues to give the full body length (as HTTP requires). See an extended example with the request headers, and multiple images in the batch: ``` POST /v2/models/my_model/infer HTTP/1.1 Host: localhost:5000 +``` +```JSON Content-Type: application/octet-stream Inference-Header-Content-Length: Content-Length: @@ -112,6 +115,8 @@ Getting back to the example from the previous section with 3 images in a batch, ``` POST /v2/models/my_model/infer HTTP/1.1 Host: localhost:5000 +``` +```JSON Content-Type: application/octet-stream Inference-Header-Content-Length: Content-Length: @@ -132,6 +137,47 @@ Content-Length: <3240000 bytes of the whole data batch for model_input tensor> ``` +For the Raw Data binary inputs `binary_data_size` parameter can be omitted since the size of particular input can be calculated from its shape. + +### Binary Outputs + +Outputs of response can be send in binary format similar to the binary inputs. To force a output to be sent in binary format you need to use "binary_data" : true parameter in request JSON. For example: +```JSON +{ + "model_name" : "mymodel", + "inputs" : [...], + "outputs" : [ + { + "name" : "output0", + "parameters" : { + "binary_data" : true + } + } + ] +} +``` + +Assuming the output datatype is FP32 and shape is [ 2, 2 ] response to this request would be: + +```JSON +HTTP/1.1 200 OK +Content-Type: application/octet-stream +Inference-Header-Content-Length: +Content-Length: +{ + "outputs" : [ + { + "name" : "output0", + "shape" : [ 2, 2 ], + "datatype" : "FP32", + "parameters" : { + "binary_data_size" : 16 + } + } + ] +} +<16 bytes of data for output0 tensor> +``` ## API Reference diff --git a/docs/model_server_grpc_api_kfs.md b/docs/model_server_grpc_api_kfs.md index ab0a172a02..87a182488f 100644 --- a/docs/model_server_grpc_api_kfs.md +++ b/docs/model_server_grpc_api_kfs.md @@ -45,7 +45,9 @@ Run inference with requested model or [DAG](./dag_scheduler.md). Check KServe documentation for more [details](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/required_api.md#inference-1). -> **NOTE**: Inference supports putting tensor buffers either in `ModelInferRequest`'s [InferTensorContents](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/grpc_predict_v2.proto#L155) and [raw_input_contents](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/grpc_predict_v2.proto#L202). There is no support for BF16 data type and there is no support for using FP16 in `InferTensorContents`. In case of sending raw images jpeg files BYTES data type should be used and data should be put in `InferTensorContents`'s `bytes_contents`. +> **NOTE**: Inference supports putting tensor buffers either in `ModelInferRequest`'s [InferTensorContents](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/grpc_predict_v2.proto#L155) and [raw_input_contents](https://github.com/kserve/kserve/blob/master/docs/predict-api/v2/grpc_predict_v2.proto#L202). There is no support for BF16 data type and there is no support for using FP16 in `InferTensorContents`. In case of sending raw images jpeg files BYTES data type should be used and data should be put in `InferTensorContents`'s `bytes_contents` or `raw_input_contents` for batch size equal to 1. + +Check [how binary data is handled in OpenVINO Model Server](./binary_input.md) ## See Also diff --git a/docs/model_server_rest_api_kfs.md b/docs/model_server_rest_api_kfs.md index 0ffc04f99a..b51775812f 100644 --- a/docs/model_server_rest_api_kfs.md +++ b/docs/model_server_rest_api_kfs.md @@ -225,6 +225,28 @@ $request_output = } ``` +Besides numerical values, it is possible to pass binary inputs using Binary Data extension: + +```JSON +{ +"model_name" : "my_model", +"inputs" : [ + { + "name" : "model_input", + "shape" : [ 1 ], + "datatype" : "BYTES", + "parameters" : { + "binary_data_size" : "9472" + } + } +] +} +<9472 bytes of data for model_input tensor> +``` + +Check [how binary data is handled in OpenVINO Model Server](./binary_input.md) + + **Response Format** If successful: diff --git a/docs/model_server_rest_api_tfs.md b/docs/model_server_rest_api_tfs.md index ed6b899990..dc5e62889f 100644 --- a/docs/model_server_rest_api_tfs.md +++ b/docs/model_server_rest_api_tfs.md @@ -49,6 +49,9 @@ If successful, returns a JSON of following format : **Usage Example** ``` $ curl http://localhost:8001/v1/models/person-detection/versions/1 +``` + +```JSON { 'model_version_status':[ { @@ -81,7 +84,7 @@ GET http://${REST_URL}:${REST_PORT}/v1/models/${MODEL_NAME}/versions/${MODEL_VER If successful, returns a JSON representation of [GetModelMetadataResponse](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/get_model_metadata.proto#L23) protobuf. **Usage example** -``` +```JSON $ curl http://localhost:8001/v1/models/person-detection/versions/1/metadata { "modelSpec": { @@ -158,7 +161,7 @@ POST http://${REST_URL}:${REST_PORT}/v1/models/${MODEL_NAME}/versions/${MODEL_VE ``` **Request Header** -``` +```JSON { // (Optional) Serving signature to use. // If unspecifed default serving signature is used. @@ -177,14 +180,14 @@ Read [How to specify input tensors in row format](https://www.tensorflow.org/tfx A request in [row format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_row_format) has response formatted as follows : -``` +```JSON { "predictions": |<(nested)list>| } ``` A request in [column format](https://www.tensorflow.org/tfx/serving/api_rest#specifying_input_tensors_in_column_format) has response formatted as follows : -``` +```JSON { "outputs": |<(nested)list>| } @@ -192,7 +195,7 @@ A request in [column format](https://www.tensorflow.org/tfx/serving/api_rest#spe Besides numerical values, it is possible to pass binary inputs. They must be Base64 encoded in passed in `b64` key like below: -``` +```JSON { "instances": [ { From 288d7b14ddd81a4cca84a25eeb9b4989c583315f Mon Sep 17 00:00:00 2001 From: michalkulakowski Date: Wed, 21 Dec 2022 12:04:34 +0100 Subject: [PATCH 127/130] Fix performance tuning (#1584) --- docs/accelerators.md | 2 +- docs/performance_tuning.md | 32 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/docs/accelerators.md b/docs/accelerators.md index 0adde47da0..3da5758de5 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -221,7 +221,7 @@ Below is an example of the command with AUTO Plugin as target device. It include The `Auto Device` plugin can also use the [PERFORMANCE_HINT](performance_tuning.md) plugin config property that enables you to specify a performance mode for the plugin. -> **NOTE**: CPU_THROUGHPUT_STREAMS and PERFORMANCE_HINT should not be used together. +> **NOTE**: NUM_STREAMS and PERFORMANCE_HINT should not be used together. To enable Performance Hints for your application, use the following command: diff --git a/docs/performance_tuning.md b/docs/performance_tuning.md index 1bc3682296..5e55ce1e4a 100644 --- a/docs/performance_tuning.md +++ b/docs/performance_tuning.md @@ -67,7 +67,7 @@ GPU --target_device GPU ``` -> **NOTE**: CPU_THROUGHPUT_STREAMS and PERFORMANCE_HINT should not be used together. +> **NOTE**: NUM_STREAMS and PERFORMANCE_HINT should not be used together. ## Adjusting the number of streams in CPU and GPU target devices @@ -75,27 +75,26 @@ OpenVINO™ Model Server can be tuned to a single client use case or a high execution streams. They split the available resources to perform parallel execution of multiple requests. It is particularly efficient for models which cannot effectively consume all CPU cores or for CPUs with high number of cores. -By default, for `CPU` target device, OpenVINO Model Server sets the value CPU_THROUGHPUT_AUTO and GPU_THROUGHPUT_AUTO for `GPU` target device. It calculates the number of streams based on number of available CPUs. It gives a compromise between the single client scenario and the high concurrency. - -If this default configuration is not suitable, adjust it with the `CPU_THROUGHPUT_STREAMS` parameter defined as part +By default, number of streams is calculated based on number of available CPUs. It gives a compromise between the single client scenario and the high concurrency. +If this default configuration is not suitable, adjust it with the `NUM_STREAMS` parameter defined as part of the device plugin configuration. In a scenario where the number of parallel connections is close to 1, set the following parameter: -`--plugin_config '{"CPU_THROUGHPUT_STREAMS": "1"}'` +`--plugin_config '{"NUM_STREAMS": "1"}'` When the number of concurrent requests is higher, increase the number of streams. Make sure, however, that the number of streams is lower than the average volume of concurrent inference operations. Otherwise, the server might not be fully utilized. Number of streams should not exceed the number of CPU cores. For example, with ~50 clients sending the requests to the server with 48 cores, set the number of streams to 24: -`--plugin_config '{"CPU_THROUGHPUT_STREAMS": "24"}'` +`--plugin_config '{"NUM_STREAMS": "24"}'` ## Input data in REST API calls While using REST API, you can adjust the data format to optimize the communication and deserialization from json format. Here are some tips to effectively use REST interface when working with OpenVINO Model Server: -- use [binary data format](binary_input.md) when possible - binary data representation is smaller in terms of request size and easier to process on the server side. +- use [binary data format](binary_input.md) when possible(for TFS API binary data format is support ony for JPEG/PNG inputs, for KFS API there are no such limitations ) - binary data representation is smaller in terms of request size and easier to process on the server side. - when working with images, consider sending JPEG/PNG directly - compressed data will greatly reduce the traffic and speed up the communication. - with JPEG/PNG it is the most efficient to send the images with the resolution of the configured model. It will avoid image resizing on the server to fit the model. - if you decide to send data inside JSON object, try to adjust the numerical data type to reduce the message size i.e. reduce the numbers precisions in the json message with a command similar to `np.round(imgs.astype(np.float),decimals=2)`. @@ -113,21 +112,20 @@ In case of using CPU plugin to run the inference, it might be also beneficial to | Parameters | Description | | :--- | :---- | -| CPU_THREADS_NUM | Specifies the number of threads that CPU plugin should use for inference. | -| CPU_BIND_THREAD | Binds inference threads to CPU cores. | -| CPU_THROUGHPUT_STREAMS | Specifies number of CPU "execution" streams for the throughput mode | +| INFERENCE_NUM_THREADS | Specifies the number of threads that CPU plugin should use for inference. | +| AFFINITY | Binds inference threads to CPU cores. | +| NUM_STREAMS | Specifies number of execution streams for the throughput mode | -> **NOTE:** For additional information about all parameters read [OpenVINO supported plugins](https://docs.openvino.ai/2022.2/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). +> **NOTE:** For additional information about all parameters read [OpenVINO supported plugins](https://docs.openvino.ai/latest/groupov_runtime_cpp_prop_api.html?#detailed-documentation). - Example: -1. While passing the plugin configuration, omit the `KEY_` phase. -2. Following docker command will set `KEY_CPU_THROUGHPUT_STREAMS` parameter to a value `KEY_CPU_THROUGHPUT_NUMA`: +Following docker command will set `NUM_STREAMS` parameter to a value `1`: ```bash docker run --rm -d --cpuset-cpus 0,1,2,3 -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest \ --model_path /opt/model --model_name resnet --port 9001 \ ---plugin_config '{"CPU_THROUGHPUT_STREAMS": "1"}' +--plugin_config '{"NUM_STREAMS": "1"}' ``` @@ -157,16 +155,16 @@ The default value is 1 second which ensures prompt response to creating new mode Depending on the device employed to run the inference operation, you can tune the execution behavior with a set of parameters. Each device is handled by its OpenVINO plugin. -> **NOTE**: For additional information, read [supported configuration parameters for all plugins](https://docs.openvino.ai/2022.2/namespaceInferenceEngine_1_1PluginConfigParams.html?#detailed-documentation). +> **NOTE**: For additional information, read [supported configuration parameters for all plugins](https://docs.openvino.ai/latest/groupov_runtime_cpp_prop_api.html?#detailed-documentation). Model's plugin configuration is a dictionary of param:value pairs passed to OpenVINO Plugin on network load. It can be set with `plugin_config` parameter. -Following docker command sets a parameter `KEY_CPU_THROUGHPUT_STREAMS` to a value `32` and `KEY_CPU_BIND_THREAD` to `NUMA`. +Following docker command sets a parameter `NUM_STREAMS` to a value `32` and `AFFINITY` to `NUMA`. ```bash docker run --rm -d -v ${PWD}/models/public/resnet-50-tf:/opt/model -p 9001:9001 openvino/model_server:latest \ --model_path /opt/model --model_name resnet --port 9001 --grpc_workers 8 --nireq 32 \ ---plugin_config '{"CPU_THROUGHPUT_STREAMS": "32", "CPU_BIND_THREAD": "NUMA"}' +--plugin_config '{"NUM_STREAMS": "32", "AFFINITY": "NUMA"}' ``` ## Analyzing performance issues From fe2590d511b63b9ed4de4ac9cddf45940e4f5897 Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 21 Dec 2022 13:04:22 +0100 Subject: [PATCH 128/130] Document about building from source (#1616) Co-authored-by: Trawinski, Dariusz --- docs/accelerators.md | 25 -------- docs/build_from_source.md | 129 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 25 deletions(-) create mode 100644 docs/build_from_source.md diff --git a/docs/accelerators.md b/docs/accelerators.md index 3da5758de5..945934d1f6 100644 --- a/docs/accelerators.md +++ b/docs/accelerators.md @@ -56,31 +56,6 @@ Check out our recommendations for [throughput optimization on HDDL](performance_ > It requires RW permissions in the docker container security context. > It is recommended to start the docker container in the same context as the account starting _hddldaemon_. For example, if you start the _hddldaemon_ as root, add `--user root` to the `docker run` command. -## Model Server image with Intel® Data Center GPU Flex Series and Intel® Arc™ GPU support (Ubuntu 20.04) - -To build the image, you need to have NEO Runtime packages available. Contact Intel representative to get the access to the preproduction drivers. - -Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2`. Expected structure is like below: -``` -drivers -└── dg2 - ├── intel-igc-core__amd64.deb - ├── intel-igc-opencl__amd64.deb - ├── intel-level-zero-gpu-dbgsym__amd64.ddeb - ├── intel-level-zero-gpu__amd64.deb - ├── intel-opencl-icd-dbgsym__amd64.ddeb - ├── intel-opencl-icd__amd64.deb - ├── libigdgmm12__amd64.deb - └── libigdgmm12__amd64.deb -``` - -and run `make docker_build` with parameter: `INSTALL_DRIVER_VERSION=dg2`. - -Example: -``` -make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 -``` - ## Starting a Docker Container with Intel integrated GPU, Intel® Data Center GPU Flex Series and Intel® Arc™ GPU The [GPU plugin](https://docs.openvino.ai/2022.2/openvino_docs_OV_UG_supported_plugins_GPU.html) uses the Intel Compute Library for diff --git a/docs/build_from_source.md b/docs/build_from_source.md new file mode 100644 index 0000000000..196f6b4007 --- /dev/null +++ b/docs/build_from_source.md @@ -0,0 +1,129 @@ +# Building from source + +This document gives information how to build docker images and the binary package from source with different variants. + +## Prerequisites + +1. [Docker Engine](https://docs.docker.com/engine/) +1. Ubuntu 20.04 or RedHat 8.7 host +1. make +1. bash + +## Makefile and building + +Makefile located in root directory of this repository contains all targets needed to build docker images and binary packages. + +It contains `docker_build` target which by default builds multiple docker images: +- `openvino/model_server:latest` - smallest release image containing only neccessary files to run model server on CPU, NCS and HDDL +- `openvino/model_server:latest-gpu` - release image containing support for Intel GPU +- `openvino/model_server:latest-nginx-mtls` - release image containing examplary NGINX MTLS configuration +- `openvino/model_server-build:latest` - image with builder environment containing all the tools to build OVMS + +The `docker_build` target also prepares binary package to run OVMS as standalone application and shared library to link against user written C/C++ applications. + +```bash +git clone https://github.com/openvinotoolkit/model_server +cd model_server +make docker_build +tree dist/ubuntu +```` + +``` +dist/ubuntu +├── Dockerfile.redhat +├── Dockerfile.ubuntu +├── drivers +├── LICENSE +├── Makefile +├── ovms.tar.gz +├── ovms.tar.gz.metadata.json +├── ovms.tar.gz.sha256 +├── ovms.tar.xz +├── ovms.tar.xz.metadata.json +├── ovms.tar.xz.sha256 +└── thirdparty-licenses +``` + +## Building Options + +### `BASE_OS` + +Select base OS: +- `ubuntu` for Ubuntu 20.04 (default) +- `redhat` for Red Hat UBI 8.7 + +```bash +make docker_build BASE_OS=redhat +``` + +
+ +Example: + +### `INSTALL_DRIVER_VERSION` + +Parameter used to control which GPU driver version will be installed. Supported versions: +| OS | Versions | +|---|---| +| Ubuntu | 22.35.24055 (default),
22.10.22597,
21.48.21782,
20.35.17767 | +| RedHat | 22.28.23726 (default),
22.10.22597,
21.38.21026,
20.35.17767 | + +Additionally it is possible to specify custom (pre-production) drivers by providing location to NEO Runtime packages on local disk. Contact Intel representative to get the access to the pre-production drivers. + +Put NEO Runtime deb packages in the catalog `/release_files/drivers/dg2`. Expected structure is like below: + +``` +drivers +└── dg2 + ├── intel-igc-core__amd64.deb + ├── intel-igc-opencl__amd64.deb + ├── intel-level-zero-gpu-dbgsym__amd64.deb + ├── intel-level-zero-gpu__amd64.deb + ├── intel-opencl-icd-dbgsym__amd64.deb + ├── intel-opencl-icd__amd64.deb + ├── libigdgmm12__amd64.deb + └── libigdgmm12__amd64.deb +``` +and run make docker_build with parameter: INSTALL_DRIVER_VERSION=dg2. + +Example: +``` +make docker_build BASE_OS=ubuntu INSTALL_DRIVER_VERSION=dg2 +``` + +
+ +### `DLDT_PACKAGE_URL` + +Parameter used to specify URL to OpenVINO package. By default set to latest release. + +
+ +### `NVIDIA` + +By default set to `0`. When set to `1`, there will be additional docker image prepared: `openvino/model_server:latest-cuda` which contains environment required to run inference on NVIDIA GPUs. Please note that such image is significantly larger than the base one. + +Hint: use together with `OV_USE_BINARY=0` to force building OpenVINO from source. Use `OV_SOURCE_BRANCH` parameter to specify which branch from [OpenVINO repository](https://github.com/openvinotoolkit/openvino) should be used. +Use together with `OV_CONTRIB_BRANCH` to specify which branch from [OpenVINO contrib](https://github.com/openvinotoolkit/openvino_contrib) repository should be used for NVIDIA plugin. + +Example: +```bash +make docker_build NVIDIA=1 OV_USE_BINARY=0 OV_SOURCE_BRANCH=releases/2022/3 OV_CONTRIB_BRANCH=releases/2022/3 +``` +```bash +docker run -it --gpus all -p 9178:9178 -v ${PWD}/models/public/resnet-50-tf:/opt/model openvino/model_server:latest-cuda --model_path /opt/model --model_name resnet --target_device NVIDIA +``` + +
+ +### `OV_USE_BINARY` + +By default set to `1`. When set to `0`, OpenVINO will be built from sources and `DLDT_PACKAGE_URL` will be omitted. +Use `OV_SOURCE_BRANCH` to select [OpenVINO repository](https://github.com/openvinotoolkit/openvino) branch. By default `master` will be used. + +Example: +```bash +make docker_build OV_USE_BINARY=0 OV_SOURCE_BRANCH= +``` + +Read more detailed usage in [developer guide](https://github.com/openvinotoolkit/model_server/blob/develop/docs/developer_guide.md). From 592ad0a115cb8fb6ec536f7bdbccbfde268629cb Mon Sep 17 00:00:00 2001 From: Damian Kalinowski Date: Wed, 21 Dec 2022 15:10:56 +0100 Subject: [PATCH 129/130] CVS-99493 Fix unit test failing on machines with lower number of cores (#1619) Since --grpc_workers need to be in a range of [1-available_cores] to ensure test will succcess we need to pass value between 1-available cores. --- src/test/c_api_tests.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index b581309cd8..ef485d2c04 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -16,6 +16,7 @@ #include #include +#include #include @@ -81,6 +82,8 @@ static void testDefaultSingleModelOptions(ModelsSettingsImpl* modelsSettings) { } \ } +const uint AVAILABLE_CORES = std::thread::hardware_concurrency(); + TEST(CApiConfigTest, MultiModelConfiguration) { OVMS_ServerSettings* _serverSettings = 0; OVMS_ModelsSettings* _modelsSettings = 0; @@ -121,7 +124,7 @@ TEST(CApiConfigTest, MultiModelConfiguration) { // Set non default values ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcPort(_serverSettings, 5555)); ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestPort(_serverSettings, 6666)); - ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcWorkers(_serverSettings, 30)); + ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcWorkers(_serverSettings, AVAILABLE_CORES)); ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetGrpcBindAddress(_serverSettings, "2.2.2.2")); ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestWorkers(_serverSettings, 31)); ASSERT_CAPI_STATUS_NULL(OVMS_ServerSettingsSetRestBindAddress(_serverSettings, "3.3.3.3")); @@ -166,7 +169,7 @@ TEST(CApiConfigTest, MultiModelConfiguration) { // Test non default values EXPECT_EQ(serverSettings->grpcPort, 5555); EXPECT_EQ(serverSettings->restPort, 6666); - EXPECT_EQ(serverSettings->grpcWorkers, 30); + EXPECT_EQ(serverSettings->grpcWorkers, AVAILABLE_CORES); EXPECT_EQ(serverSettings->grpcBindAddress, "2.2.2.2"); EXPECT_EQ(serverSettings->restWorkers, 31); EXPECT_EQ(serverSettings->restBindAddress, "3.3.3.3"); @@ -190,7 +193,7 @@ TEST(CApiConfigTest, MultiModelConfiguration) { ASSERT_TRUE(cfg.parse(serverSettings, modelsSettings)); EXPECT_EQ(cfg.port(), 5555); EXPECT_EQ(cfg.restPort(), 6666); - EXPECT_EQ(cfg.grpcWorkers(), 30); + EXPECT_EQ(cfg.grpcWorkers(), AVAILABLE_CORES); EXPECT_EQ(cfg.grpcBindAddress(), "2.2.2.2"); EXPECT_EQ(cfg.restWorkers(), 31); EXPECT_EQ(cfg.restBindAddress(), "3.3.3.3"); From 12a0eb16b82e0b426e883e23e0af4008280a2984 Mon Sep 17 00:00:00 2001 From: Adrian Tobiszewski Date: Wed, 21 Dec 2022 15:17:59 +0100 Subject: [PATCH 130/130] Change OV packages to public (#1620) Source: https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/ JIRA:CVS-98971 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 09abb504b2..835993bacc 100644 --- a/Makefile +++ b/Makefile @@ -81,14 +81,14 @@ ifeq ($(BASE_OS),ubuntu) BASE_IMAGE ?= ubuntu:$(BASE_OS_TAG_UBUNTU) endif INSTALL_DRIVER_VERSION ?= "22.35.24055" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/releases/2022/3/commit/b84161848eaf02d8004f92cb530c1f092eac5782/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2022.3.0.9038.b84161848ea_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_ubuntu20_2022.3.0.9052.9752fafe8eb_x86_64.tgz endif ifeq ($(BASE_OS),redhat) BASE_OS_TAG=$(BASE_OS_TAG_REDHAT) BASE_IMAGE ?= registry.access.redhat.com/ubi8/ubi:$(BASE_OS_TAG_REDHAT) DIST_OS=redhat INSTALL_DRIVER_VERSION ?= "22.28.23726" - DLDT_PACKAGE_URL ?= http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/releases/2022/3/commit/b84161848eaf02d8004f92cb530c1f092eac5782/swf_drop/packages/releases/l_openvino_toolkit_rhel8_2022.3.0.9038.b84161848ea_x86_64.tgz + DLDT_PACKAGE_URL ?= https://storage.openvinotoolkit.org/repositories/openvino/packages/2022.3/linux/l_openvino_toolkit_rhel8_2022.3.0.9052.9752fafe8eb_x86_64.tgz endif OVMS_CPP_DOCKER_IMAGE ?= openvino/model_server