diff --git a/python/aiffairness/poetry.lock b/python/aiffairness/poetry.lock index 5f0cae3c7b1..09e61fe1104 100644 --- a/python/aiffairness/poetry.lock +++ b/python/aiffairness/poetry.lock @@ -936,6 +936,7 @@ tritonclient = "^2.18.0" uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/alibiexplainer/poetry.lock b/python/alibiexplainer/poetry.lock index 3dfc49b76e5..5126e1a47ee 100644 --- a/python/alibiexplainer/poetry.lock +++ b/python/alibiexplainer/poetry.lock @@ -2059,6 +2059,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] @@ -3917,6 +3918,58 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "scikit-image" +version = "0.19.3" +description = "Image processing in Python" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "scikit-image-0.19.3.tar.gz", hash = "sha256:24b5367de1762da6ee126dd8f30cc4e7efda474e0d7d70685433f0e3aa2ec450"}, + {file = "scikit_image-0.19.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:3a01372ae4bca223873304b0bff79b9d92446ac6d6177f73d89b45561e2d09d8"}, + {file = "scikit_image-0.19.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:fdf48d9b1f13af69e4e2c78e05067e322e9c8c97463c315cd0ecb47a94e259fc"}, + {file = "scikit_image-0.19.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b6a8f98f2ac9bb73706461fd1dec875f6a5141759ed526850a5a49e90003d19"}, + {file = "scikit_image-0.19.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cfbb073f23deb48e0e60c47f8741d8089121d89cc78629ea8c5b51096efc5be7"}, + {file = "scikit_image-0.19.3-cp310-cp310-win_amd64.whl", hash = "sha256:cc24177de3fdceca5d04807ad9c87d665f0bf01032ed94a9055cd1ed2b3f33e9"}, + {file = "scikit_image-0.19.3-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:fd9dd3994bb6f9f7a35f228323f3c4dc44b3cf2ff15fd72d895216e9333550c6"}, + {file = "scikit_image-0.19.3-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad5d8000207a264d1a55681a9276e6a739d3f05cf4429004ad00d61d1892235f"}, + {file = "scikit_image-0.19.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:84baa3179f3ae983c3a5d81c1e404bc92dcf7daeb41bfe9369badcda3fb22b92"}, + {file = "scikit_image-0.19.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f9f8a1387afc6c70f2bed007c3854a2d7489f9f7713c242f16f32ee05934bc2"}, + {file = "scikit_image-0.19.3-cp37-cp37m-win32.whl", hash = "sha256:9fb0923a3bfa99457c5e17888f27b3b8a83a3600b4fef317992e7b7234764732"}, + {file = "scikit_image-0.19.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ce3d2207f253b8eb2c824e30d145a9f07a34a14212d57f3beca9f7e03c383cbe"}, + {file = "scikit_image-0.19.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:2a02d1bd0e2b53e36b952bd5fd6118d9ccc3ee51de35705d63d8eb1f2e86adef"}, + {file = "scikit_image-0.19.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:03779a7e1736fdf89d83c0ba67d44110496edd736a3bfce61a2b5177a1c8a099"}, + {file = "scikit_image-0.19.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19a21a101a20c587a3b611a2cf6f86c35aae9f8d9563279b987e83ee1c9a9790"}, + {file = "scikit_image-0.19.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f50b923f8099c1045fcde7418d86b206c87e333e43da980f41d8577b9605245"}, + {file = "scikit_image-0.19.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e207c6ce5ce121d7d9b9d2b61b9adca57d1abed112c902d8ffbfdc20fb42c12b"}, + {file = "scikit_image-0.19.3-cp38-cp38-win32.whl", hash = "sha256:a7c3985c68bfe05f7571167ee021d14f5b8d1a4a250c91f0b13be7fb07e6af34"}, + {file = "scikit_image-0.19.3-cp38-cp38-win_amd64.whl", hash = "sha256:651de1c2ce1fbee834753b46b8e7d81cb12a5594898babba63ac82b30ddad49d"}, + {file = "scikit_image-0.19.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:8d8917fcf85b987b1f287f823f3a1a7dac38b70aaca759bc0200f3bc292d5ced"}, + {file = "scikit_image-0.19.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:0b0a199157ce8487c77de4fde0edc0b42d6d42818881c11f459262351d678b2d"}, + {file = "scikit_image-0.19.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33dfd463ee6cc509defa279b963829f2230c9e0639ccd3931045be055878eea6"}, + {file = "scikit_image-0.19.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8714348ddd671f819457a797c97d4c672166f093def66d66c3254cbd1d43f83"}, + {file = "scikit_image-0.19.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3b1025356508d41f4fe48528e509d95f9e4015e90cf158cd58c56dc63e0ac5"}, + {file = "scikit_image-0.19.3-cp39-cp39-win32.whl", hash = "sha256:9439e5294de3f18d6e82ec8eee2c46590231cf9c690da80545e83a0733b7a69e"}, + {file = "scikit_image-0.19.3-cp39-cp39-win_amd64.whl", hash = "sha256:32fb88cc36203b99c9672fb972c9ef98635deaa5fc889fe969f3e11c44f22919"}, +] + +[package.dependencies] +imageio = ">=2.4.1" +networkx = ">=2.2" +numpy = ">=1.17.0" +packaging = ">=20.0" +pillow = ">=6.1.0,<7.1.0 || >7.1.0,<7.1.1 || >7.1.1,<8.3.0 || >8.3.0" +PyWavelets = ">=1.1.1" +scipy = ">=1.4.1" +tifffile = ">=2019.7.26" + +[package.extras] +data = ["pooch (>=1.3.0)"] +docs = ["cloudpickle (>=0.2.1)", "dask[array] (>=0.15.0,!=2.17.0)", "ipywidgets", "kaleido", "matplotlib (>=3.3)", "myst-parser", "numpydoc (>=1.0)", "pandas (>=0.23.0)", "plotly (>=4.14.0)", "pooch (>=1.3.0)", "pytest-runner", "scikit-learn", "seaborn (>=0.7.1)", "sphinx (>=1.8)", "sphinx-copybutton", "sphinx-gallery (>=0.10.1)", "tifffile (>=2020.5.30)"] +optional = ["SimpleITK", "astropy (>=3.1.2)", "cloudpickle (>=0.2.1)", "dask[array] (>=1.0.0,!=2.17.0)", "matplotlib (>=3.0.3)", "pooch (>=1.3.0)", "pyamg", "qtpy"] +test = ["asv", "codecov", "flake8", "matplotlib (>=3.0.3)", "pooch (>=1.3.0)", "pytest (>=5.2.0)", "pytest-cov (>=2.7.0)", "pytest-faulthandler", "pytest-localserver"] + [[package]] name = "scikit-image" version = "0.20.0" @@ -3956,10 +4009,7 @@ numpy = ">=1.21.1" packaging = ">=20.0" pillow = ">=9.0.1" PyWavelets = ">=1.1.1" -scipy = [ - {version = ">=1.8,<1.9.2", markers = "python_version <= \"3.9\""}, - {version = ">=1.8", markers = "python_version > \"3.9\""}, -] +scipy = {version = ">=1.8", markers = "python_version > \"3.9\""} tifffile = ">=2019.7.26" [package.extras] @@ -4025,42 +4075,6 @@ docs = ["Pillow (>=7.1.2)", "matplotlib (>=2.2.3)", "memory-profiler (>=0.57.0)" examples = ["matplotlib (>=2.2.3)", "pandas (>=0.25.0)", "scikit-image (>=0.14.5)", "seaborn (>=0.9.0)"] tests = ["black (>=21.6b0)", "flake8 (>=3.8.2)", "matplotlib (>=2.2.3)", "mypy (>=0.770)", "pandas (>=0.25.0)", "pyamg (>=4.0.0)", "pytest (>=5.0.1)", "pytest-cov (>=2.9.0)", "scikit-image (>=0.14.5)"] -[[package]] -name = "scipy" -version = "1.9.1" -description = "SciPy: Scientific Library for Python" -category = "main" -optional = false -python-versions = ">=3.8,<3.12" -files = [ - {file = "scipy-1.9.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c61b4a91a702e8e04aeb0bfc40460e1f17a640977c04dda8757efb0199c75332"}, - {file = "scipy-1.9.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d79da472015d0120ba9b357b28a99146cd6c17b9609403164b1a8ed149b4dfc8"}, - {file = "scipy-1.9.1-cp310-cp310-macosx_12_0_universal2.macosx_10_9_x86_64.whl", hash = "sha256:825951b88f56765aeb6e5e38ac9d7d47407cfaaeb008d40aa1b45a2d7ea2731e"}, - {file = "scipy-1.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f950a04b33e17b38ff561d5a0951caf3f5b47caa841edd772ffb7959f20a6af0"}, - {file = "scipy-1.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc81ac25659fec73599ccc52c989670e5ccd8974cf34bacd7b54a8d809aff1a"}, - {file = "scipy-1.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:8d3faa40ac16c6357aaf7ea50394ea6f1e8e99d75e927a51102b1943b311b4d9"}, - {file = "scipy-1.9.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a412c476a91b080e456229e413792bbb5d6202865dae963d1e6e28c2bb58691"}, - {file = "scipy-1.9.1-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:eb954f5aca4d26f468bbebcdc5448348eb287f7bea536c6306f62ea062f63d9a"}, - {file = "scipy-1.9.1-cp38-cp38-macosx_12_0_universal2.macosx_10_9_x86_64.whl", hash = "sha256:3c6f5d1d4b9a5e4fe5e14f26ffc9444fc59473bbf8d45dc4a9a15283b7063a72"}, - {file = "scipy-1.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bc4e2c77d4cd015d739e75e74ebbafed59ba8497a7ed0fd400231ed7683497c4"}, - {file = "scipy-1.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0419485dbcd0ed78c0d5bf234c5dd63e86065b39b4d669e45810d42199d49521"}, - {file = "scipy-1.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34441dfbee5b002f9e15285014fd56e5e3372493c3e64ae297bae2c4b9659f5a"}, - {file = "scipy-1.9.1-cp38-cp38-win32.whl", hash = "sha256:b97b479f39c7e4aaf807efd0424dec74bbb379108f7d22cf09323086afcd312c"}, - {file = "scipy-1.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8fe305d9d67a81255e06203454729405706907dccbdfcc330b7b3482a6c371d"}, - {file = "scipy-1.9.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:39ab9240cd215a9349c85ab908dda6d732f7d3b4b192fa05780812495536acc4"}, - {file = "scipy-1.9.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:71487c503e036740635f18324f62a11f283a632ace9d35933b2b0a04fd898c98"}, - {file = "scipy-1.9.1-cp39-cp39-macosx_12_0_universal2.macosx_10_9_x86_64.whl", hash = "sha256:3bc1ab68b9a096f368ba06c3a5e1d1d50957a86665fc929c4332d21355e7e8f4"}, - {file = "scipy-1.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f7c39f7dbb57cce00c108d06d731f3b0e2a4d3a95c66d96bce697684876ce4d4"}, - {file = "scipy-1.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47d1a95bd9d37302afcfe1b84c8011377c4f81e33649c5a5785db9ab827a6ade"}, - {file = "scipy-1.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96d7cf7b25c9f23c59a766385f6370dab0659741699ecc7a451f9b94604938ce"}, - {file = "scipy-1.9.1-cp39-cp39-win32.whl", hash = "sha256:09412eb7fb60b8f00b328037fd814d25d261066ebc43a1e339cdce4f7502877e"}, - {file = "scipy-1.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:90c805f30c46cf60f1e76e947574f02954d25e3bb1e97aa8a07bc53aa31cf7d1"}, - {file = "scipy-1.9.1.tar.gz", hash = "sha256:26d28c468900e6d5fdb37d2812ab46db0ccd22c63baa095057871faa3a498bc9"}, -] - -[package.dependencies] -numpy = ">=1.18.5,<1.25.0" - [[package]] name = "scipy" version = "1.10.1" @@ -4891,7 +4905,6 @@ python-versions = "*" files = [ {file = "tritonclient-2.33.0-py3-none-any.whl", hash = "sha256:8fd7db59c76a6e3e4506e682a3d5ba549685b70baf7c7ff560701852774ba0f5"}, {file = "tritonclient-2.33.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:92c162dac8ed25724bc80d8d8cc6dd77d8518cd6d8fa0903dae75261a609c24c"}, - {file = "tritonclient-2.33.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:959b0108cb39f83d500fe27f83741ec5222149612db69565ad8aa44ffd1274af"}, ] [package.dependencies] diff --git a/python/artexplainer/poetry.lock b/python/artexplainer/poetry.lock index ab378ea4bf8..dd8cbe64192 100644 --- a/python/artexplainer/poetry.lock +++ b/python/artexplainer/poetry.lock @@ -1058,6 +1058,7 @@ tritonclient = "^2.18.0" uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/custom_model/poetry.lock b/python/custom_model/poetry.lock index cee57242c6d..7b3f1fa6a58 100644 --- a/python/custom_model/poetry.lock +++ b/python/custom_model/poetry.lock @@ -809,6 +809,7 @@ tritonclient = "^2.18.0" uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/custom_transformer/poetry.lock b/python/custom_transformer/poetry.lock index 2884921815b..fd2feae4b53 100644 --- a/python/custom_transformer/poetry.lock +++ b/python/custom_transformer/poetry.lock @@ -836,6 +836,7 @@ tritonclient = "^2.18.0" uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/kserve/kserve/model.py b/python/kserve/kserve/model.py index 0ec6ef460c2..2ea9f35059b 100644 --- a/python/kserve/kserve/model.py +++ b/python/kserve/kserve/model.py @@ -31,7 +31,6 @@ from .protocol.grpc.grpc_predict_v2_pb2 import ModelInferRequest, ModelInferResponse from .errors import InvalidInput -from .utils.utils import is_structured_cloudevent PREDICTOR_URL_FORMAT = "http://{0}/v1/models/{1}:predict" EXPLAINER_URL_FORMAT = "http://{0}/v1/models/{1}:explain" @@ -200,27 +199,8 @@ async def preprocess(self, payload: Union[Dict, CloudEvent, InferRequest], Returns: Dict|InferRequest: Transformed inputs to ``predict`` handler or return InferRequest for predictor call. """ - response = payload - - if isinstance(payload, CloudEvent): - response = payload.data - # Try to decode and parse JSON UTF-8 if possible, otherwise - # just pass the CloudEvent data on to the predict function. - # This is for the cases that CloudEvent encoding is protobuf, avro etc. - try: - response = orjson.loads(response.decode('UTF-8')) - except (orjson.JSONDecodeError, UnicodeDecodeError) as e: - # If decoding or parsing failed, check if it was supposed to be JSON UTF-8 - if "content-type" in payload._attributes and \ - (payload._attributes["content-type"] == "application/cloudevents+json" or - payload._attributes["content-type"] == "application/json"): - raise InvalidInput(f"Failed to decode or parse binary json cloudevent: {e}") - - elif isinstance(payload, dict): - if is_structured_cloudevent(payload): - response = payload["data"] - return response + return payload def postprocess(self, response: Union[Dict, InferResponse, ModelInferResponse], headers: Dict[str, str] = None) \ -> Union[Dict, ModelInferResponse]: diff --git a/python/kserve/kserve/protocol/dataplane.py b/python/kserve/kserve/protocol/dataplane.py index 67edfce3482..5f62fe55fd3 100644 --- a/python/kserve/kserve/protocol/dataplane.py +++ b/python/kserve/kserve/protocol/dataplane.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from importlib import metadata from typing import Dict, Union, Tuple, Optional import cloudevents.exceptions as ce import orjson -import pkg_resources from cloudevents.http import CloudEvent, from_http from cloudevents.sdk.converters.util import has_binary_headers from ray.serve.api import RayServeHandle @@ -25,7 +25,7 @@ from ..errors import InvalidInput, ModelNotFound from ..model import ModelType from ..model_repository import ModelRepository -from ..utils.utils import create_response_cloudevent +from ..utils.utils import create_response_cloudevent, is_structured_cloudevent from .infer_type import InferRequest from ..constants import constants from .grpc import grpc_predict_v2_pb2 as pb @@ -43,7 +43,7 @@ def __init__(self, model_registry: ModelRepository): # Dynamically fetching version of the installed 'kserve' distribution. The assumption is # that 'kserve' will already be installed by the time this class is instantiated. - self._server_version = pkg_resources.get_distribution("kserve").version + self._server_version = metadata.version("kserve") @property def model_registry(self): @@ -205,23 +205,50 @@ def model_ready(self, model_name: str) -> bool: return self._model_registry.is_model_ready(model_name) - def decode(self, body, headers) -> Union[Dict, InferRequest]: + def decode(self, body, headers) -> Tuple[Union[Dict, InferRequest, CloudEvent], Dict]: t1 = time.time() + decoded_body = body + attributes = {} if isinstance(body, InferRequest): - return body + return decoded_body, attributes if headers and has_binary_headers(headers): body = self.get_binary_cloudevent(body, headers) - else: - if type(body) is bytes: - try: - body = orjson.loads(body) - except orjson.JSONDecodeError as e: - raise InvalidInput(f"Unrecognized request format: {e}") + elif type(body) is bytes: + try: + body = orjson.loads(body) + except orjson.JSONDecodeError as e: + raise InvalidInput(f"Unrecognized request format: {e}") + + decoded_body, attributes = self.decode_cloudevent(body) t2 = time.time() logging.debug(f"decoded request in {round((t2 - t1) * 1000, 9)}ms") - return body - - def encode(self, model_name, body, response, headers) -> Tuple[Dict, Dict[str, str]]: + return decoded_body, attributes + + def decode_cloudevent(self, body) -> Tuple[Union[Dict, InferRequest, CloudEvent], Dict]: + decoded_body = body + attributes = {} + if isinstance(body, CloudEvent): + # Try to decode and parse JSON UTF-8 if possible, otherwise + # just pass the CloudEvent data on to the predict function. + # This is for the cases that CloudEvent encoding is protobuf, avro etc. + try: + decoded_body = orjson.loads(body.data.decode('UTF-8')) + attributes = body._get_attributes() + except (orjson.JSONDecodeError, UnicodeDecodeError) as e: + # If decoding or parsing failed, check if it was supposed to be JSON UTF-8 + if "content-type" in body._attributes and \ + (body._attributes["content-type"] == "application/cloudevents+json" or + body._attributes["content-type"] == "application/json"): + raise InvalidInput(f"Failed to decode or parse binary json cloudevent: {e}") + + elif isinstance(body, dict): + if is_structured_cloudevent(body): + decoded_body = body["data"] + attributes = body + del attributes["data"] + return decoded_body, attributes + + def encode(self, model_name, response, headers, req_attributes: Dict) -> Tuple[Dict, Dict[str, str]]: response_headers = {} # if we received a cloudevent, then also return a cloudevent is_cloudevent = False @@ -233,7 +260,7 @@ def encode(self, model_name, body, response, headers) -> Tuple[Dict, Dict[str, s if headers.get("content-type", "") == "application/cloudevents+json": is_cloudevent = True if is_cloudevent: - response_headers, response = create_response_cloudevent(model_name, body, response, + response_headers, response = create_response_cloudevent(model_name, response, req_attributes, is_binary_cloudevent) if is_binary_cloudevent: @@ -268,7 +295,7 @@ async def infer( .. _CloudEvent: https://cloudevents.io/ """ - body = self.decode(body, headers) + body, req_attributes = self.decode(body, headers) # call model locally or remote model workers model = self.get_model(model_name) @@ -278,7 +305,7 @@ async def infer( model_handle: RayServeHandle = model response = await model_handle.remote(body, headers=headers) - response, response_headers = self.encode(model_name, body, response, headers) + response, response_headers = self.encode(model_name, response, headers, req_attributes) return response, response_headers async def explain(self, model_name: str, @@ -298,7 +325,7 @@ async def explain(self, model_name: str, Raises: InvalidInput: An error when the body bytes can't be decoded as JSON. """ - body = self.decode(body, headers) + body, req_attributes = self.decode(body, headers) # call model locally or remote model workers model = self.get_model(model_name) @@ -307,5 +334,5 @@ async def explain(self, model_name: str, else: model_handle = model response = await model_handle.remote(body, model_type=ModelType.EXPLAINER) - response, response_headers = self.encode(model_name, body, response, headers) + response, response_headers = self.encode(model_name, response, headers, req_attributes) return response, response_headers diff --git a/python/kserve/kserve/protocol/infer_type.py b/python/kserve/kserve/protocol/infer_type.py index 2c3d8903fd2..627056851be 100644 --- a/python/kserve/kserve/protocol/infer_type.py +++ b/python/kserve/kserve/protocol/infer_type.py @@ -31,7 +31,7 @@ class InferInput: _datatype: str _parameters: Dict - def __init__(self, name, shape, datatype, data=None, parameters={}): + def __init__(self, name, shape, datatype, data=None, parameters=None): """An object of InferInput class is used to describe input tensor for an inference request. Parameters @@ -42,9 +42,13 @@ def __init__(self, name, shape, datatype, data=None, parameters={}): The shape of the associated input. datatype : str The datatype of the associated input. - data: Union[List, InferTensorContents] + data : Union[List, InferTensorContents] The data of the REST/gRPC input. When data is not set, raw_data is used for gRPC for numpy array bytes. + parameters : dict + The additional server-specific parameters. """ + if parameters is None: + parameters = {} self._name = name self._shape = shape self._datatype = datatype @@ -236,7 +240,9 @@ class InferRequest: from_grpc: bool def __init__(self, model_name: str, infer_inputs: List[InferInput], - request_id=None, raw_inputs=None, from_grpc=False, parameters={}): + request_id=None, raw_inputs=None, from_grpc=False, parameters=None): + if parameters is None: + parameters = {} self.id = request_id self.model_name = model_name self.inputs = infer_inputs @@ -326,7 +332,7 @@ def as_dataframe(self) -> pd.DataFrame: class InferOutput: - def __init__(self, name, shape, datatype, data=None, parameters={}): + def __init__(self, name, shape, datatype, data=None, parameters=None): """An object of InferOutput class is used to describe input tensor for an inference request. Parameters @@ -337,9 +343,13 @@ def __init__(self, name, shape, datatype, data=None, parameters={}): The shape of the associated input. datatype : str The datatype of the associated input. - data: Union[List, InferTensorContents] + data : Union[List, InferTensorContents] The data of the REST/gRPC input. When data is not set, raw_data is used for gRPC for numpy array bytes. + parameters : dict + The additional server-specific parameters. """ + if parameters is None: + parameters = {} self._name = name self._shape = shape self._datatype = datatype @@ -514,7 +524,9 @@ class InferResponse: from_grpc: bool def __init__(self, response_id: str, model_name: str, infer_outputs: List[InferOutput], - raw_outputs=None, from_grpc=False, parameters={}): + raw_outputs=None, from_grpc=False, parameters=None): + if parameters is None: + parameters = {} self.id = response_id self.model_name = model_name self.outputs = infer_outputs diff --git a/python/kserve/kserve/protocol/rest/server.py b/python/kserve/kserve/protocol/rest/server.py index 206963a88c4..e6228cf0fee 100644 --- a/python/kserve/kserve/protocol/rest/server.py +++ b/python/kserve/kserve/protocol/rest/server.py @@ -14,10 +14,10 @@ import asyncio import socket +from importlib import metadata from typing import List, Optional from prometheus_client import REGISTRY, exposition -import pkg_resources import uvicorn from fastapi import FastAPI, Request, Response from fastapi.routing import APIRoute as FastAPIRoute @@ -70,7 +70,7 @@ def create_application(self) -> FastAPI: return FastAPI( title="KServe ModelServer", - version=pkg_resources.get_distribution("kserve").version, + version=metadata.version("kserve"), docs_url="/docs" if self.enable_docs_url else None, redoc_url=None, default_response_class=ORJSONResponse, diff --git a/python/kserve/kserve/utils/utils.py b/python/kserve/kserve/utils/utils.py index 070fd05051a..9eca490e566 100644 --- a/python/kserve/kserve/utils/utils.py +++ b/python/kserve/kserve/utils/utils.py @@ -94,18 +94,17 @@ def is_structured_cloudevent(body: Dict) -> bool: and "data" in body -def create_response_cloudevent(model_name: str, body: Union[Dict, CloudEvent], response: Dict, +def create_response_cloudevent(model_name: str, response: Dict, req_attributes: Dict, binary_event=False) -> tuple: ce_attributes = {} if os.getenv("CE_MERGE", "false").lower() == "true": if binary_event: - ce_attributes = body._attributes + ce_attributes = req_attributes if "datacontenttype" in ce_attributes: # Optional field so must check del ce_attributes["datacontenttype"] else: - ce_attributes = body - del ce_attributes["data"] + ce_attributes = req_attributes # Remove these fields so we generate new ones del ce_attributes["id"] diff --git a/python/kserve/poetry.lock b/python/kserve/poetry.lock index 3ce3f893903..cc1c75a3853 100644 --- a/python/kserve/poetry.lock +++ b/python/kserve/poetry.lock @@ -254,9 +254,6 @@ files = [ {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, ] -[package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} - [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] dev = ["attrs[docs,tests]", "pre-commit"] @@ -3052,4 +3049,4 @@ storage = ["azure-identity", "azure-storage-blob", "azure-storage-file-share", " [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "136659c633799e4d26eab95e854332df5c8d407423d221f1f2c3d7bfe0995a92" +content-hash = "21b8e2aaa7ac6c6834a9fa7c6a2714480513982e85dd251942abee53c1dd52fd" diff --git a/python/kserve/test/test_dataplane.py b/python/kserve/test/test_dataplane.py index fa8fa1028b6..9fbb5630809 100644 --- a/python/kserve/test/test_dataplane.py +++ b/python/kserve/test/test_dataplane.py @@ -317,7 +317,7 @@ async def test_infer_ce_avro_binary(self, dataplane_with_ce_model): writer.write(msg, encoder) data = bytes_writer.getvalue() - event = dummy_cloud_event(data, set_contenttype=True) + event = dummy_cloud_event(data, set_contenttype=True, contenttype="application/avro") # Creates the HTTP request representation of the CloudEvent in binary content mode headers, body = to_binary(event) diff --git a/python/kserve/test/test_server.py b/python/kserve/test/test_server.py index e8aa36cba35..818c6368099 100644 --- a/python/kserve/test/test_server.py +++ b/python/kserve/test/test_server.py @@ -49,7 +49,8 @@ ''' -def dummy_cloud_event(data, set_contenttype=False, add_extension=False): +def dummy_cloud_event(data, set_contenttype: bool = False, add_extension: bool = False, + contenttype: str = "application/json"): # This data defines a binary cloudevent attributes = { "type": "com.example.sampletype1", @@ -59,7 +60,7 @@ def dummy_cloud_event(data, set_contenttype=False, add_extension=False): "time": "2021-01-28T21:04:43.144141+00:00" } if set_contenttype: - attributes["content-type"] = "application/json" + attributes["content-type"] = contenttype if add_extension: attributes["custom-extension"] = "custom-value" @@ -150,7 +151,7 @@ def preprocess(self, request, headers: Dict[str, str] = None): assert attributes["specversion"] == "1.0" assert attributes["source"] == "https://example.com/event-producer" assert attributes["type"] == "com.example.sampletype1" - assert attributes["content-type"] == "application/json" + assert attributes["content-type"] == "application/avro" return self._parserequest(request.data) async def predict(self, request, headers=None): @@ -507,7 +508,7 @@ def test_predict_ce_avro_binary(self, http_server_client): writer.write(msg, encoder) data = bytes_writer.getvalue() - event = dummy_cloud_event(data, set_contenttype=True) + event = dummy_cloud_event(data, set_contenttype=True, contenttype="application/avro") # Creates the HTTP request representation of the CloudEvent in binary content mode headers, body = to_binary(event) resp = http_server_client.post('/v1/models/TestModel:predict', headers=headers, data=body) diff --git a/python/lgbserver/poetry.lock b/python/lgbserver/poetry.lock index a542758c954..17d4bde62f0 100644 --- a/python/lgbserver/poetry.lock +++ b/python/lgbserver/poetry.lock @@ -1404,6 +1404,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/paddleserver/poetry.lock b/python/paddleserver/poetry.lock index acbba844093..2c0abc56cd8 100644 --- a/python/paddleserver/poetry.lock +++ b/python/paddleserver/poetry.lock @@ -1414,6 +1414,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/pmmlserver/poetry.lock b/python/pmmlserver/poetry.lock index 8383a65dde7..91c861b1719 100644 --- a/python/pmmlserver/poetry.lock +++ b/python/pmmlserver/poetry.lock @@ -1459,6 +1459,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/sklearnserver/poetry.lock b/python/sklearnserver/poetry.lock index 5a5e5a990cc..db8074a9042 100644 --- a/python/sklearnserver/poetry.lock +++ b/python/sklearnserver/poetry.lock @@ -1332,6 +1332,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source] diff --git a/python/xgbserver/poetry.lock b/python/xgbserver/poetry.lock index c027eeea874..5cfd4075441 100644 --- a/python/xgbserver/poetry.lock +++ b/python/xgbserver/poetry.lock @@ -1404,6 +1404,7 @@ urllib3 = {version = "^1.26.8", optional = true} uvicorn = "^0.16.0" [package.extras] +logging = ["asgi-logger (>=0.1.0,<0.2.0)"] storage = ["azure-identity (>=1.8.0,<2.0.0)", "azure-storage-blob (>=12.10.0,<13.0.0)", "azure-storage-file-share (>=12.7.0,<12.8.0)", "boto3 (>=1.21.0,<2.0.0)", "google-cloud-storage (>=1.20.0,<2.0.0)", "requests (>=2.20.0,<3.0.0)", "urllib3 (>=1.26.8,<2.0.0)"] [package.source]