diff --git a/examples/istio/shadow/.gitignore b/examples/istio/shadow/.gitignore new file mode 100644 index 0000000000..2b158f424d --- /dev/null +++ b/examples/istio/shadow/.gitignore @@ -0,0 +1,2 @@ +model.yaml +shadow.yaml \ No newline at end of file diff --git a/examples/istio/shadow/README.md b/examples/istio/shadow/README.md new file mode 100644 index 0000000000..035a655017 --- /dev/null +++ b/examples/istio/shadow/README.md @@ -0,0 +1,3 @@ +# Shadow Rollout with Seldon and Istio + +Follow the [notebook](istio_shadow.ipynb) to show how you can deploy Shadow Seldon Deployment with Seldon and Istio. These are useful when you want to test a new model or higher latency inference piepline (e.g., with explanation components) with production traffic but without affecting the live deployment. diff --git a/examples/istio/shadow/istio_shadow.ipynb b/examples/istio/shadow/istio_shadow.ipynb new file mode 100644 index 0000000000..b4aa213b3d --- /dev/null +++ b/examples/istio/shadow/istio_shadow.ipynb @@ -0,0 +1,477 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Shadow Rollout with Seldon and Istio\n", + "\n", + "This notebook shows how you can deploy \"shadow\" deployments to direct traffic not only to the main Seldon Deployment but also to a shadow deployment whose response will be dicarded. This allows you to test new models in a production setting and with production traffic and anlalyse how they perform before putting them live.\n", + "\n", + "These are useful when you want to test a new model or higher latency inference piepline (e.g., with explanation components) with production traffic but without affecting the live deployment.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Seldon Core\n", + "\n", + "Use the setup notebook to [Setup Cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Setup-Cluster) with [Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Istio) and [Install Seldon Core](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Install-Seldon-Core). Instructions [also online](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/seldon created\n" + ] + } + ], + "source": [ + "!kubectl create namespace seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\n" + ] + } + ], + "source": [ + "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, \"w\") as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ensure the istio ingress gatewaty is port-forwarded to localhost:8004\n", + "\n", + "\n", + "\n", + "* Istio: `kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8004:8080`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.7.0-dev'" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ISTIO_GATEWAY = \"localhost:8004\"\n", + "\n", + "VERSION = !cat ../../../version.txt\n", + "VERSION = VERSION[0]\n", + "VERSION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch main model\n", + "\n", + "We will create a very simple Seldon Deployment with a dummy model image `seldonio/mock_classifier:1.0`. This deployment is named `example`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate model.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: production-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: default\n", + " replicas: 1\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example configured\n" + ] + } + ], + "source": [ + "!kubectl apply -f model.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-default-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get predictions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from seldon_core.seldon_client import SeldonClient\n", + "\n", + "sc = SeldonClient(\n", + " deployment_name=\"example\", namespace=\"seldon\", gateway_endpoint=ISTIO_GATEWAY\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REST Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.6670563912281003\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.09538308704053941]}}, 'meta': {'requestPath': {'classifier': 'seldonio/mock_classifier:1.7.0-dev'}}}\n" + ] + } + ], + "source": [ + "r = sc.predict(gateway=\"istio\", transport=\"rest\")\n", + "assert r.success == True\n", + "print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch Shadow\n", + "\n", + "We will now create a new Seldon Deployment for our Shadow deployment with a new model." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate shadow.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: shadow-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: default\n", + " replicas: 1\n", + " traffic: 100\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: shadow\n", + " replicas: 1\n", + " shadow: true\n", + " traffic: 100\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "You can use the traffic field under the shadow componentSpecs to mirror a fraction of the traffic, instead of mirroring all requests. If this field is absent, all traffic will be mirrored. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example configured\n" + ] + } + ], + "source": [ + "!kubectl apply -f shadow.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-default-0-classifier\" successfully rolled out\n", + "Waiting for deployment \"example-shadow-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-shadow-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')\n", + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[1].metadata.name}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's send a bunch of requests to the endpoint." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range(10):\n", + " r = sc.predict(gateway=\"istio\", transport=\"rest\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "default_count = !kubectl logs $(kubectl get pod -lseldon-app=example-default -o jsonpath='{.items[0].metadata.name}') classifier | grep \"root.predict\" | wc -l" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "shadow_count = !kubectl logs $(kubectl get pod -lseldon-app=example-shadow -o jsonpath='{.items[0].metadata.name}') classifier | grep \"root.predict\" | wc -l" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['10']\n", + "['11']\n" + ] + } + ], + "source": [ + "print(shadow_count)\n", + "print(default_count)\n", + "assert int(shadow_count[0]) == 10\n", + "assert int(default_count[0]) == 11" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TearDown" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete -f model.yaml" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.6" + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go index 83c5c6bfd6..da6d31e5dc 100644 --- a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go +++ b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go @@ -128,13 +128,17 @@ func checkTraffic(spec *SeldonDeploymentSpec, fldPath *field.Path, allErrs field var shadows int = 0 for i := 0; i < len(spec.Predictors); i++ { p := spec.Predictors[i] - trafficSum = trafficSum + p.Traffic if p.Shadow == true { shadows += 1 if shadows > 1 { allErrs = append(allErrs, field.Invalid(fldPath, spec.Predictors[i].Name, "Multiple shadows are not allowed")) } + if p.Traffic < 0 || p.Traffic > 100 { + allErrs = append(allErrs, field.Invalid(fldPath, spec.Predictors[i].Name, "shadow traffic is illegal, the traffic number should be between [0, 100]")) + } + } else { + trafficSum = trafficSum + p.Traffic } } diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go index d2cb2bf6fb..1f122d9259 100644 --- a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go +++ b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go @@ -1384,3 +1384,37 @@ func TestValidateTwoShadows(t *testing.T) { err = spec.ValidateSeldonDeployment() g.Expect(err).ToNot(BeNil()) } + +func TestValidateShadowTraffic(t *testing.T) { + g := NewGomegaWithT(t) + err := setupTestConfigMap() + g.Expect(err).To(BeNil()) + impl := PredictiveUnitImplementation(constants.PrePackedServerTensorflow) + spec := &SeldonDeploymentSpec{ + Protocol: ProtocolTensorflow, + Predictors: []PredictorSpec{ + { + Name: "p1", + Graph: PredictiveUnit{ + Name: "classifier", + Implementation: &impl, + ModelURI: "s3://mybucket/model", + }, + Traffic: 100, + }, + { + Name: "p2", + Graph: PredictiveUnit{ + Name: "classifier", + Implementation: &impl, + ModelURI: "s3://mybucket/model", + }, + Shadow: true, + Traffic: 101, + }, + }, + } + spec.DefaultSeldonDeployment("mydep", "default") + err = spec.ValidateSeldonDeployment() + g.Expect(err).ToNot(BeNil()) +} diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go index a5014c4142..7717676061 100644 --- a/operator/controllers/seldondeployment_controller.go +++ b/operator/controllers/seldondeployment_controller.go @@ -327,6 +327,17 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, }, } + if p.Traffic > 0 { + //if shadow predictor's traffic is greater than 0, set the mirror percentage (like https://istio.io/latest/docs/tasks/traffic-management/mirroring/#mirroring-traffic-to-v2) in VirtualService + vsvc.Spec.Http[0].MirrorPercentage = &istio_networking.Percent{ + Value: float64(p.Traffic), + } + + vsvc.Spec.Http[1].MirrorPercentage = &istio_networking.Percent{ + Value: float64(p.Traffic), + } + } + continue }