From 62719e8ee33c661de7dd11ab94faf98848b1c98f Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Thu, 23 Jul 2020 18:56:56 +0100 Subject: [PATCH 1/2] Allow tensorflow protocol explanations --- executor/api/rest/server.go | 2 + notebooks/explainer_examples.ipynb | 147 +++++++++++++++++- notebooks/protocol_examples.ipynb | 61 ++------ notebooks/resources/.gitignore | 3 + notebooks/seldon_core_setup.ipynb | 139 ++--------------- .../seldondeployment_explainers.go | 15 +- 6 files changed, 188 insertions(+), 179 deletions(-) diff --git a/executor/api/rest/server.go b/executor/api/rest/server.go index 482da1c78c..36f3598e7a 100644 --- a/executor/api/rest/server.go +++ b/executor/api/rest/server.go @@ -153,7 +153,9 @@ func (r *SeldonRestApi) Initialise() { case api.ProtocolTensorflow: r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}/:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) + r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) r.Router.NewRoute().Path("/v1/models/:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) // Nonstandard path - Seldon extension + r.Router.NewRoute().Path("/v1/models:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) // Nonstandard path - Seldon extension r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}").Methods("GET", "OPTIONS").HandlerFunc(r.wrapMetrics(metric.StatusHttpServiceName, r.status)) r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}/metadata").Methods("GET", "OPTIONS").HandlerFunc(r.wrapMetrics(metric.MetadataHttpServiceName, r.metadata)) } diff --git a/notebooks/explainer_examples.ipynb b/notebooks/explainer_examples.ipynb index 39ee7f3b43..93fdf6f185 100644 --- a/notebooks/explainer_examples.ipynb +++ b/notebooks/explainer_examples.ipynb @@ -607,6 +607,151 @@ "source": [ "!kubectl delete -f resources/imagenet_explainer_grpc.yaml" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tensorflow CIFAR10 Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile resources/cifar10_explainer.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: cifar10-classifier\n", + "spec:\n", + " protocol: tensorflow\n", + " annotations:\n", + " seldon.io/rest-timeout: \"100000\"\n", + " predictors:\n", + " - componentSpecs:\n", + " graph:\n", + " implementation: TENSORFLOW_SERVER\n", + " modelUri: gs://seldon-models/tfserving/cifar10/resnet32\n", + " name: cifar10-classifier\n", + " logger:\n", + " mode: all\n", + " explainer:\n", + " type: AnchorImages\n", + " modelUri: gs://seldon-models/tfserving/cifar10/explainer\n", + " name: default\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl apply -f resources/cifar10_explainer.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=cifar10-classifier -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl rollout status deploy/cifar10-classifier-default-explainer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tensorflow as tf\n", + "import matplotlib.pyplot as plt\n", + "import os\n", + "\n", + "url = 'https://storage.googleapis.com/seldon-models/alibi-detect/classifier/'\n", + "path_model = os.path.join(url, \"cifar10\", \"resnet32\", 'model.h5')\n", + "save_path = tf.keras.utils.get_file(\"resnet32\", path_model)\n", + "model = tf.keras.models.load_model(save_path)\n", + "\n", + "train, test = tf.keras.datasets.cifar10.load_data()\n", + "X_train, y_train = train\n", + "X_test, y_test = test\n", + "\n", + "X_train = X_train.astype('float32') / 255\n", + "X_test = X_test.astype('float32') / 255\n", + "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)\n", + "class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',\n", + " 'dog', 'frog', 'horse', 'ship', 'truck']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from subprocess import run, Popen, PIPE\n", + "import json\n", + "import numpy as np\n", + "idx=12\n", + "test_example=X_test[idx:idx+1].tolist()\n", + "payload='{\"instances\":'+f\"{test_example}\"+' }'\n", + "cmd=f\"\"\"curl -d '{payload}' \\\n", + " http://localhost:8003/seldon/seldon/cifar10-classifier/v1/models/cifar10-classifier/:predict \\\n", + " -H \"Content-Type: application/json\"\n", + "\"\"\"\n", + "ret = Popen(cmd, shell=True,stdout=PIPE)\n", + "raw = ret.stdout.read().decode(\"utf-8\")\n", + "print(raw)\n", + "res=json.loads(raw)\n", + "arr=np.array(res[\"predictions\"])\n", + "X = X_test[idx].reshape(1, 32, 32, 3)\n", + "plt.imshow(X.reshape(32, 32, 3))\n", + "plt.axis('off')\n", + "plt.show()\n", + "print(\"class:\",class_names[y_test[idx][0]])\n", + "print(\"prediction:\",class_names[arr[0].argmax()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_example=X_test[idx:idx+1].tolist()\n", + "payload='{\"instances\":'+f\"{test_example}\"+' }'\n", + "cmd=f\"\"\"curl -d '{payload}' \\\n", + " http://localhost:8003/seldon/seldon/cifar10-classifier-explainer/default/v1/models/cifar10-classifier:explain \\\n", + " -H \"Content-Type: application/json\"\n", + "\"\"\"\n", + "ret = Popen(cmd, shell=True,stdout=PIPE)\n", + "raw = ret.stdout.read().decode(\"utf-8\")\n", + "explanation = json.loads(raw)\n", + "arr = np.array(explanation[\"anchor\"])\n", + "plt.imshow(arr)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -626,7 +771,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/notebooks/protocol_examples.ipynb b/notebooks/protocol_examples.ipynb index 98b008139d..d336fa2fa9 100644 --- a/notebooks/protocol_examples.ipynb +++ b/notebooks/protocol_examples.ipynb @@ -304,10 +304,6 @@ " ports:\n", " - containerPort: 8501\n", " name: http\n", - " tolerations:\n", - " - key: model\n", - " operator: Exists\n", - " effect: NoSchedule\n", " graph:\n", " name: halfplustwo\n", " type: MODEL\n", @@ -440,18 +436,9 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waiting for deployment \"grpc-tfserving-model-0-halfplustwo\" rollout to finish: 0 of 1 updated replicas are available...\n", - "deployment \"grpc-tfserving-model-0-halfplustwo\" successfully rolled out\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-tfserving \\\n", " -o jsonpath='{.items[0].metadata.name}')" @@ -459,17 +446,9 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Available\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "for i in range(60):\n", " state=!kubectl get sdep grpc-tfserving -o jsonpath='{.status.state}'\n", @@ -490,17 +469,9 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'outputs': {'x': {'dtype': 'DT_FLOAT', 'tensorShape': {'dim': [{'size': '3'}]}, 'floatVal': [2.5, 3, 3.5]}}, 'modelSpec': {'name': 'halfplustwo', 'version': '123', 'signatureName': 'serving_default'}}\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "X=!cd ../executor/proto && grpcurl \\\n", " -d '{\"model_spec\":{\"name\":\"halfplustwo\"},\"inputs\":{\"x\":{\"dtype\": 1, \"tensor_shape\": {\"dim\":[{\"size\": 3}]}, \"floatVal\" : [1.0, 2.0, 3.0]}}}' \\\n", @@ -514,17 +485,9 @@ }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seldondeployment.machinelearning.seldon.io \"grpc-tfserving\" deleted\r\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!kubectl delete -f resources/model_tfserving_grpc.yaml" ] diff --git a/notebooks/resources/.gitignore b/notebooks/resources/.gitignore index 737846d521..f142320581 100644 --- a/notebooks/resources/.gitignore +++ b/notebooks/resources/.gitignore @@ -4,3 +4,6 @@ fixed_v1_rep2.yaml fixed_v1_rep4.yaml fixed_v2_2predictors.yaml fixed_v2_rep.yaml +cifar10_explainer.yaml +model_v2_grpc.yaml +model_v2_rest.yaml \ No newline at end of file diff --git a/notebooks/seldon_core_setup.ipynb b/notebooks/seldon_core_setup.ipynb index 5214252b21..9247c49542 100644 --- a/notebooks/seldon_core_setup.ipynb +++ b/notebooks/seldon_core_setup.ipynb @@ -29,36 +29,20 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "namespace/seldon created\r\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Context \"do-lon1-k8s-1-16-10-do-0-lon1-1594477430912\" modified.\r\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -74,17 +58,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "namespace/seldon-system created\r\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl create namespace seldon-system" ] @@ -98,22 +74,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME: seldon-core\r\n", - "LAST DEPLOYED: Sat Jul 11 15:37:46 2020\r\n", - "NAMESPACE: seldon-system\r\n", - "STATUS: deployed\r\n", - "REVISION: 1\r\n", - "TEST SUITE: None\r\n" - ] - } - ], + "outputs": [], "source": [ "!helm install seldon-core seldon-core-operator --repo https://storage.googleapis.com/seldon-charts --set ambassador.enabled=true --set usageMetrics.enabled=true --namespace seldon-system" ] @@ -172,99 +135,27 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\"datawire\" has been added to your repositories\r\n" - ] - } - ], + "outputs": [], "source": [ "!helm repo add datawire https://www.getambassador.io" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hang tight while we grab the latest from your chart repositories...\n", - "...Successfully got an update from the \"seldonio\" chart repository\n", - "...Successfully got an update from the \"seldon-staging\" chart repository\n", - "...Successfully got an update from the \"flagger\" chart repository\n", - "...Successfully got an update from the \"strimzi\" chart repository\n", - "...Successfully got an update from the \"jaegertracing\" chart repository\n", - "...Successfully got an update from the \"datawire\" chart repository\n", - "...Successfully got an update from the \"stable\" chart repository\n", - "Update Complete. ⎈ Happy Helming!⎈ \n" - ] - } - ], + "outputs": [], "source": [ "!helm repo update" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "manifest_sorter.go:192: info: skipping unknown hook: \"crd-install\"\n", - "NAME: ambassador\n", - "LAST DEPLOYED: Sat Jul 11 15:40:09 2020\n", - "NAMESPACE: seldon-system\n", - "STATUS: deployed\n", - "REVISION: 1\n", - "NOTES:\n", - "-------------------------------------------------------------------------------\n", - " Congratulations! You've successfully installed Ambassador!\n", - "\n", - "-------------------------------------------------------------------------------\n", - "To get the IP address of Ambassador, run the following commands:\n", - "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n", - " You can watch the status of by running 'kubectl get svc -w --namespace seldon-system ambassador'\n", - "\n", - " On GKE/Azure:\n", - " export SERVICE_IP=$(kubectl get svc --namespace seldon-system ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n", - "\n", - " On AWS:\n", - " export SERVICE_IP=$(kubectl get svc --namespace seldon-system ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n", - "\n", - " echo http://$SERVICE_IP:\n", - "\n", - "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n" - ] - } - ], + "outputs": [], "source": [ "!helm install ambassador datawire/ambassador \\\n", " --set image.repository=quay.io/datawire/ambassador \\\n", @@ -295,7 +186,7 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080" + "!kubectl port-forward $(kubectl get pods -n seldon-system -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon-system 8003:8080" ] }, { diff --git a/operator/controllers/seldondeployment_explainers.go b/operator/controllers/seldondeployment_explainers.go index a08f9c65c4..e9a5e8283b 100644 --- a/operator/controllers/seldondeployment_explainers.go +++ b/operator/controllers/seldondeployment_explainers.go @@ -116,7 +116,7 @@ func (ei *ExplainerInitialiser) createExplainer(mlDep *machinelearningv1.SeldonD var httpPort = 0 var grpcPort = 0 var portNum int32 = 9000 - var explainerProtocol string + var explainerTransport string if p.Explainer.Endpoint != nil && p.Explainer.Endpoint.ServicePort != 0 { portNum = p.Explainer.Endpoint.ServicePort } @@ -126,14 +126,19 @@ func (ei *ExplainerInitialiser) createExplainer(mlDep *machinelearningv1.SeldonD httpPort = int(portNum) customPort := getPort(portType, explainerContainer.Ports) - if p.Explainer.Endpoint != nil && p.Explainer.Endpoint.Type == machinelearningv1.GRPC { - explainerProtocol = "grpc" + if mlDep.Spec.Transport == machinelearningv1.TransportGrpc || (p.Explainer.Endpoint != nil && p.Explainer.Endpoint.Type == machinelearningv1.GRPC) { + explainerTransport = "grpc" pSvcEndpoint = c.serviceDetails[pSvcName].GrpcEndpoint } else { - explainerProtocol = "http" + explainerTransport = "http" pSvcEndpoint = c.serviceDetails[pSvcName].HttpEndpoint } + explainerProtocol := string(machinelearningv1.ProtocolSeldon) + if mlDep.Spec.Protocol == machinelearningv1.ProtocolTensorflow { + explainerProtocol = string(machinelearningv1.ProtocolTensorflow) + } + if customPort == nil { explainerContainer.Ports = append(explainerContainer.Ports, corev1.ContainerPort{Name: portType, ContainerPort: portNum, Protocol: corev1.ProtocolTCP}) } else { @@ -156,7 +161,7 @@ func (ei *ExplainerInitialiser) createExplainer(mlDep *machinelearningv1.SeldonD explainerContainer.Args = []string{ "--model_name=" + mlDep.Name, "--predictor_host=" + pSvcEndpoint, - "--protocol=" + "seldon." + explainerProtocol, + "--protocol=" + explainerProtocol + "." + explainerTransport, "--http_port=" + strconv.Itoa(int(portNum)), } From d205fd05f211d35b3dc88a172159ef2ecf4e14ad Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 24 Jul 2020 11:44:24 +0100 Subject: [PATCH 2/2] Add comment on :predict path addition --- executor/api/rest/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/executor/api/rest/server.go b/executor/api/rest/server.go index 36f3598e7a..11f1b93091 100644 --- a/executor/api/rest/server.go +++ b/executor/api/rest/server.go @@ -154,6 +154,7 @@ func (r *SeldonRestApi) Initialise() { case api.ProtocolTensorflow: r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}/:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) + // Allow both :predict before and after final / in path. r.Router.NewRoute().Path("/v1/models/:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) // Nonstandard path - Seldon extension r.Router.NewRoute().Path("/v1/models:predict").Methods("OPTIONS", "POST").HandlerFunc(r.wrapMetrics(metric.PredictionHttpServiceName, r.predictions)) // Nonstandard path - Seldon extension r.Router.NewRoute().Path("/v1/models/{"+ModelHttpPathVariable+"}").Methods("GET", "OPTIONS").HandlerFunc(r.wrapMetrics(metric.StatusHttpServiceName, r.status))