From f6f46ac56998642bbd71c1a59692a5bff6459152 Mon Sep 17 00:00:00 2001 From: Lun-Kai Hsu Date: Mon, 22 Oct 2018 16:20:17 -0700 Subject: [PATCH] Roll out model with istio (#1823) * roll out model with istio * fix test * fix test * fix test * fix test --- .../prototypes/tf-serving-gcp.jsonnet | 5 +- .../prototypes/tf-serving-service.jsonnet | 20 ++++ .../tf-serving-service-template.libsonnet | 96 +++++++++++++++++++ .../tf-serving/tf-serving-template.libsonnet | 61 +----------- testing/test_deploy.py | 18 ++-- .../workflows/components/tfserving.libsonnet | 2 - 6 files changed, 134 insertions(+), 68 deletions(-) create mode 100644 kubeflow/tf-serving/prototypes/tf-serving-service.jsonnet create mode 100644 kubeflow/tf-serving/tf-serving-service-template.libsonnet diff --git a/kubeflow/tf-serving/prototypes/tf-serving-gcp.jsonnet b/kubeflow/tf-serving/prototypes/tf-serving-gcp.jsonnet index 913ed0b2a47..936de249e72 100644 --- a/kubeflow/tf-serving/prototypes/tf-serving-gcp.jsonnet +++ b/kubeflow/tf-serving/prototypes/tf-serving-gcp.jsonnet @@ -1,13 +1,13 @@ // @apiVersion 0.1 -// @name io.ksonnet.pkg.tf-serving-gcp +// @name io.ksonnet.pkg.tf-serving-deployment-gcp // @description TensorFlow serving // @shortDescription A TensorFlow serving deployment // @param name string Name to give to each of the components -// @optionalParam serviceType string ClusterIP The k8s service type for tf serving. // @optionalParam numGpus string 0 Number of gpus to use // @optionalParam deployHttpProxy string false Whether to deploy http proxy // @optionalParam modelBasePath string gs://kubeflow-examples-data/mnist The model path // @optionalParam modelName string null The model name +// @optionalParam versionName string v1 The version name // @optionalParam defaultCpuImage string tensorflow/serving:1.8.0 The default model server image (cpu) // @optionalParam defaultGpuImage string tensorflow/serving:1.10.0-gpu The default model server image (gpu) // @optionalParam httpProxyImage string gcr.io/kubeflow-images-public/tf-model-server-http-proxy:v20180723 Http proxy image @@ -56,5 +56,4 @@ local tfDeployment = base.tfDeployment + ); util.list([ tfDeployment, - base.tfService, ],) diff --git a/kubeflow/tf-serving/prototypes/tf-serving-service.jsonnet b/kubeflow/tf-serving/prototypes/tf-serving-service.jsonnet new file mode 100644 index 00000000000..770d6a12d85 --- /dev/null +++ b/kubeflow/tf-serving/prototypes/tf-serving-service.jsonnet @@ -0,0 +1,20 @@ +// @apiVersion 0.1 +// @name io.ksonnet.pkg.tf-serving-service +// @description TensorFlow serving +// @shortDescription A TensorFlow serving model +// @param name string Name to give to each of the components +// @optionalParam serviceType string ClusterIP The k8s service type for tf serving. +// @optionalParam modelName string null The model name +// @optionalParam trafficRule string v1:100 The traffic rule, in the format of version:percentage,version:percentage,.. +// @optionalParam injectIstio string false Whether to inject istio sidecar; should be true or false. + +local k = import "k.libsonnet"; +local tfservingService = import "kubeflow/tf-serving/tf-serving-service-template.libsonnet"; +local util = import "kubeflow/tf-serving/util.libsonnet"; + +local base = tfservingService.new(env, params); +util.list( + [ + base.tfService, + ] + if util.toBool(params.injectIstio) then [base.virtualService] else [], +) diff --git a/kubeflow/tf-serving/tf-serving-service-template.libsonnet b/kubeflow/tf-serving/tf-serving-service-template.libsonnet new file mode 100644 index 00000000000..859cba85b91 --- /dev/null +++ b/kubeflow/tf-serving/tf-serving-service-template.libsonnet @@ -0,0 +1,96 @@ +{ + local k = import "k.libsonnet", + local util = import "kubeflow/tf-serving/util.libsonnet", + new(_env, _params):: { + local params = _env + _params, + local namespace = params.namespace, + local name = params.name, + local modelName = + if params.modelName == "null" then + params.name + else + params.modelName, + + local tfService = { + apiVersion: "v1", + kind: "Service", + metadata: { + labels: { + app: modelName, + }, + name: name, + namespace: namespace, + annotations: { + "getambassador.io/config": + std.join("\n", [ + "---", + "apiVersion: ambassador/v0", + "kind: Mapping", + "name: tfserving-predict-mapping-" + modelName, + "prefix: /tfserving/models/" + modelName, + "rewrite: /v1/models/" + modelName + ":predict", + "method: POST", + "service: " + name + "." + namespace + ":8500", + "---", + "apiVersion: ambassador/v0", + "kind: Mapping", + "name: tfserving-predict-mapping-" + modelName + "-get", + "prefix: /tfserving/models/" + modelName, + "rewrite: /v1/models/" + modelName, + "method: GET", + "service: " + name + "." + namespace + ":8500", + ]), + }, //annotations + }, + spec: { + ports: [ + { + name: "grpc-tf-serving", + port: 9000, + targetPort: 9000, + }, + { + name: "http-tf-serving", + port: 8500, + targetPort: 8500, + }, + ], + selector: { + app: modelName, + }, + type: params.serviceType, + }, + }, // tfService + tfService:: tfService, + + local versionWeights = std.split(params.trafficRule, ","), + local virtualService = { + apiVersion: "networking.istio.io/v1alpha3", + kind: "VirtualService", + metadata: { + name: name, + namespace: namespace, + }, + spec: { + hosts: [ + name, + ], + http: [ + { + route: [ + { + destination: { + host: name, + subset: std.split(versionWeight, ":")[0], + }, + weight: std.parseInt(std.split(versionWeight, ":")[1]), + } + for versionWeight in versionWeights + ], + }, + ], + }, + }, + virtualService:: virtualService, + }, // new +} diff --git a/kubeflow/tf-serving/tf-serving-template.libsonnet b/kubeflow/tf-serving/tf-serving-template.libsonnet index 7491f6e1140..0dddd1baa8e 100644 --- a/kubeflow/tf-serving/tf-serving-template.libsonnet +++ b/kubeflow/tf-serving/tf-serving-template.libsonnet @@ -10,6 +10,7 @@ params.name else params.modelName, + local versionName = params.versionName, local modelServerImage = if params.numGpus == "0" then params.defaultCpuImage @@ -17,61 +18,8 @@ params.defaultGpuImage, // Optional features. - // TODO(lunkai): Add Istio // TODO(lunkai): Add request logging - local tfService = { - apiVersion: "v1", - kind: "Service", - metadata: { - labels: { - app: name, - }, - name: name, - namespace: namespace, - annotations: { - "getambassador.io/config": - std.join("\n", [ - "---", - "apiVersion: ambassador/v0", - "kind: Mapping", - "name: tfserving-predict-mapping-" + name, - "prefix: /tfserving/models/" + name, - "rewrite: /v1/models/" + name + ":predict", - "method: POST", - "service: " + name + "." + namespace + ":8500", - "---", - "apiVersion: ambassador/v0", - "kind: Mapping", - "name: tfserving-predict-mapping-" + name + "-get", - "prefix: /tfserving/models/" + name, - "rewrite: /v1/models/" + name, - "method: GET", - "service: " + name + "." + namespace + ":8500", - ]), - }, //annotations - }, - spec: { - ports: [ - { - name: "grpc-tf-serving", - port: 9000, - targetPort: 9000, - }, - { - name: "http-tf-serving", - port: 8500, - targetPort: 8500, - }, - ], - selector: { - app: name, - }, - type: params.serviceType, - }, - }, // tfService - tfService:: tfService, - local modelServerContainer = { command: [ "/usr/bin/tensorflow_model_server", @@ -84,7 +32,7 @@ ], image: modelServerImage, imagePullPolicy: "IfNotPresent", - name: name, + name: modelName, ports: [ { containerPort: 9000, @@ -122,7 +70,7 @@ kind: "Deployment", metadata: { labels: { - app: name, + app: modelName, }, name: name, namespace: namespace, @@ -131,7 +79,8 @@ template: { metadata: { labels: { - app: name, + app: modelName, + version: versionName, }, annotations: { "sidecar.istio.io/inject": if util.toBool(params.injectIstio) then "true", diff --git a/testing/test_deploy.py b/testing/test_deploy.py index f0f928bba71..1119ca89797 100644 --- a/testing/test_deploy.py +++ b/testing/test_deploy.py @@ -164,22 +164,26 @@ def deploy_model(args): api_client = create_k8s_client(args) app_dir = setup_kubeflow_ks_app(args, api_client) - component = "modelServer" logging.info("Deploying tf-serving.") - generate_command = ["ks", "generate", "tf-serving-gcp", component] - - util.run(generate_command, cwd=app_dir) - params = {} for pair in args.params.split(","): k, v = pair.split("=", 1) params[k] = v - if "namespace" not in params: raise ValueError("namespace must be supplied via --params.") namespace = params["namespace"] - ks_deploy(app_dir, component, params, env=None, account=None) + # deployment component + deployComponent = "modelServer" + generate_command = ["ks", "generate", "tf-serving-deployment-gcp", deployComponent] + util.run(generate_command, cwd=app_dir) + ks_deploy(app_dir, deployComponent, params, env=None, account=None) + + # service component + serviceComponent = "modelServer-service" + generate_command = ["ks", "generate", "tf-serving-service", serviceComponent] + util.run(generate_command, cwd=app_dir) + ks_deploy(app_dir, serviceComponent, params, env=None, account=None) core_api = k8s_client.CoreV1Api(api_client) deploy = core_api.read_namespaced_service(args.deploy_name, args.namespace) diff --git a/testing/workflows/components/tfserving.libsonnet b/testing/workflows/components/tfserving.libsonnet index a86671503a0..c41a1f659c4 100644 --- a/testing/workflows/components/tfserving.libsonnet +++ b/testing/workflows/components/tfserving.libsonnet @@ -87,14 +87,12 @@ modelName: "mnist", namespace: stepsNamespace, modelPath: "gs://kubeflow-examples-data/mnist", - deployHttpProxy: true, }; local deployGpuParams = { name: "mnist-gpu", modelName: "mnist", namespace: stepsNamespace, modelPath: "gs://kubeflow-examples-data/mnist", - deployHttpProxy: true, numGpus: 1, };