Roll out model with istio (kubeflow#1823)

* roll out model with istio

* fix test

* fix test

* fix test

* fix test

kubeflow/tf-serving/prototypes/tf-serving-gcp.jsonnet

@@ -1,13 +1,13 @@
 // @apiVersion 0.1
-// @name io.ksonnet.pkg.tf-serving-gcp
+// @name io.ksonnet.pkg.tf-serving-deployment-gcp
 // @description TensorFlow serving
 // @shortDescription A TensorFlow serving deployment
 // @param name string Name to give to each of the components
-// @optionalParam serviceType string ClusterIP The k8s service type for tf serving.
 // @optionalParam numGpus string 0 Number of gpus to use
 // @optionalParam deployHttpProxy string false Whether to deploy http proxy
 // @optionalParam modelBasePath string gs://kubeflow-examples-data/mnist The model path
 // @optionalParam modelName string null The model name
+// @optionalParam versionName string v1 The version name
 // @optionalParam defaultCpuImage string tensorflow/serving:1.8.0 The default model server image (cpu)
 // @optionalParam defaultGpuImage string tensorflow/serving:1.10.0-gpu The default model server image (gpu)
 // @optionalParam httpProxyImage string gcr.io/kubeflow-images-public/tf-model-server-http-proxy:v20180723 Http proxy image
@@ -56,5 +56,4 @@ local tfDeployment = base.tfDeployment +
                      );
 util.list([
   tfDeployment,
-  base.tfService,
 ],)

kubeflow/tf-serving/prototypes/tf-serving-service.jsonnet

@@ -0,0 +1,20 @@
+// @apiVersion 0.1
+// @name io.ksonnet.pkg.tf-serving-service
+// @description TensorFlow serving
+// @shortDescription A TensorFlow serving model
+// @param name string Name to give to each of the components
+// @optionalParam serviceType string ClusterIP The k8s service type for tf serving.
+// @optionalParam modelName string null The model name
+// @optionalParam trafficRule string v1:100 The traffic rule, in the format of version:percentage,version:percentage,..
+// @optionalParam injectIstio string false Whether to inject istio sidecar; should be true or false.
+
+local k = import "k.libsonnet";
+local tfservingService = import "kubeflow/tf-serving/tf-serving-service-template.libsonnet";
+local util = import "kubeflow/tf-serving/util.libsonnet";
+
+local base = tfservingService.new(env, params);
+util.list(
+  [
+    base.tfService,
+  ] + if util.toBool(params.injectIstio) then [base.virtualService] else [],
+)

kubeflow/tf-serving/tf-serving-service-template.libsonnet

@@ -0,0 +1,96 @@
+{
+  local k = import "k.libsonnet",
+  local util = import "kubeflow/tf-serving/util.libsonnet",
+  new(_env, _params):: {
+    local params = _env + _params,
+    local namespace = params.namespace,
+    local name = params.name,
+    local modelName =
+      if params.modelName == "null" then
+        params.name
+      else
+        params.modelName,
+
+    local tfService = {
+      apiVersion: "v1",
+      kind: "Service",
+      metadata: {
+        labels: {
+          app: modelName,
+        },
+        name: name,
+        namespace: namespace,
+        annotations: {
+          "getambassador.io/config":
+            std.join("\n", [
+              "---",
+              "apiVersion: ambassador/v0",
+              "kind:  Mapping",
+              "name: tfserving-predict-mapping-" + modelName,
+              "prefix: /tfserving/models/" + modelName,
+              "rewrite: /v1/models/" + modelName + ":predict",
+              "method: POST",
+              "service: " + name + "." + namespace + ":8500",
+              "---",
+              "apiVersion: ambassador/v0",
+              "kind:  Mapping",
+              "name: tfserving-predict-mapping-" + modelName + "-get",
+              "prefix: /tfserving/models/" + modelName,
+              "rewrite: /v1/models/" + modelName,
+              "method: GET",
+              "service: " + name + "." + namespace + ":8500",
+            ]),
+        },  //annotations
+      },
+      spec: {
+        ports: [
+          {
+            name: "grpc-tf-serving",
+            port: 9000,
+            targetPort: 9000,
+          },
+          {
+            name: "http-tf-serving",
+            port: 8500,
+            targetPort: 8500,
+          },
+        ],
+        selector: {
+          app: modelName,
+        },
+        type: params.serviceType,
+      },
+    },  // tfService
+    tfService:: tfService,
+
+    local versionWeights = std.split(params.trafficRule, ","),
+    local virtualService = {
+      apiVersion: "networking.istio.io/v1alpha3",
+      kind: "VirtualService",
+      metadata: {
+        name: name,
+        namespace: namespace,
+      },
+      spec: {
+        hosts: [
+          name,
+        ],
+        http: [
+          {
+            route: [
+              {
+                destination: {
+                  host: name,
+                  subset: std.split(versionWeight, ":")[0],
+                },
+                weight: std.parseInt(std.split(versionWeight, ":")[1]),
+              }
+              for versionWeight in versionWeights
+            ],
+          },
+        ],
+      },
+    },
+    virtualService:: virtualService,
+  },  // new
+}

kubeflow/tf-serving/tf-serving-template.libsonnet

@@ -10,68 +10,16 @@
         params.name
       else
         params.modelName,
+    local versionName = params.versionName,
     local modelServerImage =
       if params.numGpus == "0" then
         params.defaultCpuImage
       else
         params.defaultGpuImage,
 
     // Optional features.
-    // TODO(lunkai): Add Istio
     // TODO(lunkai): Add request logging
 
-    local tfService = {
-      apiVersion: "v1",
-      kind: "Service",
-      metadata: {
-        labels: {
-          app: name,
-        },
-        name: name,
-        namespace: namespace,
-        annotations: {
-          "getambassador.io/config":
-            std.join("\n", [
-              "---",
-              "apiVersion: ambassador/v0",
-              "kind:  Mapping",
-              "name: tfserving-predict-mapping-" + name,
-              "prefix: /tfserving/models/" + name,
-              "rewrite: /v1/models/" + name + ":predict",
-              "method: POST",
-              "service: " + name + "." + namespace + ":8500",
-              "---",
-              "apiVersion: ambassador/v0",
-              "kind:  Mapping",
-              "name: tfserving-predict-mapping-" + name + "-get",
-              "prefix: /tfserving/models/" + name,
-              "rewrite: /v1/models/" + name,
-              "method: GET",
-              "service: " + name + "." + namespace + ":8500",
-            ]),
-        },  //annotations
-      },
-      spec: {
-        ports: [
-          {
-            name: "grpc-tf-serving",
-            port: 9000,
-            targetPort: 9000,
-          },
-          {
-            name: "http-tf-serving",
-            port: 8500,
-            targetPort: 8500,
-          },
-        ],
-        selector: {
-          app: name,
-        },
-        type: params.serviceType,
-      },
-    },  // tfService
-    tfService:: tfService,
-
     local modelServerContainer = {
       command: [
         "/usr/bin/tensorflow_model_server",
@@ -84,7 +32,7 @@
       ],
       image: modelServerImage,
       imagePullPolicy: "IfNotPresent",
-      name: name,
+      name: modelName,
       ports: [
         {
           containerPort: 9000,
@@ -122,7 +70,7 @@
       kind: "Deployment",
       metadata: {
         labels: {
-          app: name,
+          app: modelName,
         },
         name: name,
         namespace: namespace,
@@ -131,7 +79,8 @@
         template: {
           metadata: {
             labels: {
-              app: name,
+              app: modelName,
+              version: versionName,
             },
             annotations: {
               "sidecar.istio.io/inject": if util.toBool(params.injectIstio) then "true",

testing/test_deploy.py

@@ -164,22 +164,26 @@ def deploy_model(args):
   api_client = create_k8s_client(args)
   app_dir = setup_kubeflow_ks_app(args, api_client)
 
-  component = "modelServer"
   logging.info("Deploying tf-serving.")
-  generate_command = ["ks", "generate", "tf-serving-gcp", component]
-
-  util.run(generate_command, cwd=app_dir)
-
   params = {}
   for pair in args.params.split(","):
     k, v = pair.split("=", 1)
     params[k] = v
-
   if "namespace" not in params:
     raise ValueError("namespace must be supplied via --params.")
   namespace = params["namespace"]
 
-  ks_deploy(app_dir, component, params, env=None, account=None)
+  # deployment component
+  deployComponent = "modelServer"
+  generate_command = ["ks", "generate", "tf-serving-deployment-gcp", deployComponent]
+  util.run(generate_command, cwd=app_dir)
+  ks_deploy(app_dir, deployComponent, params, env=None, account=None)
+
+  # service component
+  serviceComponent = "modelServer-service"
+  generate_command = ["ks", "generate", "tf-serving-service", serviceComponent]
+  util.run(generate_command, cwd=app_dir)
+  ks_deploy(app_dir, serviceComponent, params, env=None, account=None)
 
   core_api = k8s_client.CoreV1Api(api_client)
   deploy = core_api.read_namespaced_service(args.deploy_name, args.namespace)

testing/workflows/components/tfserving.libsonnet

@@ -87,14 +87,12 @@
         modelName: "mnist",
         namespace: stepsNamespace,
         modelPath: "gs://kubeflow-examples-data/mnist",
-        deployHttpProxy: true,
       };
       local deployGpuParams = {
         name: "mnist-gpu",
         modelName: "mnist",
         namespace: stepsNamespace,
         modelPath: "gs://kubeflow-examples-data/mnist",
-        deployHttpProxy: true,
         numGpus: 1,
       };