gcp-marketplace: Updating metadata deployment to use gRPC server

.release.cloudbuild.yaml

@@ -127,12 +127,19 @@ steps:
   waitFor: ['pullVisualizationServer']
 
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/kubeflow-images-public/metadata:v0.1.8']
+  args: ['pull', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.14.0']
   id:   'PullMetadataServer'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['tag', 'gcr.io/kubeflow-images-public/metadata:v0.1.8', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME']
+  args: ['tag', 'gcr.io/tfx-oss-public/ml_metadata_store_server:0.14.0', 'gcr.io/ml-pipeline/google/pipelines/metadataserver:$TAG_NAME']
   waitFor: ['PullMetadataServer']
 
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['pull', 'gcr.io/ml-pipeline/envoy:initial']
+  id:   'PullMetadataEnvoy'
+- name: 'gcr.io/cloud-builders/docker'
+  args: ['tag', 'gcr.io/ml-pipeline/envoy:initial', 'gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$TAG_NAME']
+  waitFor: ['PullMetadataEnvoy']
+
 - name: 'gcr.io/cloud-builders/docker'
   args: ['pull', 'gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z']
   id:   'PullMinio'
@@ -222,5 +229,6 @@ images:
 - 'gcr.io/ml-pipeline/google/pipelines/cloudsqlproxy:$TAG_NAME'
 - 'gcr.io/ml-pipeline/google/pipelines/argoexecutor:$TAG_NAME'
 - 'gcr.io/ml-pipeline/google/pipelines/argoworkflowcontroller:$TAG_NAME'
+- 'gcr.io/ml-pipeline/google/pipelines/metadataenvoy:$TAG_NAME'
 
 timeout: '1200s'

manifests/gcp_marketplace/chart/kubeflow-pipelines/templates/metadata.yaml

@@ -34,19 +34,73 @@ spec:
         app.kubernetes.io/name: {{ .Release.Name }}
     spec:
       containers:
-        - command:
-            - ./server/server
-            - --http_port=8080
-            - --mysql_service_host=mysql
-            - --mysql_service_port=3306
-            {{ if .Values.managedstorage.databaseNamePrefix }}
-            - --mlmd_db_name={{ .Values.managedstorage.databaseNamePrefix }}_metadata
-            {{ else }}
-            - --mlmd_db_name={{ .Release.Name | replace "-" "_" | replace "." "_" | trunc 50 }}_metadata
-            {{ end }}
-
-          image: {{ .Values.images.metadataserver }}
-          name: container
+      - name: container
+        image: {{ .Values.images.metadataserver }}
+        env:
+        - name: DBCONFIG_USER
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: username
+        - name: DBCONFIG_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mysql-credential
+                key: password
+        command: ["/bin/metadata_store_server"]
+        args: ["--grpc_port=8080",
+                 "--mysql_config_host=mysql",
+                 {{ if .Values.managedstorage.databaseNamePrefix }}
+                 '--mysql_config_database={{ .Values.managedstorage.databaseNamePrefix }}_metadata',
+                 {{ else }}
+                 '--mysql_config_database={{ .Release.Name | replace "-" "_" | replace "." "_" | trunc 50 }}_metadata',
+                 {{ end }}
+                 "--mysql_config_port=3306",
+                 "--mysql_config_user=$(DBCONFIG_USER)",
+                 "--mysql_config_password=$(DBCONFIG_PASSWORD)"
+                ]
+        ports:
+        - containerPort: 8080
+          name: md-backendapi
+---
+kind: Service
+apiVersion: v1
+metadata:
+  labels:
+    app: metadata
+    app.kubernetes.io/name: {{ .Release.Name }}
+  name: metadata-envoy-service
+spec:
+  selector:
+    component: metadata-envoy
+  type: ClusterIP
+  ports:
+    - port: 9090
+      protocol: TCP
+      name: md-envoy
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: metadata-envoy
+  labels:
+    component: metadata-envoy
+    app.kubernetes.io/name: {{ .Release.Name }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: metadata-envoy
+  template:
+    metadata:
+      labels:
+        component: metadata-envoy
+    spec:
+      containers:
+        - name: container
+          image: {{ .Values.images.metadataenvoy }}
           ports:
-            - containerPort: 8080
-              name: md-backendapi
+            - name: md-envoy
+              containerPort: 9090
+            - name: envoy-admin
+              containerPort: 9901

manifests/gcp_marketplace/chart/kubeflow-pipelines/values.yaml

@@ -4,14 +4,15 @@ images:
   argoworkflowcontroller: gcr.io/ml-pipeline/workflow-controller:v2.3.0
   cloudsqlproxy: gcr.io/cloudsql-docker/gce-proxy:1.14
   frontend: gcr.io/ml-pipeline/frontend:0.1.27
-  metadataserver: gcr.io/kubeflow-images-public/metadata:v0.1.8
+  metadataserver: gcr.io/tfx-oss-public/ml_metadata_store_server:0.14.0
   minio: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z
   mysql: gcr.io/ml-pipeline/mysql:5.6
   persistenceagent: gcr.io/ml-pipeline/persistenceagent:0.1.27
   proxyagent: gcr.io/ml-pipeline/inverse-proxy-agent:0.1.27
   scheduledworkflow: gcr.io/ml-pipeline/scheduledworkflow:0.1.27
   viewercrd: gcr.io/ml-pipeline/viewer-crd-controller:0.1.27
   visualizationserver: gcr.io/ml-pipeline/visualization-server:0.1.27
+  metadataenvoy: gcr.io/ml-pipeline/envoy:initial
 
 gcpSecretName: "user-gcp-sa"
 serviceAccountCredential: null

manifests/gcp_marketplace/schema.yaml

@@ -69,6 +69,10 @@ x-google-marketplace:
       properties:
         images.cloudsqlproxy:
           type: FULL
+    metadataenvoy:
+      properties:
+        images.metadataenvoy:
+          type: FULL
   deployerServiceAccount:
     roles:
       - type: ClusterRole        # This is a cluster-wide ClusterRole