Update to KFP 1.0.0-rc.3 (kubeflow#1341)

* updated to kfp 1.0.0-rc.3

* Adapt to kfp 1.0rc3 refactoring

* update test snapshots

hack/pull_kfp_upstream.sh

@@ -8,7 +8,7 @@ set -ex
 
 # Please edit the following version before running the script to pull new
 # pipelines version.
-export PIPELINES_VERSION=1.0.0-rc.2
+export PIPELINES_VERSION=1.0.0-rc.3
 export PIPELINES_SRC_REPO=https://github.com/kubeflow/pipelines.git
 # Pulling for the first time
 # kpt pkg get $PIPELINES_SRC_REPO/manifests/kustomize@$PIPELINES_VERSION pipeline/upstream

pipeline/installs/generic/kustomization.yaml

@@ -5,28 +5,7 @@ resources:
 - ../../upstream/base/pipeline/cluster-scoped
 - ../../upstream/base/pipeline
 - virtual-service.yaml
-- ../../metadata-writer/base_v3
 - ../../cache/base_v3
-images:
-- name: gcr.io/ml-pipeline/api-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/persistenceagent
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/scheduledworkflow
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/frontend
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/viewer-crd-controller
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/visualization-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/metadata-writer
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/cache-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/cache-deployer
-  newTag: 1.0.0-rc.2
-# Used by Kustomize
 configMapGenerator:
 - name: pipeline-install-config
   envs:

pipeline/minio/installs/gcp-pd/kustomization.yaml

@@ -5,7 +5,6 @@ commonLabels:
   app.kubernetes.io/name: minio
 resources:
 - ../../../upstream/env/platform-agnostic/minio/
-- ../../../upstream/base/argo/minio-artifact-secret.yaml # TODO: move it to minio/ folder
 - persistent-volume.yaml
 - ../../overlays/application/application.yaml
 patchesStrategicMerge:

pipeline/minio/installs/generic/kustomization.yaml

@@ -5,5 +5,4 @@ commonLabels:
   app.kubernetes.io/name: minio
 resources:
 - ../../../upstream/env/platform-agnostic/minio/
-- ../../../upstream/base/argo/minio-artifact-secret.yaml # TODO: move it to minio/ folder
 - ../../overlays/application/application.yaml

pipeline/minio/installs/ibm/kustomization.yaml

@@ -5,7 +5,6 @@ commonLabels:
   app.kubernetes.io/name: minio
 resources:
 - ../../../upstream/env/platform-agnostic/minio/
-- ../../../upstream/base/argo/minio-artifact-secret.yaml # TODO: move it to minio/ folder
 - ../../overlays/application/application.yaml
 - persistent-volume-claim.yaml
 patchesStrategicMerge:

pipeline/upstream/Kptfile

@@ -5,7 +5,7 @@ metadata:
 upstream:
     type: git
     git:
-        commit: 079634019eb2cf92a0536f344bde8aebe340bf21
+        commit: 7a0df42fa5555110f2ada71a2728efc32d5a8110
         repo: https://github.com/kubeflow/pipelines
         directory: /manifests/kustomize
-        ref: 1.0.0-rc.2
+        ref: 1.0.0-rc.3

pipeline/upstream/base/argo/kustomization.yaml

@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- minio-artifact-secret.yaml
 - workflow-controller-configmap.yaml
 - workflow-controller-deployment.yaml
 - workflow-controller-role.yaml

pipeline/upstream/base/cache-deployer/kustomization.yaml

@@ -1,7 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 resources:
-  - cache-deployer-role.yaml
-  - cache-deployer-rolebinding.yaml
-  - cache-deployer-deployment.yaml
+- cache-deployer-role.yaml
+- cache-deployer-rolebinding.yaml
+- cache-deployer-deployment.yaml
+images:
+- name: gcr.io/ml-pipeline/cache-deployer
+  newTag: 1.0.0-rc.3

pipeline/upstream/base/cache/kustomization.yaml

@@ -1,9 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 resources:
-  - cache-deployment.yaml
-  - cache-service.yaml
-  - cache-role.yaml
-  - cache-rolebinding.yaml
-  - cache-sa.yaml
+- cache-deployment.yaml
+- cache-service.yaml
+- cache-role.yaml
+- cache-rolebinding.yaml
+- cache-sa.yaml
+images:
+- name: gcr.io/ml-pipeline/cache-server
+  newTag: 1.0.0-rc.3

pipeline/upstream/base/kustomization.yaml

@@ -10,25 +10,6 @@ bases:
 - cache-deployer
 resources:
 - pipeline-application.yaml
-images:
-- name: gcr.io/ml-pipeline/api-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/persistenceagent
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/scheduledworkflow
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/frontend
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/viewer-crd-controller
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/visualization-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/metadata-writer
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/cache-server
-  newTag: 1.0.0-rc.2
-- name: gcr.io/ml-pipeline/cache-deployer
-  newTag: 1.0.0-rc.2
 # Used by Kustomize
 configMapGenerator:
 - name: pipeline-install-config

pipeline/upstream/base/metadata/kustomization.yaml

@@ -1,13 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 resources:
-  - metadata-configmap.yaml
-  - metadata-grpc-deployment.yaml
-  - metadata-grpc-service.yaml
-  - metadata-envoy-deployment.yaml
-  - metadata-envoy-service.yaml
-  - metadata-writer-deployment.yaml
-  - metadata-writer-role.yaml
-  - metadata-writer-rolebinding.yaml
-  - metadata-writer-sa.yaml
+- metadata-grpc-configmap.yaml
+- metadata-grpc-deployment.yaml
+- metadata-grpc-service.yaml
+- metadata-envoy-deployment.yaml
+- metadata-envoy-service.yaml
+images:
+- name: gcr.io/ml-pipeline/metadata-envoy
+  newTag: 1.0.0-rc.3

pipeline/upstream/base/metadata/metadata-envoy-deployment.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
       - name: container
-        image: gcr.io/ml-pipeline/envoy:metadata-grpc
+        image: gcr.io/ml-pipeline/metadata-envoy:dummy
         ports:
         - name: md-envoy
           containerPort: 9090

pipeline/upstream/base/params.env

@@ -1,5 +1,5 @@
 appName=pipeline
-appVersion=1.0.0-rc.2
+appVersion=1.0.0-rc.3
 dbHost=mysql
 dbPort=3306
 mlmdDb=metadb

pipeline/upstream/base/pipeline/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
+bases:
+- metadata-writer
 resources:
 - ml-pipeline-apiserver-deployment.yaml
 - ml-pipeline-apiserver-role.yaml
@@ -31,3 +32,16 @@ resources:
 - pipeline-runner-rolebinding.yaml
 - pipeline-runner-sa.yaml
 - container-builder-sa.yaml
+images:
+- name: gcr.io/ml-pipeline/api-server
+  newTag: 1.0.0-rc.3
+- name: gcr.io/ml-pipeline/persistenceagent
+  newTag: 1.0.0-rc.3
+- name: gcr.io/ml-pipeline/scheduledworkflow
+  newTag: 1.0.0-rc.3
+- name: gcr.io/ml-pipeline/frontend
+  newTag: 1.0.0-rc.3
+- name: gcr.io/ml-pipeline/viewer-crd-controller
+  newTag: 1.0.0-rc.3
+- name: gcr.io/ml-pipeline/visualization-server
+  newTag: 1.0.0-rc.3

pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- metadata-writer-deployment.yaml
+- metadata-writer-role.yaml
+- metadata-writer-rolebinding.yaml
+- metadata-writer-sa.yaml
+images:
+- name: gcr.io/ml-pipeline/metadata-writer
+  newTag: 1.0.0-rc.3

pipeline/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml

@@ -51,6 +51,16 @@ spec:
             configMapKeyRef:
               name: pipeline-install-config
               key: dbPort
+        - name: OBJECTSTORECONFIG_ACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: accesskey
+        - name: OBJECTSTORECONFIG_SECRETACCESSKEY
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: secretkey
         image: gcr.io/ml-pipeline/api-server:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-api-server

pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml

@@ -24,6 +24,16 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: metadata.namespace
+          - name: MINIO_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: mlpipeline-minio-artifact
+                key: accesskey
+          - name: MINIO_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: mlpipeline-minio-artifact
+                key: secretkey
           - name: ALLOW_CUSTOM_VISUALIZATIONS
             value: "true"
         readinessProbe:

pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: gcr.io/ml-pipeline/inverse-proxy-agent
-  newTag: 1.0.0-rc.2
+  newTag: 1.0.0-rc.3
 resources:
 - proxy-configmap.yaml
 - proxy-deployment.yaml

pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml

@@ -4,3 +4,10 @@ kind: Kustomization
 resources:
 - minio-gcs-gateway-deployment.yaml
 - minio-gcs-gateway-service.yaml
+
+secretGenerator:
+- name: mlpipeline-minio-artifact
+  envs: ["minio-artifact-secret.env"]
+generatorOptions:
+  # mlpipeline-minio-artifact needs to be referred by exact name
+  disableNameSuffixHash: true

pipeline/upstream/env/gcp/minio-gcs-gateway/minio-artifact-secret.env

@@ -0,0 +1,2 @@
+accesskey=minio
+secretkey=minio123

pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml

@@ -29,8 +29,14 @@ spec:
                   name: pipeline-install-config
                   key: gcsProjectId
             - name: MINIO_ACCESS_KEY
-              value: "minio"
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: accesskey
             - name: MINIO_SECRET_KEY
-              value: "minio123"
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: secretkey
           ports:
             - containerPort: 9000

pipeline/upstream/env/platform-agnostic/minio/kustomization.yaml

@@ -5,3 +5,10 @@ resources:
 - minio-deployment.yaml
 - minio-pvc.yaml
 - minio-service.yaml
+
+secretGenerator:
+- name: mlpipeline-minio-artifact
+  envs: ["minio-artifact-secret.env"]
+generatorOptions:
+  # mlpipeline-minio-artifact needs to be referred by exact name
+  disableNameSuffixHash: true

pipeline/upstream/env/platform-agnostic/minio/minio-artifact-secret.env

@@ -0,0 +1,2 @@
+accesskey=minio
+secretkey=minio123

pipeline/upstream/env/platform-agnostic/minio/minio-deployment.yaml

@@ -21,9 +21,15 @@ spec:
         - /data
         env:
         - name: MINIO_ACCESS_KEY
-          value: minio
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: accesskey
         - name: MINIO_SECRET_KEY
-          value: minio123
+          valueFrom:
+            secretKeyRef:
+              name: mlpipeline-minio-artifact
+              key: secretkey
         image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance
         name: minio
         ports:
@@ -35,4 +41,4 @@ spec:
       volumes:
       - name: data
         persistentVolumeClaim:
-          claimName: minio-pvc
+          claimName: minio-pvc

pipeline/upstream/hack/format.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null && pwd)"
+
+function format_yaml {
+    local path=$1
+    local tmp=$(mktemp)
+    yq r "$path" > "$tmp"
+    cp "$tmp" "$path"
+}
+echo "This formatting script uses yq, it can be downloaded at https://github.com/mikefarah/yq/releases/tag/3.3.0"
+kustomization_yamls_with_images=(
+  "base/cache-deployer/kustomization.yaml"
+  "base/cache/kustomization.yaml"
+  "base/metadata/kustomization.yaml"
+  "base/pipeline/metadata-writer/kustomization.yaml"
+  "base/pipeline/kustomization.yaml"
+  "env/gcp/inverse-proxy/kustomization.yaml"
+)
+for path in "${kustomization_yamls_with_images[@]}"
+do
+  format_yaml "$DIR/../$path"
+done