Cherry pick of "feat: update KFP to 1.0.0 #1397" (#1400)

* feat: update KFP to 1.0.0

* update test snapshot

* pull new 1.0.0

* fix and update test snapshot

hack/pull_kfp_upstream.sh

@@ -8,7 +8,7 @@ set -ex
 
 # Please edit the following version before running the script to pull new
 # pipelines version.
-export PIPELINES_VERSION=1.0.0-rc.3
+export PIPELINES_VERSION=1.0.0
 export PIPELINES_SRC_REPO=https://github.com/kubeflow/pipelines.git
 
 if [ -d pipeline/upstream ]; then

pipeline/installs/multi-user/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
 patchesStrategicMerge:
 - api-service/deployment-patch.yaml
 - pipelines-ui/deployment-patch.yaml
+- pipelines-ui/configmap-patch.yaml
 - scheduled-workflow/deployment-patch.yaml
 - viewer-controller/deployment-patch.yaml
 - persistence-agent/deployment-patch.yaml

pipeline/installs/multi-user/pipelines-ui/kustomization.yaml

@@ -6,4 +6,3 @@ commonLabels:
 resources:
 - cluster-role.yaml
 - cluster-role-binding.yaml
-- configmap.yaml

pipeline/upstream/Kptfile

@@ -5,7 +5,7 @@ metadata:
 upstream:
     type: git
     git:
-        commit: 7a0df42fa5555110f2ada71a2728efc32d5a8110
+        commit: 181c35002490cf7f1b5af8c88cb8b7cf29332f2b
         repo: https://github.com/kubeflow/pipelines
         directory: /manifests/kustomize
-        ref: 1.0.0-rc.3
+        ref: 1.0.0

pipeline/upstream/base/cache-deployer/kustomization.yaml

@@ -6,4 +6,4 @@ resources:
 - cache-deployer-deployment.yaml
 images:
 - name: gcr.io/ml-pipeline/cache-deployer
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0

pipeline/upstream/base/cache/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
 - cache-sa.yaml
 images:
 - name: gcr.io/ml-pipeline/cache-server
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0

pipeline/upstream/base/metadata/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
 - metadata-envoy-service.yaml
 images:
 - name: gcr.io/ml-pipeline/metadata-envoy
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0

pipeline/upstream/base/params.env

@@ -1,5 +1,5 @@
 appName=pipeline
-appVersion=1.0.0-rc.3
+appVersion=1.0.0
 dbHost=mysql
 dbPort=3306
 mlmdDb=metadb

pipeline/upstream/base/pipeline/kustomization.yaml

@@ -17,6 +17,7 @@ resources:
 - ml-pipeline-scheduledworkflow-rolebinding.yaml
 - ml-pipeline-scheduledworkflow-sa.yaml
 - ml-pipeline-ui-deployment.yaml
+- ml-pipeline-ui-configmap.yaml
 - ml-pipeline-ui-role.yaml
 - ml-pipeline-ui-rolebinding.yaml
 - ml-pipeline-ui-sa.yaml
@@ -32,16 +33,17 @@ resources:
 - pipeline-runner-rolebinding.yaml
 - pipeline-runner-sa.yaml
 - container-builder-sa.yaml
+- viewer-sa.yaml
 images:
 - name: gcr.io/ml-pipeline/api-server
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 - name: gcr.io/ml-pipeline/persistenceagent
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 - name: gcr.io/ml-pipeline/scheduledworkflow
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 - name: gcr.io/ml-pipeline/frontend
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 - name: gcr.io/ml-pipeline/viewer-crd-controller
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 - name: gcr.io/ml-pipeline/visualization-server
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0

pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml

@@ -7,4 +7,4 @@ resources:
 - metadata-writer-sa.yaml
 images:
 - name: gcr.io/ml-pipeline/metadata-writer
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0

pipeline/upstream/base/pipeline/ml-pipeline-ui-configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ml-pipeline-ui-configmap
+data:
+   viewer-pod-template.json:  |-
+    {
+        "spec": {
+            "serviceAccountName": "kubeflow-pipelines-viewer"
+        }
+    }

pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml

@@ -13,13 +13,23 @@ spec:
       labels:
         app: ml-pipeline-ui
     spec:
+      volumes:
+      - name: config-volume
+        configMap:
+          name: ml-pipeline-ui-configmap
       containers:
       - image: gcr.io/ml-pipeline/frontend:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-ui
         ports:
         - containerPort: 3000
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/config
+          readOnly: true
         env:
+          - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
+            value: /etc/config/viewer-pod-template.json
           - name: MINIO_NAMESPACE
             valueFrom:
               fieldRef:

pipeline/upstream/base/pipeline/viewer-sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-viewer

pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml

@@ -14,6 +14,7 @@ spec:
       labels:
         app: cloudsqlproxy
     spec:
+      serviceAccountName: kubeflow-pipelines-cloudsql-proxy
       containers:
         - image: gcr.io/cloudsql-docker/gce-proxy:1.14
           name: cloudsqlproxy

pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-cloudsql-proxy

pipeline/upstream/env/gcp/cloudsql-proxy/kustomization.yaml

@@ -3,4 +3,5 @@ kind: Kustomization
 
 resources:
 - cloudsql-proxy-deployment.yaml
+- cloudsql-proxy-sa.yaml
 - mysql-service.yaml

pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: gcr.io/ml-pipeline/inverse-proxy-agent
-  newTag: 1.0.0-rc.3
+  newTag: 1.0.0
 resources:
 - proxy-configmap.yaml
 - proxy-deployment.yaml

pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 
 resources:
 - minio-gcs-gateway-deployment.yaml
+- minio-gcs-gateway-sa.yaml
 - minio-gcs-gateway-service.yaml
 
 secretGenerator:

pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml

@@ -15,6 +15,7 @@ spec:
       labels:
         app: minio
     spec:
+      serviceAccountName: kubeflow-pipelines-minio-gcs-gateway
       containers:
         - name: minio
           image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance

pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-sa.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubeflow-pipelines-minio-gcs-gateway

pipeline/upstream/gcp-workload-identity-setup.sh

@@ -16,44 +16,59 @@
 
 set -e
 
+# Kubernetes Namespace
+NAMESPACE=${NAMESPACE:-kubeflow}
+
 # Google service Account (GSA)
-SYSTEM_GSA=${SYSTEM_GSA:-$CLUSTER_NAME-kfp-system}
-USER_GSA=${USER_GSA:-$CLUSTER_NAME-kfp-user}
+SYSTEM_GSA=${SYSTEM_GSA:-$RESOURCE_PREFIX-kfp-system}
+USER_GSA=${USER_GSA:-$RESOURCE_PREFIX-kfp-user}
 
 # Kubernetes Service Account (KSA)
+# Note, if deploying manifests/kustomize/env/gcp, you can add the following KSAs
+# to the array of SYSTEM_KSA:
+# * kubeflow-pipelines-minio-gcs-gateway needs gcs permissions
+# * kubeflow-pipelines-cloudsql-proxy needs cloudsql permissions
 SYSTEM_KSA=(ml-pipeline-ui ml-pipeline-visualizationserver)
-USER_KSA=(pipeline-runner kubeflow-pipelines-container-builder)
+USER_KSA=(pipeline-runner kubeflow-pipelines-container-builder kubeflow-pipelines-viewer)
 
-cat <<EOF
+if [ -n $USE_GCP_MANAGED_STORAGE ]; then
+  SYSTEM_KSA+=(kubeflow-pipelines-minio-gcs-gateway)
+  SYSTEM_KSA+=(kubeflow-pipelines-cloudsql-proxy)
+fi
 
-It is recommended to first review introduction to workload identity: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity.
+cat <<EOF
 
-This script sets up Google service accounts and workload identity bindings for a Kubeflow Pipelines (KFP) standalone deployment.
+This script sets up Google service accounts, Kubernetes service accounts and workload identity bindings for a Kubeflow Pipelines (KFP) standalone deployment.
 You can also choose to manually set these up based on documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity.
 
 Before you begin, please check the following list:
+* Please first review introduction to workload identity: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity.
+* KFP is already or will be deployed by standalone deployment: https://www.kubeflow.org/docs/pipelines/installation/standalone-deployment/
 * gcloud is configured following steps: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#before_you_begin.
-* KFP is already deployed by standalone deployment: https://www.kubeflow.org/docs/pipelines/standalone-deployment-gcp/.
 * kubectl talks to the cluster KFP is deployed to: https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl.
+* The namespace you specified by NAMESPACE env var already exists on the cluster. You can create it by "kubectl create namespace \$NAMESPACE".
+
+The following resources will be created or updated to create workload identity bindings between GSAs and KSAs:
+* Google service accounts (GSAs)
+* Service account IAM policy bindings on these GSAs
+* Kubernetes service accounts with annotations in namespace "$NAMESPACE".
 
-The following resources will be created to bind workload identity between GSAs and KSAs:
-* Google service accounts (GSAs): $SYSTEM_GSA and $USER_GSA.
-* Service account IAM policy bindings.
-* Kubernetes service account annotations.
+Note, this script is designed to be idempotent. If something went wrong, you can safely fix the error and rerun this script.
 
 EOF
 
-NAMESPACE=${NAMESPACE:-kubeflow}
 function usage {
 cat <<\EOF
 Usage:
 ```
-PROJECT_ID=<your-gcp-project-id> CLUSTER_NAME=<your-gke-cluster-name> NAMESPACE=<your-k8s-namespace> ./gcp-workload-identity-setup.sh
+PROJECT_ID=<your-gcp-project-id> RESOURCE_PREFIX=<your-chosen-prefix> NAMESPACE=<your-k8s-namespace> ./gcp-workload-identity-setup.sh
 ```
 
 PROJECT_ID: GCP project ID your cluster belongs to.
-CLUSTER_NAME: your GKE cluster's name.
-NAMESPACE: Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to (default is kubeflow).
+RESOURCE_PREFIX: Your preferred resource prefix for GCP resources this script creates.
+NAMESPACE: Optional. Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to. (Defaults to kubeflow)
+USE_GCP_MANAGED_STORAGE: Optional. Defaults to "false", specify "true" if you intend to use GCP managed storage (Google Cloud Storage and Cloud SQL) following instructions in:
+https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize/sample
 EOF
 }
 if [ -z "$PROJECT_ID" ]; then
@@ -62,18 +77,39 @@ if [ -z "$PROJECT_ID" ]; then
   echo "Error: PROJECT_ID env variable is empty!"
   exit 1
 fi
-if [ -z "$CLUSTER_NAME" ]; then
+if [ -z "$RESOURCE_PREFIX" ]; then
   usage
   echo
-  echo "Error: CLUSTER_NAME env variable is empty!"
+  echo "Error: RESOURCE_PREFIX env variable is empty!"
   exit 1
 fi
 echo "Env variables set:"
 echo "* PROJECT_ID=$PROJECT_ID"
-echo "* CLUSTER_NAME=$CLUSTER_NAME"
+echo "* RESOURCE_PREFIX=$RESOURCE_PREFIX"
 echo "* NAMESPACE=$NAMESPACE"
+echo "* USE_GCP_MANAGED_STORAGE=${USE_GCP_MANAGED_STORAGE:-false}"
 echo
 
+SYSTEM_GSA_FULL="$SYSTEM_GSA@$PROJECT_ID.iam.gserviceaccount.com"
+USER_GSA_FULL="$USER_GSA@$PROJECT_ID.iam.gserviceaccount.com"
+
+cat <<EOF
+
+The following resources will be created or updated to create workload identity bindings between GSAs and KSAs:
+* Google service accounts (GSAs):
+  * $SYSTEM_GSA_FULL
+  * $USER_GSA_FULL
+* Service account IAM policy bindings on these GSAs to grant "Workload Identity User" role.
+* Kubernetes service accounts with annotations in namespace "$NAMESPACE".
+* $SYSTEM_GSA_FULL will be bound to these KSAs:
+  ${SYSTEM_KSA[@]}.
+* $USER_GSA_FULL will be bound to these KSAs:
+  ${USER_KSA[@]}.
+
+Note: if you prefer more granular workload identity bindings, you can modify this script to suit your needs.
+
+EOF
+
 read -p "Continue? (Y/n) " -n 1 -r
 echo
 if [[ ! $REPLY =~ ^[Yy]$ ]]; then
@@ -93,14 +129,15 @@ function create_gsa_if_not_present {
 create_gsa_if_not_present $SYSTEM_GSA
 create_gsa_if_not_present $USER_GSA
 
-# You can optionally choose to add iam policy bindings to grant project permissions to these GSAs.
-# You can also set these up later.
-# gcloud projects add-iam-policy-binding $PROJECT_ID \
-#   --member="serviceAccount:$SYSTEM_GSA@$PROJECT_ID.iam.gserviceaccount.com" \
-#   --role="roles/editor"
-# gcloud projects add-iam-policy-binding $PROJECT_ID \
-#   --member="serviceAccount:$USER_GSA@$PROJECT_ID.iam.gserviceaccount.com" \
-#   --role="roles/editor"
+function create_ksa_if_not_present {
+  local name=${1}
+  if kubectl get serviceaccount $name -n $NAMESPACE >/dev/null; then
+    echo "KSA $name already exists"
+  else
+    kubectl create serviceaccount $name -n $NAMESPACE --save-config
+    echo "KSA $name created"
+  fi
+}
 
 # Bind KSA to GSA through workload identity.
 # Documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
@@ -112,6 +149,8 @@ function bind_gsa_and_ksa {
     --member="serviceAccount:$PROJECT_ID.svc.id.goog[$NAMESPACE/$ksa]" \
     --role="roles/iam.workloadIdentityUser" \
     > /dev/null # hide verbose output
+
+  create_ksa_if_not_present $ksa
   kubectl annotate serviceaccount \
     --namespace $NAMESPACE \
     --overwrite \
@@ -129,3 +168,35 @@ echo "Binding each kfp user KSA to $USER_GSA"
 for ksa in ${USER_KSA[@]}; do
   bind_gsa_and_ksa $USER_GSA $ksa
 done
+
+echo
+echo "All the workload identity bindings have succeeded!"
+cat <<EOF
+
+
+=============
+Next steps:
+* This script won't add IAM policies to grant these GSAs with permissions KFP needs, you need to do that by yourself.
+
+### If **NOT** using GCP managed storage, you can:
+* Give $SYSTEM_GSA_FULL "Storage Object Viewer" role to allow KFP UI load data in GCS in the same project:
+gcloud projects add-iam-policy-binding $PROJECT_ID \\
+  --member="serviceAccount:$SYSTEM_GSA_FULL" \\
+  --role="roles/storage.objectViewer"
+
+* Give $USER_GSA_FULL any permissions your pipelines, container builder and tensorboard need. For **QUICK** tryouts, you can give it Project Editor role for all permissions, but **WARNING** be aware this overgrants too much permission:
+gcloud projects add-iam-policy-binding $PROJECT_ID \\
+  --member="serviceAccount:$USER_GSA_FULL" \\
+  --role="roles/editor"
+
+### If using GCP managed storage, you **ALSO** need to give $SYSTEM_GSA_FULL these roles:
+* "Storage Admin" role on specified GCS bucket to allow writing to specified GCS artifact bucket:
+gsutil iam ch serviceAccount:$SYSTEM_GSA_FULL:roles/storage.admin gs://[BUCKET_NAME]
+
+Or you can find other ways in https://cloud.google.com/storage/docs/access-control/using-iam-permissions#bucket-add.
+
+* "Cloud SQL Client" role to allow connecting to Cloud SQL instances:
+gcloud projects add-iam-policy-binding $PROJECT_ID \\
+  --member="serviceAccount:$SYSTEM_GSA_FULL" \\
+  --role="roles/cloudsql.client"
+EOF