diff --git a/hack/pull_kfp_upstream.sh b/hack/pull_kfp_upstream.sh index 0d986871df..c1b4501ecc 100755 --- a/hack/pull_kfp_upstream.sh +++ b/hack/pull_kfp_upstream.sh @@ -8,7 +8,7 @@ set -ex # Please edit the following version before running the script to pull new # pipelines version. -export PIPELINES_VERSION=1.0.0-rc.3 +export PIPELINES_VERSION=1.0.0 export PIPELINES_SRC_REPO=https://github.com/kubeflow/pipelines.git if [ -d pipeline/upstream ]; then diff --git a/pipeline/installs/multi-user/kustomization.yaml b/pipeline/installs/multi-user/kustomization.yaml index ecf633d9e6..dc5fc4607b 100644 --- a/pipeline/installs/multi-user/kustomization.yaml +++ b/pipeline/installs/multi-user/kustomization.yaml @@ -18,6 +18,7 @@ resources: patchesStrategicMerge: - api-service/deployment-patch.yaml - pipelines-ui/deployment-patch.yaml +- pipelines-ui/configmap-patch.yaml - scheduled-workflow/deployment-patch.yaml - viewer-controller/deployment-patch.yaml - persistence-agent/deployment-patch.yaml diff --git a/pipeline/installs/multi-user/pipelines-ui/configmap.yaml b/pipeline/installs/multi-user/pipelines-ui/configmap-patch.yaml similarity index 100% rename from pipeline/installs/multi-user/pipelines-ui/configmap.yaml rename to pipeline/installs/multi-user/pipelines-ui/configmap-patch.yaml diff --git a/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml b/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml index 33ad4fc768..c84d97c05c 100644 --- a/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml +++ b/pipeline/installs/multi-user/pipelines-ui/kustomization.yaml @@ -6,4 +6,3 @@ commonLabels: resources: - cluster-role.yaml - cluster-role-binding.yaml -- configmap.yaml diff --git a/pipeline/upstream/Kptfile b/pipeline/upstream/Kptfile index 350c65c2b9..e20130ddf1 100644 --- a/pipeline/upstream/Kptfile +++ b/pipeline/upstream/Kptfile @@ -5,7 +5,7 @@ metadata: upstream: type: git git: - commit: 7a0df42fa5555110f2ada71a2728efc32d5a8110 + commit: 181c35002490cf7f1b5af8c88cb8b7cf29332f2b repo: https://github.com/kubeflow/pipelines directory: /manifests/kustomize - ref: 1.0.0-rc.3 + ref: 1.0.0 diff --git a/pipeline/upstream/base/cache-deployer/kustomization.yaml b/pipeline/upstream/base/cache-deployer/kustomization.yaml index 9a95905720..fadd6a3836 100644 --- a/pipeline/upstream/base/cache-deployer/kustomization.yaml +++ b/pipeline/upstream/base/cache-deployer/kustomization.yaml @@ -6,4 +6,4 @@ resources: - cache-deployer-deployment.yaml images: - name: gcr.io/ml-pipeline/cache-deployer - newTag: 1.0.0-rc.3 + newTag: 1.0.0 diff --git a/pipeline/upstream/base/cache/kustomization.yaml b/pipeline/upstream/base/cache/kustomization.yaml index f7479e89b1..a7d8c3dc07 100644 --- a/pipeline/upstream/base/cache/kustomization.yaml +++ b/pipeline/upstream/base/cache/kustomization.yaml @@ -8,4 +8,4 @@ resources: - cache-sa.yaml images: - name: gcr.io/ml-pipeline/cache-server - newTag: 1.0.0-rc.3 + newTag: 1.0.0 diff --git a/pipeline/upstream/base/metadata/kustomization.yaml b/pipeline/upstream/base/metadata/kustomization.yaml index 517d6eb23a..5bd67c2555 100644 --- a/pipeline/upstream/base/metadata/kustomization.yaml +++ b/pipeline/upstream/base/metadata/kustomization.yaml @@ -8,4 +8,4 @@ resources: - metadata-envoy-service.yaml images: - name: gcr.io/ml-pipeline/metadata-envoy - newTag: 1.0.0-rc.3 + newTag: 1.0.0 diff --git a/pipeline/upstream/base/params.env b/pipeline/upstream/base/params.env index 2a95710eda..a274506fc7 100644 --- a/pipeline/upstream/base/params.env +++ b/pipeline/upstream/base/params.env @@ -1,5 +1,5 @@ appName=pipeline -appVersion=1.0.0-rc.3 +appVersion=1.0.0 dbHost=mysql dbPort=3306 mlmdDb=metadb diff --git a/pipeline/upstream/base/pipeline/kustomization.yaml b/pipeline/upstream/base/pipeline/kustomization.yaml index a8c518780e..75585a442f 100644 --- a/pipeline/upstream/base/pipeline/kustomization.yaml +++ b/pipeline/upstream/base/pipeline/kustomization.yaml @@ -17,6 +17,7 @@ resources: - ml-pipeline-scheduledworkflow-rolebinding.yaml - ml-pipeline-scheduledworkflow-sa.yaml - ml-pipeline-ui-deployment.yaml +- ml-pipeline-ui-configmap.yaml - ml-pipeline-ui-role.yaml - ml-pipeline-ui-rolebinding.yaml - ml-pipeline-ui-sa.yaml @@ -32,16 +33,17 @@ resources: - pipeline-runner-rolebinding.yaml - pipeline-runner-sa.yaml - container-builder-sa.yaml +- viewer-sa.yaml images: - name: gcr.io/ml-pipeline/api-server - newTag: 1.0.0-rc.3 + newTag: 1.0.0 - name: gcr.io/ml-pipeline/persistenceagent - newTag: 1.0.0-rc.3 + newTag: 1.0.0 - name: gcr.io/ml-pipeline/scheduledworkflow - newTag: 1.0.0-rc.3 + newTag: 1.0.0 - name: gcr.io/ml-pipeline/frontend - newTag: 1.0.0-rc.3 + newTag: 1.0.0 - name: gcr.io/ml-pipeline/viewer-crd-controller - newTag: 1.0.0-rc.3 + newTag: 1.0.0 - name: gcr.io/ml-pipeline/visualization-server - newTag: 1.0.0-rc.3 + newTag: 1.0.0 diff --git a/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml b/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml index 51503e59d1..6a4a3b21c5 100644 --- a/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml +++ b/pipeline/upstream/base/pipeline/metadata-writer/kustomization.yaml @@ -7,4 +7,4 @@ resources: - metadata-writer-sa.yaml images: - name: gcr.io/ml-pipeline/metadata-writer - newTag: 1.0.0-rc.3 + newTag: 1.0.0 diff --git a/pipeline/upstream/base/pipeline/ml-pipeline-ui-configmap.yaml b/pipeline/upstream/base/pipeline/ml-pipeline-ui-configmap.yaml new file mode 100644 index 0000000000..85b6422976 --- /dev/null +++ b/pipeline/upstream/base/pipeline/ml-pipeline-ui-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ml-pipeline-ui-configmap +data: + viewer-pod-template.json: |- + { + "spec": { + "serviceAccountName": "kubeflow-pipelines-viewer" + } + } diff --git a/pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml b/pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml index 70664377c7..cbb03c5232 100644 --- a/pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml +++ b/pipeline/upstream/base/pipeline/ml-pipeline-ui-deployment.yaml @@ -13,13 +13,23 @@ spec: labels: app: ml-pipeline-ui spec: + volumes: + - name: config-volume + configMap: + name: ml-pipeline-ui-configmap containers: - image: gcr.io/ml-pipeline/frontend:dummy imagePullPolicy: IfNotPresent name: ml-pipeline-ui ports: - containerPort: 3000 + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true env: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: /etc/config/viewer-pod-template.json - name: MINIO_NAMESPACE valueFrom: fieldRef: diff --git a/pipeline/upstream/base/pipeline/viewer-sa.yaml b/pipeline/upstream/base/pipeline/viewer-sa.yaml new file mode 100644 index 0000000000..932133c82a --- /dev/null +++ b/pipeline/upstream/base/pipeline/viewer-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflow-pipelines-viewer diff --git a/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml b/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml index 434afd2ee8..10e1f6aafe 100644 --- a/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml +++ b/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml @@ -14,6 +14,7 @@ spec: labels: app: cloudsqlproxy spec: + serviceAccountName: kubeflow-pipelines-cloudsql-proxy containers: - image: gcr.io/cloudsql-docker/gce-proxy:1.14 name: cloudsqlproxy diff --git a/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-sa.yaml b/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-sa.yaml new file mode 100644 index 0000000000..a4cc9c43df --- /dev/null +++ b/pipeline/upstream/env/gcp/cloudsql-proxy/cloudsql-proxy-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflow-pipelines-cloudsql-proxy diff --git a/pipeline/upstream/env/gcp/cloudsql-proxy/kustomization.yaml b/pipeline/upstream/env/gcp/cloudsql-proxy/kustomization.yaml index 7c4a112164..a336cb50f9 100644 --- a/pipeline/upstream/env/gcp/cloudsql-proxy/kustomization.yaml +++ b/pipeline/upstream/env/gcp/cloudsql-proxy/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization resources: - cloudsql-proxy-deployment.yaml +- cloudsql-proxy-sa.yaml - mysql-service.yaml diff --git a/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml b/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml index f91249a5e2..dd5b29598b 100644 --- a/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml +++ b/pipeline/upstream/env/gcp/inverse-proxy/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: gcr.io/ml-pipeline/inverse-proxy-agent - newTag: 1.0.0-rc.3 + newTag: 1.0.0 resources: - proxy-configmap.yaml - proxy-deployment.yaml diff --git a/pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml b/pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml index 08876d3990..e2d9cf597f 100644 --- a/pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml +++ b/pipeline/upstream/env/gcp/minio-gcs-gateway/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - minio-gcs-gateway-deployment.yaml +- minio-gcs-gateway-sa.yaml - minio-gcs-gateway-service.yaml secretGenerator: diff --git a/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml b/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml index e8dd6e2509..f26d27cc61 100644 --- a/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml +++ b/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-deployment.yaml @@ -15,6 +15,7 @@ spec: labels: app: minio spec: + serviceAccountName: kubeflow-pipelines-minio-gcs-gateway containers: - name: minio image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance diff --git a/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-sa.yaml b/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-sa.yaml new file mode 100644 index 0000000000..2aa4f93768 --- /dev/null +++ b/pipeline/upstream/env/gcp/minio-gcs-gateway/minio-gcs-gateway-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubeflow-pipelines-minio-gcs-gateway diff --git a/pipeline/upstream/gcp-workload-identity-setup.sh b/pipeline/upstream/gcp-workload-identity-setup.sh index e749ccbc0d..6fac04c570 100755 --- a/pipeline/upstream/gcp-workload-identity-setup.sh +++ b/pipeline/upstream/gcp-workload-identity-setup.sh @@ -16,44 +16,59 @@ set -e +# Kubernetes Namespace +NAMESPACE=${NAMESPACE:-kubeflow} + # Google service Account (GSA) -SYSTEM_GSA=${SYSTEM_GSA:-$CLUSTER_NAME-kfp-system} -USER_GSA=${USER_GSA:-$CLUSTER_NAME-kfp-user} +SYSTEM_GSA=${SYSTEM_GSA:-$RESOURCE_PREFIX-kfp-system} +USER_GSA=${USER_GSA:-$RESOURCE_PREFIX-kfp-user} # Kubernetes Service Account (KSA) +# Note, if deploying manifests/kustomize/env/gcp, you can add the following KSAs +# to the array of SYSTEM_KSA: +# * kubeflow-pipelines-minio-gcs-gateway needs gcs permissions +# * kubeflow-pipelines-cloudsql-proxy needs cloudsql permissions SYSTEM_KSA=(ml-pipeline-ui ml-pipeline-visualizationserver) -USER_KSA=(pipeline-runner kubeflow-pipelines-container-builder) +USER_KSA=(pipeline-runner kubeflow-pipelines-container-builder kubeflow-pipelines-viewer) -cat < CLUSTER_NAME= NAMESPACE= ./gcp-workload-identity-setup.sh +PROJECT_ID= RESOURCE_PREFIX= NAMESPACE= ./gcp-workload-identity-setup.sh ``` PROJECT_ID: GCP project ID your cluster belongs to. -CLUSTER_NAME: your GKE cluster's name. -NAMESPACE: Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to (default is kubeflow). +RESOURCE_PREFIX: Your preferred resource prefix for GCP resources this script creates. +NAMESPACE: Optional. Kubernetes namespace your Kubeflow Pipelines standalone deployment belongs to. (Defaults to kubeflow) +USE_GCP_MANAGED_STORAGE: Optional. Defaults to "false", specify "true" if you intend to use GCP managed storage (Google Cloud Storage and Cloud SQL) following instructions in: +https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize/sample EOF } if [ -z "$PROJECT_ID" ]; then @@ -62,18 +77,39 @@ if [ -z "$PROJECT_ID" ]; then echo "Error: PROJECT_ID env variable is empty!" exit 1 fi -if [ -z "$CLUSTER_NAME" ]; then +if [ -z "$RESOURCE_PREFIX" ]; then usage echo - echo "Error: CLUSTER_NAME env variable is empty!" + echo "Error: RESOURCE_PREFIX env variable is empty!" exit 1 fi echo "Env variables set:" echo "* PROJECT_ID=$PROJECT_ID" -echo "* CLUSTER_NAME=$CLUSTER_NAME" +echo "* RESOURCE_PREFIX=$RESOURCE_PREFIX" echo "* NAMESPACE=$NAMESPACE" +echo "* USE_GCP_MANAGED_STORAGE=${USE_GCP_MANAGED_STORAGE:-false}" echo +SYSTEM_GSA_FULL="$SYSTEM_GSA@$PROJECT_ID.iam.gserviceaccount.com" +USER_GSA_FULL="$USER_GSA@$PROJECT_ID.iam.gserviceaccount.com" + +cat </dev/null; then + echo "KSA $name already exists" + else + kubectl create serviceaccount $name -n $NAMESPACE --save-config + echo "KSA $name created" + fi +} # Bind KSA to GSA through workload identity. # Documentation: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity @@ -112,6 +149,8 @@ function bind_gsa_and_ksa { --member="serviceAccount:$PROJECT_ID.svc.id.goog[$NAMESPACE/$ksa]" \ --role="roles/iam.workloadIdentityUser" \ > /dev/null # hide verbose output + + create_ksa_if_not_present $ksa kubectl annotate serviceaccount \ --namespace $NAMESPACE \ --overwrite \ @@ -129,3 +168,35 @@ echo "Binding each kfp user KSA to $USER_GSA" for ksa in ${USER_KSA[@]}; do bind_gsa_and_ksa $USER_GSA $ksa done + +echo +echo "All the workload identity bindings have succeeded!" +cat <