feat: refactor KFP managed storage a separate package. Fixes #275

Makefile

@@ -1,43 +1,19 @@
-#***********************************************************************************************************************
-# Edit this section to set the values specific to your deployment
-
-.PHONY: set-cfg-values
-set-cfg-values:
-	kpt cfg set -R .  gke.private false
-
-	kpt cfg set -R .  mgmt-ctxt <YOUR_MANAGEMENT_CTXT>
-
-	kpt cfg set -R .  name <YOUR_KF_NAME>
-	kpt cfg set -R .  gcloud.project.projectNumber <PROJECT_NUMBER_TO_DEPLOY_IN>
-	kpt cfg set -R .  gcloud.core.project <PROJECT_ID_TO_DEPLOY_IN>
-	kpt cfg set -R .  gcloud.compute.zone <ZONE>
-	kpt cfg set -R .  location <REGION OR ZONE>
-	kpt cfg set -R .  log-firewalls false
-
-	kpt cfg set -R .  email <YOUR_EMAIL_ADDRESS>
-
-# Reset various kpt values to default values and remove other
-# files that shouldn't be included in PRs
-# TODO(jlewi): We should add a test to make sure changed values don't get checked in
-# We don't run it in generate because we don't want to force all developers to install kpt
-clean-for-pr: reset-cfg-values
-	rm -rf kubeflow/.build
-	rm -rf management/.build
-
-	rm -rf kubeflow/upstream/manifests
-	rm -rf management/upstream/management
-
 .PHONY: reset-cfg-values
 reset-cfg-values:
-	kpt cfg set -R . gke.private false
-
+	kpt cfg set -R kubeflow name KUBEFLOW-NAME
+
+	kpt cfg set -R management name MANAGEMENT-NAME
 	kpt cfg set -R . mgmt-ctxt MANAGEMENT-CTXT
 
-	kpt cfg set -R .  name KUBEFLOW-NAME
-	kpt cfg set -R .  gcloud.core.project PROJECT
-	kpt cfg set -R .  gcloud.project.projectNumber PROJECT_NUMBER
-	kpt cfg set -R .  gcloud.compute.zone ZONE
-	kpt cfg set -R .  location LOCATION
-	kpt cfg set -R .  log-firewalls false
+	kpt cfg set -R . gcloud.core.project PROJECT
+	kpt cfg set -R . gcloud.project.projectNumber PROJECT_NUMBER
+	kpt cfg set -R . location LOCATION
+	kpt cfg set -R . gcloud.compute.zone ZONE
+	kpt cfg set -R . gcloud.compute.region REGION
+	kpt cfg set -R . bucket-name BUCKET-NAME
+	kpt cfg set -R . cloudsql-name CLOUDSQL-NAME
 
-	kpt cfg set -R .  email EMAIL
+	kpt cfg set -R . email EMAIL
+
+	kpt cfg set -R . gke.private false
+	kpt cfg set -R . log-firewalls false

kubeflow/Makefile

@@ -47,6 +47,7 @@ check-name:
 	PROJECT=$(PROJECT) NAME=$(NAME) ./hack/check_domain_length.sh
 
 components= \
+common/managed-storage \
 common/cnrm \
 common/asm \
 common/kubeflow-namespace \
@@ -78,20 +79,20 @@ clean-build:
 	done
 
 .PHONY: hydrate
-hydrate: validate-values check-name 
+hydrate: validate-values check-name
 	@for component in $(components) ; do \
 	(BUILD_DIR=$(BUILD_DIR) NAME=$(NAME) KFCTXT=$(KFCTXT) LOCATION=$(LOCATION) PROJECT=$(PROJECT) ./hack/apply_component.sh -path $$component -hydrate_only) || exit 1; \
 	done
 
 .PHONY: apply
-apply: validate-values check-name 
+apply: validate-values check-name
 	@for component in $(components) ; do \
 	(BUILD_DIR=$(BUILD_DIR) NAME=$(NAME) KFCTXT=$(KFCTXT) LOCATION=$(LOCATION) PROJECT=$(PROJECT) ./hack/apply_component.sh -path $$component) || exit 1; \
 	done
 
 	KFCTXT=$(KFCTXT) $(MAKE) -C common/iap-ingress pod-reset
 
-# Delete gcp resources
 .PHONY: delete
 delete:
+	# Delete gcp resources except managed storage
 	BUILD_DIR=$(BUILD_DIR) MGMTCTXT=$(MGMTCTXT) ./hack/delete_gcp.sh

kubeflow/README.md

@@ -9,14 +9,14 @@ Install the necessary tools if not already.
 
 1. Install gcloud SDK and deployment tools:
 
-```
+```bash
 gcloud components install kubectl kpt beta
 gcloud components update
 ```
 
-2. Install Kustomize
+2. Install Kustomize:
 
-```
+```bash
 # Detect your OS and download corresponding latest Kustomize binary
 curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash
 
@@ -28,120 +28,117 @@ sudo mv ./kustomize /usr/local/bin/kustomize
 
 Follow the yq v3 [installation instruction](https://github.com/mikefarah/yq#install). For example, if using wget, you can run following commands: 
 
-```
+```bash
 sudo wget https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
 ```
 
 4. Install jq https://stedolan.github.io/jq/, for example, we can run the following command on Linux:
 
-```
+```bash
 sudo apt install jq
 ```
 
 ### Fetch packages
 
 Go to Kubeflow Cluster
 
-```
+```bash
 cd kubeflow
 ```
 
 `kubeflow/gcp-blueprints` utilizes upstream repositority `kubeflow/manifests` for versioned manifests of multiple Kubeflow components. We need to first fetch upstream manifests by running command:
 
-```
+```bash
 bash ./pull_upstream.sh
 ```
 
-
 ### Environment Variables
 
+Provide actual value for the following variables in `env.sh`, refer to detailed
+documentation in env.sh.
 
+Set the environment variables in your shell:
 
-Provide actual value for the following variables in `env.sh`:
-
-```
-KF_NAME=<kubeflow-cluster-name>
-KF_PROJECT=<gcp-project-id>
-KF_DIR=<current-kubeflow-directory-path>
-MGMT_NAME=<management-cluster-name>
-MGMTCTXT=${MGMT_NAME}
-LOCATION=<zone>
-```
-
-Provide the actual value for the following variables in `kpt-set.sh`:
-
-```
-kpt cfg set -R .  gcloud.project.projectNumber <KF_PROJECT_NUMBER>
-kpt cfg set -R .  email <YOUR_EMAIL_ADDRESS>
-```
-
-Run the following commands to set environment variables and kpt setter
-
-```
+```bash
 source env.sh
 ```
 
-```
+Configure kpt setters as environement variables in packages:
+
+```bash
 bash kpt-set.sh
 ```
 
-Set the Client ID and Secret from IAP Oauth step:
+Set the Client ID and Secret from IAP OAuth step:
 
-```
+```bash
 export CLIENT_ID=<Your CLIENT_ID>
 export CLIENT_SECRET=<Your CLIENT_SECRET>
 ```
 
 ### Deploy Kubeflow Cluster
 
 
-Run following command to login
+Run following command to login:
 
-```
+```bash
 gcloud auth login
 ```
 
-
 Set the google project you want to deploy.
-```
+```bash
 gcloud config set project $KF_PROJECT
 ```
 
-
-Set default cluster location
-```
-gcloud config set compute/zone $LOCATION
-```
-
 Deploy Kubeflow cluster, required Google Cloud resources and all Kubeflow components:
 
-```
+```bash
 make apply
 ```
 
 ## Other Commands
 
+Reminder, all the following commands assume you already set up env vars by:
+
+```bash
+source env.sh
+```
 
 ### Hydrate all manifests but not apply them
 
-If you want to check the resource in `/build` directories before applying them. You can use `hydrate-all` before running `apply-all`:
+If you want to check the resources in `build` directories first, run the
+following command before `make apply`:
 
-```
+```bash
 make hydrate
 ```
 
 ### Clean up the hydration result from all components
 
-After hydration or apply, you will have `build` folder in each component for manifest yaml files. If you want to cleean them up, you can run:
+After hydration or apply, you will have `build` folder in each component for manifest yaml files. If you want to clean them up, you can run:
 
-```
+```bash
 make clean-build
 ```
 
 ### Uninstall the whole Kubeflow cluster
 
 Deleting cluster itself doesn't necessarily remove all resources created by this instruction. You can run the following command to clean them up:
 
+```bash
+make delete
 ```
+
+#### Delete managed storage
+
+Managed storage -- CloudSQL and Cloud Storage (GCS) bucket contains Kubeflow
+Pipelines data, they are not deleted by default when deleting the Kubeflow
+cluster, because you can re-deploy a new Kubeflow cluster using existing managed
+storages.
+
+Run the following commands to delete managed storage:
+
+```bash
+cd common/managed-storage
 make delete
 ```

kubeflow/apps/pipelines/Kptfile

@@ -170,11 +170,11 @@ openAPI:
       x-k8s-cli:
         setter:
           name: cloudsql-name
-          value: dev-4-28-kfp-4
+          value: CLOUDSQL-NAME
           isSet: true
     io.k8s.cli.setters.bucket-name:
       x-k8s-cli:
         setter:
           name: bucket-name
-          value: dev-4-28-kfp-artifacts
+          value: BUCKET-NAME
           isSet: true

kubeflow/apps/pipelines/cloudsql/cnrm/kustomization.yaml

@@ -14,10 +14,7 @@
 
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-commonLabels:
-  component: db
 resources:
 - iam.yaml
-- sql-instance.yaml
 - sql-user-root.yaml
 - proxy-gsa.yaml

kubeflow/apps/pipelines/cloudsql/cnrm/sql-user-root.yaml

@@ -19,6 +19,6 @@ metadata:
 spec:
   resourceID: root
   instanceRef:
-    name: dev-4-28-kfp-4 # {"$kpt-set":"cloudsql-name"}
+    name: CLOUDSQL-NAME # {"$kpt-set":"cloudsql-name"}
   password:
     value: "" # override with your own password, TODO(Bobgy): document how to change password

kubeflow/apps/pipelines/cloudsql/pipeline-install-config-patch.yaml

@@ -19,4 +19,4 @@ metadata:
 # Reference:
 # https://github.com/kubeflow/pipelines/blob/1.5.0/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml
 data:
-  gcsCloudSqlInstanceName: PROJECT:REGION:dev-4-28-kfp-4 # {"$kpt-set": "cloudsql-instance-connection-name"}
+  gcsCloudSqlInstanceName: PROJECT:REGION:CLOUDSQL-NAME # {"$kpt-set": "cloudsql-instance-connection-name"}

kubeflow/apps/pipelines/gcs/cnrm/iam.yaml

@@ -4,7 +4,7 @@ metadata:
   name: KUBEFLOW-NAME-kfp-gcs-acl # {"$kpt-set":"gcs-acl"}
 spec:
   bucketRef:
-    name: dev-4-28-kfp-artifacts # {"$kpt-set":"bucket-name"}
+    name: BUCKET-NAME # {"$kpt-set":"bucket-name"}
   entity: user-KUBEFLOW-NAME-user@PROJECT.iam.gserviceaccount.com # {"$kpt-set": "user-sa-entity"}
   role: OWNER
 ---

kubeflow/apps/pipelines/gcs/cnrm/kustomization.yaml

@@ -14,8 +14,5 @@
 
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-commonLabels:
-  component: gcs
 resources:
-- bucket.yaml
 - iam.yaml

kubeflow/apps/pipelines/gcs/pipeline-install-config-patch.yaml

@@ -19,5 +19,5 @@ metadata:
 # Reference:
 # https://github.com/kubeflow/pipelines/blob/1.5.0/manifests/kustomize/base/installs/generic/pipeline-install-config.yaml
 data:
-  bucketName: dev-4-28-kfp-artifacts # {"$kpt-set":"bucket-name"}
+  bucketName: BUCKET-NAME # {"$kpt-set":"bucket-name"}
   gcsProjectId: PROJECT # {"$kpt-set": "gcloud.core.project"}

kubeflow/apps/profiles/Kptfile

@@ -16,16 +16,19 @@ openAPI:
         setter:
           name: name
           value: KUBEFLOW-NAME
+          isSet: true
     io.k8s.cli.setters.gcloud.core.project:
       x-k8s-cli:
         setter:
           name: gcloud.core.project
           value: PROJECT
+          isSet: true
     io.k8s.cli.setters.email:
       x-k8s-cli:
         setter:
           name: email
           value: EMAIL
+          isSet: true
     io.k8s.cli.substitutions.admin-sa-ref:
       x-k8s-cli:
         substitution:

kubeflow/common/Kptfile

@@ -29,6 +29,7 @@ openAPI:
         setter:
           name: gcloud.compute.region
           value: REGION
+          isSet: true
     io.k8s.cli.setters.location:
       x-k8s-cli:
         setter:

kubeflow/common/cluster/upstream/Kptfile

@@ -29,6 +29,7 @@ openAPI:
         setter:
           name: gcloud.compute.region
           value: REGION
+          isSet: true
     io.k8s.cli.setters.location:
       x-k8s-cli:
         setter:

kubeflow/common/iam/upstream/Kptfile

@@ -29,6 +29,7 @@ openAPI:
         setter:
           name: gcloud.compute.region
           value: REGION
+          isSet: true
     io.k8s.cli.setters.location:
       x-k8s-cli:
         setter:

kubeflow/common/ingress/upstream/Kptfile

@@ -29,6 +29,7 @@ openAPI:
         setter:
           name: gcloud.compute.region
           value: REGION
+          isSet: true
     io.k8s.cli.setters.location:
       x-k8s-cli:
         setter: