[Manifest] Apply kustomize best practices to standalone manifest (#3978)

* Use configMapKeyRef for env vars

* Allow easy customization of cluster-scoped resources namespace

* clean up

* Clean up

* Simplify var replacement with direct configmap value ref

* clean up params.env

manifests/kustomize/base/argo/workflow-controller-configmap.yaml

@@ -5,14 +5,14 @@ metadata:
 data:
   config: |
     {
-    namespace: $(NAMESPACE),
+    namespace: $(kfp-namespace),
     executorImage: gcr.io/ml-pipeline/argoexec:v2.7.5-license-compliance,
     artifactRepository:
     {
         s3: {
-            bucket: $(BUCKET_NAME),
+            bucket: $(kfp-artifact-bucket-name),
             keyPrefix: artifacts,
-            endpoint: minio-service.$(NAMESPACE):9000,
+            endpoint: minio-service.$(kfp-namespace):9000,
             insecure: true,
             accessKeySecret: {
                 name: mlpipeline-minio-artifact,

manifests/kustomize/base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml

@@ -9,4 +9,4 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: kubeflow-pipelines-cache-deployer-sa
-  namespace: $(NAMESPACE)
+# namespace will be added by kustomize automatically according to the namespace field in kustomization.yaml

manifests/kustomize/base/cache-deployer/cache-deployer-sa.yaml → manifests/kustomize/base/cache-deployer/cluster-scoped/cache-deployer-sa.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: kubeflow-pipelines-cache-deployer-sa
+  name: kubeflow-pipelines-cache-deployer-sa

manifests/kustomize/base/cache-deployer/cluster-scoped/kustomization.yaml

@@ -4,4 +4,8 @@ kind: Kustomization
 resources:
   - cache-deployer-clusterrole.yaml
   - cache-deployer-clusterrolebinding.yaml
+  # HACK: although a service account(SA) is not a cluster-scoped resource.
+  # Presence of a SA referred by a clusterrolebinding allows kustomize to auto-add
+  # namespace for the clusterrolebinding's SA ref.
+  - cache-deployer-sa.yaml
 

manifests/kustomize/base/cache-deployer/kustomization.yaml

@@ -4,6 +4,4 @@ kind: Kustomization
 resources:
   - cache-deployer-role.yaml
   - cache-deployer-rolebinding.yaml
-  - cache-deployer-sa.yaml
   - cache-deployer-deployment.yaml
-

manifests/kustomize/base/cache/cache-deployment.yaml

@@ -23,18 +23,18 @@ spec:
           - name: DBCONFIG_DB_NAME
             valueFrom:
               configMapKeyRef:
-                name: mysql-configmap
-                key: cache_db
+                name: pipeline-install-config
+                key: cacheDb
           - name: DBCONFIG_HOST_NAME
             valueFrom:
               configMapKeyRef:
-                name: mysql-configmap
-                key: host
+                name: pipeline-install-config
+                key: dbHost
           - name: DBCONFIG_PORT
             valueFrom:
               configMapKeyRef:
-                name: mysql-configmap
-                key: port
+                name: pipeline-install-config
+                key: dbPort
           - name: DBCONFIG_USER
             valueFrom:
               secretKeyRef:

manifests/kustomize/base/kustomization.yaml

@@ -1,11 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: kubeflow
 bases:
 - application
 - argo
 - pipeline
 - metadata
-- mysql
 - cache
 - cache-deployer
 resources:
@@ -37,63 +37,28 @@ secretGenerator:
 - name: mysql-secret
   env: params-db-secret.env
 vars:
-- name: NAMESPACE
+- name: kfp-namespace
   objref:
     kind: Deployment
     apiVersion: apps/v1
     name: ml-pipeline
   fieldref:
     fieldpath: metadata.namespace
-- name: APP_NAME
+- name: kfp-app-name
   objref:
     kind: ConfigMap
     name: pipeline-install-config
     apiVersion: v1
   fieldref:
     fieldpath: data.appName
-- name: APP_VERSION
+- name: kfp-app-version
   objref:
     kind: ConfigMap
     name: pipeline-install-config
     apiVersion: v1
   fieldref:
     fieldpath: data.appVersion
-- name: DBSERVICE_HOST
-  objref:
-    kind: ConfigMap
-    name: pipeline-install-config
-    apiVersion: v1
-  fieldref:
-    fieldpath: data.dbHost
-- name: DBSERVICE_PORT
-  objref:
-    kind: ConfigMap
-    name: pipeline-install-config
-    apiVersion: v1
-  fieldref:
-    fieldpath: data.dbPort
-- name: DBNAME_MLMD
-  objref:
-    kind: ConfigMap
-    name: pipeline-install-config
-    apiVersion: v1
-  fieldref:
-    fieldpath: data.mlmdDb
-- name: DBNAME_CACHE
-  objref:
-    kind: ConfigMap
-    name: pipeline-install-config
-    apiVersion: v1
-  fieldref:
-    fieldpath: data.cacheDb
-- name: DBNAME_PIPELINE
-  objref:
-    kind: ConfigMap
-    name: pipeline-install-config
-    apiVersion: v1
-  fieldref:
-    fieldpath: data.pipelineDb
-- name: BUCKET_NAME
+- name: kfp-artifact-bucket-name
   objref:
     kind: ConfigMap
     name: pipeline-install-config

manifests/kustomize/base/metadata/metadata-grpc-deployment.yaml

@@ -31,18 +31,18 @@ spec:
         - name: MYSQL_DATABASE
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: mlmd_db
+              name: pipeline-install-config
+              key: mlmdDb
         - name: MYSQL_HOST
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: host
+              name: pipeline-install-config
+              key: dbHost
         - name: MYSQL_PORT
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: port
+              name: pipeline-install-config
+              key: dbPort
         command: ["/bin/metadata_store_server"]
         args: ["--grpc_port=8080",
                "--mysql_config_database=$(MYSQL_DATABASE)",

manifests/kustomize/base/params.yaml

@@ -2,23 +2,7 @@
 varReference:
 - path: data/config
   kind: ConfigMap
-- path: data/bucket_name
-  kind: ConfigMap
-- path: data/project_id
-  kind: ConfigMap
-- path: data/host
-  kind: ConfigMap
-- path: data/port
-  kind: ConfigMap
-- path: data/mlmd_db
-  kind: ConfigMap
-- path: data/cache_db
-  kind: ConfigMap
-- path: data/pipeline_db
-  kind: ConfigMap
 - path: metadata/name
   kind: Application
 - path: spec/descriptor/version
   kind: Application
-- path: spec/template/spec/containers/image
-  kind: Deployment

manifests/kustomize/base/pipeline-application.yaml

@@ -1,7 +1,7 @@
 apiVersion: app.k8s.io/v1beta1
 kind: Application
 metadata:
-  name: $(APP_NAME)
+  name: $(kfp-app-name)
   annotations:
     kubernetes-engine.cloud.google.com/icon: >-
       data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADMAAAAyCAYAAADx/eOPAAALuUlEQVRogd2afWxd9XnHP99bK4pS3yhiGUIRiiJUVVVqbq8ppdR20ibqpuIMtDRkUYERp29J57gMZVuxsrZiK7oZXVv5re1AiOuoG+N1DMkuytprsGPEVMouxqQZJVHEWIdQlGVxZlmZdb/747zcc869jpMO+seOdPz7nd/L83tev89zzjX8Bq795Rq9o17zXp+Tey+Ijkyboela29DRWkhffyT733pH/Z3este9F2cC6N0kNjxtjD+FdRD8UF9X7u97y7UbQFPAivC0BdllS381slun3s3z3xVhhqeds90tqR/oMB7u68z19ZZra0E/l1if3WOziPx3skrDPTr+bvDxfxImEIJbgX6gGBJ7EfHJX/ySReDHwO9KYAenyWCMFKw21GSeslwa2Z17+TcuzPBRr7B8m6Df5oOJqdPAR/u6cm/2lmv3At+IT3GiXZqbcaxSLsfRoTsvn7XL2jE87ZXGnwf+VGiDY86ETM1wU1+XjvSW/RlgTJADQ2QaCZKWcX1/aDIcjE8i3SdzZLjn0lm8pJXD02417BM+gLmq2Rqjr/d16Vu95dp6wc8Ra5O8NrPIcoZCvIR1H+KZkd2qLcfnRYUZOuorJO+3uQt0RerolGYZR7r5+C9ZATwPviGyQprd6Liszy3bnwVKwGMjPbnFyxJmeNpX2T4gaR/QmmSpyYZTho/2depMb9k/kNh3KawuJ1bWauHzUcyXRpZAv5Zmg7aHBLcmNN9ECAFeAO3s69KZ3nLtDuF9dnBs0IT9JO24rbPb0JfP2syCZpFfE5q1mRWcvlgMNcwMTRq9z/+OWXdx4AGjvX1deqC37DbwPwOrMgsufol5mWMWs1ivEbjTrOCtLNNb+udygqsNbUBtopR/NkuuwTJ6Hxsw67KSuvH5MPDA/nJttfGTdUFCMUlp/ALwOtIs9muBxpnFnBzuSQf21oP/BbXclVvumWuTaDN8WNBm2GizJkxPM0CDMA2WGZ72bbb/Njue2TRj9Il/PcG87SeBz4ZTNaSTsmctHcO8SqDp14d7dCFLZ2v/3OpQ023Ah4n65kohvETUCdcsfmuilD+bpNdgGZvOODuHqYGIVGCec9g7+7o031v2jaBTiD0ysxbHRnZrPktzyz1zK7f0z10nh5pWwLRhvZro1KqznVJhNB8UyDeSsU4zAOiIXV1OuEqQ2AR79nflXgcY6dGLwIvR8q39cy1b+uc2Emo6dI824BpMSxz8iVhy4m/2WiYHdV5UmOHp2mpwm52ESCdwRn+9v0tPAWzpn9sAFAQbMdc60PaHsFZEWd9uxk4z8G3seykECfObTEd2KmuZG4CWyLXkYLMwtiYt+hMsTUdAEZQzjs9apv66SHJRk73ZjBQ+iRu29s+1VEr5OImmXs4MHUahVoLWgK23wbv6OrU4OulcuHYehWsVHhpXwpE2FNRayTszX2cwDpQEzTB+QvrJHCXUaigk+c++aXZiE98YmUVgV19X7u3ypH/fgfUA5h2usY2jNjmWoGVn50nvC9T2NviA5OPBGPW91OlG+0Xa1WJhhqadk3WjpKCilQIQFP19XZocnfIHgIeFWyNh6goXyX6gdNWfU8aJ5tNjEheAHZVS/ruGj0s8k6VPhh6ms6kwgoLl1aGuCEuSpwXfHZ2qrTJ+HHkNCpOjmbdFcEcGUIhUSj/H65rPO6j+766U8i/QXV0z8cqJc4btwF8AtWgtMb1wj+j41Df/s1EYQwdEDiqM3hDes9quGY3IKoYOvCrU7HlCoZtEWapPkzEpsU8uq8b36a6uBqaBv5l45URLpZT/pmGH8LnkvlAdAOt1oeXqRsuYTjlEMJiXvWN/Z+5szfqioKcOKo7qr/nAEesKiOyv2A/q88rOx8+8bPhK5dUTAA8jbUT6MuKnbKteNVHKP23xCeD1LC0F2TWOmzoAKEiWxmC+sr8rN1OerF2HGaqXFcZhDWaYj11S4ZxcXxVqyKqPZOeNTwM7Jkr5BeDPQJ8NFQaoC/gZ26rXT5TyxxAfRx6P94d0gU0pYYama+tsbwix/AHM4fKUrwAeB68kRJ5AZsWWieGTjLipsVCgrKCwKHF7pZQ/RXf104j76i4ZMmquxkzRXb2zUsqfxdxsfCiA70hRjZbpCDHmJcRdeZPDHkVck0Ul5PeHZ81DgHxKtglXaHCxVN9fr5TyR9hW3QA8Amqp5526SyKtBEbZVv1eZeZkbqKU7xfsFJwPqRW29s+11oUxnUhnkHf2dWoB+R5Jv5dNaGHh1wog8d/ZAI+0GgVpFPTp4AfJT2Hup7u6EvMk0tpkboutEz0HMPzHyD+mu3pFpZR/Aug0Pgm0RLkvFzLWYfjDvs7cqfKUt2LuXTLhue5mdWhVDJdEzxDDcRKawceN9lRePVkDfgBcR/LKVqNpz/s08DO6q4VKKT8j8zHgJ1HyzA1P11YZjfV1arw85auBR4RalDB5lEjDKi0CgPPphKZ0QiNRwUQeg88B2ydKreew9yH1NCxe/r4GaZpt1Vsrh/JnDDcBLwPkbLVgf6s86RXYj4KvtJKJM8KsGLkSlsmUL6mSg1RJY1xD7KmU8sfprnYgBqJsGVsiEfupsca7FfMo26p/OfHKiVqllB8HyPV16VxfV66G/G1QBwY5xvCgTT7X3/MTaBbFVr0fJvqw2ASZ+yul/FN0V68CHsesiDl3UopM3CwhDZDD/Dnwj3S/sjoYAMqTtc1YX02jVqYOiuuqsAKIkqZCfFIz/IrfFY8gDrKt2gI8irSuwQezyTeNaOl+6qYb+fpYGKEXJE9GSTObK5ItrheaLHE5/XRKcHul+kYN8x2kzWlLNNuVtUqibzKW5CBjxUoszO7NWrS1E/xWvMeJjck2WQHEKJeMD+qH4gWCSvg00m3AVxv5TMRKsp9Cs0Q/Ka/1BOZQNBSXMz2b9Q5oO9JCKgkqg2aKofl8uvTPeE1w3t5KKf8y26pFxINhLRa5R9JV6huT/aZuFu7Ds+A9jBdj+VIvZz2b9BL2Xi5yJQEgUFqinI9SZBDx358o5Q/HiRGtquOEmxJu6DcbC/afQWxnvHg+Odrwm2bP5txh5OEYjOM3vaiu8qqHJw1mPmK/Xs7HJf0LRncDMF5cAL6NWUxDrX/duwbczljxjSzvTX+gtXU3MBlrRCltrsxBTgorACKrRGf5bczOiVLrhUL74B2F9oHVjBd/iLwTWEhr+CIWaLYumDjIWLHha+aSwvRs1iJmJ9Kb9ZJRETS3ACsMC8i1ZNwgXZDYWTmU/1WhfeAW8Cjo+UL7wDrGik8jfid0kYz/Z2ODepv+GPIY+FAznpcUJhAo9w5mh81CFtEsWieCTzwXkogmfKBSyh8ttA98EDPqoPouYqYLxYEPMVY8itmEeTM+KEaqZhVAkiPPIL6QDPhLFiYQSC9J7M3mGlF/24zWSvwIM1xoH2gF/sFiTcSPxQakqUJxsIPx4jGCr0AzCUYTbROJ7DPAdsbSAX9ZwgDs3qTDiMGUOxF/1DgfekLVsPf0sw8DPARsDNwy8iYBXov4p0L7wC2MF99CfBJ4rqmbJbO/qYE+x1jx5HK8Xtp/aFgHDM/FX+RM9FFjHjjj4NV3HvlPsP4g+SqQgm6zCuvJQnHgi4wVz2JuAj8RnLGEVaCf8Y8cuRQ2L0mYEBB2Gb8ZHKD4NQBx+0Qpf7LQPrAVVGqiiWTpCcEn4QcLxcF7C7+aXMDahT1YX5IS5DHE/ZfC4yULEwr0DtIOWwuuvwZ8rVLKP1soDqzHPGJoyRao9b4SXiQQ30A8eO1/PJ8D7gK+BtQSJcQM8AXGlg747LUkmi91lad8J3CuZ5OeBii0D64ET2FdH1N0omWJvgLPkvwM8LmZf7lrnm3VO4CHsM4DH2P8I8vGSfK67P9q8v9wWPAcQLH4PbBHbK6Pq+3M9+Ml+6FL2dyC+WmhOLiWseKPMDeDd12uIPBrWCZ5Xds++AHsAwGlBKnoB5747c2J+aSJEuvRL8CDv/2Zz+cqh/LL/gPD//vrfwFjcI5oX6jDBwAAAABJRU5ErkJggg==
@@ -12,7 +12,7 @@ spec:
     matchLabels:
       application-crd-id: kubeflow-pipelines
   descriptor:
-    version: $(APP_VERSION)
+    version: $(kfp-app-version)
     type: Kubeflow Pipelines
     description: |-
       Reusable end-to-end ML workflow

manifests/kustomize/base/pipeline/ml-pipeline-apiserver-deployment.yaml

@@ -22,7 +22,10 @@ spec:
         - name: OBJECTSTORECONFIG_SECURE
           value: "false"
         - name: OBJECTSTORECONFIG_BUCKETNAME
-          value: $(BUCKET_NAME)
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: bucketName
         - name: DBCONFIG_USER
           valueFrom:
             secretKeyRef:
@@ -36,18 +39,18 @@ spec:
         - name: DBCONFIG_DBNAME
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: pipeline_db
+              name: pipeline-install-config
+              key: pipelineDb
         - name: DBCONFIG_HOST
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: host
+              name: pipeline-install-config
+              key: dbHost
         - name: DBCONFIG_PORT
           valueFrom:
             configMapKeyRef:
-              name: mysql-configmap
-              key: port
+              name: pipeline-install-config
+              key: dbPort
         image: gcr.io/ml-pipeline/api-server:dummy
         imagePullPolicy: IfNotPresent
         name: ml-pipeline-api-server

manifests/kustomize/cluster-scoped-resources/kustomization.yaml

@@ -1,28 +1,27 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: kubeflow
+
+resources:
+- namespace.yaml
 bases:
 - ../base/application/cluster-scoped
 - ../base/argo/cluster-scoped
 - ../base/pipeline/cluster-scoped
 - ../base/cache-deployer/cluster-scoped
-
-resources:
-  - namespace.yaml
-
-# Used by Kustomize
-configMapGenerator:
-  - name: pipeline-cluster-scoped-install-config
-    env: params.env
-
 vars:
-  - name: NAMESPACE
-    objref:
-      kind: ConfigMap
-      name: pipeline-cluster-scoped-install-config
-      apiVersion: v1
-    fieldref:
-      fieldpath: data.namespace
-
+# NOTE: var name must be unique globally to allow composition of multiple kustomize
+# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from
+# others.
+- name: kfp-cluster-scoped-namespace
+  objref:
+    # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml
+    # so that we only need to change kustomization.yaml's namespace field for namespace customization.
+    kind: ServiceAccount
+    name: kubeflow-pipelines-cache-deployer-sa
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.namespace
 configurations:
-  - params.yaml
+- params.yaml

manifests/kustomize/cluster-scoped-resources/namespace.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: $(NAMESPACE)
+  name: '$(kfp-cluster-scoped-namespace)'

manifests/kustomize/cluster-scoped-resources/params.yaml

@@ -2,5 +2,3 @@
 varReference:
 - path: metadata/name
   kind: Namespace
-- path: subjects/namespace
-  kind: ClusterRoleBinding

manifests/kustomize/env/gcp/cloudsql-proxy/cloudsql-proxy-deployment.yaml

@@ -17,6 +17,12 @@ spec:
       containers:
         - image: gcr.io/cloudsql-docker/gce-proxy:1.14
           name: cloudsqlproxy
+          env:            
+            - name: GCP_CLOUDSQL_INSTANCE_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: gcsCloudSqlInstanceName
           command: ["/cloud_sql_proxy",
                     "-dir=/cloudsql",                    
                     "-instances=$(GCP_CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306",
@@ -37,4 +43,4 @@ spec:
               name: cloudsql
       volumes:
         - name: cloudsql
-          emptyDir:
+          emptyDir:

manifests/kustomize/env/gcp/gcp-configurations-patch.yaml

@@ -9,17 +9,14 @@ spec:
         - name: ml-pipeline-api-server
           env:
             - name: HAS_DEFAULT_BUCKET
-              valueFrom:
-                configMapKeyRef:
-                  name: gcp-default-config
-                  key: "has_default_bucket"
+              value: 'true'
             - name: BUCKET_NAME
               valueFrom:
                 configMapKeyRef:
-                  name: gcp-default-config
-                  key: "bucket_name"
+                  name: pipeline-install-config
+                  key: bucketName
             - name: PROJECT_ID
               valueFrom:
                 configMapKeyRef:
-                  name: gcp-default-config
-                  key: "project_id"
+                  name: pipeline-install-config
+                  key: gcsProjectId