awslabs · ryansteakley · Mar 14, 2023 · Feb 24, 2023 · Feb 28, 2023 · Mar 2, 2023
diff --git a/Makefile b/Makefile
@@ -91,22 +91,27 @@ port-forward:
 bootstrap-ack: verify-cluster-variables connect-to-eks-cluster
 	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
 	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
-	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/setup_sm_controller_req.py
+	cd tests/e2e && PYTHONPATH=.. python3 utils/ack_sm_controller_bootstrap/setup_sm_controller_req.py
+
+bootstrap-pipelines: verify-cluster-variables connect-to-eks-cluster
+	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/pipelines/config.yaml
+	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/pipelines/config.yaml
+	cd tests/e2e && PYTHONPATH=.. python3 utils/pipelines/setup_pipelines_irsa.py
 
 cleanup-ack-req: verify-cluster-variables
 	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
 	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
-	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
+	cd tests/e2e && PYTHONPATH=.. python3 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
 
-deploy-kubeflow: bootstrap-ack
+deploy-kubeflow: bootstrap-ack bootstrap-pipelines
 	$(eval DEPLOYMENT_OPTION:=vanilla)
 	$(eval INSTALLATION_OPTION:=kustomize)
-	cd tests/e2e && PYTHONPATH=.. python3.8 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --cluster_name $(CLUSTER_NAME)
+	cd tests/e2e && PYTHONPATH=.. python3 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --cluster_name $(CLUSTER_NAME)
 
 delete-kubeflow:
 	$(eval DEPLOYMENT_OPTION:=vanilla)
 	$(eval INSTALLATION_OPTION:=kustomize)
-	cd tests/e2e && PYTHONPATH=.. python3.8 utils/kubeflow_uninstallation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION)
+	cd tests/e2e && PYTHONPATH=.. python3 utils/kubeflow_uninstallation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION)
 
 helmify:
-	PYTHONPATH=. python3.8 tools/helmify/src/kustomize_to_helm_automation.py
+	PYTHONPATH=. python3 tools/helmify/src/kustomize_to_helm_automation.py
diff --git a/awsconfigs/apps/pipeline/s3-static/aws-configuration-patch.yaml b/awsconfigs/apps/pipeline/s3-static/aws-configuration-patch.yaml
@@ -0,0 +1,61 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline-ui
+spec:
+  template:
+    metadata:
+      labels:
+        app: ml-pipeline-ui
+    spec:
+      volumes:
+        - name: config-volume
+          configMap:
+            name: ml-pipeline-ui-configmap
+      containers:
+        - name: ml-pipeline-ui
+          env:
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: accesskey
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: mlpipeline-minio-artifact
+                  key: secretkey
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    metadata:
+      labels:
+        app: ml-pipeline
+    spec:
+      containers:
+        - env:
+            - name: OBJECTSTORECONFIG_SECURE
+              value: "true"
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: bucketName
+            - name: OBJECTSTORECONFIG_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: minioServiceHost
+            - name: OBJECTSTORECONFIG_REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: pipeline-install-config
+                  key: minioServiceRegion
+            - name: OBJECTSTORECONFIG_PORT
+              value: ""
+          name: ml-pipeline-api-server
diff --git a/awsconfigs/apps/pipeline/s3-static/config b/awsconfigs/apps/pipeline/s3-static/config
@@ -0,0 +1,20 @@
+{
+artifactRepository:
+{
+    s3: {
+        bucket: $(kfp-artifact-bucket-name),
+        keyPrefix: artifacts,
+        endpoint: $(kfp-artifact-storage-endpoint),
+        insecure: false,
+        accessKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: accesskey
+        },
+        secretKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: secretkey
+        }
+    },
+    archiveLogs: true
+}
+}
diff --git a/awsconfigs/apps/pipeline/s3-static/deployment_patch.yaml b/awsconfigs/apps/pipeline/s3-static/deployment_patch.yaml
@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kubeflow-pipelines-profile-controller
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: profile-controller
+        env:
+        - name: MINIO_SERVICE_REGION
+          valueFrom:
+            configMapKeyRef:
+              name: pipeline-install-config
+              key: minioServiceRegion
diff --git a/awsconfigs/apps/pipeline/s3-static/disable-default-secret.yaml b/awsconfigs/apps/pipeline/s3-static/disable-default-secret.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mlpipeline-minio-artifact
+$patch: delete
+
diff --git a/awsconfigs/apps/pipeline/s3-static/disable-minio-server-resources.yaml b/awsconfigs/apps/pipeline/s3-static/disable-minio-server-resources.yaml
@@ -0,0 +1,20 @@
+$patch: delete
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: kubeflow
+---
+$patch: delete
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: minio-pvc
+  namespace: kubeflow
+---
+$patch: delete
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+  namespace: kubeflow
diff --git a/awsconfigs/apps/pipeline/s3-static/kustomization.yaml b/awsconfigs/apps/pipeline/s3-static/kustomization.yaml
@@ -0,0 +1,41 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../base
+configMapGenerator:
+- name: pipeline-install-config
+  env: params.env
+  behavior: merge
+- name: workflow-controller-configmap
+  behavior: replace
+  files:
+  - config
+- name: ml-pipeline-ui-configmap
+  behavior: replace
+  files:
+  - viewer-pod-template.json
+- name: kubeflow-pipelines-profile-controller-code
+  behavior: replace
+  files:
+  - sync.py
+generatorOptions:
+  disableNameSuffixHash: true
+patchesStrategicMerge:
+- disable-minio-server-resources.yaml
+- deployment_patch.yaml
+- disable-default-secret.yaml
+- aws-configuration-patch.yaml
+# Identifier for application manager to apply ownerReference.
+# The ownerReference ensures the resources get garbage collected
+# when application is deleted.
+commonLabels:
+  application-crd-id: kubeflow-pipelines
+vars:
+- name: kfp-artifact-storage-endpoint
+  objref:
+    kind: ConfigMap
+    name: pipeline-install-config
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.minioServiceHost
diff --git a/awsconfigs/apps/pipeline/s3-static/params.env b/awsconfigs/apps/pipeline/s3-static/params.env
@@ -0,0 +1,3 @@
+bucketName=
+minioServiceHost=s3.amazonaws.com
+minioServiceRegion=