kustomize manifests for irsa

Makefile

@@ -93,15 +93,29 @@ bootstrap-ack: verify-cluster-variables connect-to-eks-cluster
 	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
 	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/setup_sm_controller_req.py
 
+bootstrap-pipelines: verify-cluster-variables connect-to-eks-cluster
+	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/pipelines/config.yaml
+	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/pipelines/config.yaml
+	cd tests/e2e && PYTHONPATH=.. python3.8 utils/pipelines/setup_pipelines_irsa.py
+
 cleanup-ack-req: verify-cluster-variables
 	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
 	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
 	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
 
+cleanup-pipelines-req: verify-cluster-variables
+	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/pipelines/config.yaml
+	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/pipelines/config.yaml
+	cd tests/e2e && PYTHONPATH=.. python3.8 utils/pipelines/cleanup_pipelines_irsa.py
+
 deploy-kubeflow: bootstrap-ack
 	$(eval DEPLOYMENT_OPTION:=vanilla)
 	$(eval INSTALLATION_OPTION:=kustomize)
-	cd tests/e2e && PYTHONPATH=.. python3.8 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --cluster_name $(CLUSTER_NAME)
+	$(eval CREDENTIAL_OPTION:=irsa)
+	if [ "$(CREDENTIAL_OPTION)" = "irsa" ]; then \
+		make bootstrap-pipelines; \
+	fi
+	cd tests/e2e && PYTHONPATH=.. python3.8 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --credential_option $(CREDENTIAL_OPTION) --cluster_name $(CLUSTER_NAME)
 
 delete-kubeflow:
 	$(eval DEPLOYMENT_OPTION:=vanilla)

awsconfigs/apps/pipeline-static/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../pipeline/
+configMapGenerator:
+- name: workflow-controller-configmap
+  behavior: replace
+  files:
+  - ./s3/config
+generatorOptions:
+  disableNameSuffixHash: true
+patchesStrategicMerge:
+- ./s3/disable-default-secret.yaml

awsconfigs/apps/pipeline-static/s3/config

@@ -0,0 +1,20 @@
+{
+artifactRepository:
+{
+    s3: {
+        bucket: $(kfp-artifact-bucket-name),
+        keyPrefix: artifacts,
+        endpoint: $(kfp-artifact-storage-endpoint),
+        insecure: false,
+        accessKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: accesskey
+        },
+        secretKeySecret: {
+            name: mlpipeline-minio-artifact,
+            key: secretkey
+        }
+    },
+    archiveLogs: true
+}
+}

awsconfigs/apps/pipeline-static/s3/disable-default-secret.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mlpipeline-minio-artifact
+$patch: delete
+

awsconfigs/apps/pipeline-static/s3/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../../pipeline/s3/
+configMapGenerator:
+- name: workflow-controller-configmap
+  behavior: replace
+  files:
+  - config
+generatorOptions:
+  disableNameSuffixHash: true
+patchesStrategicMerge:
+- disable-default-secret.yaml

awsconfigs/apps/pipeline/kustomization.yaml

@@ -31,6 +31,7 @@ patchesStrategicMerge:
 - ./s3/deployment_patch.yaml
 - ./s3/disable-default-secret.yaml
 - ./s3/aws-configuration-patch.yaml
+- ./s3/service-account.yaml
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected
 # when application is deleted.

awsconfigs/apps/pipeline/s3/config

@@ -6,14 +6,7 @@ artifactRepository:
         keyPrefix: artifacts,
         endpoint: $(kfp-artifact-storage-endpoint),
         insecure: false,
-        accessKeySecret: {
-            name: mlpipeline-minio-artifact,
-            key: accesskey
-        },
-        secretKeySecret: {
-            name: mlpipeline-minio-artifact,
-            key: secretkey
-        }
+        useSDKCreds: true,
     },
     archiveLogs: true
 }

awsconfigs/apps/pipeline/s3/disable-default-secret.yaml

@@ -1,5 +1,10 @@
 apiVersion: v1
 kind: Secret
 metadata:
+  labels:
+    application-crd-id: kubeflow-pipelines
   name: mlpipeline-minio-artifact
-$patch: delete
+  namespace: kubeflow
+stringData:
+  accesskey: ""
+  secretkey: ""

awsconfigs/apps/pipeline/s3/kustomization.yaml

@@ -25,6 +25,7 @@ patchesStrategicMerge:
 - disable-minio-server-resources.yaml
 - deployment_patch.yaml
 - disable-default-secret.yaml
+- service-account.yaml
 - aws-configuration-patch.yaml
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected

awsconfigs/apps/pipeline/s3/service-account.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ml-pipeline
+  namespace: kubeflow
+  annotations:
+    eks.amazonaws.com/role-arn: '{{ .Values.irsa.roleName }}'

tests/e2e/utils/kubeflow_installation.py

@@ -17,7 +17,16 @@
 INSTALLATION_CONFIG_RDS_S3 = "./resources/installation_config/rds-s3.yaml"
 INSTALLATION_CONFIG_RDS_ONLY = "./resources/installation_config/rds-only.yaml"
 INSTALLATION_CONFIG_S3_ONLY = "./resources/installation_config/s3-only.yaml"
-INSTALLATION_CONFIG_COGNITO_RDS_S3 = "./resources/installation_config/cognito-rds-s3.yaml"
+INSTALLATION_CONFIG_COGNITO_RDS_S3 = (
+    "./resources/installation_config/cognito-rds-s3.yaml"
+)
+INSTALLATION_CONFIG_S3_ONLY_STATIC = (
+    "./resources/installation_config/s3-only-static.yaml"
+)
+INSTALLATION_CONFIG_RDS_S3_STATIC = "./resources/installation_config/rds-s3-static.yaml"
+INSTALLATION_CONFIG_COGNITO_RDS_S3_STATIC = (
+    "./resources/installation_config/cognito-rds-s3-static.yaml"
+)
 
 
 Install_Sequence = [
@@ -55,13 +64,23 @@
 
 
 def install_kubeflow(
-    installation_option, deployment_option, cluster_name, aws_telemetry=True
+    installation_option,
+    deployment_option,
+    cluster_name,
+    credentials_option,
+    aws_telemetry=True,
 ):
     print(cluster_name)
     if deployment_option == "vanilla":
         installation_config = load_yaml_file(INSTALLATION_CONFIG_VANILLA)
     elif deployment_option == "cognito":
         installation_config = load_yaml_file(INSTALLATION_CONFIG_COGNITO)
+    elif deployment_option == "rds-s3" and credentials_option == "static":
+        installation_config = load_yaml_file(INSTALLATION_CONFIG_RDS_S3_STATIC)
+    elif deployment_option == "s3" and credentials_option == "static":
+        installation_config = load_yaml_file(INSTALLATION_CONFIG_S3_ONLY_STATIC)
+    elif deployment_option == "cognito-rds-s3" and credentials_option == "static":
+        installation_config = load_yaml_file(INSTALLATION_CONFIG_COGNITO_RDS_S3_STATIC)
     elif deployment_option == "rds-s3":
         installation_config = load_yaml_file(INSTALLATION_CONFIG_RDS_S3)
     elif deployment_option == "rds-only":
@@ -71,16 +90,18 @@ def install_kubeflow(
     elif deployment_option == "cognito-rds-s3":
         installation_config = load_yaml_file(INSTALLATION_CONFIG_COGNITO_RDS_S3)
 
+
     print_banner(
-        f"Installing kubeflow {deployment_option} deployment with {installation_option}"
+        f"Installing kubeflow {deployment_option} deployment with {installation_option} with {credentials_option}"
     )
 
     for component in Install_Sequence:
         install_component(
             installation_option,
             component,
             installation_config,
-            cluster_name
+            cluster_name,
+            credentials_option,
         )
 
     if aws_telemetry == True:
@@ -89,6 +110,7 @@ def install_kubeflow(
             "aws-telemetry",
             installation_config,
             cluster_name,
+            credentials_option,
         )
 
 
@@ -97,6 +119,7 @@ def install_component(
     component_name,
     installation_config,
     cluster_name,
+    credentials_option,
     crd_established=True,
 ):
     # component not applicable for deployment option
@@ -105,26 +128,52 @@ def install_component(
     else:
         print(f"==========Installing {component_name}==========")
         # remote repo
-        if "repo"in installation_config[component_name]["installation_options"][installation_option]:
+        if (
+            "repo"
+            in installation_config[component_name]["installation_options"][
+                installation_option
+            ]
+        ):
             install_remote_component(component_name, cluster_name)
         # local repo
         else:
-            installation_paths = installation_config[component_name]["installation_options"][installation_option]["paths"]
+            installation_paths = installation_config[component_name][
+                "installation_options"
+            ][installation_option]["paths"]
             # helm
             if installation_option == "helm":
                 ##deal with namespace already exist issue for rds-s3 auto set-up script
                 if component_name == "kubeflow-namespace":
-                    for kustomize_path in installation_config[component_name]["installation_options"]["kustomize"]["paths"]:
+                    for kustomize_path in installation_config[component_name][
+                        "installation_options"
+                    ]["kustomize"]["paths"]:
                         apply_kustomize(kustomize_path)
                 else:
+                    if component_name == "kubeflow-pipelines":
+                        configure_kubeflow_pipelines(
+                            component_name,
+                            installation_paths,
+                            installation_option,
+                            credentials_option,
+                        )
                     install_helm(component_name, installation_paths)
             # kustomize
             else:
                 # crd required to established for installation
-                if "validations" in installation_config[component_name] and "crds" in installation_config[component_name]["validations"]:
+                if (
+                    "validations" in installation_config[component_name]
+                    and "crds" in installation_config[component_name]["validations"]
+                ):
                     print("need to wait for crds....")
                     crds = installation_config[component_name]["validations"]["crds"]
                     crd_established = False
+                if component_name == "kubeflow-pipelines":
+                    configure_kubeflow_pipelines(
+                        component_name,
+                        installation_paths,
+                        installation_option,
+                        credentials_option,
+                    )
                 for kustomize_path in installation_paths:
                     if not crd_established:
                         apply_kustomize(kustomize_path, crds)
@@ -172,7 +221,7 @@ def install_certmanager():
         f"helm upgrade --install cert-manager jetstack/cert-manager \
                         --namespace cert-manager \
                         --create-namespace \
-                        --version v1.5.0 \
+                        --version v1.10.1 \
                         --set installCRDs=true"
     )
 
@@ -230,6 +279,29 @@ def install_ack_controller():
     )
 
 
+def configure_kubeflow_pipelines(
+    component_name, installation_paths, installation_option, credentials_option
+):
+    if credentials_option == "static":
+        return
+    cfg = load_yaml_file(file_path="./utils/pipelines/config.yaml")
+    IAM_ROLE_ARN_FOR_IRSA = cfg["pipeline_oidc_role"]
+    if installation_option == "kustomize":
+        CHART_EXPORT_PATH = "../../apps/pipeline/s3/service-account.yaml"
+        exec_shell(
+            f'yq e \'.metadata.annotations."eks.amazonaws.com/role-arn"="{IAM_ROLE_ARN_FOR_IRSA}"\' '
+            + f"-i {CHART_EXPORT_PATH}"
+        )
+
+    else:
+        IAM_ROLE_ARN_FOR_IRSA = cfg["pipeline_oidc_role"]
+        CHART_EXPORT_PATH = f"{installation_paths}/templates/ServiceAccount/ml-pipeline-kubeflow-ServiceAccount.yaml"
+        exec_shell(
+            f'yq e \'.metadata.annotations."eks.amazonaws.com/role-arn"="{IAM_ROLE_ARN_FOR_IRSA}"\' '
+            + f"-i {CHART_EXPORT_PATH}"
+        )
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     INSTALLATION_OPTION_DEFAULT = "kustomize"
@@ -272,12 +344,22 @@ def install_ack_controller():
         help=f"EKS cluster Name",
         required=True,
     )
+    CREDENTIAL_OPTION_DEFAULT = "irsa"
+    parser.add_argument(
+        "--credentials_option",
+        type=str,
+        default=CREDENTIAL_OPTION_DEFAULT,
+        choices=["irsa", "static"],
+        help=f"Kubeflow default credential option default is set to irsa",
+        required=False,
+    )
 
     args, _ = parser.parse_known_args()
 
     install_kubeflow(
         args.installation_option,
         args.deployment_option,
         args.cluster_name,
+        args.credentials_option,
         args.aws_telemetry,
     )

tests/e2e/utils/pipelines/cleanup_pipeline_irsa.py

@@ -0,0 +1,54 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import json
+import boto3
+
+from e2e.utils.utils import (
+    load_json_file,
+    get_iam_client,
+    get_eks_client,
+)
+from e2e.fixtures.cluster import (
+    associate_iam_oidc_provider,
+)
+from e2e.utils.aws.iam import IAMPolicy
+from e2e.utils.pipelines import common
+from e2e.utils.config import configure_env_file
+from e2e.utils.utils import print_banner, load_yaml_file
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_account_id():
+    return boto3.client("sts").get_caller_identity().get("Account")
+
+def delete_iam_role(role_name, region):
+    iam_client = get_iam_client(region=region)
+    try:
+        iam_client.detach_role_policy(
+            RoleName=role_name, PolicyArn="arn:aws:iam::aws:policy/AmazonS3FullAccess"
+        )
+    except:
+        logger.log("Failed to detach role policy, it may not exist anymore.")
+
+    iam_client.delete_role(RoleName=role_name)
+    print(f"Deleted IAM Role : {role_name}")
+
+
+if __name__ == "__main__":
+    print_banner("Reading Config")
+    config_file_path = common.CONFIG_FILE_PATH
+    cfg = load_yaml_file(file_path=config_file_path)
+    cluster_region = cfg["cluster"]["region"]
+    cluster_name = cfg["cluster"]["name"]
+
+    print_banner("Deleting all resources created for Pipeline IRSA")
+    role_name = f"{common.PIPELINE_OIDC_ROLE_NAME_PREFIX}-{cluster_name}"
+    delete_iam_role(role_name, cluster_region)
+
+    print_banner("CLEANUP SUCCESSFUL")
+