kustomize manifests for irsa

Makefile

@@ -93,15 +93,24 @@ bootstrap-ack: verify-cluster-variables connect-to-eks-cluster
  yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
  cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/setup_sm_controller_req.py
 
+bootstrap-pipelines: verify-cluster-variables connect-to-eks-cluster
+ yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/pipelines/config.yaml
+ yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/pipelines/config.yaml
+ cd tests/e2e && PYTHONPATH=.. python3 utils/pipelines/setup_pipelines_irsa.py
+
 cleanup-ack-req: verify-cluster-variables
  yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
  yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
- cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
+ cd tests/e2e && PYTHONPATH=.. python3 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
 
 deploy-kubeflow: bootstrap-ack
  $(eval DEPLOYMENT_OPTION:=vanilla)
  $(eval INSTALLATION_OPTION:=kustomize)
- cd tests/e2e && PYTHONPATH=.. python3.8 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --cluster_name $(CLUSTER_NAME)
+ $(eval CREDENTIAL_OPTION:=irsa)
+ if [ "$(CREDENTIAL_OPTION)" = "irsa" ]; then \
+ make bootstrap-pipelines; \
+ fi
+ cd tests/e2e && PYTHONPATH=.. python3 utils/kubeflow_installation.py --deployment_option $(DEPLOYMENT_OPTION) --installation_option $(INSTALLATION_OPTION) --credential_option $(CREDENTIAL_OPTION) --cluster_name $(CLUSTER_NAME)
 
 delete-kubeflow:
  $(eval DEPLOYMENT_OPTION:=vanilla)

awsconfigs/apps/pipeline-static/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../pipeline/
+configMapGenerator:
+- name: workflow-controller-configmap
+ behavior: replace
+ files:
+ - ./s3/config
+generatorOptions:
+ disableNameSuffixHash: true
+patchesStrategicMerge:
+- ./s3/disable-default-secret.yaml

awsconfigs/apps/pipeline-static/s3/config

@@ -0,0 +1,20 @@
+{
+artifactRepository:
+{
+ s3: {
+ bucket: $(kfp-artifact-bucket-name),
+ keyPrefix: artifacts,
+ endpoint: $(kfp-artifact-storage-endpoint),
+ insecure: false,
+ accessKeySecret: {
+ name: mlpipeline-minio-artifact,
+ key: accesskey
+ },
+ secretKeySecret: {
+ name: mlpipeline-minio-artifact,
+ key: secretkey
+ }
+ },
+ archiveLogs: true
+}
+}

awsconfigs/apps/pipeline-static/s3/disable-default-secret.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: mlpipeline-minio-artifact
+$patch: delete
+

awsconfigs/apps/pipeline-static/s3/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kubeflow
+bases:
+- ../../pipeline/s3/
+configMapGenerator:
+- name: workflow-controller-configmap
+ behavior: replace
+ files:
+ - config
+generatorOptions:
+ disableNameSuffixHash: true
+patchesStrategicMerge:
+- disable-default-secret.yaml

awsconfigs/apps/pipeline/s3/config

@@ -6,14 +6,7 @@ artifactRepository:
  keyPrefix: artifacts,
  endpoint: $(kfp-artifact-storage-endpoint),
  insecure: false,
- accessKeySecret: {
- name: mlpipeline-minio-artifact,
- key: accesskey
- },
- secretKeySecret: {
- name: mlpipeline-minio-artifact,
- key: secretkey
- }
+ useSDKCreds: true,
  },
  archiveLogs: true
 }

awsconfigs/apps/pipeline/s3/disable-default-secret.yaml

@@ -1,5 +1,11 @@
 apiVersion: v1
 kind: Secret
 metadata:
+ labels:
+ application-crd-id: kubeflow-pipelines
  name: mlpipeline-minio-artifact
-$patch: delete
+ namespace: kubeflow
+stringData:
+ accesskey: ""
+ secretkey: ""
+$patch: replace

awsconfigs/apps/pipeline/s3/disable-minio-server-resources.yaml

@@ -17,4 +17,4 @@ apiVersion: v1
 kind: Service
 metadata:
  name: minio-service
- namespace: kubeflow
+ namespace: kubeflow

awsconfigs/apps/pipeline/s3/kustomization.yaml

@@ -25,6 +25,7 @@ patchesStrategicMerge:
 - disable-minio-server-resources.yaml
 - deployment_patch.yaml
 - disable-default-secret.yaml
+- service-account.yaml
 - aws-configuration-patch.yaml
 # Identifier for application manager to apply ownerReference.
 # The ownerReference ensures the resources get garbage collected

awsconfigs/apps/pipeline/s3/service-account.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: ml-pipeline
+ namespace: kubeflow
+ annotations:
+ eks.amazonaws.com/role-arn: '{{ .Values.irsa.roleName }}'

tests/e2e/utils/kubeflow_installation.py

@@ -17,7 +17,16 @@
 INSTALLATION_CONFIG_RDS_S3 = "./resources/installation_config/rds-s3.yaml"
 INSTALLATION_CONFIG_RDS_ONLY = "./resources/installation_config/rds-only.yaml"
 INSTALLATION_CONFIG_S3_ONLY = "./resources/installation_config/s3-only.yaml"
-INSTALLATION_CONFIG_COGNITO_RDS_S3 = "./resources/installation_config/cognito-rds-s3.yaml"
+INSTALLATION_CONFIG_COGNITO_RDS_S3 = (
+ "./resources/installation_config/cognito-rds-s3.yaml"
+)
+INSTALLATION_CONFIG_S3_ONLY_STATIC = (
+ "./resources/installation_config/s3-only-static.yaml"
+)
+INSTALLATION_CONFIG_RDS_S3_STATIC = "./resources/installation_config/rds-s3-static.yaml"
+INSTALLATION_CONFIG_COGNITO_RDS_S3_STATIC = (
+ "./resources/installation_config/cognito-rds-s3-static.yaml"
+)
 
 
 Install_Sequence = [
@@ -55,7 +64,11 @@
 
 
 def install_kubeflow(
- installation_option, deployment_option, cluster_name, aws_telemetry=True
+ installation_option,
+ deployment_option,
+ cluster_name,
+ credentials_option,
+ aws_telemetry=True,
 ):
  print(cluster_name)
  if deployment_option == "vanilla":
@@ -70,6 +83,12 @@ def install_kubeflow(
  installation_config = load_yaml_file(INSTALLATION_CONFIG_S3_ONLY)
  elif deployment_option == "cognito-rds-s3":
  installation_config = load_yaml_file(INSTALLATION_CONFIG_COGNITO_RDS_S3)
+ elif deployment_option == "rds-s3" and credentials_option == "static":
+ installation_config = load_yaml_file(INSTALLATION_CONFIG_RDS_S3_STATIC)
+ elif deployment_option == "s3" and credentials_option == "static":
+ installation_config = load_yaml_file(INSTALLATION_CONFIG_S3_ONLY_STATIC)
+ elif deployment_option == "cognito-rds-s3" and credentials_option == "static":
+ installation_config = load_yaml_file(INSTALLATION_CONFIG_COGNITO_RDS_S3_STATIC)
 
  print_banner(
  f"Installing kubeflow {deployment_option} deployment with {installation_option}"
@@ -80,7 +99,8 @@ def install_kubeflow(
  installation_option,
  component,
  installation_config,
- cluster_name
+ cluster_name,
+ credentials_option,
  )
 
  if aws_telemetry == True:
@@ -89,6 +109,7 @@ def install_kubeflow(
  "aws-telemetry",
  installation_config,
  cluster_name,
+ credentials_option,
  )
 
 
@@ -97,6 +118,7 @@ def install_component(
  component_name,
  installation_config,
  cluster_name,
+ credentials_option,
  crd_established=True,
 ):
  # component not applicable for deployment option
@@ -105,26 +127,52 @@ def install_component(
  else:
  print(f"==========Installing {component_name}==========")
  # remote repo
- if "repo"in installation_config[component_name]["installation_options"][installation_option]:
+ if (
+ "repo"
+ in installation_config[component_name]["installation_options"][
+ installation_option
+ ]
+ ):
  install_remote_component(component_name, cluster_name)
  # local repo
  else:
- installation_paths = installation_config[component_name]["installation_options"][installation_option]["paths"]
+ installation_paths = installation_config[component_name][
+ "installation_options"
+ ][installation_option]["paths"]
  # helm
  if installation_option == "helm":
  ##deal with namespace already exist issue for rds-s3 auto set-up script
  if component_name == "kubeflow-namespace":
- for kustomize_path in installation_config[component_name]["installation_options"]["kustomize"]["paths"]:
+ for kustomize_path in installation_config[component_name][
+ "installation_options"
+ ]["kustomize"]["paths"]:
  apply_kustomize(kustomize_path)
  else:
+ if component_name == "kubeflow-pipelines":
+ configure_kubeflow_pipelines(
+ component_name,
+ installation_paths,
+ installation_option,
+ credentials_option,
+ )
  install_helm(component_name, installation_paths)
  # kustomize
  else:
  # crd required to established for installation
- if "validations" in installation_config[component_name] and "crds" in installation_config[component_name]["validations"]:
+ if (
+ "validations" in installation_config[component_name]
+ and "crds" in installation_config[component_name]["validations"]
+ ):
  print("need to wait for crds....")
  crds = installation_config[component_name]["validations"]["crds"]
  crd_established = False
+ if component_name == "kubeflow-pipelines":
+ configure_kubeflow_pipelines(
+ component_name,
+ installation_paths,
+ installation_option,
+ credentials_option,
+ )
  for kustomize_path in installation_paths:
  if not crd_established:
  apply_kustomize(kustomize_path, crds)
@@ -172,7 +220,7 @@ def install_certmanager():
  f"helm upgrade --install cert-manager jetstack/cert-manager \
  --namespace cert-manager \
  --create-namespace \
- --version v1.5.0 \
+ --version v1.10.1 \
  --set installCRDs=true"
  )
 
@@ -230,6 +278,27 @@ def install_ack_controller():
  )
 
 
+def configure_kubeflow_pipelines(
+ component_name, installation_paths, installation_option, credentials_option
+):
+ cfg = load_yaml_file(file_path="./utils/pipelines/config.yaml")
+ IAM_ROLE_ARN_FOR_IRSA = cfg["pipeline_oidc_role"]
+ if installation_option == "kustomize":
+ CHART_EXPORT_PATH = "../../apps/pipeline/s3/service-account.yaml"
+ exec_shell(
+ f'yq e \'.metadata.annotations."eks.amazonaws.com/role-arn"="{IAM_ROLE_ARN_FOR_IRSA}"\' '
+ + f"-i {CHART_EXPORT_PATH}"
+ )
+
+ else:
+ IAM_ROLE_ARN_FOR_IRSA = cfg["pipeline_oidc_role"]
+ CHART_EXPORT_PATH = f"{installation_paths}/templates/ServiceAccount/ml-pipeline-kubeflow-ServiceAccount.yaml"
+ exec_shell(
+ f'yq e \'.metadata.annotations."eks.amazonaws.com/role-arn"="{IAM_ROLE_ARN_FOR_IRSA}"\' '
+ + f"-i {CHART_EXPORT_PATH}"
+ )
+
+
 if __name__ == "__main__":
  parser = argparse.ArgumentParser()
  INSTALLATION_OPTION_DEFAULT = "kustomize"
@@ -272,12 +341,22 @@ def install_ack_controller():
  help=f"EKS cluster Name",
  required=True,
  )
+ CREDENTIAL_OPTION_DEFAULT = "irsa"
+ parser.add_argument(
+ "--credentials_option",
+ type=str,
+ default=CREDENTIAL_OPTION_DEFAULT,
+ choices=["irsa", "static"],
+ help=f"Kubeflow default credential option default is set to irsa",
+ required=False,
+ )
 
  args, _ = parser.parse_known_args()
 
  install_kubeflow(
  args.installation_option,
  args.deployment_option,
  args.cluster_name,
+ args.credentials_option,
  args.aws_telemetry,
  )

tests/e2e/utils/pipelines/cleanup_pipeline_irsa.py

@@ -0,0 +1,54 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import json
+import boto3
+
+from e2e.utils.utils import (
+ load_json_file,
+ get_iam_client,
+ get_eks_client,
+)
+from e2e.fixtures.cluster import (
+ associate_iam_oidc_provider,
+)
+from e2e.utils.aws.iam import IAMPolicy
+from e2e.utils.pipelines import common
+from e2e.utils.config import configure_env_file
+from e2e.utils.utils import print_banner, load_yaml_file
+
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_account_id():
+ return boto3.client("sts").get_caller_identity().get("Account")
+
+def delete_iam_role(role_name, region):
+ iam_client = get_iam_client(region=region)
+ try:
+ iam_client.detach_role_policy(
+ RoleName=role_name, PolicyArn="arn:aws:iam::aws:policy/AmazonS3FullAccess"
+ )
+ except:
+ logger.log("Failed to detach role policy, it may not exist anymore.")
+
+ iam_client.delete_role(RoleName=role_name)
+ print(f"Deleted IAM Role : {role_name}")
+
+
+if __name__ == "__main__":
+ print_banner("Reading Config")
+ config_file_path = common.CONFIG_FILE_PATH
+ cfg = load_yaml_file(file_path=config_file_path)
+ cluster_region = cfg["cluster"]["region"]
+ cluster_name = cfg["cluster"]["name"]
+
+ print_banner("Deleting all resources created for Pipeline IRSA")
+ role_name = f"{common.PIPELINE_OIDC_ROLE_NAME_PREFIX}-{cluster_name}"
+ delete_iam_role(role_name, cluster_region)
+
+ print_banner("CLEANUP SUCCESSFUL")
+

tests/e2e/utils/pipelines/common.py

@@ -0,0 +1,12 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import yaml
+import logging
+
+logger = logging.getLogger(__name__)
+
+CONFIG_FILE_PATH = "./utils/pipelines/config.yaml"
+OUTPUT_FILE_PATH = "../../awsconfigs/apps/pipeline/s3/service-account.yaml"
+
+PIPELINE_OIDC_ROLE_NAME_PREFIX = "kf-pipeline-role"

tests/e2e/utils/pipelines/config.yaml

@@ -0,0 +1,4 @@
+cluster:
+ name: 
+ region: 
+pipeline_oidc_role: