Merge branch 'main' into update-notebooks-1.6-rebased

.github/workflows/build_push_website_folder.yml

@@ -3,6 +3,8 @@ name: hugo-build-push-folder
 on:
   # Trigger the workflow on git push to a branch where this workflow file exists 
   push:
+    paths:
+      - website/**
     branches:
       - main
       - 'release-v[0-9].[0-9]+.[0-9]+-aws-b[0-9].[0-9]+.[0-9]+'

.github/workflows/terraform-cognito-test.yaml

@@ -5,6 +5,7 @@ on:
     paths:
       - deployments/cognito/terraform/**
       - iaac/terraform/aws-infra/cognito/**
+      - iaac/terraform/aws-infra/subdomain/**
       # TODO: Add relevant helm chart path
     branches:
       - main
@@ -25,31 +26,44 @@ jobs:
       # needed to checkout repository
       contents: read
     env:
-      TF_VAR_cognito_user_pool_name: testpool-${{ github.run_id }}-${{ github.run_attempt }}
-      TF_VAR_aws_route53_zone_id: {{ secrets.HOSTED_ZONE_ID }}
+      TF_VAR_cognito_user_pool_name: ${{ github.run_id }}-${{ github.run_attempt }}-testpool
+      TF_VAR_aws_route53_root_zone_name: ${{ secrets.ROOT_HOSTED_ZONE_NAME }}
+      TF_VAR_aws_route53_subdomain_zone_name: ${{ github.run_id }}-${{ github.run_attempt }}.${{ secrets.ROOT_HOSTED_ZONE_NAME }}
     steps:
     - name: Checkout
       uses: actions/checkout@v3
 
     - name: Configure AWS credentials from Test account
       uses: aws-actions/configure-aws-credentials@v1
       with:
-        role-to-assume: {{ secrets.PR_BUILD_ROLE }}
+        role-to-assume: ${{ secrets.PR_BUILD_ROLE }}
         role-session-name: prrolesession-${{ github.run_id }}-${{ github.run_attempt }}
-        aws-region: {{ secrets.AWS_REGION }}
+        aws-region: ${{ secrets.AWS_REGION }}
 
     - name: Install Terraform
       uses: hashicorp/setup-terraform@v2
 
-    - name: Plan and apply terraform
+    - name: Create subdomain
+      run: |
+        cd iaac/terraform/aws-infra/subdomain
+        terraform init
+        terraform plan
+        terraform apply -auto-approve
+        terraform state list
+    - name: Create userpool
       run: |
         cd iaac/terraform/aws-infra/cognito
         terraform init
         terraform plan
         terraform apply -auto-approve
         terraform state list
-    - name: Clean up terraform
+    - name: Clean up terraform cognito
       if: success() || failure()
       run: |
         cd iaac/terraform/aws-infra/cognito
+        terraform destroy -auto-approve
+    - name: Clean up terraform subdomain
+      if: success() || failure()
+      run: |
+        cd iaac/terraform/aws-infra/subdomain
         terraform destroy -auto-approve

Makefile

@@ -0,0 +1,103 @@
+SHELL := /bin/bash # Use bash syntax
+
+install-awscli:
+	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+	unzip -o -q awscliv2.zip
+	sudo ./aws/install --update
+	rm -r ./aws
+	rm -r awscliv2.zip
+	aws --version
+
+install-eksctl:
+	$(eval EKSCTL_VERSION:=v0.111.0)
+	curl --silent --location "https://github.com/weaveworks/eksctl/releases/download/$(EKSCTL_VERSION)/eksctl_Linux_amd64.tar.gz" | tar xz -C /tmp
+	sudo mv /tmp/eksctl /usr/local/bin
+	eksctl version
+
+install-kubectl:
+	$(eval KUBECTL_VERSION:=v1.25.0)
+	curl -LO "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/linux/amd64/kubectl"
+	sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+	rm kubectl
+	kubectl version --client
+
+install-kustomize:
+	$(eval KUSTOMIZE_VERSION:=3.2.0)
+	wget https://github.com/kubernetes-sigs/kustomize/releases/download/v$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_linux_amd64
+	chmod +x kustomize_$(KUSTOMIZE_VERSION)_linux_amd64
+	sudo mv kustomize_$(KUSTOMIZE_VERSION)_linux_amd64 /usr/local/bin/kustomize
+	kustomize version
+
+install-yq:
+	$(eval YQ_VERSION:=v4.26.1)
+	wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64.tar.gz -O - | tar xz 
+	sudo mv yq_linux_amd64 /usr/bin/yq
+	rm install-man-page.sh
+	rm yq.1
+	yq --version
+
+install-jq:
+	$(eval JQ_VERSION:=1.5+dfsg-2)
+	sudo apt-get install jq=$(JQ_VERSION) -y
+
+install-terraform:
+	$(eval TERRAFORM_VERSION:=1.2.7)
+	curl "https://releases.hashicorp.com/terraform/$(TERRAFORM_VERSION)/terraform_$(TERRAFORM_VERSION)_linux_amd64.zip" -o "terraform.zip"
+	unzip -o -q terraform.zip
+	sudo install -o root -g root -m 0755 terraform /usr/local/bin/terraform
+	rm terraform.zip
+	rm terraform
+	terraform --version
+
+install-helm: 
+	curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+	helm version
+
+install-python:
+	sudo apt install -q python3.8 -y 
+	sudo apt install -q python3-pip -y
+
+install-python-packages:
+	python3.8 -m pip install -r tests/e2e/requirements.txt
+
+install-all-prerequisites: install-awscli install-eksctl install-kubectl install-kustomize install-yq install-jq install-terraform install-helm install-python install-python-packages
+
+verify-cluster-variables:
+	test $(CLUSTER_NAME) || (echo Please export CLUSTER_NAME variable ; exit 1)
+	test $(CLUSTER_REGION) || (echo Please export CLUSTER_REGION variable ; exit 1)
+
+create-eks-cluster: verify-cluster-variables
+	eksctl create cluster \
+	--name $(CLUSTER_NAME) \
+	--version 1.23 \
+	--region $(CLUSTER_REGION) \
+	--nodegroup-name linux-nodes \
+	--node-type m5.xlarge \
+	--nodes 5 \
+	--nodes-min 5 \
+	--nodes-max 10 \
+	--managed \
+	--with-oidc
+
+connect-to-eks-cluster: verify-cluster-variables
+	aws eks update-kubeconfig --name $(CLUSTER_NAME) --region $(CLUSTER_REGION)
+
+port-forward:
+	$(eval IP_ADDRESS:=127.0.0.1)
+	$(eval PORT:=8080)
+	kubectl port-forward svc/istio-ingressgateway --address $(IP_ADDRESS) -n istio-system $(PORT):80
+
+bootstrap-ack: verify-cluster-variables connect-to-eks-cluster
+	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
+	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
+	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/setup_sm_controller_req.py
+
+cleanup-ack-req: verify-cluster-variables
+	yq e '.cluster.name=env(CLUSTER_NAME)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
+	yq e '.cluster.region=env(CLUSTER_REGION)' -i tests/e2e/utils/ack_sm_controller_bootstrap/config.yaml
+	cd tests/e2e && PYTHONPATH=.. python3.8 utils/ack_sm_controller_bootstrap/cleanup_sm_controller_req.py
+
+
+deploy-kf-vanilla: bootstrap-ack
+	while ! kustomize build deployments/vanilla | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 30; done
+

awsconfigs/infra_configs/iam_ack_oidc_sm_studio_policy.json

@@ -0,0 +1,17 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Sid": "SagemakerStudioAccess",
+      "Effect": "Allow",
+      "Action": [
+        "sagemaker:*"
+      ],
+      "Resource": [
+        "arn:aws:sagemaker:*:*:domain/*",
+        "arn:aws:sagemaker:*:*:user-profile/*",
+        "arn:aws:sagemaker:*:*:app/*"
+      ]
+    }
+  ]
+}

charts/common/ingress/cognito/templates/ConfigMap/istio-ingress-cognito-parameters-istio-system-ConfigMap.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 data:
-  CognitoAppClientId: '{{ .Values.cognito.appClientId }}'
-  CognitoUserPoolArn: '{{ .Values.cognito.UserPoolArn }}'
-  CognitoUserPoolDomain: '{{ .Values.cognito.UserPoolDomain }}'
+  CognitoAppClientId: '{{ .Values.alb.cognito.appClientId }}'
+  CognitoUserPoolArn: '{{ .Values.alb.cognito.UserPoolArn }}'
+  CognitoUserPoolDomain: '{{ .Values.alb.cognito.UserPoolDomain }}'
   certArn: '{{ .Values.alb.certArn }}'
 kind: ConfigMap
 metadata:

charts/common/ingress/cognito/templates/ConfigMap/istio-ingress-parameters-istio-system-ConfigMap.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 data:
-  loadBalancerScheme: {{ .Values.alb.schema }}
+  loadBalancerScheme: {{ .Values.alb.scheme }}
 kind: ConfigMap
 metadata:
   labels:

charts/common/ingress/cognito/templates/Ingress/istio-ingress-istio-system-Ingress.yaml

@@ -2,9 +2,9 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
-    alb.ingress.kubernetes.io/auth-idp-cognito: '{"UserPoolArn":"{{ .Values.cognito.UserPoolArn
-      }}","UserPoolClientId":"{{ .Values.cognito.appClientId }}", "UserPoolDomain":"{{
-      .Values.cognito.UserPoolDomain }}"}'
+    alb.ingress.kubernetes.io/auth-idp-cognito: '{"UserPoolArn":"{{ .Values.alb.cognito.UserPoolArn
+      }}","UserPoolClientId":"{{ .Values.alb.cognito.appClientId }}", "UserPoolDomain":"{{
+      .Values.alb.cognito.UserPoolDomain }}"}'
     alb.ingress.kubernetes.io/auth-type: cognito
     alb.ingress.kubernetes.io/certificate-arn: '{{ .Values.alb.certArn }}'
     alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]'

iaac/terraform/aws-infra/cognito/app_client.tf

@@ -0,0 +1,13 @@
+resource "aws_cognito_user_pool_client" "platform" {
+  name            = "kubeflow"
+  user_pool_id    = aws_cognito_user_pool.platform.id
+  generate_secret = true
+
+  callback_urls = ["https://kubeflow.${data.aws_route53_zone.platform.name}/oauth2/idpresponse"]
+  logout_urls   = ["https://kubeflow.${data.aws_route53_zone.platform.name}"]
+
+  allowed_oauth_flows_user_pool_client = true
+  allowed_oauth_flows                  = ["code"]
+  allowed_oauth_scopes                 = ["email", "openid", "profile", "aws.cognito.signin.user.admin"]
+  supported_identity_providers         = ["COGNITO"]
+}

iaac/terraform/aws-infra/cognito/custom_domain.tf

@@ -0,0 +1,86 @@
+# Add a custom domain to the user pool
+data "aws_route53_zone" "platform" {
+  name = var.aws_route53_subdomain_zone_name
+}
+
+# In order to use a custom domain, its root(i.e. platform.example.com) must have an valid A type record
+resource "aws_route53_record" "pre_cognito_domain_a_record" {
+  allow_overwrite = true
+  zone_id = data.aws_route53_zone.platform.zone_id
+  name    = data.aws_route53_zone.platform.name
+  type    = "A"
+  ttl     = "300"
+  # This record will be updated after ALB creation
+  records = ["127.0.0.1"]
+
+  lifecycle {
+    ignore_changes = [records, alias, ttl]
+  }
+}
+
+# Cognito requires a certificate in N.Virginia in order to have a custom domain for a user pool
+# https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-pools-add-custom-domain.html
+provider "aws" {
+  region = "us-east-1"
+  alias = "virginia"
+}
+
+resource "aws_acm_certificate" "cognito_domain_cert" {
+  domain_name       = "*.${data.aws_route53_zone.platform.name}"
+  validation_method = "DNS"
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  provider = aws.virginia
+}
+
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/acm_certificate_validation
+resource "aws_route53_record" "certificate_validation_cognito_domain" {
+  for_each = {
+    for dvo in aws_acm_certificate.cognito_domain_cert.domain_validation_options : dvo.domain_name => {
+      name   = dvo.resource_record_name
+      record = dvo.resource_record_value
+      type   = dvo.resource_record_type
+    }
+  }
+
+  allow_overwrite = true
+  name            = each.value.name
+  records         = [each.value.record]
+  ttl             = 60
+  type            = each.value.type
+  zone_id         = data.aws_route53_zone.platform.zone_id
+}
+
+resource "aws_acm_certificate_validation" "cognito_domain" {
+  provider                = aws.virginia
+  certificate_arn         = aws_acm_certificate.cognito_domain_cert.arn
+  validation_record_fqdns = [for record in aws_route53_record.certificate_validation_cognito_domain : record.fqdn]
+}
+
+resource "aws_cognito_user_pool_domain" "platform" {
+  domain          = "auth.${data.aws_route53_zone.platform.name}"
+  certificate_arn = aws_acm_certificate.cognito_domain_cert.arn
+  user_pool_id    = aws_cognito_user_pool.platform.id
+
+  depends_on = [
+    aws_route53_record.pre_cognito_domain_a_record
+  ]
+}
+
+resource "aws_route53_record" "auth_cognito_domain_record" {
+  allow_overwrite = true
+  name    = aws_cognito_user_pool_domain.platform.domain
+  type    = "A"
+  zone_id = data.aws_route53_zone.platform.zone_id
+  alias {
+    evaluate_target_health = false
+    name                   = aws_cognito_user_pool_domain.platform.cloudfront_distribution_arn
+    # For creating an alias record to other AWS resource, route53 needs hosted zone id and DNS name.
+    # Since CloudFront is a global service, there is only one hosted zone id
+    # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-route53-aliastarget.html
+    zone_id = "Z2FDTNDATAQYW2"
+  }
+}

iaac/terraform/aws-infra/cognito/outputs.tf

@@ -0,0 +1,19 @@
+output "user_pool_arn" {
+  description = "Cognito User Pool ARN"
+  value       = aws_cognito_user_pool.platform.arn
+}
+
+output "app_client_id" {
+  description = "Cognito App client Id"
+  value       = aws_cognito_user_pool_client.platform.id
+}
+
+output "user_pool_domain" {
+  description = "Cognito User Pool Domain"
+  value       = aws_cognito_user_pool_domain.platform.domain
+}
+
+output "logout_url" {
+  description = "Logout URL"
+  value       = "${aws_cognito_user_pool_domain.platform.domain}/logout?client_id=${aws_cognito_user_pool_client.platform.id}&logout_uri=https://kubeflow.${data.aws_route53_zone.platform.name}"
+}

iaac/terraform/aws-infra/cognito/userpool.tf

@@ -0,0 +1,20 @@
+resource "aws_cognito_user_pool" "platform" {
+  name                = var.cognito_user_pool_name
+
+  schema {
+    name                     = "email"
+    attribute_data_type      = "String"
+    mutable                  = true
+    required                 = true
+    string_attribute_constraints {
+      min_length = "1"
+      max_length = "2048"
+    }
+  }
+  admin_create_user_config {
+    allow_admin_create_user_only = true
+  }
+
+  auto_verified_attributes = ["email"]
+
+}

iaac/terraform/aws-infra/cognito/variables.tf

@@ -0,0 +1,9 @@
+variable "cognito_user_pool_name" {
+  description = "Cognito User Pool name"
+  type        = string
+}
+
+variable "aws_route53_subdomain_zone_name" {
+  description = "SUBDOMAIN Route 53 hosted zone name(e.g. platform.example.com) which will be used for Kubeflow Platform. Must match exactly one zone"
+  type        = string
+}

iaac/terraform/aws-infra/cognito/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.1.0"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.4"
+    }
+  }
+}