feat: support additional bootstrap resource

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 All notable changes to this module will be documented in this file.
 
+## [1.1.0] - 2023-08-12
+
+### Added
+
+- Support additional worker node policies with variable `additional_worker_polices`
+- Support bootstrap to config map role, clusterrole, clusterrole binding
+  - variables: `additional_cluster_role`, `additional_cluster_role_binding`, `additional_map_roles` (sub-module, main)
+- Option to select kubectl version in worker node
+
+### Changed
+
+- Fix bootstrap script failed `modules/bootstrap/templates/scripts.sh`
+- Constraint on tls provider version `version = ">= 3.2.1"`
+
 ## [1.0.17] - 2022-04-17
 
 Here we would have the update steps for 1.0.17 for people to follow.

bootstrap.tf

@@ -1,20 +1,23 @@
 module "bootstrap" {
-  count                    = var.is_create_bootstrap ? 1 : 0
-  source                   = "./modules/bootstrap"
-  subnet_id                = var.subnets_ids[0]
-  cluster_name             = aws_eks_cluster.this.name
-  ami                      = var.bootstrap_ami
-  aws_account              = var.aws_account
-  karpenter_node_role_arns = var.karpenter_node_role_arns
-  admin_role_arns          = var.admin_role_arns
-  admin_iam_arns           = var.admin_iam_arns
-  dev_role_arns            = var.dev_role_arns
-  readonly_role_arns       = var.readonly_role_arns
-  node_group_role_arn      = aws_iam_role.node_group_role.arn
-  vpc_id                   = var.vpc_id
-  is_config_aws_auth       = var.is_config_aws_auth
-  prefix                   = var.prefix
-  environment              = var.environment
-  kms_key_id               = var.bootstrap_kms_key_id
-  tags                     = var.tags
+  count                           = var.is_create_bootstrap ? 1 : 0
+  source                          = "./modules/bootstrap"
+  subnet_id                       = var.subnets_ids[0]
+  cluster_name                    = aws_eks_cluster.this.name
+  ami                             = var.bootstrap_ami
+  aws_account                     = var.aws_account
+  karpenter_node_role_arns        = var.karpenter_node_role_arns
+  admin_role_arns                 = var.admin_role_arns
+  admin_iam_arns                  = var.admin_iam_arns
+  dev_role_arns                   = var.dev_role_arns
+  readonly_role_arns              = var.readonly_role_arns
+  additional_map_roles            = var.additional_map_roles
+  additional_cluster_role         = var.additional_cluster_role
+  additional_cluster_role_binding = var.additional_cluster_role_binding
+  node_group_role_arn             = aws_iam_role.node_group_role.arn
+  vpc_id                          = var.vpc_id
+  is_config_aws_auth              = var.is_config_aws_auth
+  prefix                          = var.prefix
+  environment                     = var.environment
+  kms_key_id                      = var.bootstrap_kms_key_id
+  tags                            = var.tags
 }

iam.tf

@@ -79,16 +79,19 @@ resource "aws_iam_role_policy_attachment" "amazon_ec2_ssm" {
 
 # Additional policies
 data "aws_iam_policy_document" "combined_policy" {
+  count                   = length(var.additional_worker_polices) > 0 ? 1 : 0
   source_policy_documents = var.additional_worker_polices
 }
 
 resource "aws_iam_policy" "combined_policy" {
+  count       = length(var.additional_worker_polices) > 0 ? 1 : 0
   name        = "${local.prefix}-node-group-additional-policy"
   description = "${local.prefix} custom policy"
-  policy      = data.aws_iam_policy_document.combined_policy.json
+  policy      = data.aws_iam_policy_document.combined_policy[0].json
 }
 
 resource "aws_iam_role_policy_attachment" "amazon_eks_worker_node_combine_policy" {
-  policy_arn = aws_iam_policy.combined_policy.arn
+  count      = length(var.additional_worker_polices) > 0 ? 1 : 0
+  policy_arn = aws_iam_policy.combined_policy[0].arn
   role       = aws_iam_role.node_group_role.name
 }

modules/bootstrap/data.tf

@@ -17,8 +17,8 @@ data "aws_ami" "ubuntu" {
 data "template_file" "eks_manifest" {
   template = file("${path.module}/templates/eks-manifest-file.yml")
   vars = {
-    node_group_role_arn      = var.node_group_role_arn
-    karpenter_node_role_arns = <<EOT
+    node_group_role_arn             = var.node_group_role_arn
+    karpenter_node_role_arns        = <<EOT
 %{for i, arn in var.karpenter_node_role_arns~}
     - groups:
       - system:bootstrappers
@@ -27,36 +27,43 @@ data "template_file" "eks_manifest" {
       username: system:node:{{EC2PrivateDNSName}}
 %{endfor~}
 EOT
-    admin_role_arns          = <<EOT
+    admin_role_arns                 = <<EOT
 %{for i, arn in var.admin_role_arns~}
     - groups: []
       rolearn: ${arn}
       username: eks-admin-${i}
 %{endfor~}
 EOT
-    dev_role_arns            = <<EOT
+    dev_role_arns                   = <<EOT
 %{for i, arn in var.dev_role_arns~}
     - groups: []
       rolearn: ${arn}
       username: eks-developer-${i}
 %{endfor~}
 EOT
-    readonly_role_arns       = <<EOT
+    readonly_role_arns              = <<EOT
 %{for i, arn in var.readonly_role_arns~}
     - groups: []
       rolearn: ${arn}
       username: eks-readonly-${i}
 %{endfor~}
 EOT
-    admin_iam_arns           = <<EOT
+    admin_iam_arns                  = <<EOT
 %{for i, arn in var.admin_iam_arns~}
     - userarn: ${arn}
       username: eks-iam-admin-${i}
       groups:
         - system:masters
 %{endfor~}
 EOT
-    admin_role_binding       = <<EOT
+    additional_map_roles            = <<EOT
+%{for map_role in var.additional_map_roles~}
+    - rolearn: ${map_role.role_arn}
+      username: ${map_role.username}
+      groups: []
+%{endfor~}
+EOT
+    admin_role_binding              = <<EOT
 %{for i, arn in var.admin_role_arns~}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -73,7 +80,7 @@ roleRef:
   name: cluster-admin
 %{endfor~}
 EOT
-    dev_role_binding         = <<EOT
+    dev_role_binding                = <<EOT
 %{for i, arn in var.dev_role_arns~}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -90,7 +97,7 @@ roleRef:
   name: edit
 %{endfor~}
 EOT
-    readonly_role_binding    = <<EOT
+    readonly_role_binding           = <<EOT
 %{for i, arn in var.readonly_role_arns~}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -106,6 +113,40 @@ roleRef:
   kind: ClusterRole
   name: view
 %{endfor~}
+EOT
+    additional_cluster_role         = <<EOT
+%{for cluster_role in var.additional_cluster_role~}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: ${cluster_role.name}
+rules:
+  %{for rule in cluster_role.rules}
+  - apiGroups: ${jsonencode(rule.apiGroups)}
+    resources: ${jsonencode(rule.resources)}
+    verbs: ${jsonencode(rule.verbs)}
+  %{~endfor~}
+%{endfor~}
+EOT
+    additional_cluster_role_binding = <<EOT
+%{for cluster_role_binding in var.additional_cluster_role_binding~}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ${cluster_role_binding.name}
+subjects:
+  %{for subject in cluster_role_binding.subjects}
+  - kind: ${jsonencode(subject.kind)}
+    name: ${jsonencode(subject.name)}
+    apiGroup: ${jsonencode(subject.apiGroup)}
+  %{~endfor~}
+roleRef:
+  apiGroup: ${cluster_role_binding.roleRef.apiGroup}
+  kind: ${cluster_role_binding.roleRef.kind}
+  name: ${cluster_role_binding.roleRef.name}
+%{endfor~}
 EOT
   }
 }
@@ -115,6 +156,7 @@ data "template_file" "scripts" {
   vars = {
     region                   = var.aws_account.region
     cluster_name             = var.cluster_name
+    kubectl_version          = var.kubectl_version
     eks_bootstrap_secret_arn = aws_secretsmanager_secret.terraform_key.arn
     is_config_aws_auth       = var.is_config_aws_auth
     eks_manifest_file        = data.template_file.eks_manifest.rendered

modules/bootstrap/templates/eks-manifest-file.yml

@@ -15,8 +15,11 @@ ${karpenter_node_role_arns}
 ${admin_role_arns}
 ${dev_role_arns}
 ${readonly_role_arns}
+${additional_map_roles}
   mapUsers: |
 ${admin_iam_arns}
 ${admin_role_binding}
 ${dev_role_binding}
 ${readonly_role_binding}
+${additional_cluster_role}
+${additional_cluster_role_binding}

modules/bootstrap/templates/scripts.bak.sh

@@ -0,0 +1,43 @@
+#!/bin/bash -e
+# install dependencies packages
+echo "starting cloud init script . . ."
+sudo su
+sudo apt-get update
+sudo apt install unzip
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+sudo ./aws/install
+sudo apt install jq -y
+
+# kubectl
+echo "install kubectl . . ."
+sudo apt-get install -y apt-transport-https ca-certificates curl
+sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
+echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+sudo apt-get update
+#sudo apt-get install -y kubectl=1.23.4-00
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+# create file
+echo "create script folder . . ."
+sudo mkdir -p /opt/scripts
+
+# configure aws
+echo "config aws account . . ."
+aws configure set region ${region}
+credential=$(aws secretsmanager get-secret-value  --secret-id ${eks_bootstrap_secret_arn} --query SecretString --output text)
+aws_access_key_id=$(echo $credential | jq '.aws_access_key_id' | tr -d '"')
+aws_secret_access_key=$(echo $credential | jq '.aws_secret_access_key' | tr -d '"')
+export AWS_ACCESS_KEY_ID=$aws_access_key_id
+export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
+aws eks update-kubeconfig --region ${region} --name ${cluster_name}
+%{ if is_config_aws_auth }
+echo "config aws-auth . . ."
+sudo touch /opt/scripts/eks-manifest-file.yml
+sudo chmod 777 /opt/scripts/eks-manifest-file.yml
+sudo echo '${eks_manifest_file}' > /opt/scripts/eks-manifest-file.yml
+sudo AWS_ACCESS_KEY_ID=$aws_access_key_id AWS_SECRET_ACCESS_KEY=$aws_secret_access_key kubectl apply -f /opt/scripts/eks-manifest-file.yml
+%{ endif }
+kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true
+sudo shutdown -h now

modules/bootstrap/templates/scripts.sh

@@ -1,25 +1,35 @@
 #!/bin/bash -e
 # install dependencies packages
-echo "starting cloud init script . . ."
+echo "/* -------------------------------------------------------------------------- */"
+echo "/*                         starting cloud init script                         */"
+echo "/* -------------------------------------------------------------------------- */"
 sudo su
+curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/kubernetes-archive-keyring.gpg --yes
 sudo apt-get update
-sudo apt install awscli -y
+sudo apt install unzip
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip -o awscliv2.zip
+sudo ./aws/install --update
 sudo apt install jq -y
 
 # kubectl
-echo "install kubectl . . ."
-sudo apt-get install -y apt-transport-https ca-certificates curl
-sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
-echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
-sudo apt-get update
-sudo apt-get install -y kubectl=1.23.4-00
+echo /* -------------------------------------------------------------------------- */
+echo /*                 Install Kubectl version ${kubectl_version}                 */
+echo /* -------------------------------------------------------------------------- */
+curl -LO https://dl.k8s.io/release/${kubectl_version}/bin/linux/amd64/kubectl
+sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+kubectl version --client
 
 # create file
-echo "create script folder . . ."
+echo "/* -------------------------------------------------------------------------- */"
+echo "/*                            create script folder                            */"
+echo "/* -------------------------------------------------------------------------- */"
 sudo mkdir -p /opt/scripts
 
 # configure aws
-echo "config aws account . . ."
+echo "/* -------------------------------------------------------------------------- */"
+echo "/*                             config aws account                             */"
+echo "/* -------------------------------------------------------------------------- */"
 aws configure set region ${region}
 credential=$(aws secretsmanager get-secret-value  --secret-id ${eks_bootstrap_secret_arn} --query SecretString --output text)
 aws_access_key_id=$(echo $credential | jq '.aws_access_key_id' | tr -d '"')
@@ -28,11 +38,16 @@ export AWS_ACCESS_KEY_ID=$aws_access_key_id
 export AWS_SECRET_ACCESS_KEY=$aws_secret_access_key
 aws eks update-kubeconfig --region ${region} --name ${cluster_name}
 %{ if is_config_aws_auth }
-echo "config aws-auth . . ."
+echo "/* -------------------------------------------------------------------------- */"
+echo "/*                               config aws-auth                              */"
+echo "/* -------------------------------------------------------------------------- */"
 sudo touch /opt/scripts/eks-manifest-file.yml
 sudo chmod 777 /opt/scripts/eks-manifest-file.yml
 sudo echo '${eks_manifest_file}' > /opt/scripts/eks-manifest-file.yml
 sudo AWS_ACCESS_KEY_ID=$aws_access_key_id AWS_SECRET_ACCESS_KEY=$aws_secret_access_key kubectl apply -f /opt/scripts/eks-manifest-file.yml
 %{ endif }
-
+echo "/* -------------------------------------------------------------------------- */"
+echo "/*                              set env daemonset                             */"
+echo "/* -------------------------------------------------------------------------- */"
+sudo AWS_ACCESS_KEY_ID=$aws_access_key_id AWS_SECRET_ACCESS_KEY=$aws_secret_access_key kubectl set env daemonset aws-node -n kube-system ENABLE_PREFIX_DELEGATION=true
 sudo shutdown -h now