karpenter 0.32

.pre-commit-config.yaml

@@ -1,18 +1,18 @@
 repos:
   - repo: https://github.com/gruntwork-io/pre-commit
-    rev: v0.1.17
+    rev: v0.1.22
     hooks:
       - id: shellcheck
 
   - repo: https://github.com/tcort/markdown-link-check
-    rev: v3.9.3
+    rev: v3.11.2
     hooks:
       - id: markdown-link-check
         args:
           - "--config=mlc_config.json"
 
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.77.0
+    rev: v1.83.3
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -29,6 +29,8 @@ repos:
         args:
           - --args=--exclude-downloaded-modules
       - id: terraform_checkov
+        args:
+          - "--args=--skip-check CKV_TF_1"
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.2.0
@@ -46,5 +48,5 @@ repos:
 
       # Security
       - id: detect-aws-credentials
-        args: ['--allow-missing-credentials']
+        args: ["--allow-missing-credentials"]
       - id: detect-private-key

.tflint.hcl

@@ -1,6 +1,6 @@
 plugin "aws" {
   enabled = true
-  version = "0.22.1"
+  version = "0.27.0"
   source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 

README.md

@@ -151,10 +151,6 @@ locals {
           timeAdded = timestamp() # required if not terraform plan complains
         }
       ]
-
-      karpenter_instance_types_list = ["m5a.large"]
-      karpenter_capacity_type_list  = ["on-demand"]
-      karpenter_arch_list           = ["amd64"]
     },
   ]
   # Karpenter Nodetemplate Config
@@ -197,8 +193,10 @@ module "karpenter" {
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.4 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.47 |
+| <a name="requirement_helm"></a> [helm](#requirement\_helm) | ~> 2.6 |
+| <a name="requirement_kubectl"></a> [kubectl](#requirement\_kubectl) | 1.14.0 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
 
 ## Providers
@@ -215,6 +213,7 @@ module "karpenter" {
 | <a name="module_ebs_csi_irsa_role"></a> [ebs\_csi\_irsa\_role](#module\_ebs\_csi\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.11.2 |
 | <a name="module_eks"></a> [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.17.0 |
 | <a name="module_fargate_profiles"></a> [fargate\_profiles](#module\_fargate\_profiles) | ./modules/fargate_profile | n/a |
+| <a name="module_karpenter"></a> [karpenter](#module\_karpenter) | ./modules/karpenter | n/a |
 | <a name="module_kms_ebs"></a> [kms\_ebs](#module\_kms\_ebs) | SPHTech-Platform/kms/aws | ~> 0.1.0 |
 | <a name="module_kms_secret"></a> [kms\_secret](#module\_kms\_secret) | SPHTech-Platform/kms/aws | ~> 0.1.0 |
 | <a name="module_node_groups"></a> [node\_groups](#module\_node\_groups) | ./modules/eks_managed_nodes | n/a |
@@ -232,6 +231,7 @@ module "karpenter" {
 | [aws_iam_service_linked_role.autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource |
 | [kubernetes_config_map_v1.amazon_vpc_cni](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1) | resource |
 | [kubernetes_manifest.fargate_node_security_group_policy](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.fargate_node_security_group_policy_for_karpenter](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_arn.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -246,6 +246,7 @@ module "karpenter" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_autoscaling_mode"></a> [autoscaling\_mode](#input\_autoscaling\_mode) | Autoscaling mode: cluster\_autoscaler or karpenter | `string` | `"karpenter"` | no |
 | <a name="input_aws_auth_fargate_profile_pod_execution_role_arns"></a> [aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns](#input\_aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns) | List of Fargate profile pod execution role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no |
 | <a name="input_cluster_additional_security_group_ids"></a> [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no |
 | <a name="input_cluster_addons"></a> [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no |
@@ -265,9 +266,13 @@ module "karpenter" {
 | <a name="input_cluster_version"></a> [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.27"` | no |
 | <a name="input_create_aws_auth_configmap"></a> [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no |
 | <a name="input_create_aws_observability_ns"></a> [create\_aws\_observability\_ns](#input\_create\_aws\_observability\_ns) | Whether to create AWS Observability Namespace. | `bool` | `true` | no |
+| <a name="input_create_aws_observability_ns_for_karpenter"></a> [create\_aws\_observability\_ns\_for\_karpenter](#input\_create\_aws\_observability\_ns\_for\_karpenter) | Create aws-observability namespace flag | `bool` | `false` | no |
 | <a name="input_create_cluster_security_group"></a> [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no |
 | <a name="input_create_cni_ipv6_iam_policy"></a> [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Whether to create CNI IPv6 IAM policy. | `bool` | `false` | no |
+| <a name="input_create_fargate_log_group_for_karpenter"></a> [create\_fargate\_log\_group\_for\_karpenter](#input\_create\_fargate\_log\_group\_for\_karpenter) | value for create\_fargate\_log\_group | `bool` | `false` | no |
 | <a name="input_create_fargate_logger_configmap"></a> [create\_fargate\_logger\_configmap](#input\_create\_fargate\_logger\_configmap) | Whether to create AWS Fargate logger configmap. | `bool` | `true` | no |
+| <a name="input_create_fargate_logger_configmap_for_karpenter"></a> [create\_fargate\_logger\_configmap\_for\_karpenter](#input\_create\_fargate\_logger\_configmap\_for\_karpenter) | create\_fargate\_logger\_configmap flag | `bool` | `false` | no |
+| <a name="input_create_fargate_logging_policy_for_karpenter"></a> [create\_fargate\_logging\_policy\_for\_karpenter](#input\_create\_fargate\_logging\_policy\_for\_karpenter) | value for create\_fargate\_logging\_policy | `bool` | `false` | no |
 | <a name="input_create_node_security_group"></a> [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no |
 | <a name="input_default_group_ami_id"></a> [default\_group\_ami\_id](#input\_default\_group\_ami\_id) | The AMI from which to launch the defualt group instance. If not supplied, EKS will use its own default image | `string` | `""` | no |
 | <a name="input_default_group_instance_types"></a> [default\_group\_instance\_types](#input\_default\_group\_instance\_types) | Instance type for the default node group | `list(string)` | <pre>[<br>  "m5a.xlarge",<br>  "m5.xlarge",<br>  "m5n.xlarge",<br>  "m5zn.xlarge"<br>]</pre> | no |
@@ -287,6 +292,10 @@ module "karpenter" {
 | <a name="input_force_imdsv2"></a> [force\_imdsv2](#input\_force\_imdsv2) | Force IMDSv2 metadata server. | `bool` | `true` | no |
 | <a name="input_force_irsa"></a> [force\_irsa](#input\_force\_irsa) | Force usage of IAM Roles for Service Account | `bool` | `true` | no |
 | <a name="input_iam_role_additional_policies"></a> [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `set(string)` | `[]` | no |
+| <a name="input_karpenter_chart_version"></a> [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"v0.32.1"` | no |
+| <a name="input_karpenter_default_subnet_selector_tags"></a> [karpenter\_default\_subnet\_selector\_tags](#input\_karpenter\_default\_subnet\_selector\_tags) | Subnet selector tags for Karpenter default node class | `map(string)` | <pre>{<br>  "kubernetes.io/role/internal-elb": "1"<br>}</pre> | no |
+| <a name="input_karpenter_nodeclasses"></a> [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | <pre>list(object({<br>    nodeclass_name                         = string<br>    karpenter_subnet_selector_maps         = list(map(any))<br>    karpenter_security_group_selector_maps = list(map(any))<br>    karpenter_ami_selector_maps            = list(map(any))<br>    karpenter_node_role                    = string<br>    karpenter_node_tags_map                = map(string)<br>    karpenter_ami_family                   = string<br>    karpenter_node_user_data               = string<br>    karpenter_node_metadata_options        = map(any)<br>    karpenter_block_device_mapping = list(object({<br>      deviceName = string<br>      ebs = object({<br>        encrypted           = bool<br>        volumeSize          = string<br>        volumeType          = string<br>        kmsKeyID            = optional(string)<br>        deleteOnTermination = bool<br>      })<br>    }))<br>  }))</pre> | `[]` | no |
+| <a name="input_karpenter_nodepools"></a> [karpenter\_nodepools](#input\_karpenter\_nodepools) | List of Provisioner maps | <pre>list(object({<br>    nodepool_name                     = string<br>    nodeclass_name                    = string<br>    karpenter_nodepool_node_labels    = map(string)<br>    karpenter_nodepool_annotations    = map(string)<br>    karpenter_nodepool_node_taints    = list(map(string))<br>    karpenter_nodepool_startup_taints = list(map(string))<br>    karpenter_requirements = list(object({<br>      key      = string<br>      operator = string<br>      values   = list(string)<br>      })<br>    )<br>    karpenter_nodepool_disruption = object({<br>      consolidation_policy = string<br>      consolidate_after    = optional(string)<br>      expire_after         = string<br>    })<br>    karpenter_nodepool_weight = number<br>  }))</pre> | <pre>[<br>  {<br>    "karpenter_nodepool_annotations": {},<br>    "karpenter_nodepool_disruption": {<br>      "consolidation_policy": "WhenUnderutilized",<br>      "expire_after": "168h"<br>    },<br>    "karpenter_nodepool_node_labels": {},<br>    "karpenter_nodepool_node_taints": [],<br>    "karpenter_nodepool_startup_taints": [],<br>    "karpenter_nodepool_weight": 10,<br>    "karpenter_requirements": [<br>      {<br>        "key": "karpenter.k8s.aws/instance-category",<br>        "operator": "In",<br>        "values": [<br>          "m"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-cpu",<br>        "operator": "In",<br>        "values": [<br>          "4"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.k8s.aws/instance-generation",<br>        "operator": "Gt",<br>        "values": [<br>          "5"<br>        ]<br>      },<br>      {<br>        "key": "karpenter.sh/capacity-type",<br>        "operator": "In",<br>        "values": [<br>          "on-demand"<br>        ]<br>      },<br>      {<br>        "key": "kubernetes.io/arch",<br>        "operator": "In",<br>        "values": [<br>          "amd64"<br>        ]<br>      },<br>      {<br>        "key": "kubernetes.io/os",<br>        "operator": "In",<br>        "values": [<br>          "linux"<br>        ]<br>      }<br>    ],<br>    "nodeclass_name": "default",<br>    "nodepool_name": "default"<br>  }<br>]</pre> | no |
 | <a name="input_manage_aws_auth_configmap"></a> [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the contents of the aws-auth configmap | `bool` | `true` | no |
 | <a name="input_node_security_group_additional_rules"></a> [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no |
 | <a name="input_node_security_group_enable_recommended_rules"></a> [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no |
@@ -313,6 +322,7 @@ module "karpenter" {
 | <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | EKS Cluster name created |
 | <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
 | <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version of the EKS Cluster |
+| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Primary Security Group ID of the EKS cluster |
 | <a name="output_cluster_security_group_id"></a> [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security Group ID of the master nodes |
 | <a name="output_cluster_version"></a> [cluster\_version](#output\_cluster\_version) | Version of the EKS Cluster |
 | <a name="output_ebs_kms_key_arn"></a> [ebs\_kms\_key\_arn](#output\_ebs\_kms\_key\_arn) | KMS Key ARN used for EBS encryption |

karpenter.tf

@@ -0,0 +1,102 @@
+locals {
+  # Karpenter Provisioners Config
+  # Use default var
+  karpenter_nodepools = var.karpenter_nodepools
+
+  # Karpenter Nodetemplate Config
+  karpenter_nodeclasses = coalescelist(var.karpenter_nodeclasses, [
+    {
+      nodeclass_name = "default"
+      karpenter_subnet_selector_maps = [{
+        tags = var.karpenter_default_subnet_selector_tags,
+        }
+      ]
+      karpenter_node_role = aws_iam_role.workers.name
+      karpenter_security_group_selector_maps = [{
+        "id" = module.eks.cluster_primary_security_group_id
+        },
+      ]
+      karpenter_node_metadata_options = {
+        httpEndpoint            = "enabled"
+        httpProtocolIPv6        = var.cluster_ip_family != "ipv6" ? "disabled" : "enabled"
+        httpPutResponseHopLimit = 1
+        httpTokens              = "required"
+      }
+      karpenter_ami_selector_maps = []
+      karpenter_node_user_data    = ""
+      karpenter_node_tags_map = {
+        "karpenter.sh/discovery" = module.eks.cluster_name,
+        "eks:cluster-name"       = module.eks.cluster_name,
+      }
+      karpenter_ami_family = "Bottlerocket"
+      karpenter_block_device_mapping = [
+        {
+          #karpenter_root_volume_size
+          "deviceName" = "/dev/xvda"
+          "ebs" = {
+            "encrypted"           = true
+            "volumeSize"          = "5Gi"
+            "volumeType"          = "gp3"
+            "deleteOnTermination" = true
+          }
+          }, {
+          #karpenter_ephemeral_volume_size
+          "deviceName" = "/dev/xvdb",
+          "ebs" = {
+            "encrypted"           = true
+            "volumeSize"          = "50Gi"
+            "volumeType"          = "gp3"
+            "deleteOnTermination" = true
+          }
+        }
+      ]
+    },
+  ])
+}
+
+module "karpenter" {
+  source = "./modules/karpenter"
+
+  count = var.autoscaling_mode == "karpenter" ? 1 : 0
+
+  karpenter_chart_version = var.karpenter_chart_version
+
+  cluster_name        = var.cluster_name
+  cluster_endpoint    = module.eks.cluster_endpoint
+  oidc_provider_arn   = module.eks.oidc_provider_arn
+  worker_iam_role_arn = aws_iam_role.workers.arn
+
+  karpenter_nodepools   = local.karpenter_nodepools
+  karpenter_nodeclasses = local.karpenter_nodeclasses
+
+  create_fargate_logger_configmap = var.create_fargate_logger_configmap_for_karpenter
+  create_aws_observability_ns     = var.create_aws_observability_ns_for_karpenter
+  create_fargate_log_group        = var.create_fargate_log_group_for_karpenter
+  create_fargate_logging_policy   = var.create_fargate_logging_policy_for_karpenter
+
+  # Required for Fargate profile
+  subnet_ids = var.subnet_ids
+}
+
+resource "kubernetes_manifest" "fargate_node_security_group_policy_for_karpenter" {
+  count = var.fargate_cluster && var.create_node_security_group && var.autoscaling_mode == "karpenter" ? 1 : 0
+
+  manifest = {
+    apiVersion = "vpcresources.k8s.aws/v1beta1"
+    kind       = "SecurityGroupPolicy"
+    metadata = {
+      name      = "fargate-karpenter-namespace-sg"
+      namespace = "karpenter"
+    }
+    spec = {
+      podSelector = {
+        matchLabels = {}
+      }
+      securityGroups = {
+        groupIds = [module.eks.node_security_group_id]
+      }
+    }
+  }
+
+  depends_on = [module.karpenter]
+}

main.tf

@@ -8,6 +8,19 @@ locals {
     )
   ) : var.aws_auth_fargate_profile_pod_execution_role_arns
 
+  additional_aws_auth_fargate_profile_pod_execution_role_arns = var.autoscaling_mode == "karpenter" ? concat(values(module.karpenter[0].fargate_profile_pod_execution_role_arn)) : []
+
+  additional_role_mapping = var.autoscaling_mode == "karpenter" ? [
+    {
+      rolearn = aws_iam_role.workers.arn
+      groups = [
+        "system:bootstrappers",
+        "system:nodes",
+      ]
+      username = "system:node:{{EC2PrivateDNSName}}"
+    }
+  ] : []
+
 }
 #tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr
 #tfsec:ignore:aws-eks-no-public-cluster-access
@@ -149,10 +162,10 @@ module "eks" {
   manage_aws_auth_configmap                        = var.manage_aws_auth_configmap
   aws_auth_node_iam_role_arns_non_windows          = [aws_iam_role.workers.arn]
   aws_auth_node_iam_role_arns_windows              = var.enable_cluster_windows_support ? [aws_iam_role.workers.arn] : []
-  aws_auth_roles                                   = var.role_mapping
+  aws_auth_roles                                   = concat(var.role_mapping, local.additional_role_mapping)
   aws_auth_users                                   = var.user_mapping
   aws_auth_accounts                                = []
-  aws_auth_fargate_profile_pod_execution_role_arns = local.aws_auth_fargate_profile_pod_execution_role_arns
+  aws_auth_fargate_profile_pod_execution_role_arns = concat(local.aws_auth_fargate_profile_pod_execution_role_arns, local.additional_aws_auth_fargate_profile_pod_execution_role_arns)
 
   tags = var.tags
 }