enhancement: Added support for EKS prefix delegation, allowing nodes to have many more IPs (17 -> 110), also change how the eks module accepts node group parameters. (#59)

* enhancement: Added support for EKS prefix delegation, allowing nodes to have many more IPs (17 -> 110), also change how the eks module accepts node group parameters.

breaking change: Switched back to using the worker security group instead of the "cluster primary" security group - we were only using it because we weren't creating a custom launch template and with this change we will be. If you are trying to upgrade, this map require tmeporarily adding a security group rule to allow your old node groups to access the db, bringing up new node groups with the new configuration, then removing your old node groups.

* docs: Clarified node group config

Author: bmonkman

modules/eks/README.md

@@ -14,6 +14,22 @@ See the necessary versions for each EKS version here:
 
 [https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html](https://docs.aws.amazon.com/eks/latest/userguide/managing-kube-proxy.html)
 
+*Node group configuration schema:*
+```
+{
+    <group name>: {
+        instance_types     = list(string) - List of instance types to use for nodes in the node group. In order of preference. Instance types in a group should be similar in resources.
+        asg_min_size       = string (default: "1") - Smallest size of this node group in instances.
+        asg_max_size       = string (default: "3") - Largest size of this node group in instances.
+        use_spot_instances = bool (default: false) - If true, use spot instances to save cost.
+        ami_type           = string (default: "AL2_x86_64") - The type of AMI to use. Other possibilities are AL2_x86_64_GPU for gpu instances or AL2_ARM_64 for ARM instances
+        use_large_ip_range = bool (default: true) - If true, enable the "prefix delegation" feature of EKS. This will create a custom launch template for each node group.
+        node_ip_limit      = int  (default: 110)  - If using prefix delegation, the max that can be used per node. 110 is the limit for all but the largest instance types.
+    },
+    ...
+}
+```
+
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
@@ -27,6 +43,7 @@ See the necessary versions for each EKS version here:
 | Name | Version |
 |------|---------|
 | aws | >= 3.37.0 |
+| null | n/a |
 
 ## Inputs
 
@@ -37,7 +54,7 @@ See the necessary versions for each EKS version here:
 | addon\_vpc\_cni\_version | Version of the VPC CNI to install. If empty you will need to upgrade the CNI yourself during a cluster version upgrade | `string` | `""` | no |
 | cluster\_name | Name to be given to the EKS cluster | `any` | n/a | yes |
 | cluster\_version | EKS cluster version number to use. Incrementing this will start a cluster upgrade | `any` | n/a | yes |
-| eks\_node\_groups | Map of maps of EKS node group config where keys are node group names | <pre>map(object({<br>    instance_types     = list(string)<br>    asg_min_size       = string<br>    asg_max_size       = string<br>    use_spot_instances = bool<br>    ami_type           = string<br>  }))</pre> | n/a | yes |
+| eks\_node\_groups | Map of maps of EKS node group config where keys are node group names. See the readme for details. | `any` | n/a | yes |
 | environment | The environment (stage/prod) | `any` | n/a | yes |
 | iam\_account\_id | Account ID of the current IAM user | `any` | n/a | yes |
 | iam\_role\_mapping | List of mappings of AWS Roles to Kubernetes Groups | <pre>list(object({<br>    iam_role_arn  = string<br>    k8s_role_name = string<br>    k8s_groups    = list(string)<br>  }))</pre> | n/a | yes |
@@ -52,6 +69,6 @@ See the necessary versions for each EKS version here:
 | cluster\_id | Identifier of the EKS cluster |
 | worker\_iam\_role\_arn | The ARN of the EKS worker IAM role |
 | worker\_iam\_role\_name | The name of the EKS worker IAM role |
-| worker\_security\_group\_id | The security group of the EKS cluster |
+| worker\_security\_group\_id | The security group of the EKS workers |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/eks/main.tf

@@ -8,19 +8,27 @@ data "aws_eks_cluster_auth" "cluster" {
 }
 
 locals {
+  k8s_exec_context = "--context ${data.aws_eks_cluster.cluster.name} --server ${data.aws_eks_cluster.cluster.endpoint}"
+
   # Map this module config to the upstream module config
   eks_node_group_config = { for n, config in var.eks_node_groups :
     n => {
       name = "${var.cluster_name}-${n}"
 
-      desired_capacity = config.asg_min_size
-      max_capacity     = config.asg_max_size
-      min_capacity     = config.asg_min_size
+      desired_capacity = lookup(config, "asg_min_size", 1)
+      max_capacity     = lookup(config, "asg_max_size", 3)
+      min_capacity     = lookup(config, "asg_min_size", 1)
+
+      create_launch_template  = lookup(config, "use_large_ip_range", true)
+      launch_template_version = "1"
+      # Hopefully temporary, as there is an issue with the upstream module that leads to this value being non-deterministic with the default of "$Latest"
+      # See https://github.com/terraform-aws-modules/terraform-aws-eks/pull/1447
 
-      ami_type       = config.ami_type
-      instance_types = config.instance_types
-      capacity_type  = config.use_spot_instances ? "SPOT" : "ON_DEMAND"
-      disk_size      = 100
+      ami_type           = lookup(config, "ami_type", "AL2_x86_64")
+      instance_types     = lookup(config, "instance_types", [])
+      capacity_type      = lookup(config, "use_spot_instances", false) ? "SPOT" : "ON_DEMAND"
+      disk_size          = 100
+      kubelet_extra_args = lookup(config, "use_large_ip_range", true) ? "--max-pods=${lookup(config, "node_ip_limit", 110)}" : ""
 
       k8s_labels = {
         Environment = var.environment
@@ -101,3 +109,19 @@ resource "aws_eks_addon" "coredns" {
   resolve_conflicts = "OVERWRITE"
   addon_version     = var.addon_coredns_version
 }
+
+# Enable prefix delegation - this will enable many more IPs to be allocated per-node.
+# See https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html
+resource "null_resource" "enable_prefix_delegation" {
+  count = var.addon_vpc_cni_version == "" ? 0 : 1
+
+  triggers = {
+    manifest_sha1 = sha1(var.addon_vpc_cni_version)
+  }
+
+  provisioner "local-exec" {
+    command = "kubectl set env daemonset aws-node ${local.k8s_exec_context} -n kube-system ENABLE_PREFIX_DELEGATION=true WARM_PREFIX_TARGET=1"
+  }
+
+  depends_on = [aws_eks_addon.vpc_cni]
+}

modules/eks/outputs.tf

@@ -14,6 +14,6 @@ output "worker_iam_role_name" {
 }
 
 output "worker_security_group_id" {
-  description = "The security group of the EKS cluster"
-  value       = module.eks.cluster_primary_security_group_id
+  description = "The security group of the EKS workers"
+  value       = module.eks.worker_security_group_id
 }

modules/eks/variables.tf

@@ -24,14 +24,8 @@ variable "vpc_id" {
 }
 
 variable "eks_node_groups" {
-  type = map(object({
-    instance_types     = list(string)
-    asg_min_size       = string
-    asg_max_size       = string
-    use_spot_instances = bool
-    ami_type           = string
-  }))
-  description = "Map of maps of EKS node group config where keys are node group names"
+  type        = any
+  description = "Map of maps of EKS node group config where keys are node group names. See the readme for details."
 }
 
 variable "iam_account_id" {