diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 8232b7bf..a72c34db 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,18 +1,18 @@
repos:
- repo: https://github.com/gruntwork-io/pre-commit
- rev: v0.1.17
+ rev: v0.1.22
hooks:
- id: shellcheck
- repo: https://github.com/tcort/markdown-link-check
- rev: v3.9.3
+ rev: v3.11.2
hooks:
- id: markdown-link-check
args:
- "--config=mlc_config.json"
- repo: https://github.com/antonbabenko/pre-commit-terraform
- rev: v1.77.0
+ rev: v1.83.3
hooks:
- id: terraform_fmt
- id: terraform_validate
@@ -29,6 +29,8 @@ repos:
args:
- --args=--exclude-downloaded-modules
- id: terraform_checkov
+ args:
+ - "--args=--skip-check CKV_TF_1"
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.2.0
@@ -46,5 +48,5 @@ repos:
# Security
- id: detect-aws-credentials
- args: ['--allow-missing-credentials']
+ args: ["--allow-missing-credentials"]
- id: detect-private-key
diff --git a/.tflint.hcl b/.tflint.hcl
index f0b7ce3e..fdabbdf6 100644
--- a/.tflint.hcl
+++ b/.tflint.hcl
@@ -1,6 +1,6 @@
plugin "aws" {
enabled = true
- version = "0.22.1"
+ version = "0.27.0"
source = "github.com/terraform-linters/tflint-ruleset-aws"
}
diff --git a/README.md b/README.md
index b034cbc3..4088053d 100644
--- a/README.md
+++ b/README.md
@@ -151,10 +151,6 @@ locals {
timeAdded = timestamp() # required if not terraform plan complains
}
]
-
- karpenter_instance_types_list = ["m5a.large"]
- karpenter_capacity_type_list = ["on-demand"]
- karpenter_arch_list = ["amd64"]
},
]
# Karpenter Nodetemplate Config
@@ -197,8 +193,10 @@ module "karpenter" {
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | >= 1.0 |
+| [terraform](#requirement\_terraform) | >= 1.4 |
| [aws](#requirement\_aws) | >= 4.47 |
+| [helm](#requirement\_helm) | ~> 2.6 |
+| [kubectl](#requirement\_kubectl) | 1.14.0 |
| [kubernetes](#requirement\_kubernetes) | >= 2.10 |
## Providers
@@ -215,6 +213,7 @@ module "karpenter" {
| [ebs\_csi\_irsa\_role](#module\_ebs\_csi\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.11.2 |
| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.17.0 |
| [fargate\_profiles](#module\_fargate\_profiles) | ./modules/fargate_profile | n/a |
+| [karpenter](#module\_karpenter) | ./modules/karpenter | n/a |
| [kms\_ebs](#module\_kms\_ebs) | SPHTech-Platform/kms/aws | ~> 0.1.0 |
| [kms\_secret](#module\_kms\_secret) | SPHTech-Platform/kms/aws | ~> 0.1.0 |
| [node\_groups](#module\_node\_groups) | ./modules/eks_managed_nodes | n/a |
@@ -232,6 +231,7 @@ module "karpenter" {
| [aws_iam_service_linked_role.autoscaling](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource |
| [kubernetes_config_map_v1.amazon_vpc_cni](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1) | resource |
| [kubernetes_manifest.fargate_node_security_group_policy](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.fargate_node_security_group_policy_for_karpenter](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
| [aws_arn.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source |
| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
@@ -246,6 +246,7 @@ module "karpenter" {
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
+| [autoscaling\_mode](#input\_autoscaling\_mode) | Autoscaling mode: cluster\_autoscaler or karpenter | `string` | `"karpenter"` | no |
| [aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns](#input\_aws\_auth\_fargate\_profile\_pod\_execution\_role\_arns) | List of Fargate profile pod execution role ARNs to add to the aws-auth configmap | `list(string)` | `[]` | no |
| [cluster\_additional\_security\_group\_ids](#input\_cluster\_additional\_security\_group\_ids) | List of additional, externally created security group IDs to attach to the cluster control plane | `list(string)` | `[]` | no |
| [cluster\_addons](#input\_cluster\_addons) | Map of cluster addon configurations to enable for the cluster. Addon name can be the map keys or set with `name` | `any` | `{}` | no |
@@ -265,9 +266,13 @@ module "karpenter" {
| [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.27"` | no |
| [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no |
| [create\_aws\_observability\_ns](#input\_create\_aws\_observability\_ns) | Whether to create AWS Observability Namespace. | `bool` | `true` | no |
+| [create\_aws\_observability\_ns\_for\_karpenter](#input\_create\_aws\_observability\_ns\_for\_karpenter) | Create aws-observability namespace flag | `bool` | `false` | no |
| [create\_cluster\_security\_group](#input\_create\_cluster\_security\_group) | Determines if a security group is created for the cluster. Note: the EKS service creates a primary security group for the cluster by default | `bool` | `true` | no |
| [create\_cni\_ipv6\_iam\_policy](#input\_create\_cni\_ipv6\_iam\_policy) | Whether to create CNI IPv6 IAM policy. | `bool` | `false` | no |
+| [create\_fargate\_log\_group\_for\_karpenter](#input\_create\_fargate\_log\_group\_for\_karpenter) | value for create\_fargate\_log\_group | `bool` | `false` | no |
| [create\_fargate\_logger\_configmap](#input\_create\_fargate\_logger\_configmap) | Whether to create AWS Fargate logger configmap. | `bool` | `true` | no |
+| [create\_fargate\_logger\_configmap\_for\_karpenter](#input\_create\_fargate\_logger\_configmap\_for\_karpenter) | create\_fargate\_logger\_configmap flag | `bool` | `false` | no |
+| [create\_fargate\_logging\_policy\_for\_karpenter](#input\_create\_fargate\_logging\_policy\_for\_karpenter) | value for create\_fargate\_logging\_policy | `bool` | `false` | no |
| [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no |
| [default\_group\_ami\_id](#input\_default\_group\_ami\_id) | The AMI from which to launch the defualt group instance. If not supplied, EKS will use its own default image | `string` | `""` | no |
| [default\_group\_instance\_types](#input\_default\_group\_instance\_types) | Instance type for the default node group | `list(string)` | [
"m5a.xlarge",
"m5.xlarge",
"m5n.xlarge",
"m5zn.xlarge"
]
| no |
@@ -287,6 +292,10 @@ module "karpenter" {
| [force\_imdsv2](#input\_force\_imdsv2) | Force IMDSv2 metadata server. | `bool` | `true` | no |
| [force\_irsa](#input\_force\_irsa) | Force usage of IAM Roles for Service Account | `bool` | `true` | no |
| [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `set(string)` | `[]` | no |
+| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"v0.32.1"` | no |
+| [karpenter\_default\_subnet\_selector\_tags](#input\_karpenter\_default\_subnet\_selector\_tags) | Subnet selector tags for Karpenter default node class | `map(string)` | {
"kubernetes.io/role/internal-elb": "1"
} | no |
+| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | list(object({
nodeclass_name = string
karpenter_subnet_selector_maps = list(map(any))
karpenter_security_group_selector_maps = list(map(any))
karpenter_ami_selector_maps = list(map(any))
karpenter_node_role = string
karpenter_node_tags_map = map(string)
karpenter_ami_family = string
karpenter_node_user_data = string
karpenter_node_metadata_options = map(any)
karpenter_block_device_mapping = list(object({
deviceName = string
ebs = object({
encrypted = bool
volumeSize = string
volumeType = string
kmsKeyID = optional(string)
deleteOnTermination = bool
})
}))
})) | `[]` | no |
+| [karpenter\_nodepools](#input\_karpenter\_nodepools) | List of Provisioner maps | list(object({
nodepool_name = string
nodeclass_name = string
karpenter_nodepool_node_labels = map(string)
karpenter_nodepool_annotations = map(string)
karpenter_nodepool_node_taints = list(map(string))
karpenter_nodepool_startup_taints = list(map(string))
karpenter_requirements = list(object({
key = string
operator = string
values = list(string)
})
)
karpenter_nodepool_disruption = object({
consolidation_policy = string
consolidate_after = optional(string)
expire_after = string
})
karpenter_nodepool_weight = number
})) | [
{
"karpenter_nodepool_annotations": {},
"karpenter_nodepool_disruption": {
"consolidation_policy": "WhenUnderutilized",
"expire_after": "168h"
},
"karpenter_nodepool_node_labels": {},
"karpenter_nodepool_node_taints": [],
"karpenter_nodepool_startup_taints": [],
"karpenter_nodepool_weight": 10,
"karpenter_requirements": [
{
"key": "karpenter.k8s.aws/instance-category",
"operator": "In",
"values": [
"m"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "In",
"values": [
"4"
]
},
{
"key": "karpenter.k8s.aws/instance-generation",
"operator": "Gt",
"values": [
"5"
]
},
{
"key": "karpenter.sh/capacity-type",
"operator": "In",
"values": [
"on-demand"
]
},
{
"key": "kubernetes.io/arch",
"operator": "In",
"values": [
"amd64"
]
},
{
"key": "kubernetes.io/os",
"operator": "In",
"values": [
"linux"
]
}
],
"nodeclass_name": "default",
"nodepool_name": "default"
}
]
| no |
| [manage\_aws\_auth\_configmap](#input\_manage\_aws\_auth\_configmap) | Determines whether to manage the contents of the aws-auth configmap | `bool` | `true` | no |
| [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no |
| [node\_security\_group\_enable\_recommended\_rules](#input\_node\_security\_group\_enable\_recommended\_rules) | Determines whether to enable recommended security group rules for the node security group created. This includes node-to-node TCP ingress on ephemeral ports and allows all egress traffic | `bool` | `true` | no |
@@ -313,6 +322,7 @@ module "karpenter" {
| [cluster\_name](#output\_cluster\_name) | EKS Cluster name created |
| [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
| [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version of the EKS Cluster |
+| [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Primary Security Group ID of the EKS cluster |
| [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | Security Group ID of the master nodes |
| [cluster\_version](#output\_cluster\_version) | Version of the EKS Cluster |
| [ebs\_kms\_key\_arn](#output\_ebs\_kms\_key\_arn) | KMS Key ARN used for EBS encryption |
diff --git a/karpenter.tf b/karpenter.tf
new file mode 100644
index 00000000..e455b9a1
--- /dev/null
+++ b/karpenter.tf
@@ -0,0 +1,102 @@
+locals {
+ # Karpenter Provisioners Config
+ # Use default var
+ karpenter_nodepools = var.karpenter_nodepools
+
+ # Karpenter Nodetemplate Config
+ karpenter_nodeclasses = coalescelist(var.karpenter_nodeclasses, [
+ {
+ nodeclass_name = "default"
+ karpenter_subnet_selector_maps = [{
+ tags = var.karpenter_default_subnet_selector_tags,
+ }
+ ]
+ karpenter_node_role = aws_iam_role.workers.name
+ karpenter_security_group_selector_maps = [{
+ "id" = module.eks.cluster_primary_security_group_id
+ },
+ ]
+ karpenter_node_metadata_options = {
+ httpEndpoint = "enabled"
+ httpProtocolIPv6 = var.cluster_ip_family != "ipv6" ? "disabled" : "enabled"
+ httpPutResponseHopLimit = 1
+ httpTokens = "required"
+ }
+ karpenter_ami_selector_maps = []
+ karpenter_node_user_data = ""
+ karpenter_node_tags_map = {
+ "karpenter.sh/discovery" = module.eks.cluster_name,
+ "eks:cluster-name" = module.eks.cluster_name,
+ }
+ karpenter_ami_family = "Bottlerocket"
+ karpenter_block_device_mapping = [
+ {
+ #karpenter_root_volume_size
+ "deviceName" = "/dev/xvda"
+ "ebs" = {
+ "encrypted" = true
+ "volumeSize" = "5Gi"
+ "volumeType" = "gp3"
+ "deleteOnTermination" = true
+ }
+ }, {
+ #karpenter_ephemeral_volume_size
+ "deviceName" = "/dev/xvdb",
+ "ebs" = {
+ "encrypted" = true
+ "volumeSize" = "50Gi"
+ "volumeType" = "gp3"
+ "deleteOnTermination" = true
+ }
+ }
+ ]
+ },
+ ])
+}
+
+module "karpenter" {
+ source = "./modules/karpenter"
+
+ count = var.autoscaling_mode == "karpenter" ? 1 : 0
+
+ karpenter_chart_version = var.karpenter_chart_version
+
+ cluster_name = var.cluster_name
+ cluster_endpoint = module.eks.cluster_endpoint
+ oidc_provider_arn = module.eks.oidc_provider_arn
+ worker_iam_role_arn = aws_iam_role.workers.arn
+
+ karpenter_nodepools = local.karpenter_nodepools
+ karpenter_nodeclasses = local.karpenter_nodeclasses
+
+ create_fargate_logger_configmap = var.create_fargate_logger_configmap_for_karpenter
+ create_aws_observability_ns = var.create_aws_observability_ns_for_karpenter
+ create_fargate_log_group = var.create_fargate_log_group_for_karpenter
+ create_fargate_logging_policy = var.create_fargate_logging_policy_for_karpenter
+
+ # Required for Fargate profile
+ subnet_ids = var.subnet_ids
+}
+
+resource "kubernetes_manifest" "fargate_node_security_group_policy_for_karpenter" {
+ count = var.fargate_cluster && var.create_node_security_group && var.autoscaling_mode == "karpenter" ? 1 : 0
+
+ manifest = {
+ apiVersion = "vpcresources.k8s.aws/v1beta1"
+ kind = "SecurityGroupPolicy"
+ metadata = {
+ name = "fargate-karpenter-namespace-sg"
+ namespace = "karpenter"
+ }
+ spec = {
+ podSelector = {
+ matchLabels = {}
+ }
+ securityGroups = {
+ groupIds = [module.eks.node_security_group_id]
+ }
+ }
+ }
+
+ depends_on = [module.karpenter]
+}
diff --git a/main.tf b/main.tf
index 749f63af..c0280316 100644
--- a/main.tf
+++ b/main.tf
@@ -8,6 +8,19 @@ locals {
)
) : var.aws_auth_fargate_profile_pod_execution_role_arns
+ additional_aws_auth_fargate_profile_pod_execution_role_arns = var.autoscaling_mode == "karpenter" ? concat(values(module.karpenter[0].fargate_profile_pod_execution_role_arn)) : []
+
+ additional_role_mapping = var.autoscaling_mode == "karpenter" ? [
+ {
+ rolearn = aws_iam_role.workers.arn
+ groups = [
+ "system:bootstrappers",
+ "system:nodes",
+ ]
+ username = "system:node:{{EC2PrivateDNSName}}"
+ }
+ ] : []
+
}
#tfsec:ignore:aws-eks-no-public-cluster-access-to-cidr
#tfsec:ignore:aws-eks-no-public-cluster-access
@@ -149,10 +162,10 @@ module "eks" {
manage_aws_auth_configmap = var.manage_aws_auth_configmap
aws_auth_node_iam_role_arns_non_windows = [aws_iam_role.workers.arn]
aws_auth_node_iam_role_arns_windows = var.enable_cluster_windows_support ? [aws_iam_role.workers.arn] : []
- aws_auth_roles = var.role_mapping
+ aws_auth_roles = concat(var.role_mapping, local.additional_role_mapping)
aws_auth_users = var.user_mapping
aws_auth_accounts = []
- aws_auth_fargate_profile_pod_execution_role_arns = local.aws_auth_fargate_profile_pod_execution_role_arns
+ aws_auth_fargate_profile_pod_execution_role_arns = concat(local.aws_auth_fargate_profile_pod_execution_role_arns, local.additional_aws_auth_fargate_profile_pod_execution_role_arns)
tags = var.tags
}
diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md
new file mode 100644
index 00000000..4288749b
--- /dev/null
+++ b/modules/karpenter/README.md
@@ -0,0 +1,77 @@
+# Karpenter Notes
+
+## Scenarios
+
+- Fresh Karpenter Cluster
+- Migration for v0.31.x
+- Switching from Cluster Autoscaler Nodegroups to pure Karpenter
+
+
+### Fresh Karpenter Cluster
+
+
+### Migration from Karpenter v0.31.x
+
+### Switching from Cluster Autoscaler Nodegroups to pure Karpenter
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.4 |
+| [aws](#requirement\_aws) | >= 4.47 |
+| [helm](#requirement\_helm) | >= 2.7 |
+| [kubectl](#requirement\_kubectl) | >= 1.14 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [helm](#provider\_helm) | >= 2.7 |
+| [kubectl](#provider\_kubectl) | >= 1.14 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| [karpenter](#module\_karpenter) | terraform-aws-modules/eks/aws//modules/karpenter | ~> 19.18.0 |
+| [karpenter-crds](#module\_karpenter-crds) | rpadovani/helm-crds/kubectl | ~> 0.3.0 |
+| [karpenter\_fargate\_profile](#module\_karpenter\_fargate\_profile) | ../fargate_profile | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubectl_manifest.karpenter_nodeclass](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+| [kubectl_manifest.karpenter_nodepool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| [cluster\_endpoint](#input\_cluster\_endpoint) | EKS Cluster Endpoint | `string` | n/a | yes |
+| [cluster\_name](#input\_cluster\_name) | EKS Cluster name | `string` | n/a | yes |
+| [create\_aws\_observability\_ns](#input\_create\_aws\_observability\_ns) | Create aws-observability namespace flag | `bool` | `false` | no |
+| [create\_fargate\_log\_group](#input\_create\_fargate\_log\_group) | create\_fargate\_log\_group flag | `bool` | `true` | no |
+| [create\_fargate\_logger\_configmap](#input\_create\_fargate\_logger\_configmap) | create\_fargate\_logger\_configmap flag | `bool` | `false` | no |
+| [create\_fargate\_logging\_policy](#input\_create\_fargate\_logging\_policy) | create\_fargate\_logging\_policy flag | `bool` | `true` | no |
+| [karpenter\_chart\_name](#input\_karpenter\_chart\_name) | Chart name for Karpenter | `string` | `"karpenter"` | no |
+| [karpenter\_chart\_repository](#input\_karpenter\_chart\_repository) | Chart repository for Karpenter | `string` | `"oci://public.ecr.aws/karpenter"` | no |
+| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"v0.32.1"` | no |
+| [karpenter\_fargate\_logging\_policy](#input\_karpenter\_fargate\_logging\_policy) | Name of Fargate Logging Profile Policy | `string` | `"karpenter_fargate_logging_cloudwatch"` | no |
+| [karpenter\_namespace](#input\_karpenter\_namespace) | Namespace to deploy karpenter | `string` | `"karpenter"` | no |
+| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | list(object({
nodeclass_name = string
karpenter_subnet_selector_maps = list(map(any))
karpenter_security_group_selector_maps = list(map(any))
karpenter_ami_selector_maps = list(map(any))
karpenter_node_role = string
karpenter_node_tags_map = map(string)
karpenter_ami_family = string
karpenter_node_user_data = string
karpenter_node_metadata_options = map(any)
karpenter_block_device_mapping = list(object({
deviceName = string
ebs = object({
encrypted = bool
volumeSize = string
volumeType = string
kmsKeyID = optional(string)
deleteOnTermination = bool
})
}))
})) | [
{
"karpenter_ami_family": "Bottlerocket",
"karpenter_ami_selector_maps": [],
"karpenter_block_device_mapping": [],
"karpenter_node_metadata_options": {
"httpEndpoint": "enabled",
"httpProtocolIPv6": "disabled",
"httpPutResponseHopLimit": 1,
"httpTokens": "required"
},
"karpenter_node_role": "module.eks.worker_iam_role_name",
"karpenter_node_tags_map": {},
"karpenter_node_user_data": "",
"karpenter_security_group_selector_maps": [],
"karpenter_subnet_selector_maps": [],
"nodeclass_name": "default"
}
]
| no |
+| [karpenter\_nodepools](#input\_karpenter\_nodepools) | List of Provisioner maps | list(object({
nodepool_name = string
nodeclass_name = string
karpenter_nodepool_node_labels = map(string)
karpenter_nodepool_annotations = map(string)
karpenter_nodepool_node_taints = list(map(string))
karpenter_nodepool_startup_taints = list(map(string))
karpenter_requirements = list(object({
key = string
operator = string
values = list(string)
})
)
karpenter_nodepool_disruption = object({
consolidation_policy = string
consolidate_after = optional(string)
expire_after = string
})
karpenter_nodepool_weight = number
})) | [
{
"karpenter_nodepool_annotations": {},
"karpenter_nodepool_disruption": {
"consolidation_policy": "WhenUnderutilized",
"expire_after": "168h"
},
"karpenter_nodepool_node_labels": {},
"karpenter_nodepool_node_taints": [],
"karpenter_nodepool_startup_taints": [],
"karpenter_nodepool_weight": 10,
"karpenter_requirements": [
{
"key": "karpenter.k8s.aws/instance-category",
"operator": "In",
"values": [
"m"
]
},
{
"key": "karpenter.k8s.aws/instance-cpu",
"operator": "In",
"values": [
"4,8,16"
]
},
{
"key": "karpenter.k8s.aws/instance-generation",
"operator": "Gt",
"values": [
"5"
]
},
{
"key": "karpenter.sh/capacity-type",
"operator": "In",
"values": [
"on-demand"
]
},
{
"key": "kubernetes.io/arch",
"operator": "In",
"values": [
"amd64"
]
},
{
"key": "kubernetes.io/os",
"operator": "In",
"values": [
"linux"
]
}
],
"nodeclass_name": "default",
"nodepool_name": "default"
}
]
| no |
+| [karpenter\_release\_name](#input\_karpenter\_release\_name) | Release name for Karpenter | `string` | `"karpenter"` | no |
+| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | ARN of the OIDC Provider for IRSA | `string` | n/a | yes |
+| [subnet\_ids](#input\_subnet\_ids) | For Fargate subnet selection | `list(string)` | `[]` | no |
+| [worker\_iam\_role\_arn](#input\_worker\_iam\_role\_arn) | Worker Nodes IAM Role arn | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [fargate\_profile\_pod\_execution\_role\_arn](#output\_fargate\_profile\_pod\_execution\_role\_arn) | Fargate Profile pod execution role ARN |
+
diff --git a/modules/karpenter/fargate-profile.tf b/modules/karpenter/fargate-profile.tf
index 1150d85c..7fb30f02 100644
--- a/modules/karpenter/fargate-profile.tf
+++ b/modules/karpenter/fargate-profile.tf
@@ -1,19 +1,15 @@
module "karpenter_fargate_profile" {
- source = "SPHTech-Platform/eks/aws//modules/fargate_profile"
- version = "~> 0.12.0"
+ source = "../fargate_profile"
create_aws_observability_ns = var.create_aws_observability_ns
create_fargate_logger_configmap = var.create_fargate_logger_configmap
- # cluster_name = local.cluster_name
- cluster_name = var.cluster_name
+ create_fargate_log_group = var.create_fargate_log_group
+ create_fargate_logging_policy = var.create_fargate_logging_policy
+ cluster_name = var.cluster_name
fargate_profiles = {
karpenter = {
iam_role_name = "fargate_profile_karpenter"
- iam_role_additional_policies = {
- additional = aws_iam_policy.karpenter_fargate_logging.arn
- }
- # subnet_ids = local.app_subnets
- subnet_ids = var.subnet_ids
+ subnet_ids = var.subnet_ids
selectors = [
{
namespace = var.karpenter_namespace
@@ -21,31 +17,4 @@ module "karpenter_fargate_profile" {
]
}
}
-
-}
-
-#tfsec:ignore:aws-iam-no-policy-wildcards
-data "aws_iam_policy_document" "karpenter_fargate_logging" {
- #checkov:skip=CKV_AWS_111:Restricted to Cloudwatch Actions only
- #checkov:skip=CKV_AWS_356: Only logs actions
- statement {
- sid = ""
- effect = "Allow"
- resources = ["*"]
-
- actions = [
- "logs:CreateLogStream",
- "logs:CreateLogGroup",
- "logs:DescribeLogStreams",
- "logs:PutLogEvents",
- ]
- }
-}
-
-resource "aws_iam_policy" "karpenter_fargate_logging" {
- name = var.karpenter_fargate_logging_policy
- path = "/"
- description = "AWS recommended cloudwatch perms policy"
-
- policy = data.aws_iam_policy_document.karpenter_fargate_logging.json
}
diff --git a/modules/karpenter/karpenter.tf b/modules/karpenter/karpenter.tf
index dc5d2523..3c7464a3 100644
--- a/modules/karpenter/karpenter.tf
+++ b/modules/karpenter/karpenter.tf
@@ -1,8 +1,6 @@
module "karpenter" {
source = "terraform-aws-modules/eks/aws//modules/karpenter"
- version = "~> 19.15.0"
-
- count = var.autoscaling_mode == "karpenter" ? 1 : 0
+ version = "~> 19.18.0"
cluster_name = var.cluster_name
@@ -12,12 +10,12 @@ module "karpenter" {
create_iam_role = false
iam_role_arn = var.worker_iam_role_arn
+ enable_karpenter_instance_profile_creation = true # Might be removed in later versions https://github.com/terraform-aws-modules/terraform-aws-eks/pull/2800/files
+
}
resource "helm_release" "karpenter" {
- count = var.autoscaling_mode == "karpenter" ? 1 : 0
-
namespace = var.karpenter_namespace
create_namespace = true
@@ -27,31 +25,17 @@ resource "helm_release" "karpenter" {
version = var.karpenter_chart_version
skip_crds = true # CRDs are managed by module.karpenter-crds
-
- set {
- name = "settings.aws.clusterName"
- value = var.cluster_name
- }
-
- set {
- name = "settings.aws.clusterEndpoint"
- value = var.cluster_endpoint
- }
-
- set {
- name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
- value = try(module.karpenter[0].irsa_arn, "")
- }
-
- set {
- name = "settings.aws.defaultInstanceProfile"
- value = try(module.karpenter[0].instance_profile_name, "")
- }
-
- set {
- name = "settings.aws.interruptionQueueName"
- value = try(module.karpenter[0].queue_name, "")
- }
+ values = [
+ <<-EOT
+ settings:
+ clusterName: ${var.cluster_name}
+ clusterEndpoint: ${var.cluster_endpoint}
+ interruptionQueueName: ${module.karpenter.queue_name}
+ serviceAccount:
+ annotations:
+ eks.amazonaws.com/role-arn: ${module.karpenter.irsa_arn}
+ EOT
+ ]
depends_on = [
module.karpenter[0].irsa_arn,
@@ -65,112 +49,62 @@ resource "helm_release" "karpenter" {
module "karpenter-crds" {
source = "rpadovani/helm-crds/kubectl"
- version = "0.3.0"
+ version = "~> 0.3.0"
crds_urls = [
"https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.sh_provisioners.yaml",
"https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml",
- # "https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.sh_machines.yaml", #not part of release yet
+ "https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.sh_machines.yaml",
+ "https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml",
+ "https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml",
+ "https://raw.githubusercontent.com/aws/karpenter/${var.karpenter_chart_version}/pkg/apis/crds/karpenter.sh_nodepools.yaml",
]
+
}
-################
-##### CRD ######
-################
-
-resource "kubernetes_manifest" "karpenter_provisioner" {
-
- for_each = { for provisioner in var.karpenter_provisioners : provisioner.name => provisioner if var.autoscaling_mode == "karpenter" }
-
- manifest = {
- apiVersion = "karpenter.sh/v1alpha5"
- kind = "Provisioner"
- metadata = {
- name = each.value.name
- }
- spec = {
- labels = each.value.karpenter_provisioner_node_labels
- taints = each.value.karpenter_provisioner_node_taints
-
- requirements = [
- {
- key = "node.kubernetes.io/instance-type"
- operator = "In"
- values = each.value.karpenter_instance_types_list
- },
- {
- key = "karpenter.sh/capacity-type"
- operator = "In"
- values = each.value.karpenter_capacity_type_list
- },
- {
- key = "kubernetes.io/arch"
- operator = "In"
- values = each.value.karpenter_arch_list
- },
- {
- key = "kubernetes.io/os"
- operator = "In"
- values = ["linux"]
- },
- ]
- limits = {
- resources = {
- cpu = "1k"
- }
- }
- providerRef = {
- name = each.value.provider_ref_nodetemplate_name
- }
- ttlSecondsAfterEmpty = 30
- }
- }
-
- computed_fields = ["spec.taints", "spec.requirements"]
+#########################
+## KUBECTL NODEPOOL ##
+#########################
- depends_on = [
- helm_release.karpenter
- ]
-}
+resource "kubectl_manifest" "karpenter_nodepool" {
-resource "kubernetes_manifest" "karpenter_node_template" {
-
- for_each = { for nodetemplate in var.karpenter_nodetemplates : nodetemplate.name => nodetemplate if var.autoscaling_mode == "karpenter" }
-
- manifest = {
- apiVersion = "karpenter.k8s.aws/v1alpha1"
- kind = "AWSNodeTemplate"
- metadata = {
- name = each.value.name
- }
- spec = {
- subnetSelector = each.value.karpenter_subnet_selector_map
- securityGroupSelector = each.value.karpenter_security_group_selector_map
- amiFamily = each.value.karpenter_ami_family
- blockDeviceMappings = [
- {
- deviceName = "/dev/xvda"
- ebs = {
- volumeSize = each.value.karpenter_root_volume_size
- volumeType = "gp3"
- encrypted = true
- }
- },
- {
- deviceName = "/dev/xvdb"
- ebs = {
- volumeSize = each.value.karpenter_ephemeral_volume_size
- volumeType = "gp3"
- encrypted = true
- }
- },
- ]
-
- tags = each.value.karpenter_nodetemplate_tag_map
- }
- }
+ for_each = { for nodepool in var.karpenter_nodepools : nodepool.nodepool_name => nodepool }
- depends_on = [
- helm_release.karpenter
- ]
+ yaml_body = templatefile("${path.module}/templates/nodepool.tftpl", {
+ nodepool_name = each.value.nodepool_name
+ karpenter_nodepool_node_labels_yaml = length(keys(each.value.karpenter_nodepool_node_labels)) == 0 ? "" : replace(yamlencode(each.value.karpenter_nodepool_node_labels), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")
+ karpenter_nodepool_annotations_yaml = length(keys(each.value.karpenter_nodepool_annotations)) == 0 ? "" : replace(yamlencode(each.value.karpenter_nodepool_annotations), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")
+ nodeclass_name = each.value.nodeclass_name
+ karpenter_nodepool_node_taints_yaml = length(each.value.karpenter_nodepool_node_taints) == 0 ? "" : replace(yamlencode(each.value.karpenter_nodepool_node_taints), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")
+ karpenter_nodepool_startup_taints_yaml = length(each.value.karpenter_nodepool_startup_taints) == 0 ? "" : replace(yamlencode(each.value.karpenter_nodepool_startup_taints), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")
+ karpenter_requirements_yaml = replace(yamlencode(each.value.karpenter_requirements), "/((?:^|\n)[\\s-]*)\"([\\w-]+)\":/", "$1$2:")
+ karpenter_nodepool_disruption = each.value.karpenter_nodepool_disruption
+ karpenter_nodepool_weight = each.value.karpenter_nodepool_weight
+ })
+
+ depends_on = [module.karpenter-crds]
+}
+
+##########################
+## KUBECTL NODECLASS ##
+##########################
+resource "kubectl_manifest" "karpenter_nodeclass" {
+ for_each = { for nodeclass in var.karpenter_nodeclasses : nodeclass.nodeclass_name => nodeclass }
+
+ yaml_body = templatefile("${path.module}/templates/nodeclass.tftpl", {
+ nodeclass_name = each.value.nodeclass_name
+ CLUSTER_NAME = var.cluster_name
+ karpenter_subnet_selector_map_yaml = length(each.value.karpenter_subnet_selector_maps) == 0 ? "" : yamlencode(each.value.karpenter_subnet_selector_maps)
+ karpenter_security_group_selector_map_yaml = length(each.value.karpenter_security_group_selector_maps) == 0 ? "" : yamlencode(each.value.karpenter_security_group_selector_maps)
+ karpenter_ami_selector_map_yaml = length(each.value.karpenter_ami_selector_maps) == 0 ? "" : yamlencode(each.value.karpenter_ami_selector_maps)
+ karpenter_node_role = each.value.karpenter_node_role
+ karpenter_node_user_data = each.value.karpenter_node_user_data
+ karpenter_node_tags_map_yaml = length(keys(each.value.karpenter_node_tags_map)) == 0 ? "" : yamlencode(each.value.karpenter_node_tags_map)
+ karpenter_node_metadata_options_yaml = length(keys(each.value.karpenter_node_metadata_options)) == 0 ? "" : replace(yamlencode(each.value.karpenter_node_metadata_options), "/\"([0-9]+)\"/", "$1")
+ karpenter_ami_family = each.value.karpenter_ami_family
+ karpenter_block_device_mapping_yaml = length(each.value.karpenter_block_device_mapping) == 0 ? "" : yamlencode(each.value.karpenter_block_device_mapping)
+
+ })
+
+ depends_on = [module.karpenter-crds]
}
diff --git a/modules/karpenter/templates/nodeclass.tftpl b/modules/karpenter/templates/nodeclass.tftpl
new file mode 100644
index 00000000..9c0a2a5c
--- /dev/null
+++ b/modules/karpenter/templates/nodeclass.tftpl
@@ -0,0 +1,48 @@
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+ name: ${nodeclass_name}
+spec:
+ # required, resolves a default ami and userdata
+ amiFamily: ${karpenter_ami_family}
+
+ # required, discovers subnets to attach to instances
+ subnetSelectorTerms:
+ ${indent(4,karpenter_subnet_selector_map_yaml)}
+
+ # required, discovers security groups to attach to instances
+ securityGroupSelectorTerms:
+ ${indent(4,karpenter_security_group_selector_map_yaml)}
+
+ # required, IAM role to use for the node identity
+ role: ${karpenter_node_role}
+
+ # optional, discovers amis to override the amiFamily's default
+ %{ if karpenter_ami_selector_map_yaml != ""}
+ amiSelectorTerms:
+ ${indent(4,karpenter_ami_selector_map_yaml)}
+ %{ endif }
+
+ # optional, overrides autogenerated userdata with a merge semantic
+ %{ if karpenter_node_user_data != ""}
+ userData: |
+ ${indent(4,karpenter_node_user_data)}
+ %{ endif }
+
+ # optional, propagates tags to underlying EC2 resources
+ %{ if karpenter_node_tags_map_yaml != ""}
+ tags:
+ ${indent(4,karpenter_node_tags_map_yaml)}
+ %{ endif }
+
+ # optional, configures IMDS for the instance, defaults to https://karpenter.sh/docs/concepts/nodeclasses/#specmetadataoptions
+ %{ if karpenter_node_metadata_options_yaml != ""}
+ metadataOptions:
+ ${indent(4,karpenter_node_metadata_options_yaml)}
+ %{ endif }
+
+ # optional, configures storage devices for the instance
+%{ if karpenter_block_device_mapping_yaml != ""}
+ blockDeviceMappings:
+ ${indent(4,karpenter_block_device_mapping_yaml)}
+ %{ endif }
diff --git a/modules/karpenter/templates/nodepool.tftpl b/modules/karpenter/templates/nodepool.tftpl
new file mode 100644
index 00000000..4230a76b
--- /dev/null
+++ b/modules/karpenter/templates/nodepool.tftpl
@@ -0,0 +1,112 @@
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+ name: ${nodepool_name}
+spec:
+ # Template section that describes how to template out NodeClaim resources that Karpenter will provision
+ # Karpenter will consider this template to be the minimum requirements needed to provision a Node using this NodePool
+ # It will overlay this NodePool with Pods that need to schedule to further constrain the NodeClaims
+ # Karpenter will provision to launch new Nodes for the cluster
+ template:
+ metadata:
+ # Labels are arbitrary key-values that are applied to all nodes
+ %{ if karpenter_nodepool_node_labels_yaml != ""}
+ labels:
+ ${indent(8, karpenter_nodepool_node_labels_yaml)}
+ %{ endif }
+ # Annotations are arbitrary key-values that are applied to all nodes
+ %{ if karpenter_nodepool_annotations_yaml != ""}
+ annotations:
+ ${indent(8, karpenter_nodepool_annotations_yaml)}
+ %{ endif }
+ spec:
+ # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation
+ nodeClassRef:
+ name: ${nodeclass_name}
+
+ # Provisioned nodes will have these taints
+ # Taints may prevent pods from scheduling if they are not tolerated by the pod.
+ %{ if karpenter_nodepool_node_taints_yaml != ""}
+ taints:
+ ${indent(8, karpenter_nodepool_node_taints_yaml)}
+ %{ endif }
+
+ # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this
+ # NodePool. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for
+ # removing the taint after it has finished initializing the node.
+ %{ if karpenter_nodepool_startup_taints_yaml != ""}
+ startupTaints:
+ ${indent(8, karpenter_nodepool_startup_taints_yaml)}
+ %{ endif }
+
+
+ # Requirements that constrain the parameters of provisioned nodes.
+ # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules.
+ # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported.
+ # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators
+ requirements:
+ ${indent(8, karpenter_requirements_yaml)}
+
+ # Karpenter provides the ability to specify a few additional Kubelet args.
+ # These are all optional and provide support for additional customization and use cases.
+ # kubelet:
+ # clusterDNS: ["10.0.1.100"]
+ # containerRuntime: containerd
+ # systemReserved:
+ # cpu: 100m
+ # memory: 100Mi
+ # ephemeral-storage: 1Gi
+ # kubeReserved:
+ # cpu: 200m
+ # memory: 100Mi
+ # ephemeral-storage: 3Gi
+ # evictionHard:
+ # memory.available: 5%
+ # nodefs.available: 10%
+ # nodefs.inodesFree: 10%
+ # evictionSoft:
+ # memory.available: 500Mi
+ # nodefs.available: 15%
+ # nodefs.inodesFree: 15%
+ # evictionSoftGracePeriod:
+ # memory.available: 1m
+ # nodefs.available: 1m30s
+ # nodefs.inodesFree: 2m
+ # evictionMaxPodGracePeriod: 60
+ # imageGCHighThresholdPercent: 85
+ # imageGCLowThresholdPercent: 80
+ # cpuCFSQuota: true
+ # podsPerCore: 2
+ # maxPods: 20
+
+ # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes
+ # Configuration in this section constrains how aggressive Karpenter can be with performing operations
+ # like rolling Nodes due to them hitting their maximum lifetime (expiry) or scaling down nodes to reduce cluster cost
+ disruption:
+ # Describes which types of Nodes Karpenter should consider for consolidation
+ # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost
+ # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods. Defaults to 'WhenUnderutilized'
+ consolidationPolicy: ${karpenter_nodepool_disruption.consolidation_policy}
+
+ # The amount of time Karpenter should wait after discovering a consolidation decision
+ # This value can currently only be set when the consolidationPolicy is 'WhenEmpty'
+ # You can choose to disable consolidation entirely by setting the string value 'Never' here
+ %{ if karpenter_nodepool_disruption.consolidation_policy == "WhenEmpty"}
+ consolidateAfter: ${karpenter_nodepool_disruption.consolidation_after}
+ %{ endif }
+
+ # The amount of time a Node can live on the cluster before being removed
+ # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes
+ # You can choose to disable expiration entirely by setting the string value 'Never' here. Defaults to 720h (30 days)
+ expireAfter: ${karpenter_nodepool_disruption.expire_after}
+
+ # Resource limits constrain the total size of the cluster.
+ # Limits prevent Karpenter from creating new instances once the limit is exceeded.
+ limits:
+ cpu: "1000"
+ memory: 1000Gi
+
+ # Priority given to the NodePool when the scheduler considers which NodePool
+ # to select. Higher weights indicate higher priority when comparing NodePools.
+ # Specifying no weight is equivalent to specifying a weight of 0. Defaults to 10
+ weight: ${karpenter_nodepool_weight}
diff --git a/modules/karpenter/templates/nodetemplate.tftpl b/modules/karpenter/templates/nodetemplate.tftpl
new file mode 100644
index 00000000..71022418
--- /dev/null
+++ b/modules/karpenter/templates/nodetemplate.tftpl
@@ -0,0 +1,14 @@
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+ name: ${node_template_name}
+spec:
+ amiFamily: ${karpenter_ami_family}
+ blockDeviceMappings:
+ ${indent(4,karpenter_block_device_mapping_yaml)}
+ securityGroupSelector:
+ ${indent(4,karpenter_security_group_selector_map_yaml)}
+ subnetSelector:
+ ${indent(4,karpenter_subnet_selector_map_yaml)}
+ tags:
+ ${indent(4,karpenter_nodetemplate_tag_map_yaml)}
diff --git a/modules/karpenter/templates/provisioner.tftpl b/modules/karpenter/templates/provisioner.tftpl
new file mode 100644
index 00000000..6689bb41
--- /dev/null
+++ b/modules/karpenter/templates/provisioner.tftpl
@@ -0,0 +1,21 @@
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+ name: ${provisioner_name}
+spec:
+%{ if karpenter_provisioner_node_labels_yaml != ""}
+ labels:
+ ${indent(4, karpenter_provisioner_node_labels_yaml)}
+%{ endif }
+%{ if karpenter_provisioner_node_taints_yaml != ""}
+ taints:
+ ${indent(4, karpenter_provisioner_node_taints_yaml)}
+%{ endif }
+ limits:
+ resources:
+ cpu: 1k
+ providerRef:
+ name: default
+ requirements:
+ ${indent(4, karpenter_requirements_yaml)}
+ ttlSecondsAfterEmpty: 30
diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf
index 7bec94da..fee95f25 100644
--- a/modules/karpenter/variables.tf
+++ b/modules/karpenter/variables.tf
@@ -29,58 +29,115 @@ variable "karpenter_chart_repository" {
variable "karpenter_chart_version" {
description = "Chart version for Karpenter"
type = string
- default = "v0.27.5"
+ default = "v0.32.1"
}
-variable "karpenter_provisioners" {
+variable "karpenter_nodepools" {
description = "List of Provisioner maps"
type = list(object({
- name = string
- provider_ref_nodetemplate_name = string
- karpenter_provisioner_node_labels = map(string)
- karpenter_provisioner_node_taints = list(map(string))
- karpenter_instance_types_list = list(string)
- karpenter_capacity_type_list = list(string)
- karpenter_arch_list = list(string)
+ nodepool_name = string
+ nodeclass_name = string
+ karpenter_nodepool_node_labels = map(string)
+ karpenter_nodepool_annotations = map(string)
+ karpenter_nodepool_node_taints = list(map(string))
+ karpenter_nodepool_startup_taints = list(map(string))
+ karpenter_requirements = list(object({
+ key = string
+ operator = string
+ values = list(string)
+ })
+ )
+ karpenter_nodepool_disruption = object({
+ consolidation_policy = string
+ consolidate_after = optional(string)
+ expire_after = string
+ })
+ karpenter_nodepool_weight = number
}))
- default = []
- ## Sample Below
- #[{
- # name = "default"
- # provider_ref_nodetemplate_name = "default"
- # karpenter_provisioner_node_labels = {}
- # karpenter_provisioner_node_taints = []
- # karpenter_instance_types_list = ["m5a.xlarge", "m6.xlarge"]
- # karpenter_capacity_type_list = ["on-demand"]
- # karpenter_arch_list = ["amd64"]
- # }]
-}
-
-variable "karpenter_nodetemplates" {
+ default = [{
+ nodepool_name = "default"
+ nodeclass_name = "default"
+ karpenter_nodepool_node_labels = {}
+ karpenter_nodepool_annotations = {}
+ karpenter_nodepool_node_taints = []
+ karpenter_nodepool_startup_taints = []
+ karpenter_requirements = [{
+ key = "karpenter.k8s.aws/instance-category"
+ operator = "In"
+ values = ["m"]
+ }, {
+ key = "karpenter.k8s.aws/instance-cpu"
+ operator = "In"
+ values = ["4,8,16"]
+ }, {
+ key = "karpenter.k8s.aws/instance-generation"
+ operator = "Gt"
+ values = ["5"]
+ }, {
+ key = "karpenter.sh/capacity-type"
+ operator = "In"
+ values = ["on-demand"]
+ }, {
+ key = "kubernetes.io/arch"
+ operator = "In"
+ values = ["amd64"]
+ }, {
+ key = "kubernetes.io/os"
+ operator = "In"
+ values = ["linux"]
+ }
+ ]
+ karpenter_nodepool_disruption = {
+ consolidation_policy = "WhenUnderutilized" # WhenUnderutilized or WhenEmpty
+ # consolidate_after = "10m" # Only used if consolidation_policy is WhenEmpty
+ expire_after = "168h" # 7d | 168h | 1w
+ }
+ karpenter_nodepool_weight = 10
+ }]
+}
+
+variable "karpenter_nodeclasses" {
description = "List of nodetemplate maps"
type = list(object({
- name = string
- karpenter_subnet_selector_map = map(string)
- karpenter_security_group_selector_map = map(string)
- karpenter_nodetemplate_tag_map = map(string)
- karpenter_ami_family = string
- karpenter_root_volume_size = string
- karpenter_ephemeral_volume_size = string
+ nodeclass_name = string
+ karpenter_subnet_selector_maps = list(map(any))
+ karpenter_security_group_selector_maps = list(map(any))
+ karpenter_ami_selector_maps = list(map(any))
+ karpenter_node_role = string
+ karpenter_node_tags_map = map(string)
+ karpenter_ami_family = string
+ karpenter_node_user_data = string
+ karpenter_node_metadata_options = map(any)
+ karpenter_block_device_mapping = list(object({
+ deviceName = string
+ ebs = object({
+ encrypted = bool
+ volumeSize = string
+ volumeType = string
+ kmsKeyID = optional(string)
+ deleteOnTermination = bool
+ })
+ }))
}))
- default = []
- ## sample below
- # [{
- # name = "default"
- # karpenter_subnet_selector_map = {}
- # karpenter_security_group_selector_map = {}
- # karpenter_nodetemplate_tag_map = {}
- # karpenter_ami_family = "Bottlerocket"
- # karpenter_root_volume_size = "5Gi"
- # karpenter_ephemeral_volume_size = "50Gi"
- # }]
+ default = [{
+ nodeclass_name = "default"
+ karpenter_block_device_mapping = []
+ karpenter_ami_selector_maps = []
+ karpenter_node_user_data = ""
+ karpenter_node_role = "module.eks.worker_iam_role_name"
+ karpenter_subnet_selector_maps = []
+ karpenter_security_group_selector_maps = []
+ karpenter_node_tags_map = {}
+ karpenter_node_metadata_options = {
+ httpEndpoint = "enabled"
+ httpProtocolIPv6 = "disabled"
+ httpPutResponseHopLimit = 1
+ httpTokens = "required"
+ }
+ karpenter_ami_family = "Bottlerocket"
+ }]
}
-
############################
# K8S Cluster Information
############################
@@ -104,15 +161,6 @@ variable "worker_iam_role_arn" {
type = string
}
-##########
-## MODE ##
-##########
-variable "autoscaling_mode" {
- description = "Autoscaling mode: cluster_autoscaler or karpenter"
- type = string
- default = "cluster_autoscaler"
-}
-
##############
## FARGATE ###
##############
@@ -125,17 +173,23 @@ variable "subnet_ids" {
variable "create_aws_observability_ns" {
description = "Create aws-observability namespace flag"
type = bool
- default = true
+ default = false
}
variable "create_fargate_logger_configmap" {
description = "create_fargate_logger_configmap flag"
type = bool
+ default = false
+}
+
+variable "create_fargate_log_group" {
+ description = "create_fargate_log_group flag"
+ type = bool
default = true
}
-variable "karpenter_fargate_logging_policy" {
- description = "Name of Fargate Logging Profile Policy"
- type = string
- default = "karpenter_fargate_logging_cloudwatch"
+variable "create_fargate_logging_policy" {
+ description = "create_fargate_logging_policy flag"
+ type = bool
+ default = true
}
diff --git a/modules/karpenter/versions.tf b/modules/karpenter/versions.tf
index 5c4cc3ad..837e27b4 100644
--- a/modules/karpenter/versions.tf
+++ b/modules/karpenter/versions.tf
@@ -1,18 +1,19 @@
terraform {
- required_version = ">= 1.3"
+ required_version = ">= 1.4"
required_providers {
+ # tflint-ignore: terraform_unused_required_providers
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 4.47"
+ }
helm = {
source = "hashicorp/helm"
version = ">= 2.7"
}
- kubernetes = {
- source = "hashicorp/kubernetes"
- version = ">= 2.10"
- }
- aws = {
- source = "hashicorp/aws"
- version = ">= 4.47"
+ kubectl = {
+ source = "gavinbunney/kubectl"
+ version = ">= 1.14"
}
}
}
diff --git a/outputs.tf b/outputs.tf
index e4245e1c..2ec155c1 100644
--- a/outputs.tf
+++ b/outputs.tf
@@ -18,6 +18,11 @@ output "worker_iam_role_name" {
value = aws_iam_role.workers.name
}
+output "cluster_primary_security_group_id" {
+ description = "Primary Security Group ID of the EKS cluster"
+ value = module.eks.cluster_primary_security_group_id
+}
+
output "cluster_security_group_id" {
description = "Security Group ID of the master nodes"
value = module.eks.cluster_security_group_id
diff --git a/variables.tf b/variables.tf
index 5ab1ffa5..96e84e73 100644
--- a/variables.tf
+++ b/variables.tf
@@ -382,3 +382,142 @@ variable "cluster_ip_family" {
type = string
default = "ipv4"
}
+##########
+## MODE ##
+##########
+variable "autoscaling_mode" {
+ description = "Autoscaling mode: cluster_autoscaler or karpenter"
+ type = string
+ default = "karpenter"
+}
+
+##############################
+## KARPENTER DEFAULT CONFIG ##
+##############################
+variable "karpenter_nodepools" {
+ description = "List of Provisioner maps"
+ type = list(object({
+ nodepool_name = string
+ nodeclass_name = string
+ karpenter_nodepool_node_labels = map(string)
+ karpenter_nodepool_annotations = map(string)
+ karpenter_nodepool_node_taints = list(map(string))
+ karpenter_nodepool_startup_taints = list(map(string))
+ karpenter_requirements = list(object({
+ key = string
+ operator = string
+ values = list(string)
+ })
+ )
+ karpenter_nodepool_disruption = object({
+ consolidation_policy = string
+ consolidate_after = optional(string)
+ expire_after = string
+ })
+ karpenter_nodepool_weight = number
+ }))
+ default = [{
+ nodepool_name = "default"
+ nodeclass_name = "default"
+ karpenter_nodepool_node_labels = {}
+ karpenter_nodepool_annotations = {}
+ karpenter_nodepool_node_taints = []
+ karpenter_nodepool_startup_taints = []
+ karpenter_requirements = [{
+ key = "karpenter.k8s.aws/instance-category"
+ operator = "In"
+ values = ["m"]
+ }, {
+ key = "karpenter.k8s.aws/instance-cpu"
+ operator = "In"
+ values = ["4"]
+ }, {
+ key = "karpenter.k8s.aws/instance-generation"
+ operator = "Gt"
+ values = ["5"]
+ }, {
+ key = "karpenter.sh/capacity-type"
+ operator = "In"
+ values = ["on-demand"]
+ }, {
+ key = "kubernetes.io/arch"
+ operator = "In"
+ values = ["amd64"]
+ }, {
+ key = "kubernetes.io/os"
+ operator = "In"
+ values = ["linux"]
+ }
+ ]
+ karpenter_nodepool_disruption = {
+ consolidation_policy = "WhenUnderutilized" # WhenUnderutilized or WhenEmpty
+ # consolidate_after = "10m" # Only used if consolidation_policy is WhenEmpty
+ expire_after = "168h" # 7d | 168h | 1w
+ }
+ karpenter_nodepool_weight = 10
+ }]
+}
+
+variable "karpenter_nodeclasses" {
+ description = "List of nodetemplate maps"
+ type = list(object({
+ nodeclass_name = string
+ karpenter_subnet_selector_maps = list(map(any))
+ karpenter_security_group_selector_maps = list(map(any))
+ karpenter_ami_selector_maps = list(map(any))
+ karpenter_node_role = string
+ karpenter_node_tags_map = map(string)
+ karpenter_ami_family = string
+ karpenter_node_user_data = string
+ karpenter_node_metadata_options = map(any)
+ karpenter_block_device_mapping = list(object({
+ deviceName = string
+ ebs = object({
+ encrypted = bool
+ volumeSize = string
+ volumeType = string
+ kmsKeyID = optional(string)
+ deleteOnTermination = bool
+ })
+ }))
+ }))
+ default = []
+}
+
+variable "create_aws_observability_ns_for_karpenter" {
+ description = "Create aws-observability namespace flag"
+ type = bool
+ default = false
+}
+
+variable "create_fargate_logger_configmap_for_karpenter" {
+ description = "create_fargate_logger_configmap flag"
+ type = bool
+ default = false
+}
+
+variable "create_fargate_log_group_for_karpenter" {
+ description = "value for create_fargate_log_group"
+ type = bool
+ default = false
+}
+
+variable "create_fargate_logging_policy_for_karpenter" {
+ description = "value for create_fargate_logging_policy"
+ type = bool
+ default = false
+}
+
+variable "karpenter_chart_version" {
+ description = "Chart version for Karpenter"
+ type = string
+ default = "v0.32.1"
+}
+
+variable "karpenter_default_subnet_selector_tags" {
+ description = "Subnet selector tags for Karpenter default node class"
+ type = map(string)
+ default = {
+ "kubernetes.io/role/internal-elb" = "1"
+ }
+}
diff --git a/versions.tf b/versions.tf
index 8f082ec7..9c9a2612 100644
--- a/versions.tf
+++ b/versions.tf
@@ -1,5 +1,5 @@
terraform {
- required_version = ">= 1.0"
+ required_version = ">= 1.4"
required_providers {
aws = {
source = "hashicorp/aws"
@@ -10,5 +10,15 @@ terraform {
source = "hashicorp/kubernetes"
version = ">= 2.10"
}
+ # tflint-ignore: terraform_unused_required_providers
+ kubectl = {
+ source = "gavinbunney/kubectl"
+ version = "1.14.0"
+ }
+ # tflint-ignore: terraform_unused_required_providers
+ helm = {
+ source = "hashicorp/helm"
+ version = "~> 2.6"
+ }
}
}