Karpenter will occasionally provision nodes that are way too large

[fullykubed]

Description

Observed Behavior:

Occasionally, Karpenter will provision a node that is far, far above what is being requested.

For example, notice the provisioned node below is 10x larger than what is being requested. Moreover, the generated nodeclaim only has a single entry for instance-types.

That is despite the NodePool (manifest below) having many, many instances types that would fit the scheduling request (which it normally does).

{
  "level": "INFO",
  "time": "2024-10-19T15:04:32.809Z",
  "logger": "controller",
  "message": "created nodeclaim",
  "commit": "62a726c",
  "controller": "provisioner",
  "namespace": "",
  "name": "",
  "reconcileID": "e438aaaa-f5dd-4ac9-8fd3-c8d5d4ddb230",
  "NodePool": {
    "name": "spot-arm-9468ed6c"
  },
  "NodeClaim": {
    "name": "spot-arm-9468ed6c-ckz95"
  },
  "requests": {
    "cpu": "1263m",
    "ephemeral-storage": "50Mi",
    "memory": "8289507076",
    "pods": "19"
  },
  "instance-types": "c6a.12xlarge"
}
{
  "level": "INFO",
  "time": "2024-10-19T15:04:34.710Z",
  "logger": "controller",
  "message": "launched nodeclaim",
  "commit": "62a726c",
  "controller": "nodeclaim.lifecycle",
  "controllerGroup": "karpenter.sh",
  "controllerKind": "NodeClaim",
  "NodeClaim": {
    "name": "spot-arm-9468ed6c-ckz95"
  },
  "namespace": "",
  "name": "spot-arm-9468ed6c-ckz95",
  "reconcileID": "cb294d4a-ffb6-4ed4-a5f0-caede430e7de",
  "provider-id": "aws:///us-east-2b/i-0485d92ce19cea74e",
  "instance-type": "c6a.12xlarge",
  "zone": "us-east-2b",
  "capacity-type": "spot",
  "allocatable": {
    "cpu": "47810m",
    "ephemeral-storage": "35Gi",
    "memory": "77078Mi",
    "pods": "110",
    "vpc.amazonaws.com/pod-eni": "114"
  }
}

apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  annotations:
    karpenter.sh/nodepool-hash: "1709207863625532397"
    karpenter.sh/nodepool-hash-version: v3
  creationTimestamp: "2024-09-04T23:12:01Z"
  generation: 7
  labels:
    panfactum.com/environment: production
    panfactum.com/local: "false"
    panfactum.com/module: kube_karpenter_node_pools
    panfactum.com/region: us-east-2
    panfactum.com/root-module: kube_karpenter_node_pools
    panfactum.com/stack-commit: local
    panfactum.com/stack-version: local
    test.1/2.3.4.5: test.1.2.3.4.5
    test1: foo
    test2: bar
    test3: baz
    test4: "42"
  name: spot-arm-9468ed6c
  resourceVersion: "249793223"
  uid: f43f92a5-c202-4b83-892a-4838375de78e
spec:
  disruption:
    budgets: ]
    consolidateAfter: 10s
    consolidationPolicy: WhenEmptyOrUnderutilized
  template:
    metadata:
      labels:
        panfactum.com/class: spot
    spec:
      expireAfter: 24h
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: spot-3008ed27
      requirements:
      - key: karpenter.k8s.aws/instance-family
        operator: In
        values:
        - m8g
        - m7g
        - m7i
        - m7a
        - m6g
        - m6i
        - m6a
        - c8g
        - c7g
        - c7i
        - c7a
        - c6g
        - c6gn
        - c6i
        - c6a
        - r8g
        - r7g
        - r7i
        - r7iz
        - r7a
        - r6g
        - r6i
        - r6a
      - key: karpenter.k8s.aws/instance-size
        operator: NotIn
        values:
        - metal
        - metal-24xl
        - metal-48xl
      - key: kubernetes.io/os
        operator: In
        values:
        - linux
      - key: karpenter.k8s.aws/instance-memory
        operator: Gt
        values:
        - "2500"
      - key: karpenter.sh/capacity-type
        operator: In
        values:
        - spot
        - on-demand
      - key: kubernetes.io/arch
        operator: In
        values:
        - arm64
        - amd64
      startupTaints:
      - effect: NoSchedule
        key: node.cilium.io/agent-not-ready
        value: "true"
      taints:
      - effect: NoSchedule
        key: spot
        value: "true"
      - effect: NoSchedule
        key: arm64
        value: "true"
      terminationGracePeriod: 2m0s
  weight: 20
status:
  conditions:
  - lastTransitionTime: "2024-09-04T23:12:01Z"
    message: ""
    reason: NodeClassReady
    status: "True"
    type: NodeClassReady
  - lastTransitionTime: "2024-09-04T23:12:02Z"
    message: ""
    reason: Ready
    status: "True"
    type: Ready
  - lastTransitionTime: "2024-09-04T23:12:02Z"
    message: ""
    reason: ValidationSucceeded
    status: "True"
    type: ValidationSucceeded
  resources:
    cpu: "8"
    ephemeral-storage: 40894Mi
    hugepages-1Gi: "0"
    hugepages-2Mi: "0"
    hugepages-32Mi: "0"
    hugepages-64Ki: "0"
    memory: 32247340Ki
    nodes: "1"
    pods: "110"

Expected Behavior:

When a set of pods is pending and needs a new node, the generated node claim includes all applicable
instance-types and an appropriately sized node is created.

This normally works correctly and generates logs as follows:

{
  "level": "INFO",
  "time": "2024-10-19T14:57:53.863Z",
  "logger": "controller",
  "message": "created nodeclaim",
  "commit": "62a726c",
  "controller": "provisioner",
  "namespace": "",
  "name": "",
  "reconcileID": "826e7b10-5052-4dcf-8688-40bdbbc4283a",
  "NodePool": {
    "name": "spot-arm-9468ed6c"
  },
  "NodeClaim": {
    "name": "spot-arm-9468ed6c-fqvlb"
  },
  "requests": {
    "cpu": "1310m",
    "memory": "3214608968",
    "pods": "6"
  },
  "instance-types": "c6g.12xlarge, c6g.16xlarge, c6g.2xlarge, c6g.4xlarge, c6g.8xlarge and 55 other(s)"
}
{
  "level": "INFO",
  "time": "2024-10-19T14:57:56.076Z",
  "logger": "controller",
  "message": "launched nodeclaim",
  "commit": "62a726c",
  "controller": "nodeclaim.lifecycle",
  "controllerGroup": "karpenter.sh",
  "controllerKind": "NodeClaim",
  "NodeClaim": {
    "name": "spot-arm-9468ed6c-fqvlb"
  },
  "namespace": "",
  "name": "spot-arm-9468ed6c-fqvlb",
  "reconcileID": "1e545dea-1bc7-4bbb-82f9-b98a29a79c96",
  "provider-id": "aws:///us-east-2a/i-08d9e7d0c1aead853",
  "instance-type": "m8g.large",
  "zone": "us-east-2a",
  "capacity-type": "spot",
  "allocatable": {
    "cpu": "1930m",
    "ephemeral-storage": "35Gi",
    "memory": "4124Mi",
    "pods": "110"
  }
}

Reproduction Steps (Please include YAML):

It is unclear to me how to reproduce. I have tried all the obvious things and am not able to reliability re-trigger the behavior (it seems to occur somewhat randomly):

• Created sets of pending pods with higher cpu, memory, and pod count requirements than the above requests
• Updated the NodePool to trigger drift detection
• Upgraded Karpenter
• Used various NodePools with different requirement settings

I have also verified that the pods do not have any scheduling constraints that would limit them to a single instance type.

In fact, which particular type is chosen for instance-types seems somewhat random. Sometimes it is appropriately sized, sometimes it is 10x too large, sometimes it is 100x too large. The instance families also differ. However, what is consistent is the the node claim is (a) created by the provisioner controller and (b) gets generated with just a single type rather than the full expected set.

After the node is created, Karpenter will then usually disrupt it shortly after and replace it with a smaller node. However, we have sometimes had PDBs prevent this which is when we noticed that this behavior was occurring.

Additionally, all of the NodePools where we have observed this behavior allow spot instances, but I do not know if that is relevant (all of our NodePools are spot-enabled).

Versions:

• Chart Version: 1.0.1
• Kubernetes Version (kubectl version): v1.29.8-eks-a737599

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• Please do not leave "+1" or "me too" comments, they generate extra noise for issue followers and do not help prioritize the request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

[fullykubed]

For additional, reference here is a EC2 fleet request when this problem occurs.

I don't think this shows any more information, but thought I would drop it here. It does show again that only a single instance is included in the request which I believe to be the core problem.

{
    "eventVersion": "1.10",
    "userIdentity": {
        "type": "AssumedRole",
        "principalId": "AROA47CRYUGV7O6HGTQFK:1729504785943908908",
        "arn": "arn:aws:sts::891377197483:assumed-role/karpenter-20240405181041887100000008/1729504785943908908",
        "accountId": "891377197483",
        "accessKeyId": "ASIA47CRYUGVSFKCURM4",
        "sessionContext": {
            "sessionIssuer": {
                "type": "Role",
                "principalId": "AROA47CRYUGV7O6HGTQFK",
                "arn": "arn:aws:iam::891377197483:role/karpenter-20240405181041887100000008",
                "accountId": "891377197483",
                "userName": "karpenter-20240405181041887100000008"
            },
            "webIdFederationData": {
                "federatedProvider": "arn:aws:iam::891377197483:oidc-provider/oidc.eks.us-east-2.amazonaws.com/id/83063DDB274B2A04B6A7DC29DCB1740E",
                "attributes": {}
            },
            "attributes": {
                "creationDate": "2024-10-21T09:59:45Z",
                "mfaAuthenticated": "false"
            }
        }
    },
    "eventTime": "2024-10-21T10:20:05Z",
    "eventSource": "ec2.amazonaws.com",
    "eventName": "CreateFleet",
    "awsRegion": "us-east-2",
    "sourceIPAddress": "18.218.214.155",
    "userAgent": "aws-sdk-go/1.55.5 (go1.22.5; linux; arm64) karpenter.sh-1.0.1",
    "requestParameters": {
        "CreateFleetRequest": {
            "TargetCapacitySpecification": {
                "DefaultTargetCapacityType": "spot",
                "TotalTargetCapacity": 1
            },
            "Type": "instant",
            "SpotOptions": {
                "AllocationStrategy": "price-capacity-optimized"
            },
            "LaunchTemplateConfigs": {
                "LaunchTemplateSpecification": {
                    "LaunchTemplateName": "karpenter.k8s.aws/12914966771093031275",
                    "Version": "$Latest"
                },
                "Overrides": {
                    "ImageId": "ami-0ce6ab0ef12b0b54c",
                    "AvailabilityZone": "us-east-2b",
                    "tag": 1,
                    "SubnetId": "subnet-046ef1097dc37648a",
                    "InstanceType": "r7iz.metal-16xl"
                },
                "tag": 1
            },
            "TagSpecification": [
                {
                    "ResourceType": "instance",
                    "tag": 1,
                    "Tag": [
                        {
                            "Value": "owned",
                            "tag": 1,
                            "Key": "kubernetes.io/cluster/production-primary"
                        },
                        {
                            "Value": "burstable-85b5e108",
                            "tag": 2,
                            "Key": "karpenter.sh/nodepool"
                        },
                        {
                            "Value": "production-primary",
                            "tag": 3,
                            "Key": "eks:eks-cluster-name"
                        },
                        {
                            "Value": "burstable-f500363d",
                            "tag": 4,
                            "Key": "karpenter.k8s.aws/ec2nodeclass"
                        }
                    ]
                },
                {
                    "ResourceType": "volume",
                    "tag": 2,
                    "Tag": [
                        {
                            "Value": "owned",
                            "tag": 1,
                            "Key": "kubernetes.io/cluster/production-primary"
                        },
                        {
                            "Value": "burstable-85b5e108",
                            "tag": 2,
                            "Key": "karpenter.sh/nodepool"
                        },
                        {
                            "Value": "production-primary",
                            "tag": 3,
                            "Key": "eks:eks-cluster-name"
                        },
                        {
                            "Value": "burstable-f500363d",
                            "tag": 4,
                            "Key": "karpenter.k8s.aws/ec2nodeclass"
                        }
                    ]
                },
                {
                    "ResourceType": "fleet",
                    "tag": 3,
                    "Tag": [
                        {
                            "Value": "burstable-f500363d",
                            "tag": 1,
                            "Key": "karpenter.k8s.aws/ec2nodeclass"
                        },
                        {
                            "Value": "owned",
                            "tag": 2,
                            "Key": "kubernetes.io/cluster/production-primary"
                        },
                        {
                            "Value": "burstable-85b5e108",
                            "tag": 3,
                            "Key": "karpenter.sh/nodepool"
                        },
                        {
                            "Value": "production-primary",
                            "tag": 4,
                            "Key": "eks:eks-cluster-name"
                        }
                    ]
                }
            ]
        }
    },
    "responseElements": {
        "CreateFleetResponse": {
            "fleetInstanceSet": {
                "item": {
                    "lifecycle": "spot",
                    "instanceIds": {
                        "item": "i-031bc83e78cd9423d"
                    },
                    "instanceType": "r7iz.metal-16xl",
                    "launchTemplateAndOverrides": {
                        "overrides": {
                            "subnetId": "subnet-046ef1097dc37648a",
                            "imageId": "ami-0ce6ab0ef12b0b54c",
                            "instanceType": "r7iz.metal-16xl",
                            "availabilityZone": "us-east-2b"
                        },
                        "launchTemplateSpecification": {
                            "launchTemplateId": "lt-0a86406a76d5b08be",
                            "version": 1
                        }
                    }
                }
            },
            "xmlns": "http://ec2.amazonaws.com/doc/2016-11-15/",
            "requestId": "6c26f26a-d723-4d6b-91ea-f996742ac34b",
            "fleetId": "fleet-03bfdd35-440e-cc8f-a6b8-a9025e0ae254",
            "errorSet": ""
        }
    },
    "requestID": "6c26f26a-d723-4d6b-91ea-f996742ac34b",
    "eventID": "7dd21712-ee0d-4266-ad55-bc2ffcf46f1d",
    "readOnly": false,
    "eventType": "AwsApiCall",
    "managementEvent": true,
    "recipientAccountId": "891377197483",
    "eventCategory": "Management",
    "tlsDetails": {
        "tlsVersion": "TLSv1.3",
        "cipherSuite": "TLS_AES_128_GCM_SHA256",
        "clientProvidedHostHeader": "ec2.us-east-2.amazonaws.com"
    }
}

[fullykubed]

Immediately before the CreateFleet API call, I do notice the following API error from Karpenter for DescribeLaunchTemplates. However, I notice this seems to occur prior to every CreateFleet call, not just the ones that are bugged.

{
    "eventVersion": "1.10",
    "userIdentity": {
        "type": "AssumedRole",
        "principalId": "AROA47CRYUGV7O6HGTQFK:1729504785943908908",
        "arn": "arn:aws:sts::891377197483:assumed-role/karpenter-20240405181041887100000008/1729504785943908908",
        "accountId": "891377197483",
        "accessKeyId": "ASIA47CRYUGVSFKCURM4",
        "sessionContext": {
            "sessionIssuer": {
                "type": "Role",
                "principalId": "AROA47CRYUGV7O6HGTQFK",
                "arn": "arn:aws:iam::891377197483:role/karpenter-20240405181041887100000008",
                "accountId": "891377197483",
                "userName": "karpenter-20240405181041887100000008"
            },
            "webIdFederationData": {
                "federatedProvider": "arn:aws:iam::891377197483:oidc-provider/oidc.eks.us-east-2.amazonaws.com/id/83063DDB274B2A04B6A7DC29DCB1740E",
                "attributes": {}
            },
            "attributes": {
                "creationDate": "2024-10-21T09:59:45Z",
                "mfaAuthenticated": "false"
            }
        }
    },
    "eventTime": "2024-10-21T10:20:04Z",
    "eventSource": "ec2.amazonaws.com",
    "eventName": "DescribeLaunchTemplates",
    "awsRegion": "us-east-2",
    "sourceIPAddress": "18.218.214.155",
    "userAgent": "aws-sdk-go/1.55.5 (go1.22.5; linux; arm64) karpenter.sh-1.0.1",
    "errorCode": "Client.InvalidLaunchTemplateName.NotFoundException",
    "errorMessage": "At least one of the launch templates specified in the request does not exist.",
    "requestParameters": {
        "DescribeLaunchTemplatesRequest": {
            "LaunchTemplateName": {
                "tag": 1,
                "content": "karpenter.k8s.aws/12914966771093031275"
            }
        }
    },
    "responseElements": null,
    "requestID": "a02a278f-e6de-4c1b-bca7-738050942304",
    "eventID": "7888c311-84c1-4829-85c9-7d5bf7c50398",
    "readOnly": true,
    "eventType": "AwsApiCall",
    "managementEvent": true,
    "recipientAccountId": "891377197483",
    "eventCategory": "Management",
    "tlsDetails": {
        "tlsVersion": "TLSv1.3",
        "cipherSuite": "TLS_AES_128_GCM_SHA256",
        "clientProvidedHostHeader": "ec2.us-east-2.amazonaws.com"
    }
}

[fullykubed]

Found the problem in the core Karpenter library. Not specific to AWS. Moving the conversation there.