diff --git a/manifests/modules/troubleshooting/cni/.workshop/cleanup.sh b/manifests/modules/troubleshooting/cni/.workshop/cleanup.sh new file mode 100755 index 000000000..8a89274a2 --- /dev/null +++ b/manifests/modules/troubleshooting/cni/.workshop/cleanup.sh @@ -0,0 +1,55 @@ +# VPC_CNI_IAM_ROLE_NAME="eksctl-eks-workshop-addon-vpc-cni-Role1-n85u3l0IhDSv" + +kubectl delete namespace cni-tshoot +attached_policies=$(aws iam list-attached-role-policies --role-name $VPC_CNI_IAM_ROLE_NAME --query 'AttachedPolicies[*].PolicyArn' --output text) + +is_policy_exist=0 + +for policy in ${attached_policies[@]}; do + if [ "$policy" == "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" ]; then + is_policy_exist=1 + else + aws iam detach-role-policy --role-name $VPC_CNI_IAM_ROLE_NAME --policy-arn $policy + fi +done + +if [ $is_policy_exist -eq 0 ]; then + logmessage "Attaching back AmazonEKS_CNI_Policy policy into VPC CNI addon role" + + aws iam attach-role-policy --role-name $VPC_CNI_IAM_ROLE_NAME --policy-arn arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy +fi + +nodes=$(aws eks list-nodegroups --cluster-name $EKS_CLUSTER_NAME --query 'nodegroups' --output text) +deleted_nodes=() + +logmessage "Reverting EKS managed nodegroup configuration" +for node in ${nodes[@]}; do + if [[ "$node" != "default" && "$node" != "cni_troubleshooting_nodes" ]]; then + logmessage "Deleting nodegroup $node" + aws eks delete-nodegroup --cluster-name $EKS_CLUSTER_NAME --nodegroup-name $node + deleted_nodes+=$node + fi +done + +logmessage "Waiting for EKS managed nodegroup to be deleted" +for deleted_node in ${deleted_nodes[@]}; do + logmessage "waiting for deletion of $deleted_node" + aws eks wait nodegroup-deleted --cluster-name $EKS_CLUSTER_NAME --nodegroup-name $deleted_node +done + +DEFAULT_CONFIG='{"enableNetworkPolicy":"true","env":{"ENABLE_POD_ENI":"true","ENABLE_PREFIX_DELEGATION":"true","POD_SECURITY_GROUP_ENFORCING_MODE":"standard"},"nodeAgent":{"enablePolicyEventLogs":"true"}}' +CURRENT_CONFIG=$(aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --query addon.configurationValues --output text | jq --sort-keys -c .) + +if [ $DEFAULT_CONFIG != $CURRENT_CONFIG ]; then + logmessage "Reverting VPC CNI config to default" + addons_status=$(aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --query addon.status --output text) + while [ $addons_status == "UPDATING" ]; do + logmessage "Waiting for VPC CNI addons status to not be in UPDATING" + sleep 60 + addons_status=$(aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --query addon.status --output text) + done + + aws eks update-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --service-account-role-arn $VPC_CNI_IAM_ROLE_ARN --configuration-values $DEFAULT_CONFIG +fi + + diff --git a/manifests/modules/troubleshooting/cni/.workshop/ssm.sh b/manifests/modules/troubleshooting/cni/.workshop/ssm.sh new file mode 100755 index 000000000..d17070fc6 --- /dev/null +++ b/manifests/modules/troubleshooting/cni/.workshop/ssm.sh @@ -0,0 +1,23 @@ +#!/bin/bash +COMMAND_ID=$(aws ssm send-command \ + --instance-ids $1 \ + --document-name "AWS-RunShellScript" \ + --comment "Demo run shell script on Linux Instances" \ + --parameters '{"commands":["sudo -Hiu root bash << END","tail -n '$3' /var/log/aws-routed-eni/'$2'.log | grep '$4'", "END"]}' \ + --output text \ + --query "Command.CommandId") + +STATUS=InProgress +while [ "$STATUS" == "InProgress" ]; do + STATUS=$(aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id $1 \ + --output text \ + --query "Status") +done + +aws ssm list-command-invocations \ + --command-id "$COMMAND_ID" \ + --details \ + --output text \ + --query "CommandInvocations[].CommandPlugins[].Output" diff --git a/manifests/modules/troubleshooting/cni/.workshop/terraform/main.tf b/manifests/modules/troubleshooting/cni/.workshop/terraform/main.tf new file mode 100644 index 000000000..833d0924f --- /dev/null +++ b/manifests/modules/troubleshooting/cni/.workshop/terraform/main.tf @@ -0,0 +1,197 @@ +locals { + tags = { + module = "troubleshooting" + } + secondary_cidr = "100.64.0.0/22" +} + +data "aws_vpc" "selected" { + tags = { + created-by = "eks-workshop-v2" + env = var.addon_context.eks_cluster_id + } +} + +data "aws_subnets" "private" { + tags = { + created-by = "eks-workshop-v2" + env = var.addon_context.eks_cluster_id + } + + filter { + name = "tag:Name" + values = ["*Private*"] + } +} + +resource "aws_vpc_ipv4_cidr_block_association" "secondary_cidr" { + vpc_id = data.aws_vpc.selected.id + cidr_block = local.secondary_cidr +} + +data "aws_subnet" "selected" { + count = length(data.aws_subnets.private.ids) + + id = data.aws_subnets.private.ids[count.index] +} + +resource "aws_subnet" "large_subnet" { + count = length(data.aws_subnets.private.ids) + + vpc_id = aws_vpc_ipv4_cidr_block_association.secondary_cidr.vpc_id + cidr_block = cidrsubnet(local.secondary_cidr, 2, count.index) + availability_zone = data.aws_subnet.selected[count.index].availability_zone + + tags = merge(local.tags, var.tags, { + AdditionalSubnet = "true" + Size = "large" + }) + + depends_on = [ + aws_vpc_ipv4_cidr_block_association.secondary_cidr + ] +} + +resource "aws_subnet" "small_subnet" { + count = length(data.aws_subnets.private.ids) + + vpc_id = aws_vpc_ipv4_cidr_block_association.secondary_cidr.vpc_id + cidr_block = cidrsubnet(local.secondary_cidr, 6, count.index + 48) + availability_zone = data.aws_subnet.selected[count.index].availability_zone + + tags = merge(local.tags, { + AdditionalSubnet = "true" + Size = "small" + }) + + depends_on = [ + aws_vpc_ipv4_cidr_block_association.secondary_cidr + ] +} + +data "aws_route_table" "private" { + count = length(data.aws_subnets.private.ids) + + vpc_id = data.aws_vpc.selected.id + subnet_id = data.aws_subnets.private.ids[count.index] +} + +resource "aws_route_table_association" "small_subnet" { + count = length(data.aws_subnets.private.ids) + + subnet_id = aws_subnet.small_subnet[count.index].id + route_table_id = data.aws_route_table.private[count.index].route_table_id +} + +resource "aws_route_table_association" "large_subnet" { + count = length(data.aws_subnets.private.ids) + + subnet_id = aws_subnet.large_subnet[count.index].id + route_table_id = data.aws_route_table.private[count.index].route_table_id +} + +resource "aws_iam_role" "node_role" { + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = [ + "ec2.amazonaws.com" + ] + } + Action = "sts:AssumeRole" + } + ] + }) + managed_policy_arns = [ + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] +} + +resource "aws_eks_access_entry" "cni_troubleshooting_nodes" { + cluster_name = var.eks_cluster_id + principal_arn = aws_iam_role.node_role.arn + type = "EC2_LINUX" +} + +resource "aws_eks_node_group" "cni_troubleshooting_nodes" { + + cluster_name = var.eks_cluster_id + node_group_name = "cni_troubleshooting_nodes" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = aws_subnet.small_subnet[*].id + instance_types = ["m5.large"] + + scaling_config { + desired_size = 0 + max_size = 6 + min_size = 0 + } + + labels = { + app = "cni_troubleshooting" + } + + taint { + key = "purpose" + value = "cni_troubleshooting" + effect = "NO_SCHEDULE" + } + + update_config { + max_unavailable = 1 + } + + tags = merge(local.tags, var.tags) + +} + + +data "aws_eks_addon" "vpc_cni" { + addon_name = "vpc-cni" + cluster_name = var.addon_context.eks_cluster_id +} + +resource "null_resource" "change_config" { + triggers = { + config = data.aws_eks_addon.vpc_cni.configuration_values, + cluster_name = var.addon_context.eks_cluster_id, + role_arn = data.aws_eks_addon.vpc_cni.service_account_role_arn, + node_group_name = aws_eks_node_group.cni_troubleshooting_nodes.node_group_name, + role_name = split("/", data.aws_eks_addon.vpc_cni.service_account_role_arn)[1], + timestamp = timestamp() + } + + provisioner "local-exec" { + command = < 91s v1.30.0-eks-036c24b cni_troubleshooting +``` + +we'll select this node as our focus + +```bash test=false +$ export NODE_NAME=$(kubectl get nodes -l app=cni_troubleshooting -L app -o custom-columns=:metadata.name --no-headers) +``` + +Now, let's investigate the reason behind this node's NotReady state. We'll use the describe command to gather detailed information about the node, paying particular attention to the Conditions section in the output. This will help us identify the specific issues preventing the node from being ready. Take a moment to run the command and analyze the results + +```bash test=false +$ kubectl describe node $NODE_NAME + +Conditions: + Type Status LastHeartbeatTime LastTransitionTime Reason Message + ---- ------ ----------------- ------------------ ------ ------- + MemoryPressure False Wed, 30 Oct 2024 19:37:02 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available + DiskPressure False Wed, 30 Oct 2024 19:37:02 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure + PIDPressure False Wed, 30 Oct 2024 19:37:02 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasSufficientPID kubelet has sufficient PID available + Ready False Wed, 30 Oct 2024 19:37:02 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletNotReady container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized +``` + +Let's analyze the situation: The node was in a NotReady state because the CNI (Container Network Interface) plugin wasn't initialized. + +:::Info +On EKS Linux nodes, the CNI plugin is initialized by a healthy VPC CNI pod called 'aws-node'. These aws-node pods run as DaemonSets, meaning each Linux worker node should have one. +::: + +Action step: Review the 'Non-terminated Pods' section in the describe node output. Look for an aws-node pod on this specific node. If it's missing or in an unhealthy state, this could explain the NotReady status. + +```bash test=false +$ kubectl describe node $NODE_NAME + +Non-terminated Pods: (2 in total) + Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age + --------- ---- ------------ ---------- --------------- ------------- --- + kube-system kube-proxy-69754 100m (5%) 0 (0%) 0 (0%) 0 (0%) 17m +``` + +Is the aws-node pod present and healthy? If not, what would be your next steps to troubleshoot or resolve this issue? + +### Step 3 + +Having identified that the aws-node is missing from the node, our next step is to investigate the aws-node daemonset. Let's use the kubectl describe command to gather more information about this daemonset and understand why it's not running on the affected node. This will help us pinpoint the root cause of the issue and determine the appropriate solution. + +```bash test=false +$ kubectl describe ds aws-node -n kube-system +Name: aws-node +Selector: k8s-app=aws-node +Node-Selector: +Labels: app.kubernetes.io/instance=aws-vpc-cni + app.kubernetes.io/managed-by=Helm + app.kubernetes.io/name=aws-node + app.kubernetes.io/version=v1.16.0 + helm.sh/chart=aws-vpc-cni-1.16.0 + k8s-app=aws-node +Annotations: deprecated.daemonset.template.generation: 5 +Desired Number of Nodes Scheduled: 4 +Current Number of Nodes Scheduled: 4 +Number of Nodes Scheduled with Up-to-date Pods: 4 +Number of Nodes Scheduled with Available Pods: 3 +Number of Nodes Misscheduled: 0 +Pods Status: 3 Running / 1 Waiting / 0 Succeeded / 0 Failed +``` + +The DaemonSet appears to be functioning correctly, as the Desired and Current pod numbers match the cluster's node count. However, we've identified one pod in a waiting state. Let's investigate further by examining the status of all aws-node pods in the cluster. This will help us pinpoint the cause of the waiting pod + +```bash +$ kubectl get pods -n kube-system -l k8s-app=aws-node -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +aws-node-5fgqh 2/2 Running 0 72m 10.42.165.112 ip-10-42-165-112.us-west-2.compute.internal +aws-node-mkjkr 0/2 Pending 0 20m +aws-node-shw94 2/2 Running 0 72m 10.42.145.11 ip-10-42-145-11.us-west-2.compute.internal +aws-node-v9dq6 2/2 Running 0 72m 10.42.102.141 ip-10-42-102-141.us-west-2.compute.internal +``` + +Let's select the Pending aws-node pod + +```bash +$ export AWS_NODE_POD=$(kubectl get pods -l k8s-app=aws-node -n kube-system | grep Pending | awk 'NR==1{print $1}') +``` + +An aws-node pod is currently in a Pending state, which is unusual for this critical system daemonset. Normally, aws-node should run even on NotReady nodes. Let's investigate further by describing the affected pod to identify the root cause of this issue. + +```bash +$ kubectl describe pod -n kube-system $AWS_NODE_POD +Name: aws-node-mkjkr +Namespace: kube-system +Priority: 2000001000 +Priority Class Name: system-node-critical +Service Account: aws-node +Node: +Labels: app.kubernetes.io/instance=aws-vpc-cni + app.kubernetes.io/name=aws-node + controller-revision-hash=774956ddb4 + k8s-app=aws-node + pod-template-generation=5 +Annotations: +Status: Pending +IP: +IPs: +Controlled By: DaemonSet/aws-node +Init Containers: + aws-vpc-cni-init: + Image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init:v1.16.0-eksbuild.1 + Port: + Host Port: + Requests: + cpu: 25m + memory: 2G + Environment: + DISABLE_TCP_EARLY_DEMUX: false + ENABLE_IPv6: false + AWS_STS_REGIONAL_ENDPOINTS: regional + AWS_DEFAULT_REGION: us-west-2 + AWS_REGION: us-west-2 + AWS_ROLE_ARN: arn:aws:iam::1234567890:role/eksctl-eks-workshop-addon-vpc-cni-Role1-rvDMIG8AaPGr + AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + Mounts: + /host/opt/cni/bin from cni-bin-dir (rw) + /var/run/secrets/eks.amazonaws.com/serviceaccount from aws-iam-token (ro) + /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2npz9 (ro) +Containers: + aws-node: + Image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.16.0-eksbuild.1 + Port: 61678/TCP + Host Port: 61678/TCP + Requests: + cpu: 25m + memory: 2G + Liveness: exec [/app/grpc-health-probe -addr=:50051 -connect-timeout=5s -rpc-timeout=5s] delay=60s timeout=10s period=10s #success=1 #failure=3 + Readiness: exec [/app/grpc-health-probe -addr=:50051 -connect-timeout=5s -rpc-timeout=5s] delay=1s timeout=10s period=10s #success=1 #failure=3 + Environment: + ADDITIONAL_ENI_TAGS: {} + ANNOTATE_POD_IP: false + AWS_VPC_CNI_NODE_PORT_SUPPORT: true + AWS_VPC_ENI_MTU: 9001 + AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG: false + AWS_VPC_K8S_CNI_EXTERNALSNAT: false + AWS_VPC_K8S_CNI_LOGLEVEL: DEBUG + AWS_VPC_K8S_CNI_LOG_FILE: /host/var/log/aws-routed-eni/ipamd.log + AWS_VPC_K8S_CNI_RANDOMIZESNAT: prng + AWS_VPC_K8S_CNI_VETHPREFIX: eni + AWS_VPC_K8S_PLUGIN_LOG_FILE: /var/log/aws-routed-eni/plugin.log + AWS_VPC_K8S_PLUGIN_LOG_LEVEL: DEBUG + CLUSTER_NAME: eks-workshop + DISABLE_INTROSPECTION: false + DISABLE_METRICS: false + DISABLE_NETWORK_RESOURCE_PROVISIONING: false + ENABLE_IPv4: true + ENABLE_IPv6: false + ENABLE_POD_ENI: true + ENABLE_PREFIX_DELEGATION: true + POD_SECURITY_GROUP_ENFORCING_MODE: standard + VPC_CNI_VERSION: v1.16.0 + VPC_ID: vpc-0be3747f7a6076d7c + WARM_ENI_TARGET: 1 + WARM_PREFIX_TARGET: 1 + MY_NODE_NAME: (v1:spec.nodeName) + MY_POD_NAME: aws-node-mkjkr (v1:metadata.name) + AWS_STS_REGIONAL_ENDPOINTS: regional + AWS_DEFAULT_REGION: us-west-2 + AWS_REGION: us-west-2 + AWS_ROLE_ARN: arn:aws:iam::1234567890:role/eksctl-eks-workshop-addon-vpc-cni-Role1-rvDMIG8AaPGr + AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + Mounts: + /host/etc/cni/net.d from cni-net-dir (rw) + /host/opt/cni/bin from cni-bin-dir (rw) + /host/var/log/aws-routed-eni from log-dir (rw) + /run/xtables.lock from xtables-lock (rw) + /var/run/aws-node from run-dir (rw) + /var/run/secrets/eks.amazonaws.com/serviceaccount from aws-iam-token (ro) + /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2npz9 (ro) + aws-eks-nodeagent: + Image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-network-policy-agent:v1.0.7-eksbuild.1 + Port: + Host Port: + Args: + --enable-ipv6=false + --enable-network-policy=true + --enable-cloudwatch-logs=false + --enable-policy-event-logs=true + --metrics-bind-addr=:8162 + --health-probe-bind-addr=:8163 + --conntrack-cache-cleanup-period=300 + Requests: + cpu: 25m + memory: 2G + Environment: + MY_NODE_NAME: (v1:spec.nodeName) + AWS_STS_REGIONAL_ENDPOINTS: regional + AWS_DEFAULT_REGION: us-west-2 + AWS_REGION: us-west-2 + AWS_ROLE_ARN: arn:aws:iam::1234567890:role/eksctl-eks-workshop-addon-vpc-cni-Role1-rvDMIG8AaPGr + AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + Mounts: + /host/opt/cni/bin from cni-bin-dir (rw) + /sys/fs/bpf from bpf-pin-path (rw) + /var/log/aws-routed-eni from log-dir (rw) + /var/run/aws-node from run-dir (rw) + /var/run/secrets/eks.amazonaws.com/serviceaccount from aws-iam-token (ro) + /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-2npz9 (ro) +Conditions: + Type Status + PodScheduled False +Volumes: + aws-iam-token: + Type: Projected (a volume that contains injected data from multiple sources) + TokenExpirationSeconds: 86400 + bpf-pin-path: + Type: HostPath (bare host directory volume) + Path: /sys/fs/bpf + HostPathType: + cni-bin-dir: + Type: HostPath (bare host directory volume) + Path: /opt/cni/bin + HostPathType: + cni-net-dir: + Type: HostPath (bare host directory volume) + Path: /etc/cni/net.d + HostPathType: + log-dir: + Type: HostPath (bare host directory volume) + Path: /var/log/aws-routed-eni + HostPathType: DirectoryOrCreate + run-dir: + Type: HostPath (bare host directory volume) + Path: /var/run/aws-node + HostPathType: DirectoryOrCreate + xtables-lock: + Type: HostPath (bare host directory volume) + Path: /run/xtables.lock + HostPathType: + kube-api-access-2npz9: + Type: Projected (a volume that contains injected data from multiple sources) + TokenExpirationSeconds: 3607 + ConfigMapName: kube-root-ca.crt + ConfigMapOptional: + DownwardAPI: true +QoS Class: Burstable +Node-Selectors: +Tolerations: op=Exists + node.kubernetes.io/disk-pressure:NoSchedule op=Exists + node.kubernetes.io/memory-pressure:NoSchedule op=Exists + node.kubernetes.io/network-unavailable:NoSchedule op=Exists + node.kubernetes.io/not-ready:NoExecute op=Exists + node.kubernetes.io/pid-pressure:NoSchedule op=Exists + node.kubernetes.io/unreachable:NoExecute op=Exists + node.kubernetes.io/unschedulable:NoSchedule op=Exists +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning FailedScheduling 3m15s (x5 over 14m) default-scheduler 0/4 nodes are available: 1 Insufficient memory. preemption: 0/4 nodes are available: 1 Insufficient memory, 3 Preemption is not helpful for scheduling. +``` + +The issue stems from insufficient memory on one of the nodes to meet the aws-node pod's 4 GB memory request. To resolve this, we have two options: + +1. Create a new nodegroup with more resources +2. Adjust the memory request for the aws-node daemonset + +We'll proceed with option 2. Since the aws-node daemonset is deployed via Amazon VPC CNI managed addons, let's examine the addon configuration. + +```bash test=false +$ aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --output text --query addon.configurationValues | jq . +{ + "env": { + "ENABLE_PREFIX_DELEGATION": "true", + "ENABLE_POD_ENI": "true", + "POD_SECURITY_GROUP_ENFORCING_MODE": "standard" + }, + "enableNetworkPolicy": "true", + "nodeAgent": { + "enablePolicyEventLogs": "true" + }, + "resources": { + "requests": { + "memory": "2G" + } + } +} +``` + +### Step 4 + +Having identified the necessary VPC CNI configuration adjustments for aws-node compatibility, let's proceed to update our setup. + +1. Remove the existing resource definitions and create a variable with the revised configuration +```bash +$ export CURRENT_CONFIG=$(aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --output text --query addon.configurationValues) +$ export REVISED_CONFIG=$(echo $CURRENT_CONFIG | jq -c 'del(.resources)') +``` +2. Maintain IRSA Configuration for VPC CNI Add-ons: When updating the configuration of VPC CNI managed add-ons, it's crucial to preserve the existing IAM Role for Service Account (IRSA) setup. Before making any changes, identify the associated IAM role to ensure it remains intact throughout the update process. +```bash +$ export CNI_ROLE_ARN=$(aws eks describe-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --output text --query addon.serviceAccountRoleArn) +``` +3. Apply the configuration changes using `aws eks update-addon` CLI command: + +```bash timeout=180 hook=fix-1 hookTimeout=600 +$ aws eks update-addon --addon-name vpc-cni --cluster-name $EKS_CLUSTER_NAME --service-account-role-arn $CNI_ROLE_ARN --configuration-values $REVISED_CONFIG +``` + +After completing the update process, verify that aws-node pods are now scheduled on all worker nodes. Check the pod distribution to ensure proper deployment across the cluster. + +```bash +$ kubectl get pods -n kube-system -l k8s-app=aws-node -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +aws-node-5fgqh 2/2 Running 0 83m 10.42.165.112 ip-10-42-165-112.us-west-2.compute.internal +aws-node-5jwzn 1/2 Running 0 32s 100.64.3.8 ip-100-64-3-8.us-west-2.compute.internal +aws-node-v9dq6 2/2 Running 0 83m 10.42.102.141 ip-10-42-102-141.us-west-2.compute.internal +aws-node-zwxhf 1/2 Running 0 31s 10.42.145.11 ip-10-42-145-11.us-west-2.compute.internal +``` diff --git a/website/docs/troubleshooting/cni/cni_fix_5.md b/website/docs/troubleshooting/cni/cni_fix_5.md new file mode 100644 index 000000000..feb36256c --- /dev/null +++ b/website/docs/troubleshooting/cni/cni_fix_5.md @@ -0,0 +1,224 @@ +--- +title: "Section 2 - Fixing Policy Issue" +sidebar_position: 30 +--- + +### Step 5 + +Well done on scheduling aws-node on all worker nodes! Now, let's tackle our main objective: getting the nginx-app pods running. They're currently still stuck in Pending status. Your next task is to investigate the cause and implement a solution to achieve full cluster functionality. Ready to continue troubleshooting? + +```bash +$ kubectl get pods -n cni-tshoot +NAME READY STATUS RESTARTS AGE +nginx-app-5cf4cbfd97-2v8tp 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-5nw9n 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-9vf6k 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-bmtb2 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-d9wcn 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-lplnw 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-n2whf 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-pxf57 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-sb4fm 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-spv2b 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-v2xp2 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-vllxz 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-wbvtv 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-wm7xd 0/1 Pending 0 16m +nginx-app-5cf4cbfd97-wn9lc 0/1 Pending 0 16m +``` + +```bash +$ kubectl get nodes -L app +NAME STATUS ROLES AGE VERSION APP +ip-10-42-102-141.us-west-2.compute.internal Ready 160m v1.30.0-eks-036c24b +ip-10-42-145-11.us-west-2.compute.internal Ready 160m v1.30.0-eks-036c24b +ip-10-42-165-112.us-west-2.compute.internal Ready 160m v1.30.0-eks-036c24b +ip-100-64-3-8.us-west-2.compute.internal NotReady 82m v1.30.4-eks-a737599 cni_troubleshooting +``` + +Despite the aws-node being scheduled, the worker nodes remain in a NotReady state. Let's investigate further by describing the node again to assess its current status. + +```bash test=false +$ kubectl describe node $NODE_NAME | grep Conditions: -A 7 +Conditions: + Type Status LastHeartbeatTime LastTransitionTime Reason Message + ---- ------ ----------------- ------------------ ------ ------- + MemoryPressure False Wed, 30 Oct 2024 20:43:45 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available + DiskPressure False Wed, 30 Oct 2024 20:43:45 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure + PIDPressure False Wed, 30 Oct 2024 20:43:45 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletHasSufficientPID kubelet has sufficient PID available + Ready False Wed, 30 Oct 2024 20:43:45 +0000 Wed, 30 Oct 2024 19:21:08 +0000 KubeletNotReady container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized +``` + +:::info +To ensure EKS worker nodes become Ready when using the default AWS VPC CNI, all containers in the aws-node pod must be ready +::: +Let's identify the aws-node pod on this node: + +1. Examine the Non-terminated Pods section in the output. +2. Locate the pod name starting with 'aws-node'. + +This step is crucial for troubleshooting node readiness issues." + +```bash test=false +$ kubectl describe node $NODE_NAME | grep Non-terminated -A 6 +Non-terminated Pods: (3 in total) + Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age + --------- ---- ------------ ---------- --------------- ------------- --- + kube-system aws-node-5jwzn 50m (2%) 0 (0%) 0 (0%) 0 (0%) 53m + kube-system kube-proxy-69754 100m (5%) 0 (0%) 0 (0%) 0 (0%) 84m +``` + +Let's capture the pod name + +```bash test=false +$ AWS_NODE_POD=$(kubectl get pods -n kube-system -l k8s-app=aws-node -o wide | grep $NODE_NAME| awk 'NR==1{print $1}') +``` + +Having identified the pod name, our next step is to examine the pod's details. Let's use the describe command to focus on the readiness status of each container within the pod. + +```bash test=false +$ kubectl describe pods -n kube-system $AWS_NODE_POD + +Containers: + aws-node: + Container ID: containerd://24af57294e0285468b03bb7e5a27a3daa7d00834c20f915c67441197ac4fc869 + Image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.16.0-eksbuild.1 + Image ID: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni@sha256:61e1a92ff2e63e3130db430c773736450fe941ed8701b77dd20ac6e8546f8255 + Port: 61678/TCP + Host Port: 61678/TCP + State: Running + Started: Wed, 30 Oct 2024 20:52:21 +0000 + Last State: Terminated + Reason: Error + Exit Code: 2 + Started: Wed, 30 Oct 2024 20:45:41 +0000 + Finished: Wed, 30 Oct 2024 20:47:11 +0000 + Ready: False + Restart Count: 18 + Requests: + cpu: 25m + Liveness: exec [/app/grpc-health-probe -addr=:50051 -connect-timeout=5s -rpc-timeout=5s] delay=60s timeout=10s period=10s #success=1 #failure=3 + Readiness: exec [/app/grpc-health-probe -addr=:50051 -connect-timeout=5s -rpc-timeout=5s] delay=1s timeout=10s period=10s #success=1 #failure=3 + + + + aws-eks-nodeagent: + Container ID: containerd://a51df446be528d4dad3649b57ad2a53ae4dc163d230dabe275e20bced4c8b5d0 + Image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-network-policy-agent:v1.0.7-eksbuild.1 + Image ID: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-network-policy-agent@sha256:0e7fd75230dee735c1636ad86d69bf38b1bc48e1b78459147957827d978e5635 + Port: + Host Port: + Args: + --enable-ipv6=false + --enable-network-policy=true + --enable-cloudwatch-logs=false + --enable-policy-event-logs=true + --metrics-bind-addr=:8162 + --health-probe-bind-addr=:8163 + --conntrack-cache-cleanup-period=300 + State: Running + Started: Wed, 30 Oct 2024 19:52:36 +0000 + Ready: True + Restart Count: 0 +``` + +The output reveals that the aws-node container within the aws-node pod is not transitioning to a ready state (Ready=False). This is likely due to a failed Readiness probe, which is configured for the container. To resolve this issue, investigate the aws-node container to identify and address the factors preventing the Readiness probe from passing. + +### Step 6 + +Now, let's examine the event section of the describe pod output. Identify this crucial information and be prepared to analyze it. + +```bash test=false +$ kubectl describe pods -n kube-system aws-node +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning MissingIAMPermissions 33m (x2 over 33m) aws-node Unauthorized operation: failed to call ec2:DescribeNetworkInterfaces due to missing permissions. Please refer https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md to attach relevant policy to IAM role + Warning MissingIAMPermissions 31m (x2 over 31m) aws-node Unauthorized operation: failed to call ec2:DescribeNetworkInterfaces due to missing permissions. Please refer https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md to attach relevant policy to IAM role + Warning MissingIAMPermissions 53m (x2 over 53m) aws-node Unauthorized operation: failed to call ec2:DescribeNetworkInterfaces due to missing permissions. Please refer https://github.com/aws/amazon-vpc-cni-k8s/blob/master/docs/iam-policy.md to attach relevant policy to IAM role +``` + +This error suggests that the aws-node/L-IPAMD component lacks the necessary permissions to access the EC2 API, specifically the ec2:DescribeNetworkInterfaces action. + +### Step 7 + +Let's verify the VPC CNI IAM role whether it has the required **AmazonEKS_CNI_Policy** managed policy. This policy should be associted with IAM role assigned during the VPC CNI addon configuration update in Step 5. Let's check the role's policies to confirm this. + +:::info +For your convenience, we have set the environment variable **VPC_CNI_IAM_ROLE_NAME** to contain the name of the IAM role associated with the VPC CNI managed add-on. +::: + +```bash +$ aws iam list-attached-role-policies --role-name $VPC_CNI_IAM_ROLE_NAME +{ + "AttachedPolicies": [] +} +``` + +We now know that the VPC CNI role is missing **AmazonEKS_CNI_Policy**. + +### Step 8 + +Having identified the missing policy, let's resolve this issue by attaching the necessary IAM managed policy. Execute the following command to implement the fix: + +```bash timeout=180 hook=fix-5 hookTimeout=600 +$ aws iam attach-role-policy --role-name $VPC_CNI_IAM_ROLE_NAME --policy-arn arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy +``` + +Let's confirm the attachment of the specified IAM managed policy to the VPC CNI add-ons role + +```bash +$ aws iam list-attached-role-policies --role-name $VPC_CNI_IAM_ROLE_NAME +{ + "AttachedPolicies": [ + { + "PolicyName": "AmazonEKS_CNI_Policy", + "PolicyArn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy" + } + ] +} +``` + +Allow a few minutes for aws-node to process the changes. Monitor the status until aws-node displays READY (2/2). Once confirmed, proceed to the next step + +```bash +$ kubectl get pods -n kube-system -l k8s-app=aws-node -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +aws-node-5jwzn 2/2 Running 20 (6m38s ago) 69m 100.64.3.8 ip-100-64-3-8.us-west-2.compute.internal +aws-node-l8gzs 2/2 Running 0 70s 10.42.102.141 ip-10-42-102-141.us-west-2.compute.internal +aws-node-nqwjc 2/2 Running 0 66s 10.42.165.112 ip-10-42-165-112.us-west-2.compute.internal +aws-node-zwxhf 2/2 Running 20 (6m18s ago) 69m 10.42.145.11 ip-10-42-145-11.us-west-2.compute.internal +``` + +Ensure all nodes are in a 'Ready' state before proceeding. + +```bash +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-10-42-102-141.us-west-2.compute.internal Ready 179m v1.30.0-eks-036c24b +ip-10-42-145-11.us-west-2.compute.internal Ready 179m v1.30.0-eks-036c24b +ip-10-42-165-112.us-west-2.compute.internal Ready 179m v1.30.0-eks-036c24b +ip-100-64-3-8.us-west-2.compute.internal Ready 101m v1.30.4-eks-a737599 +``` + +Verify that all app containers are scheduled to a node and display a 'ContainerCreating' status. This indicates proper initial deployment + +```bash +$ kubectl get pods -n cni-tshoot -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +nginx-app-5cf4cbfd97-2v8tp 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-5nw9n 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-9vf6k 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-bmtb2 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-d9wcn 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-lplnw 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-n2whf 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-pxf57 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-sb4fm 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-spv2b 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-v2xp2 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-vllxz 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-wbvtv 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-wm7xd 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +nginx-app-5cf4cbfd97-wn9lc 0/1 ContainerCreating 0 85m ip-100-64-3-8.us-west-2.compute.internal +``` diff --git a/website/docs/troubleshooting/cni/cni_fix_9.md b/website/docs/troubleshooting/cni/cni_fix_9.md new file mode 100644 index 000000000..8e174122b --- /dev/null +++ b/website/docs/troubleshooting/cni/cni_fix_9.md @@ -0,0 +1,204 @@ +--- +title: "Section 3 - Expanding Worker Node Subnet" +sidebar_position: 30 +--- + +### Step 9 + +Congratulations, The nginx-app pods have been scheduled on the new node. However, they are stuck in the ContainerCreating state. To troubleshoot this issue, describe one of the affected pods and examine the Events section of the output. This will help identify the root cause of the problem. + +```bash test=false +$ kubectl describe pod -n cni-tshoot +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning FailedScheduling 23m (x15 over 93m) default-scheduler 0/4 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }, 3 node(s) didn't match Pod's node affinity/selector. preemption: 0/4 nodes are available: 4 Preemption is not helpful for scheduling. + Normal Scheduled 23m default-scheduler Successfully assigned cni-tshoot/nginx-app-5cf4cbfd97-2v8tp to ip-100-64-3-8.us-west-2.compute.internal + Warning FailedCreatePodSandBox 23m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "14bffa7734d01abd808dead23744386135518961ab240ba48b88a9e269398126": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 23m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "1c18b4373dd31dcc13500dd1f8465bf34ce1659560b9ad430ff75d543b9d6775": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 22m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "de3888bd06bb284aafe0531c42c672a0019351f3346b64e30813007ac04b8c43": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 22m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "44223b1930b6a487866048dd918c752e6b69a5da00bc48fa24552a4446bcc1fb": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 22m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "fdbafbf1c5c543ba8667a49156116d4892d9cfaabc413fb7e4bf80e242eb137d": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 22m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "a791bb0db767a4df6a51c5b8ce3a3139683bb65785d45bb05f4f42ad6db0ca83": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 21m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "badb3b775bcde51577e9a1bceb0c38f87d7df34a6fa7282799e9febf0d715e71": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 21m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "c8472dc0407b051e786bc9bd2a3b277a514b4f4897e3ac186c349b3061b219da": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 21m kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "ba6330c1dc7d5bcf715a635569c54e1eb67f40ea26aa00db56d54e6ddb25ea7f": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container + Warning FailedCreatePodSandBox 3m7s (x84 over 21m) kubelet (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "ee02fa03d490b4d0691df0f8db608a7c5c4c7983b498eee620a9494c31eb1628": plugin type="aws-cni" name="aws-cni" failed (add): add cmd: failed to assign an IP address to container +``` + +:::info VPC CNI Overview and Troubleshooting: + +The VPC CNI (Container Network Interface) operates by placing the aws-cni binary in the /opt/cni/bin directory of each node. This binary plays a crucial role during pod creation: + +- It's executed when a pod's network namespace is created. +- It requests an available IP address from the L-IPAMD (IP Address Management Daemon), also known as aws-node. +- This request is made via a gRPC call to a local socket on port 50051. + +When troubleshooting network issues, focus on two main components: + +- The aws-cni binary logs (/var/log/aws-routed-eni/plugin.log) +- The L-IPAMD (aws-node) logs (/var/log/aws-routed-eni/ipamd.log) + +**Note**: These logs are typically only accessible from the node itself. To facilitate log collection, AWS provides a specialized tool called [the EKS Log Collector](https://github.com/awslabs/amazon-eks-ami/tree/main/log-collector-script/). You can find this script in the Amazon EKS AMI GitHub repository. +::: + +The next step, we'll utilize the AWS Systems Manager (SSM) Agent to execute commands on our EKS nodes. The SSM Agent is pre-installed on EKS Optimized Amazon Machine Images (AMIs). To enable its functionality, we need to attach the **AmazonSSMManagedInstanceCore** IAM managed policy to the IAM role associated with the node. We've prepared a script to assist you in retrieving logs from the underlying EC2 instance. + +```bash test=false +$ cat eks-workshop/modules/troubleshooting/cni/.workshop/ssm.sh + +#!/bin/bash +COMMAND_ID=$(aws ssm send-command \ + --instance-ids $1 \ + --document-name "AWS-RunShellScript" \ + --comment "Demo run shell script on Linux Instances" \ + --parameters '{"commands":["sudo -Hiu root bash << END","tail -n '$3' /var/log/aws-routed-eni/'$2'.log | grep '$4'", "END"]}' \ + --output text \ + --query "Command.CommandId") + +STATUS=InProgress +while [ "$STATUS" == "InProgress" ]; do + STATUS=$(aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id $1 \ + --output text \ + --query "Status") +done + +aws ssm list-command-invocations \ + --command-id "$COMMAND_ID" \ + --details \ + --output text \ + --query "CommandInvocations[].CommandPlugins[].Output" +``` + +### Step 10 + +First, we'll select a representative pod name as our focus + +```bash +$ export POD_NAME=$(kubectl get pods -n cni-tshoot -o custom-columns=:metadata.name --no-headers | awk 'NR==1{print $1}') +``` + +Next, we'll identify the specific node where our pod is currently running by retrieving its instance ID + +```bash +$ export NODE_NAME=$(kubectl get pod $POD_NAME -n cni-tshoot -o custom-columns=:spec.nodeName --no-headers) +$ export NODE_ID=$(kubectl get node $NODE_NAME -o=jsonpath='{.spec.providerID}' | cut -d "/" -f 5) +``` + +Now that we have completed the setup, let's confirm that the CNI plugin received the request to provision a network namespace for the pod named $POD_NAME. This verification step is crucial to ensure proper communication between the container runtime and VPC CNI plugin. + +```bash test=false +$ bash eks-workshop/modules/troubleshooting/cni/.workshop/ssm.sh $NODE_ID plugin 200 $POD_NAME +{"level":"info","ts":"2024-10-30T21:31:30.508Z","caller":"routed-eni-cni-plugin/cni.go:125","msg":"Received CNI add request: ContainerID(493b063c06e901827b21737af8d543a77f724b2d8ce97d650a6d1703b724a549) Netns(/var/run/netns/cni-5b637183-286d-95ec-1c4a-de61ed54de26) IfName(eth0) Args(K8S_POD_INFRA_CONTAINER_ID=493b063c06e901827b21737af8d543a77f724b2d8ce97d650a6d1703b724a549;K8S_POD_UID=53b79f5f-dc74-454d-bdde-5a9282ac6ace;IgnoreUnknown=1;K8S_POD_NAMESPACE=cni-tshoot;K8S_POD_NAME=nginx-app-5cf4cbfd97-2v8tp) Path(/opt/cni/bin) argsStdinData({\"cniVersion\":\"1.0.0\",\"mtu\":\"9001\",\"name\":\"aws-cni\",\"pluginLogFile\":\"/var/log/aws-routed-eni/plugin.log\",\"pluginLogLevel\":\"DEBUG\",\"podSGEnforcingMode\":\"standard\",\"type\":\"aws-cni\",\"vethPrefix\":\"eni\"})"} +{"level":"info","ts":"2024-10-30T21:31:30.545Z","caller":"routed-eni-cni-plugin/cni.go:282","msg":"Received CNI del request: ContainerID(493b063c06e901827b21737af8d543a77f724b2d8ce97d650a6d1703b724a549) Netns(/var/run/netns/cni-5b637183-286d-95ec-1c4a-de61ed54de26) IfName(eth0) Args(IgnoreUnknown=1;K8S_POD_NAMESPACE=cni-tshoot;K8S_POD_NAME=nginx-app-5cf4cbfd97-2v8tp;K8S_POD_INFRA_CONTAINER_ID=493b063c06e901827b21737af8d543a77f724b2d8ce97d650a6d1703b724a549;K8S_POD_UID=53b79f5f-dc74-454d-bdde-5a9282ac6ace) Path(/opt/cni/bin) argsStdinData({\"cniVersion\":\"1.0.0\",\"mtu\":\"9001\",\"name\":\"aws-cni\",\"pluginLogFile\":\"/var/log/aws-routed-eni/plugin.log\",\"pluginLogLevel\":\"DEBUG\",\"podSGEnforcingMode\":\"standard\",\"type\":\"aws-cni\",\"vethPrefix\":\"eni\"})"} +``` + +Excellent! We've confirmed that the CNI plugin successfully received the request to set up the network namespace for pod $POD_NAME. Our next step is to check if the L-IPAMD received an IP address request from the CNI plugin for this specific pod. + +```bash test=false +$ bash eks-workshop/modules/troubleshooting/cni/.workshop/ssm.sh $NODE_ID ipamd 200 $POD_NAME +{"level":"debug","ts":"2024-10-30T21:32:23.519Z","caller":"rpc/rpc.pb.go:713","msg":"AddNetworkRequest: K8S_POD_NAME:\"nginx-app-5cf4cbfd97-2v8tp\" K8S_POD_NAMESPACE:\"cni-tshoot\" K8S_POD_INFRA_CONTAINER_ID:\"1921d4fa98f25f481b0d5935eebd0e8e4b8c2b937b2d98277828ada327069393\" ContainerID:\"1921d4fa98f25f481b0d5935eebd0e8e4b8c2b937b2d98277828ada327069393\" IfName:\"eth0\" NetworkName:\"aws-cni\" Netns:\"/var/run/netns/cni-65045971-ebe5-d672-a6b5-f7690545d61e\""} +{"level":"debug","ts":"2024-10-30T21:32:23.556Z","caller":"rpc/rpc.pb.go:731","msg":"DelNetworkRequest: K8S_POD_NAME:\"nginx-app-5cf4cbfd97-2v8tp\" K8S_POD_NAMESPACE:\"cni-tshoot\" K8S_POD_INFRA_CONTAINER_ID:\"1921d4fa98f25f481b0d5935eebd0e8e4b8c2b937b2d98277828ada327069393\" Reason:\"PodDeleted\" ContainerID:\"1921d4fa98f25f481b0d5935eebd0e8e4b8c2b937b2d98277828ada327069393\" IfName:\"eth0\" NetworkName:\"aws-cni\""} +``` + +Great! We've confirmed that the L-IPAMD successfully received an IP address request for the pod named $POD_NAME from the CNI plugin. Our next step is to examine the IP address pool managed by the IPAMD at the time this request was processed. + +```bash test=false +$ bash eks-workshop/modules/troubleshooting/cni/.workshop/ssm.sh $NODE_ID ipamd 200 datastore/data +{"level":"debug","ts":"2024-10-30T21:32:54.531Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: IP address pool stats: total 0, assigned 0"} +{"level":"debug","ts":"2024-10-30T21:32:54.531Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: ENI eni-0ee6517834f4b39ac does not have available addresses"} +{"level":"error","ts":"2024-10-30T21:32:54.531Z","caller":"datastore/data_store.go:607","msg":"DataStore has no available IP/Prefix addresses"} +{"level":"debug","ts":"2024-10-30T21:32:54.564Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: IP address pool stats: total 0, assigned 0"} +{"level":"debug","ts":"2024-10-30T21:32:54.564Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: ENI eni-0ee6517834f4b39ac does not have available addresses"} +{"level":"error","ts":"2024-10-30T21:32:54.564Z","caller":"datastore/data_store.go:607","msg":"DataStore has no available IP/Prefix addresses"} +{"level":"debug","ts":"2024-10-30T21:32:56.559Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: IP address pool stats: total 0, assigned 0"} +{"level":"debug","ts":"2024-10-30T21:32:56.559Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: ENI eni-0ee6517834f4b39ac does not have available addresses"} +{"level":"error","ts":"2024-10-30T21:32:56.559Z","caller":"datastore/data_store.go:607","msg":"DataStore has no available IP/Prefix addresses"} +{"level":"debug","ts":"2024-10-30T21:32:56.566Z","caller":"datastore/data_store.go:607","msg":"AssignPodIPv4Address: IP address pool stats: total 0, assigned 0"} +``` + +Upon examination, we've discovered that the IP Address Management (IPAMD) warm pool is currently empty. This indicates that we should shift our focus to investigating the configuration of the Virtual Private Cloud (VPC) subnets. + +### Step 11 + +As we turn our attention to VPC subnet configuration, our first step is to locate the subnet where the worker node instances are deployed. Once identified, we'll examine the number of available IP addresses within that subnet. + +```bash +$ export NODE_SUBNET=$(aws ec2 describe-instances --instance-ids $NODE_ID --query 'Reservations[0].Instances[0].SubnetId' --output text) +$ aws ec2 describe-subnets --subnet-ids $NODE_SUBNET --output text --query 'Subnets[0].AvailableIpAddressCount' +10 +``` + +We've identified that this particular node's subnet is running low on available IP addresses. To get a comprehensive view of the situation, it's important to assess the IP address availability across all subnets associated with this nodegroup. Let's proceed to examine the remaining subnets within this nodegroup to determine their current IP address capacity. + +```bash +$ export NODE_GROUP_SUBNETS=$(aws eks describe-nodegroup --cluster-name $EKS_CLUSTER_NAME --nodegroup-name cni_troubleshooting_nodes --query 'nodegroup.subnets' --output text) +$ aws ec2 describe-subnets --subnet-ids $NODE_GROUP_SUBNETS --output text --query 'Subnets[*].AvailableIpAddressCount' +11 11 10 +``` + +Having identified that our existing node subnets are running low on available IP addresses, we need to take action. The solution is to create a new nodegroup that utilizes subnets with a larger pool of available IPs. It's important to note that we cannot modify an existing nodegroup to use different subnets, which is why creating a new nodegroup is necessary. + +### Step 12 + +When expanding your EKS cluster, it's possible to create new subnets for additional nodegroups, even if these subnets weren't part of the original cluster configuration. +:::info +In our case, we've established new subnets on a secondary VPC CIDR to accommodate our new nodegroups, they are defined in these environment variables + +1. **ADDITIONAL_SUBNET_1** +2. **ADDITIONAL_SUBNET_2** +3. **ADDITIONAL_SUBNET_3** + +::: + +Before we move forward with creating a new managed nodegroup, it's essential to confirm that there are sufficient IP addresses available in these newly created subnets. + +```bash +$ aws ec2 describe-subnets --subnet-ids $ADDITIONAL_SUBNET_1 $ADDITIONAL_SUBNET_2 $ADDITIONAL_SUBNET_3 --output text --query 'Subnets[*].AvailableIpAddressCount' +251 251 251 +``` + +Now that we've confirmed the availability of sufficient IP addresses, we can proceed to create a new managed node group. + +```bash +$ aws eks create-nodegroup --region $AWS_REGION \ + --cluster-name $EKS_CLUSTER_NAME \ + --nodegroup-name new-cni-nodes \ + --instance-types m5.large --node-role $NODEGROUP_IAM_ROLE \ + --subnets $ADDITIONAL_SUBNET_1 $ADDITIONAL_SUBNET_2 $ADDITIONAL_SUBNET_3 \ + --labels app=cni_troubleshooting \ + --taints key=purpose,value=cni_troubleshooting,effect=NO_SCHEDULE\ + --scaling-config minSize=1,maxSize=3,desiredSize=1 +``` + +During the creation of the new node group, we'll initiate a scale-in process for the existing node group. This action will encourage the automatic rescheduling of pods onto the newly created nodes, ensuring a smooth transition to the updated infrastructure. + +```bash timeout=180 hook=fix-9 hookTimeout=600 +$ aws eks update-nodegroup-config --cluster-name $EKS_CLUSTER_NAME --nodegroup-name cni_troubleshooting_nodes --scaling-config minSize=0,maxSize=1,desiredSize=0 +``` + +After the new node group becomes fully operational and the old node group has scaled down completely, you should observe that all your nginx-app pods are in a **Running** state. + +```bash +$ kubectl get pods -n cni-tshoot +NAME READY STATUS RESTARTS AGE +nginx-app-5cf4cbfd97-28lv7 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-5strx 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-6rz56 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-866kt 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-8h2dg 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-9r98g 1/1 Running 0 6m21s +nginx-app-5cf4cbfd97-f5gxn 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-kqrf2 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-pp6vd 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-pth6m 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-q7rfd 1/1 Running 0 6m21s +nginx-app-5cf4cbfd97-rl6fp 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-rptlq 1/1 Running 0 6m22s +nginx-app-5cf4cbfd97-t9cgr 1/1 Running 0 6m21s +nginx-app-5cf4cbfd97-zb8dk 1/1 Running 0 6m22s +``` + +### Congratulations! you deserve a pat on the back diff --git a/website/docs/troubleshooting/cni/index.md b/website/docs/troubleshooting/cni/index.md new file mode 100644 index 000000000..d3da694c3 --- /dev/null +++ b/website/docs/troubleshooting/cni/index.md @@ -0,0 +1,147 @@ +--- +title: "VPC CNI scenario" +sidebar_position: 20 +chapter: true +sidebar_custom_props: { "module": true } +description: "Troubleshooting issues related to EKS VPC CNI IAM policy, IP allocation and scheduling" +--- + +::required-time + +In this section, we will explore techniques for diagnosing and resolving common issues related to the VPC CNI (Virtual Private Cloud Container Network Interface). This hands-on scenario will equip you with practical troubleshooting skills for VPC CNI problems. For a comprehensive understanding of VPC CNI functionality and architecture, please refer to our [Networking module](/docs/networking/vpc-cni), which provides in-depth coverage of the topic. + +:::tip Before you start +Prepare your environment for this section: + +```bash timeout=600 wait=300 +$ prepare-environment troubleshooting/cni +``` + +**Lab Setup Overview:** + +Before we begin, please allow a few minutes for the lab environment to be prepared. This setup process will implement the following modifications: + +1. **VPC Configuration**: A secondary CIDR will be added to the existing Virtual Private Cloud. +2. **Subnet Creation**: New subnets will be established within the VPC. +3. **Application Deployment**: A base application will be pre-installed for our exercises. +4. **Managed Nodegroup Creation**: An additional Managed Nodegroup will be set up in the Amazon EKS cluster to set our working environment. +5. **Troubleshooting Scenario**: A deliberate configuration issue will be introduced. This will provide us with a practical opportunity to develop and apply troubleshooting skills specific to this environment. + +::: + +You can view the Terraform that applies these changes [here](https://github.com/VAR::MANIFESTS_OWNER/VAR::MANIFESTS_REPOSITORY/tree/VAR::MANIFESTS_REF/manifests/modules/troubleshooting/cni/.workshop/terraform). + +:::info Root Cause Analysis (RCA) Methodology + +While we wait for the scenario to finalize its configuration, lets talk about the _RCA Methodology_ really quick. + +The Root Cause Analysis (RCA) helps in identifying how and why an event or failure happened, allowing for corrective and preventive measures to be put in place and the RCA generally serves as input to a remediation process whereby corrective actions are taken to prevent the problem from reoccurring. + +**_The method steps:_** + +1. Identify and describe the problem clearly. +2. Collect data +3. Establish a timeline from the normal situation until the problem occurs. +4. Identify Root Cause +5. Distinguish between the root cause and other causal factors (e.g., using event correlation). +6. Establish a causal graph between the root cause and the problem. +7. Although the word "cause" is singular in RCA, experience shows that generally causes are plural. Therefore, look for multiple causes when carrying out RCA. + +::: + +Next, we'll confirm that our newly created Managed Nodegroup has been properly configured with the specified capacity. We'll also check to ensure that the corresponding EC2 instances are in a 'Running' state. + +```bash +$ aws eks describe-nodegroup --cluster-name $EKS_CLUSTER_NAME --nodegroup-name cni_troubleshooting_nodes +{ + "nodegroup": { + "nodegroupName": "cni_troubleshooting_nodes", + "nodegroupArn": "arn:aws:eks:us-west-2:1234567890:nodegroup/eks-workshop/cni_troubleshooting_nodes/acc9931f-58b2-77c2-7af8-a547204f05ed", + "clusterName": "eks-workshop", + "version": "1.30", + "releaseVersion": "1.30.4-20241109", + "createdAt": "2024-10-30T18:57:44.538000+00:00", + "modifiedAt": "2024-10-3018:57:44.538000+00:00", + "status": "ACTIVE", + "capacityType": "ON_DEMAND", + "scalingConfig": { + "minSize": 1, + "maxSize": 3, + "desiredSize": 1 + }, + "instanceTypes": [ + "m5.large" + ], + "subnets": [ + "subnet-0b055bfb40a357c1d", + "subnet-0b450c109787fc1a7", + "subnet-0de03cca5fade5784" + ], + "amiType": "AL2023_x86_64_STANDARD", + "nodeRole": "arn:aws:iam::1234567890:role/terraform-20241113154725667800000001", + "labels": { + "app": "cni_troubleshooting" + }, + "taints": [ + { + "key": "purpose", + "value": "cni_troubleshooting", + "effect": "NO_SCHEDULE" + } + ], + "diskSize": 20, + "health": { + "issues": [] + }, + "updateConfig": { + "maxUnavailable": 1 + }, + "tags": {} + } +} +``` + +and + +```bash +$ aws ec2 describe-instances --filters "Name=tag:kubernetes.io/cluster/eks-workshop,Values=owned" --query 'Reservations[*].Instances[*].[PrivateDnsName,State.Name]' --output text +ip-10-42-120-86.us-west-2.compute.internal running +ip-10-42-139-247.us-west-2.compute.internal running +ip-10-42-172-120.us-west-2.compute.internal running +ip-100-64-3-8.us-west-2.compute.internal running +``` + +Next, we will be verifying whether the new nodes are joining the cluster and base app was deployed + +```bash +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-10-42-120-86.us-west-2.compute.internal Ready 15m v1.30.0-eks-036c24b +ip-10-42-139-247.us-west-2.compute.internal Ready 15m v1.30.0-eks-036c24b +ip-10-42-172-120.us-west-2.compute.internal Ready 15m v1.30.0-eks-036c24b +ip-100-64-3-8.us-west-2.compute.internal NotReady 42s v1.30.4-eks-a737599 +``` + +and + +```bash +$ kubectl get pods -n cni-tshoot +NAME READY STATUS RESTARTS AGE +nginx-app-5cf4cbfd97-5xbtb 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-6pr6g 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-9td6m 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-ctf9s 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-dgr5z 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-ghx4z 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-gwwmb 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-hjld5 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-hr64c 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-jwjsz 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-l4dpk 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-lhr25 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-qn4g8 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-t8l6l 0/1 Pending 0 10m +nginx-app-5cf4cbfd97-vj56m 0/1 Pending 0 10m +``` + +**Important note**: There's no need for alarm at this point. The output you're seeing is anticipated, as the nodes are expected to be in a NotReady state while the pods remain in a Pending state. If your results match these conditions, you're on the right track and prepared to begin the troubleshooting process. Please proceed to the next section to continue with the exercise. diff --git a/website/docs/troubleshooting/cni/ssm.sh b/website/docs/troubleshooting/cni/ssm.sh new file mode 100755 index 000000000..d17070fc6 --- /dev/null +++ b/website/docs/troubleshooting/cni/ssm.sh @@ -0,0 +1,23 @@ +#!/bin/bash +COMMAND_ID=$(aws ssm send-command \ + --instance-ids $1 \ + --document-name "AWS-RunShellScript" \ + --comment "Demo run shell script on Linux Instances" \ + --parameters '{"commands":["sudo -Hiu root bash << END","tail -n '$3' /var/log/aws-routed-eni/'$2'.log | grep '$4'", "END"]}' \ + --output text \ + --query "Command.CommandId") + +STATUS=InProgress +while [ "$STATUS" == "InProgress" ]; do + STATUS=$(aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id $1 \ + --output text \ + --query "Status") +done + +aws ssm list-command-invocations \ + --command-id "$COMMAND_ID" \ + --details \ + --output text \ + --query "CommandInvocations[].CommandPlugins[].Output" diff --git a/website/docs/troubleshooting/cni/tests/hook-fix-1.sh b/website/docs/troubleshooting/cni/tests/hook-fix-1.sh new file mode 100644 index 000000000..e524553fa --- /dev/null +++ b/website/docs/troubleshooting/cni/tests/hook-fix-1.sh @@ -0,0 +1,28 @@ +set -Eeuo pipefail + +before() { + echo "noop" +} + +after() { + sleep 60 + + EXIT_CODE=0 + + timeout -s TERM 160 bash -c \ + 'while [[ $(kubectl get pods -n kube-system -l k8s-app=aws-node -o wide | grep Pending | wc -l) -gt 0 ]];\ + do sleep 30;\ + done' || EXIT_CODE=$? + + debug=$(kubectl get pods -n kube-system -l k8s-app=aws-node -o wide) + echo "$debug" + + if [ $EXIT_CODE -ne 0 ]; then + cat << EOF >&2 +One of the aws-node is still in Pending state within 160 seconds +EOF + exit 1 + fi +} + +"$@" \ No newline at end of file diff --git a/website/docs/troubleshooting/cni/tests/hook-fix-5.sh b/website/docs/troubleshooting/cni/tests/hook-fix-5.sh new file mode 100644 index 000000000..22e356321 --- /dev/null +++ b/website/docs/troubleshooting/cni/tests/hook-fix-5.sh @@ -0,0 +1,31 @@ +set -Eeuo pipefail + +before() { + echo "noop" +} + +after() { + sleep 60 + + echo "Recycle aws-node pod for faster response" + kubectl delete pods -n kube-system -l k8s-app=aws-node + + EXIT_CODE=0 + + timeout -s TERM 160 bash -c \ + 'while [[ $(kubectl get nodes | grep NotReady | wc -l) -gt 0 ]];\ + do sleep 30;\ + done' || EXIT_CODE=$? + + debug=$(kubectl get nodes) + echo "$debug" + + if [ $EXIT_CODE -ne 0 ]; then + cat << EOF >&2 +One of the worker nodes is still in NotReady state within 160 seconds +EOF + exit 1 + fi +} + +"$@" \ No newline at end of file diff --git a/website/docs/troubleshooting/cni/tests/hook-fix-9.sh b/website/docs/troubleshooting/cni/tests/hook-fix-9.sh new file mode 100644 index 000000000..c368964c7 --- /dev/null +++ b/website/docs/troubleshooting/cni/tests/hook-fix-9.sh @@ -0,0 +1,28 @@ +set -Eeuo pipefail + +before() { + echo "noop" +} + +after() { + sleep 60 + + EXIT_CODE=0 + + timeout -s TERM 160 bash -c \ + 'while [[ $(kubectl get pods -n cni-tshoot -o wide | grep Running | wc -l) -lt 15 ]];\ + do sleep 30;\ + done' || EXIT_CODE=$? + + debug=$(kubectl get pods -n cni-tshoot -o wide) + echo "$debug" + + if [ $EXIT_CODE -ne 0 ]; then + cat << EOF >&2 +Some of the pods are not transition into Running state within 160 seconds +EOF + exit 1 + fi +} + +"$@" \ No newline at end of file diff --git a/website/docs/troubleshooting/cni/tests/hook-suite.sh b/website/docs/troubleshooting/cni/tests/hook-suite.sh new file mode 100644 index 000000000..36ca4218e --- /dev/null +++ b/website/docs/troubleshooting/cni/tests/hook-suite.sh @@ -0,0 +1,11 @@ +set -e + +before() { + echo "noop" +} + +after() { + prepare-environment +} + +"$@" \ No newline at end of file