Instances failed to join cluster

[dreinhardt89]

Description

When creating a new cluster after 20-30 minutes of attempting to create the managed node group it errors out with the below message.

• Module version [Required]: 18.30

• Terraform version: 1.2.9

Code

module "eks_new" {
  providers = {
    kubernetes = kubernetes.new_eks
  }
  count  = (var.env_name == "ninja-dms" || var.env_name == "ninja") ? 1 : 0
  source = "./modules/eks-18.30.3"

  cluster_name    = local.eks_cluster_name_new
  cluster_version = var.eks_version

  iam_role_additional_policies = [
    "arn:aws:iam::aws:policy/AmazonEKSServicePolicy",
  ]

  iam_role_path = "/eks/"

  iam_role_use_name_prefix = false
  aws_auth_roles           = concat(var.eks_roles)

  cluster_timeouts = {
    create = "30m"
    delete = "15m"
    update = "60m"
  }

  cluster_enabled_log_types = var.eks_cluster_enabled_log_types

  vpc_id     = module.eks_vpc.vpc_id
  subnet_ids = module.eks_vpc.private_subnets

  create_aws_auth_configmap = true
  manage_aws_auth_configmap = true

  eks_managed_node_group_defaults = {
    # We are using the IRSA created below for permissions
    # However, we have to deploy with the policy attached FIRST (when creating a fresh cluster)
    # and then turn this off after the cluster/node group is created. Without this initial policy,
    # the VPC CNI fails to assign IPs and nodes cannot join the cluster
    # See https://github.com/aws/containers-roadmap/issues/1666 for more context
    iam_role_attach_cni_policy = true
  }

  eks_managed_node_groups = {
    node_group = {
      name                       = "${local.eks_cluster_name_new}-managed-node-group"
      enable_bootstrap_user_data = true
      use_name_prefix            = false

      min_size     = 0
      max_size     = 10
      desired_size = 4

      ami_id = var.eks_ami_id

      capacity_type        = "ON_DEMAND"
      force_update_version = true
      instance_types       = ["m5.4xlarge"]

      update_config = {
        max_unavailable = 1
      }

      ebs_optimized = true
      vpc_security_group_ids = [
        aws_security_group.eks_db_subnet_access[0].id,
      ]

      block_device_mappings = {
        xvda = {
          device_name = "/dev/xvda"
          ebs = {
            delete_on_termination = "true"
            encrypted             = "true"
            volume_size           = 100
            volume_type           = "gp2"
          }
        }
      }

      disable_api_termination = false
      enable_monitoring       = true

      create_iam_role          = true
      iam_role_use_name_prefix = false
      iam_role_path            = "/eks/"

      iam_role_additional_policies = [
        "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
        "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
      ]

      create_security_group = false

      network_interfaces = [
        {
          associate_public_ip_address = false
          delete_on_termination       = true
        }
      ]

      launch_template_name = "${local.eks_cluster_name_new}-managed-node-group"

      metadata_options = {
        http_tokens = "optional"
      }
    }
  }

Terminal Output Screenshot(s)

Additional context

Tried the solutions here: #1910 but didn't seem to help.

[bryantbiggs]

aws-iam-authenticator does not handle IAM role paths which might be the issue here kubernetes-sigs/aws-iam-authenticator#268

[dreinhardt89]

@bryantbiggs That was right. Thanks!

[github-actions]

I'm going to lock this issue because it has been closed for 30 days ⏳. This helps our maintainers find and focus on the active issues. If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.