feat(eks): managed nodegroup with custom AMI and launch template supp…

…ort (#9881)

- This PR allows you to:
1. use `LaunchTemplate` for the managed nodegroups
2. specify custom AMI in the `LaunchTemplate`

- `prop.vpc` of Pinger is now `ec2.IVpc`
- bump cluster k8s version in the integration testing from `1.16` to `1.17`

Closes: #9873 
Closes: #9924

## Note

At this moment we use the property override to make it work. When cfn spec is updated we can use `CfnNodeGroup` to specify launch template natively, which should be backward-compatible with no breaking changes.

Users will need to create `CfnLaunchTemplate` resource and pass the resource as the property to the nodegroup untill we support the LaunchTemplate L2(#6734).

## TODO

- [x] integ testing
- [x] unit testing
- [x] update README


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-eks/README.md

@@ -176,6 +176,34 @@ cluster.addNodegroup('nodegroup', {
 });
 ```
 
+#### Custom AMI and Launch Template support
+
+Specify the launch template for the nodegroup with your custom AMI. When using a custom AMI,
+Amazon EKS doesn't merge any user data. Rather, You are responsible for supplying the required
+bootstrap commands for nodes to join the cluster. In the following sample, `/ect/eks/bootstrap.sh` from the AMI will be used to bootstrap the node. See [Using a custom AMI](https://docs.aws.amazon.com/en_ca/eks/latest/userguide/launch-templates.html) for more details.
+
+```ts
+const userData = ec2.UserData.forLinux();
+userData.addCommands(
+  'set -o xtrace',
+  `/etc/eks/bootstrap.sh ${this.cluster.clusterName}`,
+);
+const lt = new ec2.CfnLaunchTemplate(this, 'LaunchTemplate', {
+  launchTemplateData: {
+    // specify your custom AMI below
+    imageId,
+    instanceType: new ec2.InstanceType('t3.small').toString(),
+    userData: Fn.base64(userData.render()),
+  },
+});
+this.cluster.addNodegroup('extra-ng', {
+  launchTemplate: {
+    id: lt.ref,
+    version: lt.attrDefaultVersionNumber,
+  },
+});
+```
+
 ### ARM64 Support
 
 Instance types with `ARM64` architecture are supported in both managed nodegroup and self-managed capacity. Simply specify an ARM64 `instanceType` (such as `m6g.medium`), and the latest 

packages/@aws-cdk/aws-eks/lib/managed-nodegroup.ts

@@ -56,6 +56,22 @@ export interface NodegroupRemoteAccess {
   readonly sourceSecurityGroups?: ISecurityGroup[];
 }
 
+/**
+ * Launch template property specification
+ */
+export interface LaunchTemplate {
+  /**
+   * The Launch template ID
+   */
+  readonly id: string;
+  /**
+   * The launch template version to be used (optional).
+   *
+   * @default - the default version of the launch template
+   */
+  readonly version?: string;
+}
+
 /**
  * The Nodegroup Options for addNodeGroup() method
  */
@@ -160,6 +176,12 @@ export interface NodegroupOptions {
    * @default - None
    */
   readonly tags?: { [name: string]: string };
+  /**
+   * Launch template used for the nodegroup
+   * @see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html
+   * @default - no launch template
+   */
+  readonly launchTemplate?: LaunchTemplate;
 }
 
 /**
@@ -268,6 +290,25 @@ export class Nodegroup extends Resource implements INodegroup {
       tags: props.tags,
     });
 
+    if (props.launchTemplate) {
+      if (props.diskSize) {
+        // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html
+        // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize
+        throw new Error('diskSize must be specified within the launch template');
+      }
+      if (props.instanceType) {
+        // see - https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html
+        // and https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-eks-nodegroup.html#cfn-eks-nodegroup-disksize
+        throw new Error('Instance types must be specified within the launch template');
+      }
+      // TODO: update this when the L1 resource spec is updated.
+      resource.addPropertyOverride('LaunchTemplate', {
+        Id: props.launchTemplate.id,
+        Version: props.launchTemplate.version,
+      });
+    }
+
+
     // managed nodegroups update the `aws-auth` on creation, but we still need to track
     // its state for consistency.
     if (this.cluster instanceof Cluster) {

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json

@@ -972,7 +972,7 @@
           ]
         },
         "Config": {
-          "version": "1.16",
+          "version": "1.17",
           "roleArn": {
             "Fn::GetAtt": [
               "ClusterRoleFA261979",
@@ -1192,6 +1192,13 @@
                   "Arn"
                 ]
               },
+              "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"",
+              {
+                "Fn::GetAtt": [
+                  "ClusterNodegroupDefaultCapacityNodeGroupRole55953B04",
+                  "Arn"
+                ]
+              },
               "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]}]\",\"mapUsers\":\"[]\",\"mapAccounts\":\"[]\"}}]"
             ]
           ]
@@ -1574,7 +1581,7 @@
       "Type": "AWS::AutoScaling::LaunchConfiguration",
       "Properties": {
         "ImageId": {
-          "Ref": "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
+          "Ref": "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
         },
         "InstanceType": "t2.medium",
         "IamInstanceProfile": {
@@ -1851,7 +1858,7 @@
       "Type": "AWS::AutoScaling::LaunchConfiguration",
       "Properties": {
         "ImageId": {
-          "Ref": "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2arm64recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
+          "Ref": "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2arm64recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
         },
         "InstanceType": "m6g.medium",
         "IamInstanceProfile": {
@@ -2419,7 +2426,7 @@
       "Type": "AWS::AutoScaling::LaunchConfiguration",
       "Properties": {
         "ImageId": {
-          "Ref": "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
+          "Ref": "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
         },
         "InstanceType": "t3.large",
         "IamInstanceProfile": {
@@ -2729,7 +2736,7 @@
       "Type": "AWS::AutoScaling::LaunchConfiguration",
       "Properties": {
         "ImageId": {
-          "Ref": "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2gpurecommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
+          "Ref": "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2gpurecommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
         },
         "InstanceType": "inf1.2xlarge",
         "IamInstanceProfile": {
@@ -3036,6 +3043,48 @@
         }
       }
     },
+    "ClusterNodegroupextrang2F1FB0D40": {
+      "Type": "AWS::EKS::Nodegroup",
+      "Properties": {
+        "ClusterName": {
+          "Ref": "Cluster9EE0221C"
+        },
+        "NodeRole": {
+          "Fn::GetAtt": [
+            "ClusterNodegroupDefaultCapacityNodeGroupRole55953B04",
+            "Arn"
+          ]
+        },
+        "Subnets": [
+          {
+            "Ref": "VpcPrivateSubnet1Subnet536B997A"
+          },
+          {
+            "Ref": "VpcPrivateSubnet2Subnet3788AAA1"
+          },
+          {
+            "Ref": "VpcPrivateSubnet3SubnetF258B56E"
+          }
+        ],
+        "ForceUpdateEnabled": true,
+        "ScalingConfig": {
+          "DesiredSize": 1,
+          "MaxSize": 1,
+          "MinSize": 1
+        },
+        "LaunchTemplate": {
+          "Id": {
+            "Ref": "LaunchTemplate"
+          },
+          "Version": {
+            "Fn::GetAtt": [
+              "LaunchTemplate",
+              "DefaultVersionNumber"
+            ]
+          }
+        }
+      }
+    },
     "ClustermanifestHelloApp078A45D8": {
       "Type": "Custom::AWSCDK-EKS-KubernetesResource",
       "Properties": {
@@ -3528,6 +3577,30 @@
         }
       }
     },
+    "LaunchTemplate": {
+      "Type": "AWS::EC2::LaunchTemplate",
+      "Properties": {
+        "LaunchTemplateData": {
+          "ImageId": {
+            "Ref": "SsmParameterValueawsserviceeksoptimizedami114amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter"
+          },
+          "InstanceType": "t3.small",
+          "UserData": {
+            "Fn::Base64": {
+              "Fn::Join": [
+                "",
+                [
+                  "#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ",
+                  {
+                    "Ref": "Cluster9EE0221C"
+                  }
+                ]
+              ]
+            }
+          }
+        }
+      }
+    },
     "AWSCDKCfnUtilsProviderCustomResourceProviderRoleFE0EE867": {
       "Type": "AWS::IAM::Role",
       "Properties": {
@@ -4206,21 +4279,25 @@
       "Type": "String",
       "Description": "Artifact hash for asset \"a298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dc\""
     },
-    "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
+    "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",
-      "Default": "/aws/service/eks/optimized-ami/1.16/amazon-linux-2/recommended/image_id"
+      "Default": "/aws/service/eks/optimized-ami/1.17/amazon-linux-2/recommended/image_id"
     },
-    "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2arm64recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
+    "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2arm64recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",
-      "Default": "/aws/service/eks/optimized-ami/1.16/amazon-linux-2-arm64/recommended/image_id"
+      "Default": "/aws/service/eks/optimized-ami/1.17/amazon-linux-2-arm64/recommended/image_id"
     },
     "SsmParameterValueawsservicebottlerocketawsk8s115x8664latestimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",
       "Default": "/aws/service/bottlerocket/aws-k8s-1.15/x86_64/latest/image_id"
     },
-    "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2gpurecommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
+    "SsmParameterValueawsserviceeksoptimizedami117amazonlinux2gpurecommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
+      "Type": "AWS::SSM::Parameter::Value<String>",
+      "Default": "/aws/service/eks/optimized-ami/1.17/amazon-linux-2-gpu/recommended/image_id"
+    },
+    "SsmParameterValueawsserviceeksoptimizedami114amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",
-      "Default": "/aws/service/eks/optimized-ami/1.16/amazon-linux-2-gpu/recommended/image_id"
+      "Default": "/aws/service/eks/optimized-ami/1.14/amazon-linux-2/recommended/image_id"
     }
   }
 }

packages/@aws-cdk/aws-eks/test/integ.eks-cluster.ts

@@ -2,16 +2,17 @@
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
 import * as kms from '@aws-cdk/aws-kms';
-import { App, CfnOutput, Duration, Token } from '@aws-cdk/core';
+import { App, CfnOutput, Duration, Token, Fn } from '@aws-cdk/core';
 import * as eks from '../lib';
 import * as hello from './hello-k8s';
 import { Pinger } from './pinger/pinger';
 import { TestStack } from './util';
 
+
 class EksClusterStack extends TestStack {
 
   private cluster: eks.Cluster;
-  private vpc: ec2.Vpc;
+  private vpc: ec2.IVpc;
 
   constructor(scope: App, id: string) {
     super(scope, id);
@@ -31,7 +32,7 @@ class EksClusterStack extends TestStack {
       vpc: this.vpc,
       mastersRole,
       defaultCapacity: 2,
-      version: eks.KubernetesVersion.V1_16,
+      version: eks.KubernetesVersion.V1_17,
       secretsEncryptionKey,
     });
 
@@ -51,6 +52,8 @@ class EksClusterStack extends TestStack {
 
     this.assertNodeGroupArm();
 
+    this.assertNodeGroupCustomAmi();
+
     this.assertSimpleManifest();
 
     this.assertSimpleHelmChart();
@@ -118,6 +121,30 @@ class EksClusterStack extends TestStack {
       nodeRole: this.cluster.defaultCapacity ? this.cluster.defaultCapacity.role : undefined,
     });
   }
+  private assertNodeGroupCustomAmi() {
+    // add a extra nodegroup
+    const userData = ec2.UserData.forLinux();
+    userData.addCommands(
+      'set -o xtrace',
+      `/etc/eks/bootstrap.sh ${this.cluster.clusterName}`,
+    );
+    const lt = new ec2.CfnLaunchTemplate(this, 'LaunchTemplate', {
+      launchTemplateData: {
+        imageId: new eks.EksOptimizedImage().getImage(this).imageId,
+        instanceType: new ec2.InstanceType('t3.small').toString(),
+        userData: Fn.base64(userData.render()),
+      },
+    });
+    this.cluster.addNodegroup('extra-ng2', {
+      minSize: 1,
+      // reusing the default capacity nodegroup instance role when available
+      nodeRole: this.cluster.defaultNodegroup?.role || this.cluster.defaultCapacity?.role,
+      launchTemplate: {
+        id: lt.ref,
+        version: lt.attrDefaultVersionNumber,
+      },
+    });
+  }
   private assertNodeGroupArm() {
     // add a extra nodegroup
     this.cluster.addNodegroup('extra-ng-arm', {

packages/@aws-cdk/aws-eks/test/pinger/pinger.ts

@@ -6,7 +6,7 @@ import * as cr from '@aws-cdk/custom-resources';
 export interface PingerProps {
   readonly url: string;
   readonly securityGroup?: ec2.SecurityGroup;
-  readonly vpc?: ec2.Vpc;
+  readonly vpc?: ec2.IVpc;
 }
 export class Pinger extends Construct {
 
@@ -40,4 +40,4 @@ export class Pinger extends Construct {
     return Token.asString(this._resource.getAtt('Value'));
   }
 
-}
+}