diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index efe50404a2ffa..205b6b99dca72 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -202,7 +202,7 @@ cluster.addNodegroupCapacity('custom-node-group', { #### Spot Instances Support Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using -Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. +Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property. > For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types). @@ -395,7 +395,7 @@ const asg = new ec2.AutoScalingGroup(...) cluster.connectAutoScalingGroupCapacity(asg); ``` -This will add the necessary user-data and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster. +This will add the necessary user-data to access the apiserver and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster. #### Spot Instances diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 6737ac1c7f604..fa5f18f3c9a79 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1259,7 +1259,7 @@ export class Cluster extends ClusterBase { if (bootstrapEnabled) { const userData = options.machineImageType === MachineImageType.BOTTLEROCKET ? renderBottlerocketUserData(this) : - renderAmazonLinuxUserData(this.clusterName, autoScalingGroup, options.bootstrapOptions); + renderAmazonLinuxUserData(this, autoScalingGroup, options.bootstrapOptions); autoScalingGroup.addUserData(...userData); } @@ -1632,6 +1632,16 @@ export interface BootstrapOptions { */ readonly dockerConfigJson?: string; + /** + + * Overrides the IP address to use for DNS queries within the + * cluster. + * + * @default - 10.100.0.10 or 172.20.0.10 based on the IP + * address of the primary interface. + */ + readonly dnsClusterIp?: string; + /** * Extra arguments to add to the kubelet. Useful for adding labels or taints. * diff --git a/packages/@aws-cdk/aws-eks/lib/user-data.ts b/packages/@aws-cdk/aws-eks/lib/user-data.ts index cf38cf7ee9761..33faa54791c86 100644 --- a/packages/@aws-cdk/aws-eks/lib/user-data.ts +++ b/packages/@aws-cdk/aws-eks/lib/user-data.ts @@ -3,7 +3,7 @@ import { Stack } from '@aws-cdk/core'; import { BootstrapOptions, ICluster } from './cluster'; // eslint-disable-next-line max-len -export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] { +export function renderAmazonLinuxUserData(cluster: ICluster, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] { const stack = Stack.of(autoScalingGroup); @@ -13,6 +13,9 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: const extraArgs = new Array(); + extraArgs.push(`--apiserver-endpoint '${cluster.clusterEndpoint}'`); + extraArgs.push(`--b64-cluster-ca '${cluster.clusterCertificateAuthorityData}'`); + extraArgs.push(`--use-max-pods ${options.useMaxPods === undefined ? true : options.useMaxPods}`); if (options.awsApiRetryAttempts) { @@ -27,6 +30,10 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: extraArgs.push(`--docker-config-json '${options.dockerConfigJson}'`); } + if (options.dnsClusterIp) { + extraArgs.push(`--dns-cluster-ip ${options.dnsClusterIp}`); + } + if (options.additionalArgs) { extraArgs.push(options.additionalArgs); } @@ -41,7 +48,7 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: return [ 'set -o xtrace', - `/etc/eks/bootstrap.sh ${clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(), + `/etc/eks/bootstrap.sh ${cluster.clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(), `/opt/aws/bin/cfn-signal --exit-code $? --stack ${stack.stackName} --resource ${asgLogicalId} --region ${stack.region}`, ]; } diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 5c0b4bf402a5b..d15344fac75fa 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -1712,7 +1712,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region" ] ] } @@ -2023,7 +2037,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region" ] ] } @@ -2660,7 +2688,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region" ] ] } @@ -3003,7 +3045,21 @@ { "Ref": "Cluster9EE0221C" }, - " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region" + " --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "Endpoint" + ] + }, + "' --b64-cluster-ca '", + { + "Fn::GetAtt": [ + "Cluster9EE0221C", + "CertificateAuthorityData" + ] + }, + "' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region" ] ] } @@ -3277,6 +3333,7 @@ } ], "AmiType": "AL2_x86_64", + "CapacityType": "SPOT", "ForceUpdateEnabled": true, "InstanceTypes": [ "c5.large", @@ -3287,8 +3344,7 @@ "DesiredSize": 3, "MaxSize": 3, "MinSize": 3 - }, - "CapacityType": "SPOT" + } } }, "ClusterNodegroupextrangarmNodeGroupRoleADF5749F": { @@ -3840,7 +3896,7 @@ }, "/", { - "Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187" + "Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3Bucket862E8D6F" }, "/", { @@ -3850,7 +3906,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243" + "Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D" } ] } @@ -3863,7 +3919,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243" + "Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D" } ] } @@ -3907,7 +3963,7 @@ }, "/", { - "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806" + "Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3BucketF9C7C3C5" }, "/", { @@ -3917,7 +3973,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970" + "Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5" } ] } @@ -3930,7 +3986,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970" + "Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5" } ] } @@ -4347,13 +4403,13 @@ ] } }, - "Handler": "index.handler", "Role": { "Fn::GetAtt": [ "ServicePingerFunctionServiceRole3120191B", "Arn" ] }, + "Handler": "index.handler", "Runtime": "python3.6", "Timeout": 600, "VpcConfig": { @@ -4480,14 +4536,12 @@ ] } }, - "Handler": "framework.onEvent", "Role": { "Fn::GetAtt": [ "ServicePingerProviderframeworkonEventServiceRole3DB083B7", "Arn" ] }, - "Runtime": "nodejs10.x", "Description": "AWS CDK resource provider framework - onEvent (aws-cdk-eks-cluster-test/ServicePinger/Provider)", "Environment": { "Variables": { @@ -4499,6 +4553,8 @@ } } }, + "Handler": "framework.onEvent", + "Runtime": "nodejs10.x", "Timeout": 900 }, "DependsOn": [ @@ -4727,29 +4783,29 @@ "Type": "String", "Description": "Artifact hash for asset \"5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636d\"" }, - "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187": { + "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3Bucket862E8D6F": { "Type": "String", - "Description": "S3 bucket for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" + "Description": "S3 bucket for asset \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\"" }, - "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243": { + "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D": { "Type": "String", - "Description": "S3 key for asset version \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" + "Description": "S3 key for asset version \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\"" }, - "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cArtifactHashBADE945D": { + "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaArtifactHashC5F5158C": { "Type": "String", - "Description": "Artifact hash for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\"" + "Description": "Artifact hash for asset \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\"" }, - "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806": { + "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3BucketF9C7C3C5": { "Type": "String", - "Description": "S3 bucket for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" + "Description": "S3 bucket for asset \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\"" }, - "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970": { + "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5": { "Type": "String", - "Description": "S3 key for asset version \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" + "Description": "S3 key for asset version \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\"" }, - "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8ArtifactHashAAFBAA4D": { + "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcArtifactHash5984E3CE": { "Type": "String", - "Description": "Artifact hash for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\"" + "Description": "Artifact hash for asset \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\"" }, "SsmParameterValueawsserviceeksoptimizedami118amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": { "Type": "AWS::SSM::Parameter::Value", diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index 7e7f01d7ede84..6d8e7a5eee5a3 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -1294,7 +1294,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, @@ -1333,7 +1333,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, @@ -1353,7 +1353,7 @@ export = { // THEN const template = app.synth().getStackByName(stack.stackName).template; const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData; - test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); + test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } }); test.done(); }, diff --git a/packages/@aws-cdk/aws-eks/test/test.user-data.ts b/packages/@aws-cdk/aws-eks/test/test.user-data.ts index 044d1ff55f7ca..b46755940c019 100644 --- a/packages/@aws-cdk/aws-eks/test/test.user-data.ts +++ b/packages/@aws-cdk/aws-eks/test/test.user-data.ts @@ -2,6 +2,7 @@ import * as autoscaling from '@aws-cdk/aws-autoscaling'; import * as ec2 from '@aws-cdk/aws-ec2'; import { App, Stack } from '@aws-cdk/core'; import { Test } from 'nodeunit'; +import { Cluster } from '../lib/cluster'; import { renderAmazonLinuxUserData } from '../lib/user-data'; /* eslint-disable max-len */ @@ -9,15 +10,15 @@ import { renderAmazonLinuxUserData } from '../lib/user-data'; export = { 'default user data'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg)); + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg)); // THEN test.deepEqual(userData, [ 'set -o xtrace', - '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true', + '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true', '/opt/aws/bin/cfn-signal --exit-code $? --stack my-stack --resource ASG46ED3070 --region us-west-33', ]); @@ -26,127 +27,142 @@ export = { '--use-max-pods=true'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { useMaxPods: true, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, '--use-max-pods=false'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { useMaxPods: false, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods false'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods false'); test.done(); }, '--aws-api-retry-attempts'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { awsApiRetryAttempts: 123, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --aws-api-retry-attempts 123'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --aws-api-retry-attempts 123'); + test.done(); + }, + + '--dns-cluster-ip'(test: Test) { + // GIVEN + const { asg, stack, cluster } = newFixtures(); + + // WHEN + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { + dnsClusterIp: '192.0.2.53', + })); + + // THEN + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --dns-cluster-ip 192.0.2.53'); test.done(); }, '--docker-config-json'(test: Test) { // GIVEN - const { asg } = newFixtures(); + const { asg, cluster } = newFixtures(); // WHEN - const userData = renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = renderAmazonLinuxUserData(cluster, asg, { dockerConfigJson: '{"docker":123}', }); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --docker-config-json \'{"docker":123}\''); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --docker-config-json \'{"docker":123}\''); test.done(); }, '--enable-docker-bridge=true'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { enableDockerBridge: true, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --enable-docker-bridge true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --enable-docker-bridge true'); test.done(); }, '--enable-docker-bridge=false'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { enableDockerBridge: false, })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, '--kubelet-extra-args'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { kubeletExtraArgs: '--extra-args-for --kubelet', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand --extra-args-for --kubelet" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand --extra-args-for --kubelet" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, 'arbitrary additional bootstrap arguments can be passed through "additionalArgs"'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(); + const { asg, stack, cluster } = newFixtures(); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { additionalArgs: '--apiserver-endpoint 1111 --foo-bar', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true --apiserver-endpoint 1111 --foo-bar'); + // NB: duplicated --apiserver-endpoint is fine. Last wins. + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true --apiserver-endpoint 1111 --foo-bar'); test.done(); }, 'if asg has spot instances, the correct label and taint is used'(test: Test) { // GIVEN - const { asg, stack } = newFixtures(true); + const { asg, stack, cluster } = newFixtures(true); // WHEN - const userData = stack.resolve(renderAmazonLinuxUserData('my-cluster-name', asg, { + const userData = stack.resolve(renderAmazonLinuxUserData(cluster, asg, { kubeletExtraArgs: '--node-labels X=y', })); // THEN - test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels X=y" --use-max-pods true'); + test.deepEqual(userData[1], '/etc/eks/bootstrap.sh my-cluster-name --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels X=y" --apiserver-endpoint \'https://mycluster:443\' --b64-cluster-ca \'CADATA\' --use-max-pods true'); test.done(); }, }; @@ -155,6 +171,12 @@ function newFixtures(spot = false) { const app = new App(); const stack = new Stack(app, 'my-stack', { env: { region: 'us-west-33' } }); const vpc = new ec2.Vpc(stack, 'vpc'); + const cluster = Cluster.fromClusterAttributes(stack, 'cluster', { + clusterName: 'my-cluster-name', + clusterCertificateAuthorityData: 'CADATA', + clusterEndpoint: 'https://mycluster:443', + vpc, + }); const asg = new autoscaling.AutoScalingGroup(stack, 'ASG', { instanceType: new ec2.InstanceType('m4.xlarge'), machineImage: new ec2.AmazonLinuxImage(), @@ -162,5 +184,5 @@ function newFixtures(spot = false) { vpc, }); - return { stack, vpc, asg }; + return { stack, vpc, cluster, asg }; }