Skip to content

Commit

Permalink
feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call
Browse files Browse the repository at this point in the history 
Pass sufficient information to bootstrap.sh in ASG user-data, to
prevent it from making a DescribeCluster AWS API call.  This per-node
API call can become an issue in large clusters, and we already have
all the information to avoid it.
  • Loading branch information
@anguslees
anguslees committed Jan 22, 2021
1 parent 2ac13a2 commit 76d4f66
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 66 deletions.
4 changes: 2 additions & 2 deletions packages/@aws-cdk/aws-eks/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ cluster.addNodegroupCapacity('custom-node-group', {
#### Spot Instances Support

Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.

> For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types).
Expand Down Expand Up @@ -395,7 +395,7 @@ const asg = new ec2.AutoScalingGroup(...)
cluster.connectAutoScalingGroupCapacity(asg);
```

This will add the necessary user-data and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.
This will add the necessary user-data to access the apiserver and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.

#### Spot Instances

Expand Down
12 changes: 11 additions & 1 deletion packages/@aws-cdk/aws-eks/lib/cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1259,7 +1259,7 @@ export class Cluster extends ClusterBase {
if (bootstrapEnabled) {
const userData = options.machineImageType === MachineImageType.BOTTLEROCKET ?
renderBottlerocketUserData(this) :
renderAmazonLinuxUserData(this.clusterName, autoScalingGroup, options.bootstrapOptions);
renderAmazonLinuxUserData(this, autoScalingGroup, options.bootstrapOptions);
autoScalingGroup.addUserData(...userData);
}

Expand Down Expand Up @@ -1632,6 +1632,16 @@ export interface BootstrapOptions {
*/
readonly dockerConfigJson?: string;

/**
* Overrides the IP address to use for DNS queries within the
* cluster.
*
* @default - 10.100.0.10 or 172.20.0.10 based on the IP
* address of the primary interface.
*/
readonly dnsClusterIp?: string;

/**
* Extra arguments to add to the kubelet. Useful for adding labels or taints.
*
Expand Down
11 changes: 9 additions & 2 deletions packages/@aws-cdk/aws-eks/lib/user-data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Stack } from '@aws-cdk/core';
import { BootstrapOptions, ICluster } from './cluster';

// eslint-disable-next-line max-len
export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {
export function renderAmazonLinuxUserData(cluster: ICluster, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {

const stack = Stack.of(autoScalingGroup);

Expand All @@ -13,6 +13,9 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

const extraArgs = new Array<string>();

extraArgs.push(`--apiserver-endpoint '${cluster.clusterEndpoint}'`);
extraArgs.push(`--b64-cluster-ca '${cluster.clusterCertificateAuthorityData}'`);

extraArgs.push(`--use-max-pods ${options.useMaxPods === undefined ? true : options.useMaxPods}`);

if (options.awsApiRetryAttempts) {
Expand All @@ -27,6 +30,10 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:
extraArgs.push(`--docker-config-json '${options.dockerConfigJson}'`);
}

if (options.dnsClusterIp) {
extraArgs.push(`--dns-cluster-ip ${options.dnsClusterIp}`);
}

if (options.additionalArgs) {
extraArgs.push(options.additionalArgs);
}
Expand All @@ -41,7 +48,7 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

return [
'set -o xtrace',
`/etc/eks/bootstrap.sh ${clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/etc/eks/bootstrap.sh ${cluster.clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/opt/aws/bin/cfn-signal --exit-code $? --stack ${stack.stackName} --resource ${asgLogicalId} --region ${stack.region}`,
];
}
Expand Down
110 changes: 83 additions & 27 deletions packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1712,7 +1712,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
]
]
}
Expand Down Expand Up @@ -2023,7 +2037,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
]
]
}
Expand Down Expand Up @@ -2660,7 +2688,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
]
]
}
Expand Down Expand Up @@ -3003,7 +3045,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
]
]
}
Expand Down Expand Up @@ -3277,6 +3333,7 @@
}
],
"AmiType": "AL2_x86_64",
"CapacityType": "SPOT",
"ForceUpdateEnabled": true,
"InstanceTypes": [
"c5.large",
Expand All @@ -3287,8 +3344,7 @@
"DesiredSize": 3,
"MaxSize": 3,
"MinSize": 3
},
"CapacityType": "SPOT"
}
}
},
"ClusterNodegroupextrangarmNodeGroupRoleADF5749F": {
Expand Down Expand Up @@ -3840,7 +3896,7 @@
},
"/",
{
"Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187"
"Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3Bucket862E8D6F"
},
"/",
{
Expand All @@ -3850,7 +3906,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243"
"Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D"
}
]
}
Expand All @@ -3863,7 +3919,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243"
"Ref": "AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D"
}
]
}
Expand Down Expand Up @@ -3907,7 +3963,7 @@
},
"/",
{
"Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806"
"Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3BucketF9C7C3C5"
},
"/",
{
Expand All @@ -3917,7 +3973,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970"
"Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5"
}
]
}
Expand All @@ -3930,7 +3986,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970"
"Ref": "AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5"
}
]
}
Expand Down Expand Up @@ -4347,13 +4403,13 @@
]
}
},
"Handler": "index.handler",
"Role": {
"Fn::GetAtt": [
"ServicePingerFunctionServiceRole3120191B",
"Arn"
]
},
"Handler": "index.handler",
"Runtime": "python3.6",
"Timeout": 600,
"VpcConfig": {
Expand Down Expand Up @@ -4480,14 +4536,12 @@
]
}
},
"Handler": "framework.onEvent",
"Role": {
"Fn::GetAtt": [
"ServicePingerProviderframeworkonEventServiceRole3DB083B7",
"Arn"
]
},
"Runtime": "nodejs10.x",
"Description": "AWS CDK resource provider framework - onEvent (aws-cdk-eks-cluster-test/ServicePinger/Provider)",
"Environment": {
"Variables": {
Expand All @@ -4499,6 +4553,8 @@
}
}
},
"Handler": "framework.onEvent",
"Runtime": "nodejs10.x",
"Timeout": 900
},
"DependsOn": [
Expand Down Expand Up @@ -4727,29 +4783,29 @@
"Type": "String",
"Description": "Artifact hash for asset \"5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636d\""
},
"AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3Bucket1CB7A187": {
"AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3Bucket862E8D6F": {
"Type": "String",
"Description": "S3 bucket for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\""
"Description": "S3 bucket for asset \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\""
},
"AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cS3VersionKey7C13F243": {
"AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaS3VersionKey9466BE3D": {
"Type": "String",
"Description": "S3 key for asset version \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\""
"Description": "S3 key for asset version \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\""
},
"AssetParametersa69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0cArtifactHashBADE945D": {
"AssetParameters264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daaArtifactHashC5F5158C": {
"Type": "String",
"Description": "Artifact hash for asset \"a69aadbed84d554dd9f2eb7987ffe5d8f76b53a86f1909059df07050e57bef0c\""
"Description": "Artifact hash for asset \"264acf17cbf0c643f47bec1f4dbaed805e3bd1bad3f018c093d16fb936227daa\""
},
"AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3Bucket0B8E3806": {
"AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3BucketF9C7C3C5": {
"Type": "String",
"Description": "S3 bucket for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\""
"Description": "S3 bucket for asset \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\""
},
"AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8S3VersionKey862F0970": {
"AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcS3VersionKey950894D5": {
"Type": "String",
"Description": "S3 key for asset version \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\""
"Description": "S3 key for asset version \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\""
},
"AssetParameters6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8ArtifactHashAAFBAA4D": {
"AssetParameters9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabcArtifactHash5984E3CE": {
"Type": "String",
"Description": "Artifact hash for asset \"6b9ad3782e5bfd49d7a58fc915b6151dbed2e24d824730d7720bc8237ba252c8\""
"Description": "Artifact hash for asset \"9f954a0baf5cb008231906c33569617ace43f4b2c804d16d0d4bae15fe9dfabc\""
},
"SsmParameterValueawsserviceeksoptimizedami118amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
"Type": "AWS::SSM::Parameter::Value<String>",
Expand Down
6 changes: 3 additions & 3 deletions packages/@aws-cdk/aws-eks/test/test.cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1294,7 +1294,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down Expand Up @@ -1333,7 +1333,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand All @@ -1353,7 +1353,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down
Loading

0 comments on commit 76d4f66

Please sign in to comment.