Skip to content

Commit

Permalink
feat(eks): Pass bootstrap.sh args to avoid DescribeCluster call
Browse files Browse the repository at this point in the history 
Pass sufficient information to bootstrap.sh in ASG user-data, to
prevent it from making a DescribeCluster AWS API call.  This per-node
API call can become an issue in large clusters, and we already have
all the information to avoid it.
  • Loading branch information
@anguslees
anguslees committed Mar 31, 2021
1 parent df5c133 commit aabeaf0
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 52 deletions.
4 changes: 2 additions & 2 deletions packages/@aws-cdk/aws-eks/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ cluster.addNodegroupCapacity('custom-node-group', {
#### Spot Instances Support

Use `capacityType` to create managed node groups comprised of spot instances. To maximize the availability of your applications while using
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.
Spot Instances, we recommend that you configure a Spot managed node group to use multiple instance types with the `instanceTypes` property.

> For more details visit [Managed node group capacity types](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types).
Expand Down Expand Up @@ -395,7 +395,7 @@ const asg = new ec2.AutoScalingGroup(...)
cluster.connectAutoScalingGroupCapacity(asg);
```

This will add the necessary user-data and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.
This will add the necessary user-data to access the apiserver and configure all connections, roles, and tags needed for the instances in the auto-scaling group to properly join the cluster.

#### Spot Instances

Expand Down
2 changes: 1 addition & 1 deletion packages/@aws-cdk/aws-eks/lib/cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1264,7 +1264,7 @@ export class Cluster extends ClusterBase {
if (bootstrapEnabled) {
const userData = options.machineImageType === MachineImageType.BOTTLEROCKET ?
renderBottlerocketUserData(this) :
renderAmazonLinuxUserData(this.clusterName, autoScalingGroup, options.bootstrapOptions);
renderAmazonLinuxUserData(this, autoScalingGroup, options.bootstrapOptions);
autoScalingGroup.addUserData(...userData);
}

Expand Down
20 changes: 18 additions & 2 deletions packages/@aws-cdk/aws-eks/lib/user-data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Stack } from '@aws-cdk/core';
import { BootstrapOptions, ICluster } from './cluster';

// eslint-disable-next-line max-len
export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {
export function renderAmazonLinuxUserData(cluster: ICluster, autoScalingGroup: autoscaling.AutoScalingGroup, options: BootstrapOptions = {}): string[] {

const stack = Stack.of(autoScalingGroup);

Expand All @@ -13,6 +13,22 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

const extraArgs = new Array<string>();

try {
extraArgs.push(`--apiserver-endpoint '${cluster.clusterEndpoint}'`);
} catch (e) {
if (!e.message.includes('is not defined for this imported cluster')) {
throw e;
}
}

try {
extraArgs.push(`--b64-cluster-ca '${cluster.clusterCertificateAuthorityData}'`);
} catch (e) {
if (!e.message.includes('is not defined for this imported cluster')) {
throw e;
}
}

extraArgs.push(`--use-max-pods ${options.useMaxPods ?? true}`);

if (options.awsApiRetryAttempts) {
Expand Down Expand Up @@ -41,7 +57,7 @@ export function renderAmazonLinuxUserData(clusterName: string, autoScalingGroup:

return [
'set -o xtrace',
`/etc/eks/bootstrap.sh ${clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/etc/eks/bootstrap.sh ${cluster.clusterName} --kubelet-extra-args "${kubeletExtraArgs}" ${commandLineSuffix}`.trim(),
`/opt/aws/bin/cfn-signal --exit-code $? --stack ${stack.stackName} --resource ${asgLogicalId} --region ${stack.region}`,
];
}
Expand Down
82 changes: 69 additions & 13 deletions packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1682,7 +1682,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesASGF172BD19 --region test-region"
]
]
}
Expand Down Expand Up @@ -1993,7 +2007,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterNodesArmASG40A593D0 --region test-region"
]
]
}
Expand Down Expand Up @@ -2630,7 +2658,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule --node-labels foo=bar,goo=far\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true --aws-api-retry-attempts 5\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterspotASG857494B6 --region test-region"
]
]
}
Expand Down Expand Up @@ -2973,7 +3015,21 @@
{
"Ref": "Cluster9EE0221C"
},
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
" --kubelet-extra-args \"--node-labels lifecycle=OnDemand\" --apiserver-endpoint '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"Endpoint"
]
},
"' --b64-cluster-ca '",
{
"Fn::GetAtt": [
"Cluster9EE0221C",
"CertificateAuthorityData"
]
},
"' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack aws-cdk-eks-cluster-test --resource ClusterInferenceInstancesASGE90717C7 --region test-region"
]
]
}
Expand Down Expand Up @@ -4146,7 +4202,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -4159,7 +4215,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD"
}
]
}
Expand All @@ -4172,7 +4228,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299"
"Ref": "AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD"
}
]
}
Expand Down Expand Up @@ -4674,17 +4730,17 @@
"Type": "String",
"Description": "Artifact hash for asset \"b7d38dc0eeb2c5d024919020e09d2590b68559eab4a5264c3b1aa7a429d1edd4\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3Bucket055DC235": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3BucketB7E1A9C0": {
"Type": "String",
"Description": "S3 bucket for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "S3 bucket for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344S3VersionKey2FFFA299": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3S3VersionKey542FDEBD": {
"Type": "String",
"Description": "S3 key for asset version \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "S3 key for asset version \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344ArtifactHash1AB042BC": {
"AssetParameters6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3ArtifactHash5E61FCA5": {
"Type": "String",
"Description": "Artifact hash for asset \"952bd1c03e8201c4c1c67d6de0f3fdaaf88fda05f89a1232c3f6364343cd5344\""
"Description": "Artifact hash for asset \"6afd8be511f58dbedd46c8a09c07db8b7340d99fd3527b6d3dfb729208060fc3\""
},
"AssetParameters5f49893093e1ad14831626016699156d48da5f0890f19eb930bc3c46cf5f636dS3BucketA6642550": {
"Type": "String",
Expand Down
6 changes: 3 additions & 3 deletions packages/@aws-cdk/aws-eks/test/test.cluster.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1294,7 +1294,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down Expand Up @@ -1333,7 +1333,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=OnDemand --node-labels FOO=42" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand All @@ -1353,7 +1353,7 @@ export = {
// THEN
const template = app.synth().getStackByName(stack.stackName).template;
const userData = template.Resources.ClusterMyCapcityLaunchConfig58583345.Properties.UserData;
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.deepEqual(userData, { 'Fn::Base64': { 'Fn::Join': ['', ['#!/bin/bash\nset -o xtrace\n/etc/eks/bootstrap.sh ', { Ref: 'Cluster9EE0221C' }, ' --kubelet-extra-args "--node-labels lifecycle=Ec2Spot --register-with-taints=spotInstance=true:PreferNoSchedule" --apiserver-endpoint \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'Endpoint'] }, '\' --b64-cluster-ca \'', { 'Fn::GetAtt': ['Cluster9EE0221C', 'CertificateAuthorityData'] }, '\' --use-max-pods true\n/opt/aws/bin/cfn-signal --exit-code $? --stack Stack --resource ClusterMyCapcityASGD4CD8B97 --region us-east-1']] } });
test.done();
},

Expand Down
Loading

0 comments on commit aabeaf0

Please sign in to comment.