-
Notifications
You must be signed in to change notification settings - Fork 4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(eks-v2-alpha): EKS Auto Mode support #33373
base: main
Are you sure you want to change the base?
Changes from 1 commit
f907aa6
e55f7c1
433d168
bb56556
bf1e9bb
993833f
318c96c
759946d
c32983f
d931ef0
6c80d10
55b1abd
b9c2c44
d349e34
f0187c8
47189ea
a25908c
9592020
f9edabe
d68baa7
a1c279d
7236626
b843221
f35213c
d67beb9
9de5fe0
d4118a5
111b3fd
a334f54
3912339
6ca090f
65944e2
df7287f
7e0215d
23cd47d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,8 +22,63 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2'; | |
import * as iam from 'aws-cdk-lib/aws-iam'; | ||
import * as kms from 'aws-cdk-lib/aws-kms'; | ||
import * as ssm from 'aws-cdk-lib/aws-ssm'; | ||
import { Annotations, CfnOutput, CfnResource, IResource, Resource, Tags, Token, Duration, ArnComponents } from 'aws-cdk-lib/core'; | ||
import { Annotations, CfnOutput, CfnResource, IResource, Resource, Tags, Token, Duration, ArnComponents, Stack } from 'aws-cdk-lib/core'; | ||
import { CfnCluster } from 'aws-cdk-lib/aws-eks'; | ||
|
||
/** | ||
* Properties for the EKS cluster auto mode configuration | ||
*/ | ||
interface AutoModeConfig { | ||
/** | ||
* Auto Mode compute configuration | ||
*/ | ||
readonly compute?: { | ||
/** | ||
* Whether to enable Auto Mode compute | ||
*/ | ||
readonly enabled?: boolean; | ||
}; | ||
|
||
/** | ||
* Auto Mode storage configuration | ||
*/ | ||
readonly storage?: { | ||
/** | ||
* Block storage configuration | ||
*/ | ||
readonly blockStorage?: { | ||
/** | ||
* Whether to enable Auto Mode block storage | ||
*/ | ||
readonly enabled?: boolean; | ||
}; | ||
}; | ||
|
||
/** | ||
* Auto Mode networking configuration | ||
*/ | ||
readonly kubernetesNetwork?: { | ||
/** | ||
* Elastic load balancing configuration | ||
*/ | ||
readonly elasticLoadBalancing?: { | ||
/** | ||
* Whether to enable Auto Mode elastic load balancing | ||
*/ | ||
readonly enabled?: boolean; | ||
}; | ||
}; | ||
} | ||
|
||
// Extend CfnClusterProps to include autoMode | ||
declare module 'aws-cdk-lib/aws-eks' { | ||
interface CfnClusterProps { | ||
/** | ||
* Auto Mode configuration for the cluster | ||
*/ | ||
readonly autoMode?: AutoModeConfig; | ||
} | ||
} | ||
import { MethodMetadata, addConstructMetadata } from 'aws-cdk-lib/core/lib/metadata-resource'; | ||
|
||
// defaults are based on https://eksctl.io | ||
|
@@ -519,11 +574,81 @@ export class EndpointAccess { | |
} | ||
} | ||
|
||
/** | ||
* Options for configuring EKS Auto Mode compute settings. | ||
* When enabled, EKS will automatically manage compute resources like node groups and Fargate profiles. | ||
*/ | ||
export interface ComputeConfig { | ||
/** | ||
* Names of nodePools to include in Auto Mode. | ||
* | ||
* @see - https://docs.aws.amazon.com/eks/latest/userguide/create-node-pool.html | ||
* | ||
* @default - ['system', 'general-purpose'] | ||
*/ | ||
readonly nodePools?: string[]; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. At this moment, EKS only allows to opt in |
||
|
||
/** | ||
* IAM role for the nodePools. | ||
* | ||
* @see - https://docs.aws.amazon.com/eks/latest/userguide/create-node-role.html | ||
* | ||
* @default - generated by the CDK | ||
*/ | ||
readonly nodeRole?: iam.IRole; | ||
xazhao marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
} | ||
|
||
/** | ||
* Options for configuring EKS Auto Mode block storage settings. | ||
* When enabled, EKS will automatically manage block storage resources. | ||
*/ | ||
export interface BlockStorageConfig { | ||
} | ||
|
||
/** | ||
* Options for configuring EKS Auto Mode storage settings. | ||
* Controls automatic management of storage resources in the cluster. | ||
*/ | ||
export interface StorageConfig { | ||
/** | ||
* Block storage configuration for Auto Mode. | ||
* Controls how EKS automatically manages EBS volumes and storage classes. | ||
* @default - Auto Mode storage disabled | ||
*/ | ||
readonly blockStorage?: BlockStorageConfig; | ||
} | ||
|
||
/** | ||
* Options for configuring EKS Auto Mode elastic load balancing settings. | ||
* When enabled, EKS will automatically manage load balancers. | ||
*/ | ||
export interface ElasticLoadBalancingConfig { | ||
/** | ||
* Whether to enable Auto Mode elastic load balancing. | ||
* When enabled, EKS will automatically provision and configure load balancers based on service requirements. | ||
* @default false | ||
*/ | ||
readonly enabled?: boolean; | ||
} | ||
|
||
/** | ||
* Options for configuring EKS Auto Mode Kubernetes networking settings. | ||
* Controls automatic management of networking resources in the cluster. | ||
*/ | ||
export interface KubernetesNetworkConfig { | ||
/** | ||
* Elastic load balancing configuration for Auto Mode. | ||
* Controls how EKS automatically manages load balancers for services. | ||
* @default - Auto Mode elastic load balancing disabled | ||
*/ | ||
readonly elasticLoadBalancing?: ElasticLoadBalancingConfig; | ||
} | ||
|
||
/** | ||
* Properties for configuring a standard EKS cluster (non-Fargate) | ||
*/ | ||
export interface ClusterProps extends ClusterCommonOptions { | ||
|
||
/** | ||
* Number of instances to allocate as an initial capacity for this cluster. | ||
* Instance type can be configured through `defaultCapacityInstanceType`, | ||
|
@@ -560,6 +685,43 @@ export interface ClusterProps extends ClusterCommonOptions { | |
* @default true | ||
*/ | ||
readonly bootstrapClusterCreatorAdminPermissions?: boolean; | ||
|
||
/** | ||
* Determines whether a CloudFormation output with the `aws eks | ||
* update-kubeconfig` command will be synthesized. This command will include | ||
* the cluster name and, if applicable, the ARN of the masters IAM role. | ||
* | ||
* @default true | ||
*/ | ||
readonly outputConfigCommand?: boolean; | ||
pahud marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
/** | ||
* Whether to enable EKS Auto Mode. When enabled, EKS will automatically manage compute, storage, and networking resources. | ||
* | ||
* @default - true | ||
*/ | ||
readonly autoMode?: boolean; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need this flag? Customers can disable auto mode by setting There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good callout. As we discussed offline, AutoMode and Nodegroup can both exist. I am thinking maybe we should make then not mutual exclusive. For example
Then we have all the use cases as below:
Make sense? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, I think this makes sense! There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've removed |
||
|
||
/** | ||
* Configuration for compute settings in Auto Mode. | ||
* When enabled, EKS will automatically manage compute resources. | ||
* @default - Auto Mode compute disabled | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: the default value should be There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this should respect There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I've removed |
||
*/ | ||
readonly compute?: ComputeConfig; | ||
|
||
/** | ||
* Configuration for storage settings in Auto Mode. | ||
* When enabled, EKS will automatically manage storage resources. | ||
* @default - Auto Mode storage disabled | ||
*/ | ||
readonly storage?: StorageConfig; | ||
|
||
/** | ||
* Configuration for Kubernetes networking settings in Auto Mode. | ||
* When enabled, EKS will automatically manage networking resources. | ||
* @default - Auto Mode networking disabled | ||
*/ | ||
readonly kubernetesNetwork?: KubernetesNetworkConfig; | ||
} | ||
|
||
/** | ||
|
@@ -1055,6 +1217,16 @@ export class Cluster extends ClusterBase { | |
], | ||
}); | ||
|
||
if (this.role instanceof iam.Role) { | ||
this.role.assumeRolePolicy?.addStatements( | ||
new iam.PolicyStatement({ | ||
effect: iam.Effect.ALLOW, | ||
principals: [new iam.ServicePrincipal('eks.amazonaws.com')], | ||
actions: ['sts:TagSession'], | ||
}), | ||
); | ||
} | ||
|
||
const securityGroup = props.securityGroup || new ec2.SecurityGroup(this, 'ControlPlaneSecurityGroup', { | ||
vpc: this.vpc, | ||
description: 'EKS Control Plane Security Group', | ||
|
@@ -1109,6 +1281,21 @@ export class Cluster extends ClusterBase { | |
authenticationMode: 'API', | ||
bootstrapClusterCreatorAdminPermissions: props.bootstrapClusterCreatorAdminPermissions, | ||
}, | ||
computeConfig: { | ||
enabled: props.autoMode, | ||
nodePools: props.compute?.nodePools ?? ['system', 'general-purpose'], | ||
nodeRoleArn: props.compute?.nodeRole?.roleArn ?? this.addNodePoolRole(`${id}nodePoolRole`).roleArn, | ||
}, | ||
storageConfig: { | ||
blockStorage: { | ||
enabled: props.autoMode, | ||
}, | ||
}, | ||
kubernetesNetworkConfig: { | ||
elasticLoadBalancing: { | ||
enabled: props.autoMode, | ||
}, | ||
}, | ||
resourcesVpcConfig: { | ||
securityGroupIds: [securityGroup.securityGroupId], | ||
subnetIds, | ||
|
@@ -1124,14 +1311,19 @@ export class Cluster extends ClusterBase { | |
resources: ['secrets'], | ||
}], | ||
} : {}), | ||
kubernetesNetworkConfig: { | ||
ipFamily: this.ipFamily, | ||
serviceIpv4Cidr: props.serviceIpv4Cidr, | ||
}, | ||
tags: Object.keys(props.tags ?? {}).map(k => ({ key: k, value: props.tags![k] })), | ||
logging: this.logging, | ||
}); | ||
|
||
if (props.autoMode != false) { | ||
// attach required managed policy for the cluster role in EKS Auto Mode | ||
// see - https://docs.aws.amazon.com/eks/latest/userguide/auto-cluster-iam-role.html | ||
['AmazonEKSComputePolicy', 'AmazonEKSBlockStoragePolicy', 'AmazonEKSLoadBalancingPolicy', | ||
'AmazonEKSNetworkingPolicy', 'AmazonEKSClusterPolicy'].forEach((policyName) => { | ||
this.role.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName(policyName)); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. move this to the same place when add There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. AmazonEKSClusterPolicy already exist in the base role, will add it throw error or just no-op? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no-op but yeah let's remove the dup There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. DONE |
||
}); | ||
} | ||
|
||
let kubectlSubnets = this._kubectlProviderOptions?.privateSubnets; | ||
|
||
if (this.endpointAccess._config.privateAccess && privateSubnets.length !== 0) { | ||
|
@@ -1177,6 +1369,11 @@ export class Cluster extends ClusterBase { | |
defaultPort: ec2.Port.tcp(443), // Control Plane has an HTTPS API | ||
}); | ||
|
||
const stack = Stack.of(this); | ||
const updateConfigCommandPrefix = `aws eks update-kubeconfig --name ${this.clusterName}`; | ||
const getTokenCommandPrefix = `aws eks get-token --cluster-name ${this.clusterName}`; | ||
const commonCommandOptions = [`--region ${stack.region}`]; | ||
|
||
if (props.kubectlProviderOptions) { | ||
this._kubectlProvider = new KubectlProvider(this, 'KubectlProvider', { | ||
cluster: this, | ||
|
@@ -1199,29 +1396,42 @@ export class Cluster extends ClusterBase { | |
|
||
// do not create a masters role if one is not provided. Trusting the accountRootPrincipal() is too permissive. | ||
if (props.mastersRole) { | ||
const mastersRole = props.mastersRole; | ||
this.grantAccess('mastersRoleAccess', props.mastersRole.roleArn, [ | ||
AccessPolicy.fromAccessPolicyName('AmazonEKSClusterAdminPolicy', { | ||
accessScopeType: AccessScopeType.CLUSTER, | ||
}), | ||
]); | ||
|
||
commonCommandOptions.push(`--role-arn ${mastersRole.roleArn}`); | ||
} | ||
|
||
if (props.albController) { | ||
this.albController = AlbController.create(this, { ...props.albController, cluster: this }); | ||
} | ||
|
||
// allocate default capacity if non-zero (or default). | ||
const minCapacity = props.defaultCapacity ?? DEFAULT_CAPACITY_COUNT; | ||
if (minCapacity > 0) { | ||
const instanceType = props.defaultCapacityInstance || DEFAULT_CAPACITY_TYPE; | ||
this.defaultCapacity = props.defaultCapacityType === DefaultCapacityType.EC2 ? | ||
this.addAutoScalingGroupCapacity('DefaultCapacity', { instanceType, minCapacity }) : undefined; | ||
// allocate default capacity if non-zero (or default) when autoMode is disabled. This means when | ||
// autoMode is enabled, the default capacity is ignored. | ||
if (props.autoMode === false) { | ||
const minCapacity = props.defaultCapacity ?? DEFAULT_CAPACITY_COUNT; | ||
if (minCapacity > 0) { | ||
const instanceType = props.defaultCapacityInstance || DEFAULT_CAPACITY_TYPE; | ||
this.defaultCapacity = props.defaultCapacityType === DefaultCapacityType.EC2 ? | ||
this.addAutoScalingGroupCapacity('DefaultCapacity', { instanceType, minCapacity }) : undefined; | ||
|
||
this.defaultNodegroup = props.defaultCapacityType !== DefaultCapacityType.EC2 ? | ||
this.addNodegroupCapacity('DefaultCapacity', { instanceTypes: [instanceType], minSize: minCapacity }) : undefined; | ||
} | ||
|
||
this.defaultNodegroup = props.defaultCapacityType !== DefaultCapacityType.EC2 ? | ||
this.addNodegroupCapacity('DefaultCapacity', { instanceTypes: [instanceType], minSize: minCapacity }) : undefined; | ||
this.defineCoreDnsComputeType(props.coreDnsComputeType ?? CoreDnsComputeType.EC2); | ||
} | ||
|
||
this.defineCoreDnsComputeType(props.coreDnsComputeType ?? CoreDnsComputeType.EC2); | ||
const outputConfigCommand = (props.outputConfigCommand ?? true) && props.mastersRole; | ||
if (outputConfigCommand) { | ||
const postfix = commonCommandOptions.join(' '); | ||
new CfnOutput(this, 'ConfigCommand', { value: `${updateConfigCommandPrefix} ${postfix}` }); | ||
new CfnOutput(this, 'GetTokenCommand', { value: `${getTokenCommandPrefix} ${postfix}` }); | ||
} | ||
} | ||
|
||
/** | ||
|
@@ -1436,6 +1646,20 @@ export class Cluster extends ClusterBase { | |
return this._fargateProfiles; | ||
} | ||
|
||
private addNodePoolRole(id: string): iam.Role { | ||
const role = new iam.Role(this, id, { | ||
assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), | ||
// to be able to access the AWSLoadBalancerController | ||
managedPolicies: [ | ||
iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKSWorkerNodePolicy'), | ||
iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly'), | ||
iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy'), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think we need this policy, referring to this doc https://docs.aws.amazon.com/eks/latest/userguide/automode-get-started-cli.html#auto-mode-create-roles There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. DONE |
||
], | ||
}); | ||
|
||
return role; | ||
} | ||
|
||
/** | ||
* Adds an access entry to the cluster's access entries map. | ||
* | ||
|
@@ -1976,4 +2200,3 @@ function clusterArnComponents(clusterName: string): ArnComponents { | |
resourceName: clusterName, | ||
}; | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
shall we specifically mention this is the behavior for v2 CDK? as v1 might differ
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
DONE