Skip to content

Commit a1b5bf6

Browse files
iliapoloiliapolo
authored
fix(eks): restricted public access breaks cluster functionality (aws#10103)
Currently, we attach the VPC to the `KubectlProvider` only when public access is not enabled. The idea was that if public access is enabled, the provider could always connect to the cluster via the internet. The problem is that public access can be restricted to specific CIDR's via the `onlyFrom` method. Solution is to switch up the logic and attach the VPC to the provider when private access is enabled. This would enable configuring `PUBLIC_AND_PRIVATE.onlyFrom(...)`. Also, using `PUBLIC.onlyFrom` is now unsupported because it will most likely break the provider since private access is disabled, and public access is restricted. Bottom line, these are the configurations that should work: - Public (with or without private subnets) - Private (with private subnets) - Private and **unrestricted** public (with or without private subents) - Private and **restricted** public (with private subnets) I also moved the `KubectlSecurityGroup` to be created only if needed. Fixes aws#9866 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 088cd48 commit a1b5bf6

File tree

4 files changed

+248
-86
lines changed

4 files changed

+248
-86
lines changed

‎packages/@aws-cdk/aws-eks/lib/cluster.ts

+45-24
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,11 @@ export class EndpointAccess {
460460
* @param cidr CIDR blocks.
461461
*/
462462
public onlyFrom(...cidr: string[]) {
463+
if (!this._config.privateAccess) {
464+
// when private access is disabled, we can't restric public
465+
// access since it will render the kubectl provider unusable.
466+
throw new Error('Cannot restric public access to endpoint when private access is disabled. Use PUBLIC_AND_PRIVATE.onlyFrom() instead.');
467+
}
463468
return new EndpointAccess({
464469
...this._config,
465470
// override CIDR
@@ -856,20 +861,23 @@ export class Cluster extends ClusterBase {
856861
this.kubectlEnvironment = props.kubectlEnvironment;
857862
this.kubectlLayer = props.kubectlLayer;
858863

859-
if (this.endpointAccess._config.privateAccess && this.vpc instanceof ec2.Vpc) {
860-
// validate VPC properties according to: https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
861-
if (!this.vpc.dnsHostnamesEnabled || !this.vpc.dnsSupportEnabled) {
862-
throw new Error('Private endpoint access requires the VPC to have DNS support and DNS hostnames enabled. Use `enableDnsHostnames: true` and `enableDnsSupport: true` when creating the VPC.');
863-
}
864-
}
864+
const privateSubents = this.selectPrivateSubnets().slice(0, 16);
865+
const publicAccessDisabled = !this.endpointAccess._config.publicAccess;
866+
const publicAccessRestricted = !publicAccessDisabled
867+
&& this.endpointAccess._config.publicCidrs
868+
&& this.endpointAccess._config.publicCidrs.length !== 0;
865869

866-
this.kubectlSecurityGroup = new ec2.SecurityGroup(this, 'KubectlProviderSecurityGroup', {
867-
vpc: this.vpc,
868-
description: 'Comminication between KubectlProvider and EKS Control Plane',
869-
});
870+
// validate endpoint access configuration
870871

871-
// grant the kubectl provider access to the cluster control plane.
872-
this.connections.allowFrom(this.kubectlSecurityGroup, this.connections.defaultPort!);
872+
if (privateSubents.length === 0 && publicAccessDisabled) {
873+
// no private subnets and no public access at all, no good.
874+
throw new Error('Vpc must contain private subnets when public endpoint access is disabled');
875+
}
876+
877+
if (privateSubents.length === 0 && publicAccessRestricted) {
878+
// no private subents and public access is restricted, no good.
879+
throw new Error('Vpc must contain private subnets when public endpoint access is restricted');
880+
}
873881

874882
const resource = this._clusterResource = new ClusterResource(this, 'Resource', {
875883
name: this.physicalName,
@@ -894,11 +902,32 @@ export class Cluster extends ClusterBase {
894902
vpc: this.vpc,
895903
});
896904

897-
this.adminRole = resource.adminRole;
905+
if (this.endpointAccess._config.privateAccess && privateSubents.length !== 0) {
906+
907+
// when private access is enabled and the vpc has private subnets, lets connect
908+
// the provider to the vpc so that it will work even when restricting public access.
909+
910+
// validate VPC properties according to: https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
911+
if (this.vpc instanceof ec2.Vpc && !(this.vpc.dnsHostnamesEnabled && this.vpc.dnsSupportEnabled)) {
912+
throw new Error('Private endpoint access requires the VPC to have DNS support and DNS hostnames enabled. Use `enableDnsHostnames: true` and `enableDnsSupport: true` when creating the VPC.');
913+
}
898914

899-
// the security group and vpc must exist in order to properly delete the cluster (since we run `kubectl delete`).
900-
// this ensures that.
901-
this._clusterResource.node.addDependency(this.kubectlSecurityGroup, this.vpc);
915+
this.kubectlPrivateSubnets = privateSubents;
916+
917+
this.kubectlSecurityGroup = new ec2.SecurityGroup(this, 'KubectlProviderSecurityGroup', {
918+
vpc: this.vpc,
919+
description: 'Comminication between KubectlProvider and EKS Control Plane',
920+
});
921+
922+
// grant the kubectl provider access to the cluster control plane.
923+
this.connections.allowFrom(this.kubectlSecurityGroup, this.connections.defaultPort!);
924+
925+
// the security group and vpc must exist in order to properly delete the cluster (since we run `kubectl delete`).
926+
// this ensures that.
927+
this._clusterResource.node.addDependency(this.kubectlSecurityGroup, this.vpc);
928+
}
929+
930+
this.adminRole = resource.adminRole;
902931

903932
// we use an SSM parameter as a barrier because it's free and fast.
904933
this._kubectlReadyBarrier = new CfnResource(this, 'KubectlReadyBarrier', {
@@ -924,14 +953,6 @@ export class Cluster extends ClusterBase {
924953
// cluster is first created, that's the only role that has "system:masters" permissions
925954
this.kubectlRole = this.adminRole;
926955

927-
// specify private subnets for kubectl only if we don't have public k8s endpoint access
928-
if (!this.endpointAccess._config.publicAccess) {
929-
this.kubectlPrivateSubnets = this.selectPrivateSubnets().slice(0, 16);
930-
if (this.kubectlPrivateSubnets.length === 0) {
931-
throw new Error('Vpc must contain private subnets to configure private endpoint access');
932-
}
933-
}
934-
935956
this._kubectlResourceProvider = this.defineKubectlProvider();
936957

937958
const updateConfigCommandPrefix = `aws eks update-kubeconfig --name ${this.clusterName}`;

‎packages/@aws-cdk/aws-eks/test/integ.eks-cluster-private-endpoint.expected.json

+25-25
Original file line numberDiff line numberDiff line change
@@ -602,22 +602,6 @@
602602
"ToPort": 443
603603
}
604604
},
605-
"ClusterKubectlProviderSecurityGroup2D90691C": {
606-
"Type": "AWS::EC2::SecurityGroup",
607-
"Properties": {
608-
"GroupDescription": "Comminication between KubectlProvider and EKS Control Plane",
609-
"SecurityGroupEgress": [
610-
{
611-
"CidrIp": "0.0.0.0/0",
612-
"Description": "Allow all outbound traffic by default",
613-
"IpProtocol": "-1"
614-
}
615-
],
616-
"VpcId": {
617-
"Ref": "Vpc8378EB38"
618-
}
619-
}
620-
},
621605
"ClusterCreationRole360249B6": {
622606
"Type": "AWS::IAM::Role",
623607
"Properties": {
@@ -896,6 +880,22 @@
896880
"UpdateReplacePolicy": "Delete",
897881
"DeletionPolicy": "Delete"
898882
},
883+
"ClusterKubectlProviderSecurityGroup2D90691C": {
884+
"Type": "AWS::EC2::SecurityGroup",
885+
"Properties": {
886+
"GroupDescription": "Comminication between KubectlProvider and EKS Control Plane",
887+
"SecurityGroupEgress": [
888+
{
889+
"CidrIp": "0.0.0.0/0",
890+
"Description": "Allow all outbound traffic by default",
891+
"IpProtocol": "-1"
892+
}
893+
],
894+
"VpcId": {
895+
"Ref": "Vpc8378EB38"
896+
}
897+
}
898+
},
899899
"ClusterKubectlReadyBarrier200052AF": {
900900
"Type": "AWS::SSM::Parameter",
901901
"Properties": {
@@ -1167,7 +1167,7 @@
11671167
},
11681168
"/",
11691169
{
1170-
"Ref": "AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbS3Bucket475C950C"
1170+
"Ref": "AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20S3Bucket12418C8C"
11711171
},
11721172
"/",
11731173
{
@@ -1177,7 +1177,7 @@
11771177
"Fn::Split": [
11781178
"||",
11791179
{
1180-
"Ref": "AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbS3VersionKeyA1911337"
1180+
"Ref": "AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20S3VersionKey8C9B24CA"
11811181
}
11821182
]
11831183
}
@@ -1190,7 +1190,7 @@
11901190
"Fn::Split": [
11911191
"||",
11921192
{
1193-
"Ref": "AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbS3VersionKeyA1911337"
1193+
"Ref": "AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20S3VersionKey8C9B24CA"
11941194
}
11951195
]
11961196
}
@@ -1334,17 +1334,17 @@
13341334
"Type": "String",
13351335
"Description": "Artifact hash for asset \"570f91ed45d0c45e8ff145969f7499419312e806c83f009b76539ce989960e51\""
13361336
},
1337-
"AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbS3Bucket475C950C": {
1337+
"AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20S3Bucket12418C8C": {
13381338
"Type": "String",
1339-
"Description": "S3 bucket for asset \"239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccb\""
1339+
"Description": "S3 bucket for asset \"daac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20\""
13401340
},
1341-
"AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbS3VersionKeyA1911337": {
1341+
"AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20S3VersionKey8C9B24CA": {
13421342
"Type": "String",
1343-
"Description": "S3 key for asset version \"239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccb\""
1343+
"Description": "S3 key for asset version \"daac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20\""
13441344
},
1345-
"AssetParameters239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccbArtifactHash3D850561": {
1345+
"AssetParametersdaac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20ArtifactHash90BA6C4A": {
13461346
"Type": "String",
1347-
"Description": "Artifact hash for asset \"239f3911452e16bf26eaf985b77bc9f361a22cb4adbc3e1d4fe5301abd724ccb\""
1347+
"Description": "Artifact hash for asset \"daac37af2b50452c854a73ef7e2c57d5229667e390db39773ffb9dfb497bbd20\""
13481348
}
13491349
}
13501350
}

‎packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json

+40-25
Original file line numberDiff line numberDiff line change
@@ -733,22 +733,6 @@
733733
"ToPort": 443
734734
}
735735
},
736-
"ClusterKubectlProviderSecurityGroup2D90691C": {
737-
"Type": "AWS::EC2::SecurityGroup",
738-
"Properties": {
739-
"GroupDescription": "Comminication between KubectlProvider and EKS Control Plane",
740-
"SecurityGroupEgress": [
741-
{
742-
"CidrIp": "0.0.0.0/0",
743-
"Description": "Allow all outbound traffic by default",
744-
"IpProtocol": "-1"
745-
}
746-
],
747-
"VpcId": {
748-
"Ref": "Vpc8378EB38"
749-
}
750-
}
751-
},
752736
"ClusterCreationRole360249B6": {
753737
"Type": "AWS::IAM::Role",
754738
"Properties": {
@@ -1067,6 +1051,22 @@
10671051
"UpdateReplacePolicy": "Delete",
10681052
"DeletionPolicy": "Delete"
10691053
},
1054+
"ClusterKubectlProviderSecurityGroup2D90691C": {
1055+
"Type": "AWS::EC2::SecurityGroup",
1056+
"Properties": {
1057+
"GroupDescription": "Comminication between KubectlProvider and EKS Control Plane",
1058+
"SecurityGroupEgress": [
1059+
{
1060+
"CidrIp": "0.0.0.0/0",
1061+
"Description": "Allow all outbound traffic by default",
1062+
"IpProtocol": "-1"
1063+
}
1064+
],
1065+
"VpcId": {
1066+
"Ref": "Vpc8378EB38"
1067+
}
1068+
}
1069+
},
10701070
"ClusterKubectlReadyBarrier200052AF": {
10711071
"Type": "AWS::SSM::Parameter",
10721072
"Properties": {
@@ -3039,7 +3039,7 @@
30393039
},
30403040
"/",
30413041
{
3042-
"Ref": "AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aS3BucketFEA73057"
3042+
"Ref": "AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcS3BucketCA7ADF01"
30433043
},
30443044
"/",
30453045
{
@@ -3049,7 +3049,7 @@
30493049
"Fn::Split": [
30503050
"||",
30513051
{
3052-
"Ref": "AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aS3VersionKey00C85273"
3052+
"Ref": "AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcS3VersionKey822F0346"
30533053
}
30543054
]
30553055
}
@@ -3062,7 +3062,7 @@
30623062
"Fn::Split": [
30633063
"||",
30643064
{
3065-
"Ref": "AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aS3VersionKey00C85273"
3065+
"Ref": "AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcS3VersionKey822F0346"
30663066
}
30673067
]
30683068
}
@@ -3090,6 +3090,21 @@
30903090
"referencetoawscdkeksclustertestAssetParametersb7d8a9750f8bfded8ac76be100e3bee1c3d4824df006766110d023f42952f5c2S3VersionKey901D947ARef": {
30913091
"Ref": "AssetParametersb7d8a9750f8bfded8ac76be100e3bee1c3d4824df006766110d023f42952f5c2S3VersionKey40FF2C4A"
30923092
},
3093+
"referencetoawscdkeksclustertestVpcPrivateSubnet1Subnet32A4EC2ARef": {
3094+
"Ref": "VpcPrivateSubnet1Subnet536B997A"
3095+
},
3096+
"referencetoawscdkeksclustertestVpcPrivateSubnet2Subnet5CC53627Ref": {
3097+
"Ref": "VpcPrivateSubnet2Subnet3788AAA1"
3098+
},
3099+
"referencetoawscdkeksclustertestVpcPrivateSubnet3Subnet7F5D6918Ref": {
3100+
"Ref": "VpcPrivateSubnet3SubnetF258B56E"
3101+
},
3102+
"referencetoawscdkeksclustertestClusterKubectlProviderSecurityGroupD167EE6BGroupId": {
3103+
"Fn::GetAtt": [
3104+
"ClusterKubectlProviderSecurityGroup2D90691C",
3105+
"GroupId"
3106+
]
3107+
},
30933108
"referencetoawscdkeksclustertestAssetParameters34131c2e554ab57ad3a47fc0a13173a5c2a4b65a7582fe9622277b3d04c8e1e1S3Bucket85526CA7Ref": {
30943109
"Ref": "AssetParameters34131c2e554ab57ad3a47fc0a13173a5c2a4b65a7582fe9622277b3d04c8e1e1S3BucketD25BCC90"
30953110
},
@@ -3765,17 +3780,17 @@
37653780
"Type": "String",
37663781
"Description": "Artifact hash for asset \"04fa2d485a51abd8261468eb6fa053d3a72242fc068fa75683232a52960b30cf\""
37673782
},
3768-
"AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aS3BucketFEA73057": {
3783+
"AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcS3BucketCA7ADF01": {
37693784
"Type": "String",
3770-
"Description": "S3 bucket for asset \"2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569a\""
3785+
"Description": "S3 bucket for asset \"a298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dc\""
37713786
},
3772-
"AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aS3VersionKey00C85273": {
3787+
"AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcS3VersionKey822F0346": {
37733788
"Type": "String",
3774-
"Description": "S3 key for asset version \"2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569a\""
3789+
"Description": "S3 key for asset version \"a298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dc\""
37753790
},
3776-
"AssetParameters2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569aArtifactHashF20FF2C0": {
3791+
"AssetParametersa298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dcArtifactHashA688F4F0": {
37773792
"Type": "String",
3778-
"Description": "Artifact hash for asset \"2944b93098bcfadbe7d696c05bc208c5956fbd7bf8bd0bc43a58410cdcee569a\""
3793+
"Description": "Artifact hash for asset \"a298dd278c9ef814ebac4c9d8b2dc8e1b8374a14c5b7d0e79f041a296668f5dc\""
37793794
},
37803795
"SsmParameterValueawsserviceeksoptimizedami116amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
37813796
"Type": "AWS::SSM::Parameter::Value<String>",

0 commit comments

Comments
 (0)