fix(eks): kubectl resources fail before fargate profiles are created (#…

…8859) When a Fargate profile is being created, the Kubernetes API server in EKS sometimes rejects requests. This means that kubectl-related resources such as KubernetesResources Helm charts may fail during deployment. To address this, we add a "barrier resource" (in the form of an SSM parameter) which waits for all fargate profiles to be created before allowing kubectl resources to continue. This is done by the barrier taking a dependency on all FargateProfile resources and all kubectl resources taking a dependency on the barrier. Fixes #8854 This commit also fixes #8574 by adding `iam:ListAttachedRolePolicies` to the cluster's creation role IAM policy. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Jul 2, 2020 · 4fad9bc · 4fad9bc
1 parent 8e055d4
commit 4fad9bc
Show file tree

Hide file tree

Showing 9 changed files with 206 additions and 35 deletions.
diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts
@@ -104,7 +104,7 @@ export class ClusterResource extends Construct {
     }));
 
     this.creationRole.addToPolicy(new iam.PolicyStatement({
-      actions: [ 'iam:GetRole' ],
+      actions: [ 'iam:GetRole', 'iam:listAttachedRolePolicies' ],
       resources: [ '*' ],
     }));
 

diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts
@@ -2,7 +2,7 @@ import * as autoscaling from '@aws-cdk/aws-autoscaling';
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
 import * as ssm from '@aws-cdk/aws-ssm';
-import { CfnOutput, Construct, IResource, Resource, Stack, Tag, Token } from '@aws-cdk/core';
+import { CfnOutput, CfnResource, Construct, IResource, Resource, Stack, Tag, Token } from '@aws-cdk/core';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as YAML from 'yaml';
@@ -392,6 +392,20 @@ export class Cluster extends Resource implements ICluster {
 
   private readonly version: string | undefined;
 
+  /**
+   * A dummy CloudFormation resource that is used as a wait barrier which
+   * represents that the cluster is ready to receive "kubectl" commands.
+   *
+   * Specifically, all fargate profiles are automatically added as a dependency
+   * of this barrier, which means that it will only be "signaled" when all
+   * fargate profiles have been successfully created.
+   *
+   * When kubectl resources call `_attachKubectlResourceScope()`, this resource
+   * is added as their dependency which implies that they can only be deployed
+   * after the cluster is ready.
+   */
+  private readonly _kubectlReadyBarrier?: CfnResource;
+
   /**
    * Initiates an EKS Cluster with the supplied arguments
    *
@@ -448,6 +462,18 @@ export class Cluster extends Resource implements ICluster {
     if (this.kubectlEnabled) {
       resource = new ClusterResource(this, 'Resource', clusterProps);
       this._clusterResource = resource;
+
+      // we use an SSM parameter as a barrier because it's free and fast.
+      this._kubectlReadyBarrier = new CfnResource(this, 'KubectlReadyBarrier', {
+        type: 'AWS::SSM::Parameter',
+        properties: {
+          Type: 'String',
+          Value: 'aws:cdk:eks:kubectl-ready',
+        },
+      });
+
+      // add the cluster resource itself as a dependency of the barrier
+      this._kubectlReadyBarrier.node.addDependency(this._clusterResource);
     } else {
       resource = new CfnCluster(this, 'Resource', clusterProps);
     }
@@ -502,7 +528,7 @@ export class Cluster extends Resource implements ICluster {
     }
 
     if (this.kubectlEnabled) {
-      this.defineCoreDnsComputeType(props.coreDnsComputeType || CoreDnsComputeType.EC2);
+      this.defineCoreDnsComputeType(props.coreDnsComputeType ?? CoreDnsComputeType.EC2);
     }
   }
 
@@ -794,33 +820,59 @@ export class Cluster extends Resource implements ICluster {
   }
 
   /**
-   * Returns the custom resource provider for kubectl-related resources.
+   * Internal API used by `FargateProfile` to keep inventory of Fargate profiles associated with
+   * this cluster, for the sake of ensuring the profiles are created sequentially.
+   *
+   * @returns the list of FargateProfiles attached to this cluster, including the one just attached.
+   * @internal
+   */
+  public _attachFargateProfile(fargateProfile: FargateProfile): FargateProfile[] {
+    this._fargateProfiles.push(fargateProfile);
+
+    // add all profiles as a dependency of the "kubectl-ready" barrier because all kubectl-
+    // resources can only be deployed after all fargate profiles are created.
+    if (this._kubectlReadyBarrier) {
+      this._kubectlReadyBarrier.node.addDependency(fargateProfile);
+    }
+
+    return this._fargateProfiles;
+  }
+
+  /**
+   * Adds a resource scope that requires `kubectl` to this cluster and returns
+   * the `KubectlProvider` which is the custom resource provider that should be
+   * used as the resource provider.
+   *
+   * Called from `HelmResource` and `KubernetesResource`
+   *
+   * @property resourceScope the construct scope in which kubectl resources are defined.
+   *
    * @internal
    */
-  public get _kubectlProvider(): KubectlProvider {
+  public _attachKubectlResourceScope(resourceScope: Construct): KubectlProvider {
+    const uid = '@aws-cdk/aws-eks.KubectlProvider';
+
     if (!this._clusterResource) {
       throw new Error('Unable to perform this operation since kubectl is not enabled for this cluster');
     }
 
-    const uid = '@aws-cdk/aws-eks.KubectlProvider';
-    const provider = this.stack.node.tryFindChild(uid) as KubectlProvider || new KubectlProvider(this.stack, uid);
+    // singleton
+    let provider = this.stack.node.tryFindChild(uid) as KubectlProvider;
+    if (!provider) {
+      // create the provider.
+      provider = new KubectlProvider(this.stack, uid);
+    }
 
     // allow the kubectl provider to assume the cluster creation role.
     this._clusterResource.addTrustedRole(provider.role);
 
-    return provider;
-  }
+    if (!this._kubectlReadyBarrier) {
+      throw new Error('unexpected: kubectl enabled clusters should have a kubectl-ready barrier resource');
+    }
 
-  /**
-   * Internal API used by `FargateProfile` to keep inventory of Fargate profiles associated with
-   * this cluster, for the sake of ensuring the profiles are created sequentially.
-   *
-   * @returns the list of FargateProfiles attached to this cluster, including the one just attached.
-   * @internal
-   */
-  public _attachFargateProfile(fargateProfile: FargateProfile): FargateProfile[] {
-    this._fargateProfiles.push(fargateProfile);
-    return this._fargateProfiles;
+    resourceScope.node.addDependency(this._kubectlReadyBarrier);
+
+    return provider;
   }
 
   /**

diff --git a/packages/@aws-cdk/aws-eks/lib/helm-chart.ts b/packages/@aws-cdk/aws-eks/lib/helm-chart.ts
@@ -89,7 +89,7 @@ export class HelmChart extends Construct {
 
     const stack = Stack.of(this);
 
-    const provider = props.cluster._kubectlProvider;
+    const provider = props.cluster._attachKubectlResourceScope(this);
 
     const timeout = props.timeout?.toSeconds();
     if (timeout && timeout > 900) {

diff --git a/packages/@aws-cdk/aws-eks/lib/k8s-patch.ts b/packages/@aws-cdk/aws-eks/lib/k8s-patch.ts
@@ -70,7 +70,7 @@ export class KubernetesPatch extends Construct {
     super(scope, id);
 
     const stack = Stack.of(this);
-    const provider = props.cluster._kubectlProvider;
+    const provider = props.cluster._attachKubectlResourceScope(this);
 
     new CustomResource(this, 'Resource', {
       serviceToken: provider.serviceToken,

diff --git a/packages/@aws-cdk/aws-eks/lib/k8s-resource.ts b/packages/@aws-cdk/aws-eks/lib/k8s-resource.ts
@@ -54,7 +54,7 @@ export class KubernetesResource extends Construct {
     super(scope, id);
 
     const stack = Stack.of(this);
-    const provider = props.cluster._kubectlProvider;
+    const provider = props.cluster._attachKubectlResourceScope(this);
 
     new CustomResource(this, 'Resource', {
       serviceToken: provider.serviceToken,

diff --git a/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts b/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts
@@ -24,6 +24,7 @@ export class KubectlProvider extends NestedStack {
       runtime: lambda.Runtime.PYTHON_3_7,
       handler: 'index.handler',
       timeout: Duration.minutes(15),
+      description: 'onEvent handler for EKS kubectl resource provider',
       layers: [ KubectlLayer.getOrCreate(this, { version: '2.0.0' }) ],
       memorySize: 256,
     });

diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json
@@ -756,7 +756,10 @@
               "Resource": "*"
             },
             {
-              "Action": "iam:GetRole",
+              "Action": [
+                "iam:GetRole",
+                "iam:listAttachedRolePolicies"
+              ],
               "Effect": "Allow",
               "Resource": "*"
             },
@@ -849,6 +852,20 @@
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
+    "ClusterKubectlReadyBarrier200052AF": {
+      "Type": "AWS::SSM::Parameter",
+      "Properties": {
+        "Type": "String",
+        "Value": "aws:cdk:eks:kubectl-ready"
+      },
+      "DependsOn": [
+        "ClusterfargateprofiledefaultPodExecutionRole09952CFF",
+        "ClusterfargateprofiledefaultEFC59F14",
+        "ClusterCreationRoleDefaultPolicyE8BDFC7B",
+        "ClusterCreationRole360249B6",
+        "Cluster9EE0221C"
+      ]
+    },
     "ClusterAwsAuthmanifestFE51F8AE": {
       "Type": "Custom::AWSCDK-EKS-KubernetesResource",
       "Properties": {
@@ -925,6 +942,9 @@
           ]
         }
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -1998,6 +2018,9 @@
         "Repository": "https://aws.github.io/eks-charts",
         "CreateNamespace": true
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2309,6 +2332,9 @@
           ]
         }
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2431,6 +2457,9 @@
           ]
         }
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2458,6 +2487,9 @@
         "Repository": "https://kubernetes.github.io/dashboard/",
         "CreateNamespace": true
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2481,6 +2513,9 @@
           ]
         }
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2510,6 +2545,7 @@
         "Repository": "https://helm.nginx.com/stable"
       },
       "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF",
         "ClustermanifestnginxnamespaceA68B4CE0"
       ],
       "UpdateReplacePolicy": "Delete",
@@ -2636,6 +2672,9 @@
           ]
         }
       },
+      "DependsOn": [
+        "ClusterKubectlReadyBarrier200052AF"
+      ],
       "UpdateReplacePolicy": "Delete",
       "DeletionPolicy": "Delete"
     },
@@ -2713,7 +2752,7 @@
               },
               "/",
               {
-                "Ref": "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762S3Bucket8B322431"
+                "Ref": "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bS3BucketBEF9DA08"
               },
               "/",
               {
@@ -2723,7 +2762,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762S3VersionKey3D736637"
+                        "Ref": "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bS3VersionKey8B401BBD"
                       }
                     ]
                   }
@@ -2736,7 +2775,7 @@
                     "Fn::Split": [
                       "||",
                       {
-                        "Ref": "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762S3VersionKey3D736637"
+                        "Ref": "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bS3VersionKey8B401BBD"
                       }
                     ]
                   }
@@ -3099,17 +3138,17 @@
       "Type": "String",
       "Description": "Artifact hash for asset \"5215f685494c7a295ec1b06b713a041a82e7ac216473965711a88e32405e9053\""
     },
-    "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762S3Bucket8B322431": {
+    "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bS3BucketBEF9DA08": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762\""
+      "Description": "S3 bucket for asset \"2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42b\""
     },
-    "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762S3VersionKey3D736637": {
+    "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bS3VersionKey8B401BBD": {
       "Type": "String",
-      "Description": "S3 key for asset version \"159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762\""
+      "Description": "S3 key for asset version \"2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42b\""
     },
-    "AssetParameters159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762ArtifactHash56F3AB07": {
+    "AssetParameters2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42bArtifactHash87F44C09": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"159827cc934bc51d62122aee0f5681239bab34e46900fceb8557c05e8109d762\""
+      "Description": "Artifact hash for asset \"2181e1ea22ea11a566260dec2f26c5f66ac77bb1b73812ba467b9c3bc564e42b\""
     },
     "SsmParameterValueawsserviceeksoptimizedami116amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": {
       "Type": "AWS::SSM::Parameter::Value<String>",