feat(eks): support adding k8s resources to imported clusters

packages/@aws-cdk/aws-eks/README.md

@@ -322,7 +322,7 @@ new KubernetesManifest(this, 'hello-kub', {
 cluster.addManifest('hello-kub', service, deployment);
 ```
 
-##### Kubectl Environment
+#### Kubectl Environment
 
 The resources are created in the cluster by running `kubectl apply` from a python lambda function. You can configure the environment of this function by specifying it at cluster instantiation. For example, this can useful in order to configure an http proxy:
 
@@ -333,7 +333,6 @@ const cluster = new eks.Cluster(this, 'hello-eks', {
     'http_proxy': 'http://proxy.myproxy.com'
   }
 });
-
 ```
 
 #### Adding resources from a URL
@@ -432,6 +431,59 @@ Specifically, since the above use-case is quite common, there is an easier way t
 const loadBalancerAddress = cluster.getServiceLoadBalancerAddress('my-service');
 ```
 
+### Kubernetes Resources in Existing Clusters
+
+The Amazon EKS library allows defining Kubernetes resources such as [Kubernetes
+manifests](#kubernetes-resources) and [Helm charts](#helm-charts) on clusters
+that are not defined as part of your CDK app.
+
+First, you'll need to "import" a cluster to your CDK app. To do that, use the
+`eks.Cluster.fromClusterAttributes()` static method:
+
+```ts
+const cluster = eks.Cluster.fromClusterAttributes(this, 'MyCluster', {
+  clusterName: 'my-cluster-name',
+  kubectlRoleArn: 'arn:aws:iam::1111111:role/iam-role-that-has-masters-access',
+});
+```
+
+Then, you can use `addManifest` or `addHelmChart` to define resources inside
+your Kubernetes cluster. For example:
+
+```ts
+cluster.addManifest('Test', {
+  apiVersion: 'v1',
+  kind: 'ConfigMap',
+  metadata: {
+    name: 'myconfigmap',
+  },
+  data: {
+    Key: 'value',
+    Another: '123454',
+  },
+});
+```
+
+At the minimum, when importing clusters for `kubectl` management, you will need
+to specify:
+
+- `clusterName` - the name of the cluster.
+- `kubectlRoleArn` - the ARN of an IAM role mapped to the `system:masters` RBAC
+  role. If the cluster you are importing was created using the AWS CDK, the
+  CloudFormation stack has an output that includes an IAM role that can be used.
+  Otherwise, you can create an IAM role and map it to `system:masters` manually.
+  The trust policy of this role should include the the
+  `arn:aws::iam::${accountId}:root` principal in order to allow the execution
+  role of the kubectl resource to assume it.
+
+If the cluster is configured with private-only or private and restricted public
+Kubernetes [endpoint access](#endpoint-access), you must also specify:
+
+- `kubectlSecurityGroupId` - the ID of an EC2 security group that is allowed
+  connections to the cluster's control security group.
+- `kubectlPrivateSubnetIds` - a list of private VPC subnets IDs that will be used
+  to access the Kubernetes endpoint.
+
 ### AWS IAM Mapping
 
 As described in the [Amazon EKS User Guide](https://docs.aws.amazon.com/en_us/eks/latest/userguide/add-user-role.html),
@@ -623,7 +675,3 @@ mypod.node.addDependency(sa);
 // print the IAM role arn for this service account
 new cdk.CfnOutput(this, 'ServiceAccountIamRole', { value: sa.role.roleArn })
 ```
-
-### Roadmap
-
-- [ ] AutoScaling (combine EC2 and Kubernetes scaling)

packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts

@@ -7,6 +7,13 @@ import * as cr from '@aws-cdk/custom-resources';
 const HANDLER_DIR = path.join(__dirname, 'cluster-resource-handler');
 const HANDLER_RUNTIME = lambda.Runtime.NODEJS_12_X;
 
+export interface ClusterResourceProviderProps {
+  /**
+   * The IAM role to assume in order to interact with the cluster.
+   */
+  readonly adminRole: iam.IRole;
+}
+
 /**
  * A custom resource provider that handles cluster operations. It serves
  * multiple custom resources such as the cluster resource and the fargate
@@ -16,23 +23,18 @@ const HANDLER_RUNTIME = lambda.Runtime.NODEJS_12_X;
  */
 export class ClusterResourceProvider extends NestedStack {
 
-  public static getOrCreate(scope: Construct) {
+  public static getOrCreate(scope: Construct, props: ClusterResourceProviderProps) {
     const stack = Stack.of(scope);
     const uid = '@aws-cdk/aws-eks.ClusterResourceProvider';
-    return stack.node.tryFindChild(uid) as ClusterResourceProvider || new ClusterResourceProvider(stack, uid);
+    return stack.node.tryFindChild(uid) as ClusterResourceProvider ?? new ClusterResourceProvider(stack, uid, props);
   }
 
   /**
    * The custom resource provider to use for custom resources.
    */
   public readonly provider: cr.Provider;
 
-  /**
-   * The IAM roles used by the provider's lambda handlers.
-   */
-  public readonly roles: iam.IRole[];
-
-  private constructor(scope: Construct, id: string) {
+  private constructor(scope: Construct, id: string, props: ClusterResourceProviderProps) {
     super(scope, id);
 
     const onEvent = new lambda.Function(this, 'OnEventHandler', {
@@ -58,7 +60,8 @@ export class ClusterResourceProvider extends NestedStack {
       queryInterval: Duration.minutes(1),
     });
 
-    this.roles = [onEvent.role!, isComplete.role!];
+    props.adminRole.grant(onEvent.role!, 'sts:AssumeRole');
+    props.adminRole.grant(isComplete.role!, 'sts:AssumeRole');
   }
 
   /**

packages/@aws-cdk/aws-eks/lib/cluster-resource.ts

@@ -1,26 +1,22 @@
+import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
-import { ArnComponents, Construct, CustomResource, Lazy, Stack, Token } from '@aws-cdk/core';
+import * as kms from '@aws-cdk/aws-kms';
+import { ArnComponents, Construct, CustomResource, Token, Stack, Lazy } from '@aws-cdk/core';
 import { CLUSTER_RESOURCE_TYPE } from './cluster-resource-handler/consts';
 import { ClusterResourceProvider } from './cluster-resource-provider';
-import { CfnClusterProps, CfnCluster } from './eks.generated';
-
-export interface ClusterResourceProps extends CfnClusterProps {
-
-  /**
-   * Enable private endpoint access to the cluster.
-   */
+import { CfnCluster } from './eks.generated';
+
+export interface ClusterResourceProps {
+  readonly resourcesVpcConfig: CfnCluster.ResourcesVpcConfigProperty;
+  readonly roleArn: string;
+  readonly encryptionConfig?: Array<CfnCluster.EncryptionConfigProperty>;
+  readonly name: string;
+  readonly version?: string;
   readonly endpointPrivateAccess: boolean;
-
-  /**
-   * Enable public endpoint access to the cluster.
-   */
   readonly endpointPublicAccess: boolean;
-
-  /**
-   * Limit public address with CIDR blocks.
-   */
   readonly publicAccessCidrs?: string[];
-
+  readonly vpc: ec2.IVpc;
+  readonly secretsEncryptionKey?: kms.IKey;
 }
 
 /**
@@ -42,34 +38,76 @@ export class ClusterResource extends Construct {
   public readonly attrOpenIdConnectIssuerUrl: string;
   public readonly attrOpenIdConnectIssuer: string;
   public readonly ref: string;
-  /**
-   * The IAM role which created the cluster. Initially this is the only IAM role
-   * that gets administrator privilages on the cluster (`system:masters`), and
-   * will be able to issue `kubectl` commands against it.
-   */
-  public readonly creationRole: iam.Role;
 
-  private readonly trustedPrincipals: string[] = [];
+  public readonly adminRole: iam.Role;
 
   constructor(scope: Construct, id: string, props: ClusterResourceProps) {
     super(scope, id);
 
-    const stack = Stack.of(this);
-    const provider = ClusterResourceProvider.getOrCreate(this);
-
     if (!props.roleArn) {
       throw new Error('"roleArn" is required');
     }
 
+    this.adminRole = this.createAdminRole(props);
+
+    const provider = ClusterResourceProvider.getOrCreate(this, {
+      adminRole: this.adminRole,
+    });
+
+    const resource = new CustomResource(this, 'Resource', {
+      resourceType: CLUSTER_RESOURCE_TYPE,
+      serviceToken: provider.serviceToken,
+      properties: {
+        // the structure of config needs to be that of 'aws.EKS.CreateClusterRequest' since its passed as is
+        // to the eks.createCluster sdk invocation.
+        Config: {
+          name: props.name,
+          version: props.version,
+          roleArn: props.roleArn,
+          encryptionConfig: props.encryptionConfig,
+          resourcesVpcConfig: {
+            subnetIds: (props.resourcesVpcConfig as CfnCluster.ResourcesVpcConfigProperty).subnetIds,
+            securityGroupIds: (props.resourcesVpcConfig as CfnCluster.ResourcesVpcConfigProperty).securityGroupIds,
+            endpointPublicAccess: props.endpointPublicAccess,
+            endpointPrivateAccess: props.endpointPrivateAccess,
+            publicAccessCidrs: props.publicAccessCidrs,
+          },
+        },
+        AssumeRoleArn: this.adminRole.roleArn,
+
+        // IMPORTANT: increment this number when you add new attributes to the
+        // resource. Otherwise, CloudFormation will error with "Vendor response
+        // doesn't contain XXX key in object" (see #8276) by incrementing this
+        // number, you will effectively cause a "no-op update" to the cluster
+        // which will return the new set of attribute.
+        AttributesRevision: 2,
+      },
+    });
+
+    resource.node.addDependency(this.adminRole);
+
+    this.ref = resource.ref;
+    this.attrEndpoint = Token.asString(resource.getAtt('Endpoint'));
+    this.attrArn = Token.asString(resource.getAtt('Arn'));
+    this.attrCertificateAuthorityData = Token.asString(resource.getAtt('CertificateAuthorityData'));
+    this.attrClusterSecurityGroupId = Token.asString(resource.getAtt('ClusterSecurityGroupId'));
+    this.attrEncryptionConfigKeyArn = Token.asString(resource.getAtt('EncryptionConfigKeyArn'));
+    this.attrOpenIdConnectIssuerUrl = Token.asString(resource.getAtt('OpenIdConnectIssuerUrl'));
+    this.attrOpenIdConnectIssuer = Token.asString(resource.getAtt('OpenIdConnectIssuer'));
+  }
+
+  private createAdminRole(props: ClusterResourceProps) {
+    const stack = Stack.of(this);
+
     // the role used to create the cluster. this becomes the administrator role
     // of the cluster.
-    this.creationRole = new iam.Role(this, 'CreationRole', {
-      assumedBy: new iam.CompositePrincipal(...provider.roles.map(x => new iam.ArnPrincipal(x.roleArn))),
+    const creationRole = new iam.Role(this, 'CreationRole', {
+      assumedBy: new iam.AccountRootPrincipal(),
     });
 
     // the CreateCluster API will allow the cluster to assume this role, so we
     // need to allow the lambda execution role to pass it.
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: ['iam:PassRole'],
       resources: [props.roleArn],
     }));
@@ -94,15 +132,15 @@ export class ClusterResource extends Construct {
         : '*',
     });
 
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: [
         'ec2:DescribeSubnets',
         'ec2:DescribeRouteTables',
       ],
       resources: ['*'],
     }));
 
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: [
         'eks:CreateCluster',
         'eks:DescribeCluster',
@@ -117,81 +155,46 @@ export class ClusterResource extends Construct {
       resources: resourceArns,
     }));
 
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: ['eks:DescribeFargateProfile', 'eks:DeleteFargateProfile'],
       resources: [fargateProfileResourceArn],
     }));
 
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: ['iam:GetRole', 'iam:listAttachedRolePolicies'],
       resources: ['*'],
     }));
 
-    this.creationRole.addToPolicy(new iam.PolicyStatement({
+    creationRole.addToPolicy(new iam.PolicyStatement({
       actions: ['iam:CreateServiceLinkedRole'],
       resources: ['*'],
     }));
 
-    const resource = new CustomResource(this, 'Resource', {
-      resourceType: CLUSTER_RESOURCE_TYPE,
-      serviceToken: provider.serviceToken,
-      properties: {
-        // the structure of config needs to be that of 'aws.EKS.CreateClusterRequest' since its passed as is
-        // to the eks.createCluster sdk invocation.
-        Config: {
-          name: props.name,
-          version: props.version,
-          roleArn: props.roleArn,
-          encryptionConfig: props.encryptionConfig,
-          resourcesVpcConfig: {
-            subnetIds: (props.resourcesVpcConfig as CfnCluster.ResourcesVpcConfigProperty).subnetIds,
-            securityGroupIds: (props.resourcesVpcConfig as CfnCluster.ResourcesVpcConfigProperty).securityGroupIds,
-            endpointPublicAccess: props.endpointPublicAccess,
-            endpointPrivateAccess: props.endpointPrivateAccess,
-            publicAccessCidrs: props.publicAccessCidrs,
-          },
-        },
-        AssumeRoleArn: this.creationRole.roleArn,
-
-        // IMPORTANT: increment this number when you add new attributes to the
-        // resource. Otherwise, CloudFormation will error with "Vendor response
-        // doesn't contain XXX key in object" (see #8276) by incrementing this
-        // number, you will effectively cause a "no-op update" to the cluster
-        // which will return the new set of attribute.
-        AttributesRevision: 2,
-      },
-    });
-
-    resource.node.addDependency(this.creationRole);
-
-    this.ref = resource.ref;
-    this.attrEndpoint = Token.asString(resource.getAtt('Endpoint'));
-    this.attrArn = Token.asString(resource.getAtt('Arn'));
-    this.attrCertificateAuthorityData = Token.asString(resource.getAtt('CertificateAuthorityData'));
-    this.attrClusterSecurityGroupId = Token.asString(resource.getAtt('ClusterSecurityGroupId'));
-    this.attrEncryptionConfigKeyArn = Token.asString(resource.getAtt('EncryptionConfigKeyArn'));
-    this.attrOpenIdConnectIssuerUrl = Token.asString(resource.getAtt('OpenIdConnectIssuerUrl'));
-    this.attrOpenIdConnectIssuer = Token.asString(resource.getAtt('OpenIdConnectIssuer'));
-  }
-
-  /**
-   * Grants `trustedRole` permissions to assume the creation role.
-   */
-  public addTrustedRole(trustedRole: iam.IRole): void {
-    if (this.trustedPrincipals.includes(trustedRole.roleArn)) {
-      return;
-    }
+    // see https://github.com/aws/aws-cdk/issues/9027
+    creationRole.addToPolicy(new iam.PolicyStatement({
+      actions: ['ec2:DescribeVpcs'],
+      resources: [stack.formatArn({
+        service: 'ec2',
+        resource: 'vpc',
+        resourceName: props.vpc.vpcId,
+      })],
+    }));
 
-    if (!this.creationRole.assumeRolePolicy) {
-      throw new Error('unexpected: cluster creation role must have trust policy');
+    // grant cluster creation role sufficient permission to access the specified key
+    // see https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html
+    if (props.secretsEncryptionKey) {
+      creationRole.addToPolicy(new iam.PolicyStatement({
+        actions: [
+          'kms:Encrypt',
+          'kms:Decrypt',
+          'kms:DescribeKey',
+          'kms:CreateGrant',
+        ],
+        resources: [props.secretsEncryptionKey.keyArn],
+      }));
     }
 
-    this.creationRole.assumeRolePolicy.addStatements(new iam.PolicyStatement({
-      actions: ['sts:AssumeRole'],
-      principals: [new iam.ArnPrincipal(trustedRole.roleArn)],
-    }));
-
-    this.trustedPrincipals.push(trustedRole.roleArn);
+    return creationRole;
   }
 }