feat(eks): kubernetes resource pruning (#11932)

In order to support deletion of kubernetes manifest resources, the EKS module now automatically allocates and injects a "prune label" to all resources. This label is then passed down to `kubectl apply` with the `--prune` option so that any resources in the cluster that do not appear in the manifest will get deleted.

The `prune` option can be set to `false` (either at the `Cluster` level or at the KubernetesResource level) to disable this.

In order to avoid needing to update all tests, many of the existing tests set `prune: false` so that their outputs are not impacted.

Resolves #10495

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-eks/README.md

@@ -807,16 +807,29 @@ or through `cluster.addManifest()`) (e.g. `cluster.addManifest('foo', r1, r2,
 r3,...)`), these resources will be applied as a single manifest via `kubectl`
 and will be applied sequentially (the standard behavior in `kubectl`).
 
-----------------------
+---
 
 Since Kubernetes manifests are implemented as CloudFormation resources in the
 CDK. This means that if the manifest is deleted from your code (or the stack is
 deleted), the next `cdk deploy` will issue a `kubectl delete` command and the
 Kubernetes resources in that manifest will be deleted.
 
-#### Caveat
+#### Resource Pruning
+
+When a resource is deleted from a Kubernetes manifest, the EKS module will
+automatically delete these resources by injecting a _prune label_ to all
+manifest resources. This label is then passed to [`kubectl apply --prune`].
+
+[`kubectl apply --prune`]: https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/#alternative-kubectl-apply-f-directory-prune-l-your-label
+
+Pruning is enabled by default but can be disabled through the `prune` option
+when a cluster is defined:
 
-If you have multiple resources in a single `KubernetesManifest`, and one of those **resources** is removed from the manifest, it will not be deleted and will remain orphan. See [Support Object pruning](https://github.com/aws/aws-cdk/issues/10495) for more details.
+```ts
+new Cluster(this, 'MyCluster', {
+  prune: false
+});
+```
 
 ### Helm Charts
 

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -122,6 +122,15 @@ export interface ICluster extends IResource, ec2.IConnectable {
    * Amount of memory to allocate to the provider's lambda function.
    */
   readonly kubectlMemory?: Size;
+
+  /**
+   * Indicates whether Kubernetes resources can be automatically pruned. When
+   * this is enabled (default), prune labels will be allocated and injected to
+   * each resource. These labels will then be used when issuing the `kubectl
+   * apply` operation with the `--prune` switch.
+   */
+  readonly prune: boolean;
+
   /**
    * Creates a new service account with corresponding IAM Role (IRSA).
    *
@@ -281,6 +290,16 @@ export interface ClusterAttributes {
    * @default Size.gibibytes(1)
    */
   readonly kubectlMemory?: Size;
+
+  /**
+   * Indicates whether Kubernetes resources added through `addManifest()` can be
+   * automatically pruned. When this is enabled (default), prune labels will be
+   * allocated and injected to each resource. These labels will then be used
+   * when issuing the `kubectl apply` operation with the `--prune` switch.
+   *
+   * @default true
+   */
+  readonly prune?: boolean;
 }
 
 /**
@@ -433,6 +452,16 @@ export interface ClusterOptions extends CommonClusterOptions {
    * @default Size.gibibytes(1)
    */
   readonly kubectlMemory?: Size;
+
+  /**
+   * Indicates whether Kubernetes resources added through `addManifest()` can be
+   * automatically pruned. When this is enabled (default), prune labels will be
+   * allocated and injected to each resource. These labels will then be used
+   * when issuing the `kubectl apply` operation with the `--prune` switch.
+   *
+   * @default true
+   */
+  readonly prune?: boolean;
 }
 
 /**
@@ -648,6 +677,7 @@ abstract class ClusterBase extends Resource implements ICluster {
   public abstract readonly kubectlSecurityGroup?: ec2.ISecurityGroup;
   public abstract readonly kubectlPrivateSubnets?: ec2.ISubnet[];
   public abstract readonly kubectlMemory?: Size;
+  public abstract readonly prune: boolean;
   public abstract readonly openIdConnectProvider: iam.IOpenIdConnectProvider;
 
   /**
@@ -865,6 +895,11 @@ export class Cluster extends ClusterBase {
    */
   public readonly kubectlMemory?: Size;
 
+  /**
+   * Determines if Kubernetes resources can be pruned automatically.
+   */
+  public readonly prune: boolean;
+
   /**
    * If this cluster is kubectl-enabled, returns the `ClusterResource` object
    * that manages it. If this cluster is not kubectl-enabled (i.e. uses the
@@ -925,6 +960,7 @@ export class Cluster extends ClusterBase {
 
     const stack = Stack.of(this);
 
+    this.prune = props.prune ?? true;
     this.vpc = props.vpc || new ec2.Vpc(this, 'DefaultVpc');
     this.version = props.version;
 
@@ -1655,6 +1691,7 @@ class ImportedCluster extends ClusterBase {
   public readonly kubectlPrivateSubnets?: ec2.ISubnet[] | undefined;
   public readonly kubectlLayer?: lambda.ILayerVersion;
   public readonly kubectlMemory?: Size;
+  public readonly prune: boolean;
 
   constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) {
     super(scope, id);
@@ -1667,6 +1704,7 @@ class ImportedCluster extends ClusterBase {
     this.kubectlPrivateSubnets = props.kubectlPrivateSubnetIds ? props.kubectlPrivateSubnetIds.map((subnetid, index) => ec2.Subnet.fromSubnetId(this, `KubectlSubnet${index}`, subnetid)) : undefined;
     this.kubectlLayer = props.kubectlLayer;
     this.kubectlMemory = props.kubectlMemory;
+    this.prune = props.prune ?? true;
 
     let i = 1;
     for (const sgid of props.securityGroupIds ?? []) {

packages/@aws-cdk/aws-eks/lib/k8s-manifest.ts

@@ -1,16 +1,48 @@
 import { CustomResource, Stack } from '@aws-cdk/core';
-import { Construct } from 'constructs';
+import { Construct, Node } from 'constructs';
 import { ICluster } from './cluster';
 import { KubectlProvider } from './kubectl-provider';
 
 // v2 - keep this import as a separate section to reduce merge conflict when forward merging with the v2 branch.
 // eslint-disable-next-line
 import { Construct as CoreConstruct } from '@aws-cdk/core';
 
+const PRUNE_LABEL_PREFIX = 'aws.cdk.eks/prune-';
+
+/**
+ * Options for `KubernetesManifest`.
+ */
+export interface KubernetesManifestOptions {
+  /**
+   * When a resource is removed from a Kubernetes manifest, it no longer appears
+   * in the manifest, and there is no way to know that this resource needs to be
+   * deleted. To address this, `kubectl apply` has a `--prune` option which will
+   * query the cluster for all resources with a specific label and will remove
+   * all the labeld resources that are not part of the applied manifest. If this
+   * option is disabled and a resource is removed, it will become "orphaned" and
+   * will not be deleted from the cluster.
+   *
+   * When this option is enabled (default), the construct will inject a label to
+   * all Kubernetes resources included in this manifest which will be used to
+   * prune resources when the manifest changes via `kubectl apply --prune`.
+   *
+   * The label name will be `aws.cdk.eks/prune-<ADDR>` where `<ADDR>` is the
+   * 42-char unique address of this construct in the construct tree. Value is
+   * empty.
+   *
+   * @see
+   * https://kubernetes.io/docs/tasks/manage-kubernetes-objects/declarative-config/#alternative-kubectl-apply-f-directory-prune-l-your-label
+   *
+   * @default - based on the prune option of the cluster, which is `true` unless
+   * otherwise specified.
+   */
+  readonly prune?: boolean;
+}
+
 /**
  * Properties for KubernetesManifest
  */
-export interface KubernetesManifestProps {
+export interface KubernetesManifestProps extends KubernetesManifestOptions {
   /**
    * The EKS cluster to apply this manifest to.
    *
@@ -62,6 +94,11 @@ export class KubernetesManifest extends CoreConstruct {
     const stack = Stack.of(this);
     const provider = KubectlProvider.getOrCreate(this, props.cluster);
 
+    const prune = props.prune ?? props.cluster.prune;
+    const pruneLabel = prune
+      ? this.injectPruneLabel(props.manifest)
+      : undefined;
+
     new CustomResource(this, 'Resource', {
       serviceToken: provider.serviceToken,
       resourceType: KubernetesManifest.RESOURCE_TYPE,
@@ -72,7 +109,42 @@ export class KubernetesManifest extends CoreConstruct {
         Manifest: stack.toJsonString(props.manifest),
         ClusterName: props.cluster.clusterName,
         RoleArn: provider.roleArn, // TODO: bake into provider's environment
+        PruneLabel: pruneLabel,
       },
     });
   }
+
+  /**
+   * Injects a generated prune label to all resources in this manifest. The
+   * label name will be `awscdk.eks/manifest-ADDR` where `ADDR` is the address
+   * of the construct in the construct tree.
+   *
+   * @returns the label name
+   */
+  private injectPruneLabel(manifest: Record<string, any>[]): string {
+    // max label name is 64 chars and addrs is always 42.
+    const pruneLabel = PRUNE_LABEL_PREFIX + Node.of(this).addr;
+
+    for (const resource of manifest) {
+      // skip resource if it's not an object or if it does not have a "kind"
+      if (typeof(resource) !== 'object' || !resource.kind) {
+        continue;
+      }
+
+      if (!resource.metadata) {
+        resource.metadata = {};
+      }
+
+      if (!resource.metadata.labels) {
+        resource.metadata.labels = {};
+      }
+
+      resource.metadata.labels = {
+        [pruneLabel]: '',
+        ...resource.metadata.labels,
+      };
+    }
+
+    return pruneLabel;
+  }
 }

packages/@aws-cdk/aws-eks/lib/kubectl-handler/apply/__init__.py

@@ -22,6 +22,7 @@ def apply_handler(event, context):
     cluster_name  = props['ClusterName']
     manifest_text = props['Manifest']
     role_arn      = props['RoleArn']
+    prune_label   = props.get('PruneLabel', None)
 
     # "log in" to the cluster
     subprocess.check_call([ 'aws', 'eks', 'update-kubeconfig',
@@ -40,20 +41,23 @@ def apply_handler(event, context):
     logger.info("manifest written to: %s" % manifest_file)
 
     if request_type == 'Create' or request_type == 'Update':
-        kubectl('apply', manifest_file)
+        opts = []
+        if prune_label is not None:
+            opts = ['--prune', '-l', prune_label]
+        kubectl('apply', manifest_file, *opts)
     elif request_type == "Delete":
         try:
             kubectl('delete', manifest_file)
         except Exception as e:
             logger.info("delete error: %s" % e)
 
 
-def kubectl(verb, file):
+def kubectl(verb, file, *opts):
     maxAttempts = 3
     retry = maxAttempts
     while retry > 0:
         try:
-            cmd = ['kubectl', verb, '--kubeconfig', kubeconfig, '-f', file]
+            cmd = ['kubectl', verb, '--kubeconfig', kubeconfig, '-f', file] + list(opts)
             logger.info(f'Running command: {cmd}')
             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as exc:

packages/@aws-cdk/aws-eks/lib/legacy-cluster.ts

@@ -147,6 +147,8 @@ export class LegacyCluster extends Resource implements ICluster {
    */
   public readonly defaultNodegroup?: Nodegroup;
 
+  public readonly prune: boolean = false;
+
   private readonly version: KubernetesVersion;
 
   /**
@@ -424,6 +426,7 @@ class ImportedCluster extends Resource implements ICluster {
   public readonly clusterName: string;
   public readonly clusterArn: string;
   public readonly connections = new ec2.Connections();
+  public readonly prune: boolean = false;
 
   constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) {
     super(scope, id);

packages/@aws-cdk/aws-eks/test/integ.eks-cluster-private-endpoint.ts

@@ -26,6 +26,7 @@ class EksClusterStack extends TestStack {
       defaultCapacity: 2,
       version: CLUSTER_VERSION,
       endpointAccess: eks.EndpointAccess.PRIVATE,
+      prune: false,
     });
 
     // this is the valdiation. it won't work if the private access is not setup properly.