feat(s3): option to auto delete objects upon bucket removal (#12090)

Use the custom resource provider from core to delete objects in the
bucket. A bucket policy gives the correct permissions to the provider's
Lambda function role.

Credits to @Chriscbr for starting the work on this.

Closes #3297
Closes #9751


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-s3/README.md

@@ -349,7 +349,7 @@ bucket.virtualHostedUrlForObject('objectname', { regional: false }); // Virtual
 
 ### Object Ownership
 
-You can use the two following properties to specify the bucket [object Ownership]. 
+You can use the two following properties to specify the bucket [object Ownership].
 
 [object Ownership]: https://docs.aws.amazon.com/AmazonS3/latest/dev/about-object-ownership.html
 
@@ -365,10 +365,28 @@ new s3.Bucket(this, 'MyBucket', {
 
 #### Bucket owner preferred
 
-The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account. 
+The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account.
 
 ```ts
 new s3.Bucket(this, 'MyBucket', {
   objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_PREFERRED,
 });
 ```
+
+### Bucket deletion
+
+When a bucket is removed from a stack (or the stack is deleted), the S3
+bucket will be removed according to its removal policy (which by default will
+simply orphan the bucket and leave it in your AWS account). If the removal
+policy is set to `RemovalPolicy.DESTROY`, the bucket will be deleted as long
+as it does not contain any objects.
+
+To override this and force all objects to get deleted during bucket deletion,
+enable the`autoDeleteObjects` option.
+
+```ts
+const bucket = new Bucket(this, 'MyTempFileBucket', {
+  removalPolicy: RemovalPolicy.DESTROY,
+  autoDeleteObjects: true,
+});
+```

packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts

@@ -0,0 +1,42 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { S3 } from 'aws-sdk';
+
+const s3 = new S3();
+
+export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent) {
+  switch (event.RequestType) {
+    case 'Create':
+    case 'Update':
+      return;
+    case 'Delete':
+      return onDelete(event);
+  }
+}
+
+/**
+ * Recursively delete all items in the bucket
+ *
+ * @param bucketName the bucket name
+ */
+async function emptyBucket(bucketName: string) {
+  const listedObjects = await s3.listObjectVersions({ Bucket: bucketName }).promise();
+  const contents = [...listedObjects.Versions ?? [], ...listedObjects.DeleteMarkers ?? []];
+  if (contents.length === 0) {
+    return;
+  };
+
+  const records = contents.map((record: any) => ({ Key: record.Key, VersionId: record.VersionId }));
+  await s3.deleteObjects({ Bucket: bucketName, Delete: { Objects: records } }).promise();
+
+  if (listedObjects?.IsTruncated) {
+    await emptyBucket(bucketName);
+  }
+}
+
+async function onDelete(deleteEvent: AWSLambda.CloudFormationCustomResourceDeleteEvent) {
+  const bucketName = deleteEvent.ResourceProperties?.BucketName;
+  if (!bucketName) {
+    throw new Error('No BucketName was provided.');
+  }
+  await emptyBucket(bucketName);
+}

packages/@aws-cdk/aws-s3/lib/bucket.ts

@@ -1,8 +1,12 @@
 import { EOL } from 'os';
+import * as path from 'path';
 import * as events from '@aws-cdk/aws-events';
 import * as iam from '@aws-cdk/aws-iam';
 import * as kms from '@aws-cdk/aws-kms';
-import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core';
+import {
+  Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token,
+  CustomResource, CustomResourceProvider, CustomResourceProviderRuntime,
+} from '@aws-cdk/core';
 import { Construct } from 'constructs';
 import { BucketPolicy } from './bucket-policy';
 import { IBucketNotificationDestination } from './destination';
@@ -12,6 +16,8 @@ import { LifecycleRule } from './rule';
 import { CfnBucket } from './s3.generated';
 import { parseBucketArn, parseBucketName } from './util';
 
+const AUTO_DELETE_OBJECTS_RESOURCE_TYPE = 'Custom::S3AutoDeleteObjects';
+
 export interface IBucket extends IResource {
   /**
    * The ARN of the bucket.
@@ -1041,6 +1047,16 @@ export interface BucketProps {
    */
   readonly removalPolicy?: RemovalPolicy;
 
+  /**
+   * Whether all objects should be automatically deleted when the bucket is
+   * removed from the stack or when the stack is deleted.
+   *
+   * Requires the `removalPolicy` to be set to `RemovalPolicy.DESTROY`.
+   *
+   * @default false
+   */
+  readonly autoDeleteObjects?: boolean;
+
   /**
    * Whether this bucket should have versioning turned on or not.
    *
@@ -1326,6 +1342,14 @@ export class Bucket extends BucketBase {
     if (props.publicReadAccess) {
       this.grantPublicAccess();
     }
+
+    if (props.autoDeleteObjects) {
+      if (props.removalPolicy !== RemovalPolicy.DESTROY) {
+        throw new Error('Cannot use \'autoDeleteObjects\' property on a bucket without setting removal policy to \'DESTROY\'.');
+      }
+
+      this.enableAutoDeleteObjects();
+    }
   }
 
   /**
@@ -1728,6 +1752,42 @@ export class Bucket extends BucketBase {
       };
     });
   }
+
+  private enableAutoDeleteObjects() {
+    const provider = CustomResourceProvider.getOrCreateProvider(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, {
+      codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'),
+      runtime: CustomResourceProviderRuntime.NODEJS_12,
+    });
+
+    // Use a bucket policy to allow the custom resource to delete
+    // objects in the bucket
+    this.addToResourcePolicy(new iam.PolicyStatement({
+      actions: [
+        ...perms.BUCKET_READ_ACTIONS, // list objects
+        ...perms.BUCKET_DELETE_ACTIONS, // and then delete them
+      ],
+      resources: [
+        this.bucketArn,
+        this.arnForObjects('*'),
+      ],
+      principals: [new iam.ArnPrincipal(provider.roleArn)],
+    }));
+
+    const customResource = new CustomResource(this, 'AutoDeleteObjectsCustomResource', {
+      resourceType: AUTO_DELETE_OBJECTS_RESOURCE_TYPE,
+      serviceToken: provider.serviceToken,
+      properties: {
+        BucketName: this.bucketName,
+      },
+    });
+
+    // Ensure bucket policy is deleted AFTER the custom resource otherwise
+    // we don't have permissions to list and delete in the bucket.
+    // (add a `if` to make TS happy)
+    if (this.policy) {
+      customResource.node.addDependency(this.policy);
+    }
+  }
 }
 
 /**

packages/@aws-cdk/aws-s3/test/auto-delete-objects-handler.test.ts

@@ -0,0 +1,168 @@
+const mockS3Client = {
+  listObjectVersions: jest.fn().mockReturnThis(),
+  deleteObjects: jest.fn().mockReturnThis(),
+  promise: jest.fn(),
+};
+
+import { handler } from '../lib/auto-delete-objects-handler';
+
+jest.mock('aws-sdk', () => {
+  return { S3: jest.fn(() => mockS3Client) };
+});
+
+beforeEach(() => {
+  mockS3Client.listObjectVersions.mockReturnThis();
+  mockS3Client.deleteObjects.mockReturnThis();
+});
+
+afterEach(() => {
+  jest.resetAllMocks();
+});
+
+test('does nothing on create event', async () => {
+  // GIVEN
+  const event: Partial<AWSLambda.CloudFormationCustomResourceCreateEvent> = {
+    RequestType: 'Create',
+    ResourceProperties: {
+      ServiceToken: 'Foo',
+      BucketName: 'MyBucket',
+    },
+  };
+
+  // WHEN
+  await invokeHandler(event);
+
+  // THEN
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0);
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('does nothing on update event', async () => {
+  // GIVEN
+  const event: Partial<AWSLambda.CloudFormationCustomResourceUpdateEvent> = {
+    RequestType: 'Update',
+    ResourceProperties: {
+      ServiceToken: 'Foo',
+      BucketName: 'MyBucket',
+    },
+  };
+
+  // WHEN
+  await invokeHandler(event);
+
+  // THEN
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0);
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('deletes no objects on delete event when bucket has no objects', async () => {
+  // GIVEN
+  mockS3Client.promise.mockResolvedValue({ Versions: [] }); // listObjectVersions() call
+
+  // WHEN
+  const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+    RequestType: 'Delete',
+    ResourceProperties: {
+      ServiceToken: 'Foo',
+      BucketName: 'MyBucket',
+    },
+  };
+  await invokeHandler(event);
+
+  // THEN
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1);
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('deletes all objects on delete event', async () => {
+  // GIVEN
+  mockS3Client.promise.mockResolvedValue({ // listObjectVersions() call
+    Versions: [
+      { Key: 'Key1', VersionId: 'VersionId1' },
+      { Key: 'Key2', VersionId: 'VersionId2' },
+    ],
+  });
+
+  // WHEN
+  const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+    RequestType: 'Delete',
+    ResourceProperties: {
+      ServiceToken: 'Foo',
+      BucketName: 'MyBucket',
+    },
+  };
+  await invokeHandler(event);
+
+  // THEN
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1);
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(1);
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledWith({
+    Bucket: 'MyBucket',
+    Delete: {
+      Objects: [
+        { Key: 'Key1', VersionId: 'VersionId1' },
+        { Key: 'Key2', VersionId: 'VersionId2' },
+      ],
+    },
+  });
+});
+
+test('delete event where bucket has many objects does recurse appropriately', async () => {
+  // GIVEN
+  mockS3Client.promise // listObjectVersions() call
+    .mockResolvedValueOnce({
+      Versions: [
+        { Key: 'Key1', VersionId: 'VersionId1' },
+        { Key: 'Key2', VersionId: 'VersionId2' },
+      ],
+      IsTruncated: true,
+    })
+    .mockResolvedValueOnce(undefined) // deleteObjects() call
+    .mockResolvedValueOnce({ // listObjectVersions() call
+      Versions: [
+        { Key: 'Key3', VersionId: 'VersionId3' },
+        { Key: 'Key4', VersionId: 'VersionId4' },
+      ],
+    });
+
+  // WHEN
+  const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+    RequestType: 'Delete',
+    ResourceProperties: {
+      ServiceToken: 'Foo',
+      BucketName: 'MyBucket',
+    },
+  };
+  await invokeHandler(event);
+
+  // THEN
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(2);
+  expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+  expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(2);
+  expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(1, {
+    Bucket: 'MyBucket',
+    Delete: {
+      Objects: [
+        { Key: 'Key1', VersionId: 'VersionId1' },
+        { Key: 'Key2', VersionId: 'VersionId2' },
+      ],
+    },
+  });
+  expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(2, {
+    Bucket: 'MyBucket',
+    Delete: {
+      Objects: [
+        { Key: 'Key3', VersionId: 'VersionId3' },
+        { Key: 'Key4', VersionId: 'VersionId4' },
+      ],
+    },
+  });
+});
+
+// helper function to get around TypeScript expecting a complete event object,
+// even though our tests only need some of the fields
+async function invokeHandler(event: Partial<AWSLambda.CloudFormationCustomResourceEvent>) {
+  return handler(event as AWSLambda.CloudFormationCustomResourceEvent);
+}