feat(s3): option to auto delete objects upon bucket removal

Use the custom resource provider from core to delete objects in the
bucket. A bucket policy gives the correct permissions to the provider's
Lambda function role.

Credits to @Chriscbr for starting the work on this.

Closes aws#3297
Closes aws#9751

packages/@aws-cdk/aws-s3/README.md

@@ -349,7 +349,7 @@ bucket.virtualHostedUrlForObject('objectname', { regional: false }); // Virtual
 
 ### Object Ownership
 
-You can use the two following properties to specify the bucket [object Ownership]. 
+You can use the two following properties to specify the bucket [object Ownership].
 
 [object Ownership]: https://docs.aws.amazon.com/AmazonS3/latest/dev/about-object-ownership.html
 
@@ -365,10 +365,28 @@ new s3.Bucket(this, 'MyBucket', {
 
 #### Bucket owner preferred
 
-The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account. 
+The bucket owner will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. Without this setting and canned ACL, the object is uploaded and remains owned by the uploading account.
 
 ```ts
 new s3.Bucket(this, 'MyBucket', {
  objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_PREFERRED,
 });
 ```
+
+### Bucket deletion
+
+When a bucket is removed from a stack (or the stack is deleted), the S3
+bucket will be removed according to its removal policy (which by default will
+simply orphan the bucket and leave it in your AWS account). If the removal
+policy is set to `RemovalPolicy.DESTROY`, the bucket will be deleted as long
+as it does not contain any objects.
+
+To override this and force all objects to get deleted during bucket deletion,
+enable the`autoDeleteObjects` option.
+
+```ts
+const bucket = new Bucket(this, 'MyTempFileBucket', {
+ removalPolicy: RemovalPolicy.DESTROY,
+ autoDeleteObjects: true,
+});
+```

packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts

@@ -0,0 +1,42 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import * as AWS from 'aws-sdk';
+
+const s3 = new AWS.S3();
+
+export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent) {
+ switch (event.RequestType) {
+ case 'Create':
+ case 'Update':
+ return;
+ case 'Delete':
+ return onDelete(event);
+ }
+}
+
+/**
+ * Recursively delete all items in the bucket
+ *
+ * @param bucketName the bucket name
+ */
+async function emptyBucket(bucketName: string) {
+ const listedObjects = await s3.listObjectVersions({ Bucket: bucketName }).promise();
+ const contents = [...listedObjects.Versions ?? [], ...listedObjects.DeleteMarkers ?? []];
+ if (contents.length === 0) {
+ return;
+ };
+
+ const records = contents.map((record: any) => ({ Key: record.Key, VersionId: record.VersionId }));
+ await s3.deleteObjects({ Bucket: bucketName, Delete: { Objects: records } }).promise();
+
+ if (listedObjects?.IsTruncated) {
+ await emptyBucket(bucketName);
+ }
+}
+
+async function onDelete(deleteEvent: AWSLambda.CloudFormationCustomResourceDeleteEvent) {
+ const bucketName = deleteEvent.ResourceProperties?.BucketName;
+ if (!bucketName) {
+ throw new Error('No BucketName was provided.');
+ }
+ await emptyBucket(bucketName);
+}

packages/@aws-cdk/aws-s3/lib/bucket.ts

@@ -1,17 +1,23 @@
 import { EOL } from 'os';
+import * as path from 'path';
 import * as events from '@aws-cdk/aws-events';
 import * as iam from '@aws-cdk/aws-iam';
 import * as kms from '@aws-cdk/aws-kms';
-import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core';
+import {
+ Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token,
+ CustomResource, CustomResourceProvider, CustomResourceProviderRuntime,
+} from '@aws-cdk/core';
 import { Construct } from 'constructs';
 import { BucketPolicy } from './bucket-policy';
 import { IBucketNotificationDestination } from './destination';
 import { BucketNotifications } from './notifications-resource';
 import * as perms from './perms';
 import { LifecycleRule } from './rule';
-import { CfnBucket } from './s3.generated';
+import { CfnBucket, CfnBucketPolicy } from './s3.generated';
 import { parseBucketArn, parseBucketName } from './util';
 
+const AUTO_DELETE_OBJECTS_RESOURCE_TYPE = 'Custom::S3AutoDeleteObjects';
+
 export interface IBucket extends IResource {
  /**
  * The ARN of the bucket.
@@ -1041,6 +1047,16 @@ export interface BucketProps {
  */
  readonly removalPolicy?: RemovalPolicy;
 
+ /**
+ * Whether all objects should be automatically deleted when the bucket is
+ * removed from the stack or when the stack is deleted.
+ *
+ * Requires the `removalPolicy` to be set to `RemovalPolicy.DESTROY`.
+ *
+ * @default false
+ */
+ readonly autoDeleteObjects?: boolean;
+
  /**
  * Whether this bucket should have versioning turned on or not.
  *
@@ -1326,6 +1342,14 @@ export class Bucket extends BucketBase {
  if (props.publicReadAccess) {
  this.grantPublicAccess();
  }
+
+ if (props.autoDeleteObjects) {
+ if (props.removalPolicy !== RemovalPolicy.DESTROY) {
+ throw new Error('Cannot use \'autoDeleteObjects\' property on a bucket without setting removal policy to \'DESTROY\'.');
+ }
+
+ this.enableAutoDeleteObjects();
+ }
  }
 
  /**
@@ -1728,6 +1752,40 @@ export class Bucket extends BucketBase {
  };
  });
  }
+
+ private enableAutoDeleteObjects() {
+ const provider = CustomResourceProvider.getOrCreateProvider(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, {
+ codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'),
+ runtime: CustomResourceProviderRuntime.NODEJS_12,
+ });
+
+ // Use a bucket policy to allow the custom resource to delete
+ // objects in the bucket
+ this.addToResourcePolicy(new iam.PolicyStatement({
+ actions: [
+ ...perms.BUCKET_READ_ACTIONS, // list objects
+ ...perms.BUCKET_DELETE_ACTIONS, // and then delete them
+ ],
+ resources: [
+ this.bucketArn,
+ this.arnForObjects('*'),
+ ],
+ principals: [new iam.ArnPrincipal(provider.roleArn)],
+ }));
+
+ const customResource = new CustomResource(this, 'AutoDeleteObjectsCustomResource', {
+ resourceType: AUTO_DELETE_OBJECTS_RESOURCE_TYPE,
+ serviceToken: provider.serviceToken,
+ properties: {
+ BucketName: this.bucketName,
+ },
+ });
+
+ // Ensure bucket policy is deleted AFTER the custom resource otherwise
+ // we don't have permissions to list and delete in the bucket.
+ const policy = this.node.tryFindChild('Policy') as CfnBucketPolicy;
+ customResource.node.addDependency(policy);
+ }
 }
 
 /**

packages/@aws-cdk/aws-s3/test/auto-delete-objects-handler.test.ts

@@ -0,0 +1,168 @@
+const mockS3Client = {
+ listObjectVersions: jest.fn().mockReturnThis(),
+ deleteObjects: jest.fn().mockReturnThis(),
+ promise: jest.fn(),
+};
+
+import { handler } from '../lib/auto-delete-objects-handler';
+
+jest.mock('aws-sdk', () => {
+ return { S3: jest.fn(() => mockS3Client) };
+});
+
+beforeEach(() => {
+ mockS3Client.listObjectVersions.mockReturnThis();
+ mockS3Client.deleteObjects.mockReturnThis();
+});
+
+afterEach(() => {
+ jest.resetAllMocks();
+});
+
+test('does nothing on create event', async () => {
+ // GIVEN
+ const event: Partial<AWSLambda.CloudFormationCustomResourceCreateEvent> = {
+ RequestType: 'Create',
+ ResourceProperties: {
+ ServiceToken: 'Foo',
+ BucketName: 'MyBucket',
+ },
+ };
+
+ // WHEN
+ await invokeHandler(event);
+
+ // THEN
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0);
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('does nothing on update event', async () => {
+ // GIVEN
+ const event: Partial<AWSLambda.CloudFormationCustomResourceUpdateEvent> = {
+ RequestType: 'Update',
+ ResourceProperties: {
+ ServiceToken: 'Foo',
+ BucketName: 'MyBucket',
+ },
+ };
+
+ // WHEN
+ await invokeHandler(event);
+
+ // THEN
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0);
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('deletes no objects on delete event when bucket has no objects', async () => {
+ // GIVEN
+ mockS3Client.promise.mockResolvedValue({ Versions: [] }); // listObjectVersions() call
+
+ // WHEN
+ const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+ RequestType: 'Delete',
+ ResourceProperties: {
+ ServiceToken: 'Foo',
+ BucketName: 'MyBucket',
+ },
+ };
+ await invokeHandler(event);
+
+ // THEN
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1);
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0);
+});
+
+test('deletes all objects on delete event', async () => {
+ // GIVEN
+ mockS3Client.promise.mockResolvedValue({ // listObjectVersions() call
+ Versions: [
+ { Key: 'Key1', VersionId: 'VersionId1' },
+ { Key: 'Key2', VersionId: 'VersionId2' },
+ ],
+ });
+
+ // WHEN
+ const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+ RequestType: 'Delete',
+ ResourceProperties: {
+ ServiceToken: 'Foo',
+ BucketName: 'MyBucket',
+ },
+ };
+ await invokeHandler(event);
+
+ // THEN
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1);
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(1);
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledWith({
+ Bucket: 'MyBucket',
+ Delete: {
+ Objects: [
+ { Key: 'Key1', VersionId: 'VersionId1' },
+ { Key: 'Key2', VersionId: 'VersionId2' },
+ ],
+ },
+ });
+});
+
+test('delete event where bucket has many objects does recurse appropriately', async () => {
+ // GIVEN
+ mockS3Client.promise // listObjectVersions() call
+ .mockResolvedValueOnce({
+ Versions: [
+ { Key: 'Key1', VersionId: 'VersionId1' },
+ { Key: 'Key2', VersionId: 'VersionId2' },
+ ],
+ IsTruncated: true,
+ })
+ .mockResolvedValueOnce(undefined) // deleteObjects() call
+ .mockResolvedValueOnce({ // listObjectVersions() call
+ Versions: [
+ { Key: 'Key3', VersionId: 'VersionId3' },
+ { Key: 'Key4', VersionId: 'VersionId4' },
+ ],
+ });
+
+ // WHEN
+ const event: Partial<AWSLambda.CloudFormationCustomResourceDeleteEvent> = {
+ RequestType: 'Delete',
+ ResourceProperties: {
+ ServiceToken: 'Foo',
+ BucketName: 'MyBucket',
+ },
+ };
+ await invokeHandler(event);
+
+ // THEN
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(2);
+ expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' });
+ expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(2);
+ expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(1, {
+ Bucket: 'MyBucket',
+ Delete: {
+ Objects: [
+ { Key: 'Key1', VersionId: 'VersionId1' },
+ { Key: 'Key2', VersionId: 'VersionId2' },
+ ],
+ },
+ });
+ expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(2, {
+ Bucket: 'MyBucket',
+ Delete: {
+ Objects: [
+ { Key: 'Key3', VersionId: 'VersionId3' },
+ { Key: 'Key4', VersionId: 'VersionId4' },
+ ],
+ },
+ });
+});
+
+// helper function to get around TypeScript expecting a complete event object,
+// even though our tests only need some of the fields
+async function invokeHandler(event: Partial<AWSLambda.CloudFormationCustomResourceEvent>) {
+ return handler(event as AWSLambda.CloudFormationCustomResourceEvent);
+}