diff --git a/packages/@aws-cdk/aws-s3/README.md b/packages/@aws-cdk/aws-s3/README.md index b3eb2e1a4707d..9de58f73e916c 100644 --- a/packages/@aws-cdk/aws-s3/README.md +++ b/packages/@aws-cdk/aws-s3/README.md @@ -341,3 +341,20 @@ bucket.urlForObject('objectname'); // Path-Style URL bucket.virtualHostedUrlForObject('objectname'); // Virtual Hosted-Style URL bucket.virtualHostedUrlForObject('objectname', { regional: false }); // Virtual Hosted-Style URL but non-regional ``` + +### Bucket deletion + +When a bucket is removed from a stack (or the stack is deleted), the S3 +bucket will be removed according to its removal policy (which by default will +simply orphan the bucket and leave it in your AWS account). If the removal +policy is set to `RemovalPolicy.DESTROY`, the bucket will be deleted as long +as it does not contain any objects. + +To override this and force all objects to get deleted during bucket deletion, +enable the`autoDeleteObjects` option. + +```ts +const bucket = new Bucket(this, 'MyTempFileBucket', { + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, +}); diff --git a/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts b/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts new file mode 100644 index 0000000000000..c27e346615775 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/auto-delete-objects-handler/index.ts @@ -0,0 +1,43 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import * as AWS from 'aws-sdk'; + +const s3 = new AWS.S3(); + +export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent) { + switch (event.RequestType) { + case 'Create': + case 'Update': + return; + case 'Delete': + return onDelete(event); + } +} + +/** + * Recursively delete all items in the bucket + * + * @param {AWS.S3} s3 the S3 client + * @param {*} bucketName the bucket name + */ +async function emptyBucket(bucketName: string) { + const listedObjects = await s3.listObjectVersions({ Bucket: bucketName }).promise(); + const contents = (listedObjects.Versions || []).concat(listedObjects.DeleteMarkers || []); + if (contents.length === 0) { + return; + }; + + const records = contents.map((record: any) => ({ Key: record.Key, VersionId: record.VersionId })); + await s3.deleteObjects({ Bucket: bucketName, Delete: { Objects: records } }).promise(); + + if (listedObjects?.IsTruncated) { + await emptyBucket(bucketName); + } +} + +async function onDelete(deleteEvent: AWSLambda.CloudFormationCustomResourceDeleteEvent) { + const bucketName = deleteEvent.ResourceProperties?.BucketName; + if (!bucketName) { + throw new Error('No BucketName was provided.'); + } + await emptyBucket(bucketName); +} diff --git a/packages/@aws-cdk/aws-s3/lib/bucket.ts b/packages/@aws-cdk/aws-s3/lib/bucket.ts index f1752cc402b5e..f70d0e7f853de 100644 --- a/packages/@aws-cdk/aws-s3/lib/bucket.ts +++ b/packages/@aws-cdk/aws-s3/lib/bucket.ts @@ -1,8 +1,9 @@ import { EOL } from 'os'; +import * as path from 'path'; import * as events from '@aws-cdk/aws-events'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; -import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token } from '@aws-cdk/core'; +import { Fn, IResource, Lazy, RemovalPolicy, Resource, Stack, Token, CustomResource, CustomResourceProvider, CustomResourceProviderRuntime } from '@aws-cdk/core'; import { Construct } from 'constructs'; import { BucketPolicy } from './bucket-policy'; import { IBucketNotificationDestination } from './destination'; @@ -12,6 +13,8 @@ import { LifecycleRule } from './rule'; import { CfnBucket } from './s3.generated'; import { parseBucketArn, parseBucketName } from './util'; +const AUTO_DELETE_OBJECTS_RESOURCE_TYPE = 'Custom::AutoDeleteObjects'; + export interface IBucket extends IResource { /** * The ARN of the bucket. @@ -1026,6 +1029,16 @@ export interface BucketProps { */ readonly removalPolicy?: RemovalPolicy; + /** + * Whether all objects should be automatically deleted when the bucket is + * removed from the stack. + * + * Requires the removal policy to be set to destroy. + * + * @default false + */ + readonly autoDeleteObjects?: boolean; + /** * Whether this bucket should have versioning turned on or not. * @@ -1301,6 +1314,20 @@ export class Bucket extends BucketBase { if (props.publicReadAccess) { this.grantPublicAccess(); } + + if (props.autoDeleteObjects) { + if (props.removalPolicy !== RemovalPolicy.DESTROY) { + throw new Error("Cannot use 'autoDeleteObjects' property on a bucket without setting removal policy to 'destroy'."); + } + + new CustomResource(this, 'AutoDeleteObjectsResource', { + resourceType: AUTO_DELETE_OBJECTS_RESOURCE_TYPE, + serviceToken: this.getOrCreateAutoDeleteObjectsResource(), + properties: { + BucketName: this.bucketName, + }, + }); + } } /** @@ -1692,6 +1719,23 @@ export class Bucket extends BucketBase { }; }); } + + private getOrCreateAutoDeleteObjectsResource() { + return CustomResourceProvider.getOrCreate(this, AUTO_DELETE_OBJECTS_RESOURCE_TYPE, { + codeDirectory: path.join(__dirname, 'auto-delete-objects-handler'), + runtime: CustomResourceProviderRuntime.NODEJS_12, + policyStatements: [ + { + Effect: 'Allow', + Resource: '*', + Action: [ + ...perms.BUCKET_READ_ACTIONS, + ...perms.BUCKET_DELETE_ACTIONS, + ], + }, + ], + }); + } } /** diff --git a/packages/@aws-cdk/aws-s3/test/auto-delete-objects.test.ts b/packages/@aws-cdk/aws-s3/test/auto-delete-objects.test.ts new file mode 100644 index 0000000000000..3c705a8de62dd --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/auto-delete-objects.test.ts @@ -0,0 +1,319 @@ +import '@aws-cdk/assert/jest'; +import * as cdk from '@aws-cdk/core'; +import * as s3 from '../lib'; + +const mockS3Client = { + listObjectVersions: jest.fn().mockReturnThis(), + deleteObjects: jest.fn().mockReturnThis(), + promise: jest.fn(), +}; + +import { handler } from '../lib/auto-delete-objects-handler'; + +jest.mock('aws-sdk', () => { + return { S3: jest.fn(() => mockS3Client) }; +}); + +test('when autoDeleteObjects is enabled, a custom resource is provisioned + a lambda handler for it', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new s3.Bucket(stack, 'MyBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // THEN + expect(stack).toHaveResource('AWS::S3::Bucket'); + expect(stack).toHaveResource('AWS::Lambda::Function'); + expect(stack).toHaveResource('Custom::AutoDeleteObjects', + { + BucketName: { Ref: 'MyBucketF68F3FF0' }, + ServiceToken: { + 'Fn::GetAtt': [ + 'CustomAutoDeleteObjectsCustomResourceProviderHandlerE060A45A', + 'Arn', + ], + }, + }); +}); + +test('two buckets with autoDeleteObjects enabled share the same cr provider', () => { + // GIVEN + const app = new cdk.App(); + const stack = new cdk.Stack(app); + + // WHEN + new s3.Bucket(stack, 'MyBucketOne', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + new s3.Bucket(stack, 'MyBucketTwo', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // THEN + const template = app.synth().getStackArtifact(stack.artifactId).template; + const resourceTypes = Object.values(template.Resources).map((r: any) => r.Type).sort(); + + expect(resourceTypes).toStrictEqual([ + // custom resource provider resources (shared) + 'AWS::IAM::Role', + 'AWS::Lambda::Function', + + // buckets + 'AWS::S3::Bucket', + 'AWS::S3::Bucket', + + // auto delete object resources + 'Custom::AutoDeleteObjects', + 'Custom::AutoDeleteObjects', + ]); +}); + +test('when only one bucket has autoDeleteObjects enabled, only that bucket gets assigned a custom resource', () => { + // GIVEN + const app = new cdk.App(); + const stack = new cdk.Stack(app); + + // WHEN + new s3.Bucket(stack, 'MyBucketOne', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + new s3.Bucket(stack, 'MyBucketTwo', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: false, + }); + + // THEN + const template = app.synth().getStackArtifact(stack.artifactId).template; + const resourceTypes = Object.values(template.Resources).map((r: any) => r.Type).sort(); + + expect(resourceTypes).toStrictEqual([ + // custom resource provider resources + 'AWS::IAM::Role', + 'AWS::Lambda::Function', + + // buckets + 'AWS::S3::Bucket', + 'AWS::S3::Bucket', + + // auto delete object resource + 'Custom::AutoDeleteObjects', + ]); + + // custom resource for MyBucket1 is present + expect(stack).toHaveResource('Custom::AutoDeleteObjects', + { BucketName: { Ref: 'MyBucketOneA6BE54C9' } }); + + // custom resource for MyBucket2 is not present + expect(stack).not.toHaveResource('Custom::AutoDeleteObjects', + { BucketName: { Ref: 'MyBucketTwoC7437026' } }); +}); + +test('iam policy is created', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new s3.Bucket(stack, 'MyBucket', { + removalPolicy: cdk.RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // THEN + expect(stack).toHaveResource('AWS::IAM::Role', { + Policies: [ + { + PolicyName: 'Inline', + PolicyDocument: { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Resource: '*', + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + ], + }, + ], + }, + }, + ], + }); +}); + +test('throws if autoDeleteObjects is enabled but if removalPolicy is not specified', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + // THEN + expect(() => new s3.Bucket(stack, 'MyBucket', { autoDeleteObjects: true })).toThrowError(/removal policy/); +}); + +describe('custom resource handler', () => { + + beforeEach(() => { + mockS3Client.listObjectVersions.mockReturnThis(); + mockS3Client.deleteObjects.mockReturnThis(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + test('does nothing on create event', async () => { + // GIVEN + const event: Partial = { + RequestType: 'Create', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + + // WHEN + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); + }); + + test('does nothing on update event', async () => { + // GIVEN + const event: Partial = { + RequestType: 'Update', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + + // WHEN + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(0); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); + }); + + test('deletes no objects on delete event when bucket has no objects', async () => { + // GIVEN + mockS3Client.promise.mockResolvedValue({ Versions: [] }); // listObjectVersions() call + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(0); + }); + + test('deletes all objects on delete event', async () => { + // GIVEN + mockS3Client.promise.mockResolvedValue({ // listObjectVersions() call + Versions: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }); + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(1); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(1); + expect(mockS3Client.deleteObjects).toHaveBeenCalledWith({ + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }, + }); + }); + + test('delete event where bucket has many objects does recurse appropriately', async () => { + // GIVEN + mockS3Client.promise // listObjectVersions() call + .mockResolvedValueOnce({ + Versions: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + IsTruncated: true, + }) + .mockResolvedValueOnce(undefined) // deleteObjects() call + .mockResolvedValueOnce({ // listObjectVersions() call + Versions: [ + { Key: 'Key3', VersionId: 'VersionId3' }, + { Key: 'Key4', VersionId: 'VersionId4' }, + ], + }); + + // WHEN + const event: Partial = { + RequestType: 'Delete', + ResourceProperties: { + ServiceToken: 'Foo', + BucketName: 'MyBucket', + }, + }; + await invokeHandler(event); + + // THEN + expect(mockS3Client.listObjectVersions).toHaveBeenCalledTimes(2); + expect(mockS3Client.listObjectVersions).toHaveBeenCalledWith({ Bucket: 'MyBucket' }); + expect(mockS3Client.deleteObjects).toHaveBeenCalledTimes(2); + expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(1, { + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key1', VersionId: 'VersionId1' }, + { Key: 'Key2', VersionId: 'VersionId2' }, + ], + }, + }); + expect(mockS3Client.deleteObjects).toHaveBeenNthCalledWith(2, { + Bucket: 'MyBucket', + Delete: { + Objects: [ + { Key: 'Key3', VersionId: 'VersionId3' }, + { Key: 'Key4', VersionId: 'VersionId4' }, + ], + }, + }); + }); +}); + +// helper function to get around TypeScript expecting a complete event object, +// even though our tests only need some of the fields +async function invokeHandler(event: Partial) { + return handler(event as AWSLambda.CloudFormationCustomResourceEvent); +}