From 8cd07e62fc8b651e6bbdbc85553f2e6dc885016e Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Mon, 13 Aug 2018 19:09:09 +0300 Subject: [PATCH] feat(aws-s3): Bucket Notifications (#201) Adds support for S3 bucket notifications. The `bucket.onEvent` method will add a notification destination for a bucket. The s3.INotificationDestination interface is used to allow SNS, SQS and Lambda to implement notification destinations. This interface inverts the control and allows the destination to prepare to receive notifications. For example, it can modify it's policy appropriately. Since CloudFormation bucket notification support require two-phase deployments (due to the fact PutBucketNotification will fail if the destination policy has not been updated, and CloudFormation cannot create the policy until the bucket is created). The reason this is a limitation in CloudFormation is that they could not model the 1:1 relationship between the bucket and the notifications using the current semantics of CloudFormation. In the CDK, we can model this relationship by encapsulating the notifications custom resource behind a bucket. This means that users don't interact with this resource directly, but rather just subscribe to notifications on a bucket, and the resource (and accompanying handler) will be created as needed. --- packages/@aws-cdk/aws-s3/README.md | 37 +++ packages/@aws-cdk/aws-s3/lib/bucket.ts | 174 +++++++++++ packages/@aws-cdk/aws-s3/lib/index.ts | 1 + .../@aws-cdk/aws-s3/lib/notification-dest.ts | 37 +++ .../lib/notifications-resource/index.ts | 1 + .../notifications-resource-handler.ts | 165 +++++++++++ .../notifications-resource.ts | 172 +++++++++++ .../test/integ.notifications.expected.json | 273 ++++++++++++++++++ .../aws-s3/test/integ.notifications.ts | 20 ++ .../aws-s3/test/notification-dests.ts | 47 +++ .../aws-s3/test/test.notifications.ts | 272 +++++++++++++++++ 11 files changed, 1199 insertions(+) create mode 100644 packages/@aws-cdk/aws-s3/lib/notification-dest.ts create mode 100644 packages/@aws-cdk/aws-s3/lib/notifications-resource/index.ts create mode 100644 packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource-handler.ts create mode 100644 packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource.ts create mode 100644 packages/@aws-cdk/aws-s3/test/integ.notifications.expected.json create mode 100644 packages/@aws-cdk/aws-s3/test/integ.notifications.ts create mode 100644 packages/@aws-cdk/aws-s3/test/notification-dests.ts create mode 100644 packages/@aws-cdk/aws-s3/test/test.notifications.ts diff --git a/packages/@aws-cdk/aws-s3/README.md b/packages/@aws-cdk/aws-s3/README.md index 2eaa3503204e6..4b51848ed16e8 100644 --- a/packages/@aws-cdk/aws-s3/README.md +++ b/packages/@aws-cdk/aws-s3/README.md @@ -143,3 +143,40 @@ new Consumer(app, 'consume', { process.stdout.write(app.run()); ``` + +### Bucket Notifications + +The Amazon S3 notification feature enables you to receive notifications when +certain events happen in your bucket as described under [S3 Bucket +Notifications] of the S3 Developer Guide. + +To subscribe for bucket notifications, use the `bucket.onEvent` method. The +`bucket.onObjectCreated` and `bucket.onObjectRemoved` can also be used for these +common use cases. + +The following example will subscribe an SNS topic to be notified of all +``s3:ObjectCreated:*` events: + +```ts +const myTopic = new sns.Topic(this, 'MyTopic'); +bucket.onEvent(s3.EventType.ObjectCreated, myTopic); +``` + +This call will also ensure that the topic policy can accept notifications for +this specific bucket. + +The following destinations are currently supported: + + * `sns.Topic` + * `sqs.Queue` + * `lambda.Function` + +It is also possible to specify S3 object key filters when subscribing. The +following example will notify `myQueue` when objects prefixed with `foo/` and +have the `.jpg` suffix are removed from the bucket. + +```ts +bucket.onEvent(s3.EventType.ObjectRemoved, myQueue, { prefix: 'foo/', suffix: '.jpg' }); +``` + +[S3 Bucket Notifications]: https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html diff --git a/packages/@aws-cdk/aws-s3/lib/bucket.ts b/packages/@aws-cdk/aws-s3/lib/bucket.ts index ffa713df2bab8..b2f9fb390c47b 100644 --- a/packages/@aws-cdk/aws-s3/lib/bucket.ts +++ b/packages/@aws-cdk/aws-s3/lib/bucket.ts @@ -2,6 +2,8 @@ import iam = require('@aws-cdk/aws-iam'); import kms = require('@aws-cdk/aws-kms'); import cdk = require('@aws-cdk/cdk'); import { BucketPolicy } from './bucket-policy'; +import { IBucketNotificationDestination } from './notification-dest'; +import { BucketNotifications } from './notifications-resource'; import perms = require('./perms'); import { LifecycleRule } from './rule'; import { BucketArn, BucketDomainName, BucketDualStackDomainName, cloudformation } from './s3.generated'; @@ -289,6 +291,7 @@ export class Bucket extends BucketRef { protected autoCreatePolicy = true; private readonly lifecycleRules: LifecycleRule[] = []; private readonly versioned?: boolean; + private readonly notifications: BucketNotifications; constructor(parent: cdk.Construct, name: string, props: BucketProps = {}) { super(parent, name); @@ -316,6 +319,10 @@ export class Bucket extends BucketRef { // Add all lifecycle rules (props.lifecycleRules || []).forEach(this.addLifecycleRule.bind(this)); + + // defines a BucketNotifications construct. Notice that an actual resource will only + // be added if there are notifications added, so we don't need to condition this. + this.notifications = new BucketNotifications(this, 'Notifications', { bucket: this }); } /** @@ -333,6 +340,53 @@ export class Bucket extends BucketRef { this.lifecycleRules.push(rule); } + /** + * Adds a bucket notification event destination. + * @param event The event to trigger the notification + * @param dest The notification destination (Lambda, SNS Topic or SQS Queue) + * + * @param filters S3 object key filter rules to determine which objects + * trigger this event. Each filter must include a `prefix` and/or `suffix` + * that will be matched against the s3 object key. Refer to the S3 Developer Guide + * for details about allowed filter rules. + * + * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-filtering + * + * @example + * + * bucket.onEvent(EventType.OnObjectCreated, myLambda, 'home/myusername/*') + * + * @see + * https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html + */ + public onEvent(event: EventType, dest: IBucketNotificationDestination, ...filters: NotificationKeyFilter[]) { + this.notifications.addNotification(event, dest, ...filters); + } + + /** + * Subscribes a destination to receive notificatins when an object is + * created in the bucket. This is identical to calling + * `onEvent(EventType.ObjectCreated)`. + * + * @param dest The notification destination (see onEvent) + * @param filters Filters (see onEvent) + */ + public onObjectCreated(dest: IBucketNotificationDestination, ...filters: NotificationKeyFilter[]) { + return this.onEvent(EventType.ObjectCreated, dest, ...filters); + } + + /** + * Subscribes a destination to receive notificatins when an object is + * removed from the bucket. This is identical to calling + * `onEvent(EventType.ObjectRemoved)`. + * + * @param dest The notification destination (see onEvent) + * @param filters Filters (see onEvent) + */ + public onObjectRemoved(dest: IBucketNotificationDestination, ...filters: NotificationKeyFilter[]) { + return this.onEvent(EventType.ObjectRemoved, dest, ...filters); + } + /** * Set up key properties and return the Bucket encryption property from the * user's configuration. @@ -485,6 +539,126 @@ export class S3Url extends cdk.Token { } +/** + * Notification event types. + */ +export enum EventType { + /** + * Amazon S3 APIs such as PUT, POST, and COPY can create an object. Using + * these event types, you can enable notification when an object is created + * using a specific API, or you can use the s3:ObjectCreated:* event type to + * request notification regardless of the API that was used to create an + * object. + */ + ObjectCreated = 's3:ObjectCreated:*', + + /** + * Amazon S3 APIs such as PUT, POST, and COPY can create an object. Using + * these event types, you can enable notification when an object is created + * using a specific API, or you can use the s3:ObjectCreated:* event type to + * request notification regardless of the API that was used to create an + * object. + */ + ObjectCreatedPut = 's3:ObjectCreated:Put', + + /** + * Amazon S3 APIs such as PUT, POST, and COPY can create an object. Using + * these event types, you can enable notification when an object is created + * using a specific API, or you can use the s3:ObjectCreated:* event type to + * request notification regardless of the API that was used to create an + * object. + */ + ObjectCreatedPost = 's3:ObjectCreated:Post', + + /** + * Amazon S3 APIs such as PUT, POST, and COPY can create an object. Using + * these event types, you can enable notification when an object is created + * using a specific API, or you can use the s3:ObjectCreated:* event type to + * request notification regardless of the API that was used to create an + * object. + */ + ObjectCreatedCopy = 's3:ObjectCreated:Copy', + + /** + * Amazon S3 APIs such as PUT, POST, and COPY can create an object. Using + * these event types, you can enable notification when an object is created + * using a specific API, or you can use the s3:ObjectCreated:* event type to + * request notification regardless of the API that was used to create an + * object. + */ + ObjectCreatedCompleteMultipartUpload = 's3:ObjectCreated:CompleteMultipartUpload', + + /** + * By using the ObjectRemoved event types, you can enable notification when + * an object or a batch of objects is removed from a bucket. + * + * You can request notification when an object is deleted or a versioned + * object is permanently deleted by using the s3:ObjectRemoved:Delete event + * type. Or you can request notification when a delete marker is created for + * a versioned object by using s3:ObjectRemoved:DeleteMarkerCreated. For + * information about deleting versioned objects, see Deleting Object + * Versions. You can also use a wildcard s3:ObjectRemoved:* to request + * notification anytime an object is deleted. + * + * You will not receive event notifications from automatic deletes from + * lifecycle policies or from failed operations. + */ + ObjectRemoved = 's3:ObjectRemoved:*', + + /** + * By using the ObjectRemoved event types, you can enable notification when + * an object or a batch of objects is removed from a bucket. + * + * You can request notification when an object is deleted or a versioned + * object is permanently deleted by using the s3:ObjectRemoved:Delete event + * type. Or you can request notification when a delete marker is created for + * a versioned object by using s3:ObjectRemoved:DeleteMarkerCreated. For + * information about deleting versioned objects, see Deleting Object + * Versions. You can also use a wildcard s3:ObjectRemoved:* to request + * notification anytime an object is deleted. + * + * You will not receive event notifications from automatic deletes from + * lifecycle policies or from failed operations. + */ + ObjectRemovedDelete = 's3:ObjectRemoved:Delete', + + /** + * By using the ObjectRemoved event types, you can enable notification when + * an object or a batch of objects is removed from a bucket. + * + * You can request notification when an object is deleted or a versioned + * object is permanently deleted by using the s3:ObjectRemoved:Delete event + * type. Or you can request notification when a delete marker is created for + * a versioned object by using s3:ObjectRemoved:DeleteMarkerCreated. For + * information about deleting versioned objects, see Deleting Object + * Versions. You can also use a wildcard s3:ObjectRemoved:* to request + * notification anytime an object is deleted. + * + * You will not receive event notifications from automatic deletes from + * lifecycle policies or from failed operations. + */ + ObjectRemovedDeleteMarkerCreated = 's3:ObjectRemoved:DeleteMarkerCreated', + + /** + * You can use this event type to request Amazon S3 to send a notification + * message when Amazon S3 detects that an object of the RRS storage class is + * lost. + */ + ReducedRedundancyLostObject = 's3:ReducedRedundancyLostObject', +} + +export interface NotificationKeyFilter { + /** + * S3 keys must have the specified prefix. + */ + prefix?: string; + + /** + * S3 keys must have the specified suffix. + */ + suffix?: string; +} + class ImportedBucketRef extends BucketRef { public readonly bucketArn: BucketArn; public readonly bucketName: BucketName; diff --git a/packages/@aws-cdk/aws-s3/lib/index.ts b/packages/@aws-cdk/aws-s3/lib/index.ts index 593c797757b3f..e0d99e5e7ebe1 100644 --- a/packages/@aws-cdk/aws-s3/lib/index.ts +++ b/packages/@aws-cdk/aws-s3/lib/index.ts @@ -1,6 +1,7 @@ export * from './bucket'; export * from './bucket-policy'; export * from './rule'; +export * from './notification-dest'; // AWS::S3 CloudFormation Resources: export * from './s3.generated'; diff --git a/packages/@aws-cdk/aws-s3/lib/notification-dest.ts b/packages/@aws-cdk/aws-s3/lib/notification-dest.ts new file mode 100644 index 0000000000000..2e16e1aa1e7f3 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/notification-dest.ts @@ -0,0 +1,37 @@ +import cdk = require('@aws-cdk/cdk'); +import { Bucket } from './bucket'; + +/** + * Implemented by constructs that can be used as bucket notification destinations. + */ +export interface IBucketNotificationDestination { + /** + * Registers this resource to receive notifications for the specified bucket. + * @param bucket The bucket. Use the `path` of the bucket as a unique ID. + */ + asBucketNotificationDestination(bucket: Bucket): BucketNotificationDestinationProps; +} + +/** + * Represents the properties of a notification destination. + */ +export interface BucketNotificationDestinationProps { + /** + * The notification type. + */ + readonly type: BucketNotificationDestinationType; + + /** + * The ARN of the destination (i.e. Lambda, SNS, SQS). + */ + readonly arn: cdk.Arn; +} + +/** + * Supported types of notification destinations. + */ +export enum BucketNotificationDestinationType { + Lambda, + Queue, + Topic +} diff --git a/packages/@aws-cdk/aws-s3/lib/notifications-resource/index.ts b/packages/@aws-cdk/aws-s3/lib/notifications-resource/index.ts new file mode 100644 index 0000000000000..6cd00a3d115d2 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/notifications-resource/index.ts @@ -0,0 +1 @@ +export * from './notifications-resource'; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource-handler.ts b/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource-handler.ts new file mode 100644 index 0000000000000..209a85b3f52b3 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource-handler.ts @@ -0,0 +1,165 @@ +import iam = require('@aws-cdk/aws-iam'); +import cdk = require('@aws-cdk/cdk'); + +/** + * A Lambda-based custom resource handler that provisions S3 bucket + * notifications for a bucket. + * + * The resource property schema is: + * + * { + * BucketName: string, NotificationConfiguration: { see + * PutBucketNotificationConfiguration } + * } + * + * For 'Delete' operations, we send an empty NotificationConfiguration as + * required. We propagate errors and results as-is. + * + * Sadly, we can't use @aws-cdk/aws-lambda as it will introduce a dependency + * cycle, so this uses raw `cdk.Resource`s. + */ +export class NotificationsResourceHandler extends cdk.Construct { + /** + * Defines a stack-singleton lambda function with the logic for a CloudFormation custom + * resource that provisions bucket notification configuration for a bucket. + * + * @returns The ARN of the custom resource lambda function. + */ + public static singleton(context: cdk.Construct) { + const root = cdk.Stack.find(context); + + // well-known logical id to ensure stack singletonity + const logicalId = 'BucketNotificationsHandler050a0587b7544547bf325f094a3db834'; + let lambda = root.tryFindChild(logicalId) as NotificationsResourceHandler; + if (!lambda) { + lambda = new NotificationsResourceHandler(root, logicalId); + } + + return lambda.functionArn; + } + + /** + * The ARN of the handler's lambda function. Used as a service token in the + * custom resource. + */ + public readonly functionArn: cdk.Arn; + + constructor(parent: cdk.Construct, id: string) { + super(parent, id); + + const role = new iam.Role(this, 'Role', { + assumedBy: new cdk.ServicePrincipal('lambda.amazonaws.com'), + managedPolicyArns: [ + cdk.Arn.fromComponents({ + service: 'iam', + region: '', // no region for managed policy + account: 'aws', // the account for a managed policy is 'aws' + resource: 'policy', + resourceName: 'service-role/AWSLambdaBasicExecutionRole', + }) + ] + }); + + // handler allows to put bucket notification on s3 buckets. + role.addToPolicy(new cdk.PolicyStatement() + .addAction('s3:PutBucketNotification') + .addResource('*')); + + const resource = new cdk.Resource(this, 'Resource', { + type: 'AWS::Lambda::Function', + properties: { + Description: 'AWS CloudFormation handler for "Custom::S3BucketNotifications" resources (@aws-cdk/aws-s3)', + Code: { ZipFile: `exports.handler = ${handler.toString()};` }, + Handler: 'index.handler', + Role: role.roleArn, + Runtime: 'nodejs8.10', + Timeout: 300, + } + }); + + this.functionArn = resource.getAtt('Arn'); + } +} + +// tslint:disable:no-console + +/** + * Lambda event handler for the custom resource. Bear in mind that we are going + * to .toString() this function and inline it as Lambda code. + * + * The function will issue a putBucketNotificationConfiguration request for the + * specified bucket. + */ +const handler = (event: any, context: any) => { + const s3 = new (require('aws-sdk').S3)(); + const https = require("https"); + const url = require("url"); + + log(JSON.stringify(event, undefined, 2)); + + const props = event.ResourceProperties; + + if (event.RequestType === 'Delete') { + props.NotificationConfiguration = { }; // this is how you clean out notifications + } + + const req = { + Bucket: props.BucketName, + NotificationConfiguration: props.NotificationConfiguration + }; + + return s3.putBucketNotificationConfiguration(req, (err: any, data: any) => { + log({ err, data }); + if (err) { + return submitResponse("FAILED", err.message + `\nMore information in CloudWatch Log Stream: ${context.logStreamName}`); + } else { + return submitResponse("SUCCESS"); + } + }); + + function log(obj: any) { + console.error(event.RequestId, event.StackId, event.LogicalResourceId, obj); + } + + // tslint:disable-next-line:max-line-length + // adapted from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html#cfn-lambda-function-code-cfnresponsemodule + // to allow sending an error messge as a reason. + function submitResponse(responseStatus: string, reason?: string) { + const responseBody = JSON.stringify({ + Status: responseStatus, + Reason: reason || "See the details in CloudWatch Log Stream: " + context.logStreamName, + PhysicalResourceId: context.logStreamName, + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + NoEcho: false, + }); + + log({ responseBody }); + + const parsedUrl = url.parse(event.ResponseURL); + const options = { + hostname: parsedUrl.hostname, + port: 443, + path: parsedUrl.path, + method: "PUT", + headers: { + "content-type": "", + "content-length": responseBody.length + } + }; + + const request = https.request(options, (r: any) => { + log({ statusCode: r.statusCode, statusMessage: r.statusMessage }); + context.done(); + }); + + request.on("error", (error: any) => { + log({ sendError: error }); + context.done(); + }); + + request.write(responseBody); + request.end(); + } +}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource.ts b/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource.ts new file mode 100644 index 0000000000000..fb14744f32346 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/lib/notifications-resource/notifications-resource.ts @@ -0,0 +1,172 @@ +import cdk = require('@aws-cdk/cdk'); +import { Bucket, EventType, NotificationKeyFilter } from '../bucket'; +import { BucketNotificationDestinationType, IBucketNotificationDestination } from '../notification-dest'; +import { NotificationsResourceHandler } from './notifications-resource-handler'; + +interface NotificationsProps { + /** + * The bucket to manage notifications for. + * + * This cannot be a `BucketRef` because the bucket maintains the 1:1 + * relationship with this resource. + */ + bucket: Bucket; +} + +/** + * A custom CloudFormation resource that updates bucket notifications for a + * bucket. The reason we need it is because the AWS::S3::Bucket notification + * configuration is defined on the bucket itself, which makes it impossible to + * provision notifications at the same time as the target (since + * PutBucketNotifications validates the targets). + * + * Since only a single BucketNotifications resource is allowed for each Bucket, + * this construct is not exported in the public API of this module. Instead, it + * is created just-in-time by `s3.Bucket.onEvent`, so a 1:1 relationship is + * ensured. + * + * @see + * https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-notificationconfig.html + */ +export class BucketNotifications extends cdk.Construct { + private readonly lambdaNotifications = new Array(); + private readonly queueNotifications = new Array(); + private readonly topicNotifications = new Array(); + private customResourceCreated = false; + private readonly bucket: Bucket; + + constructor(parent: cdk.Construct, id: string, props: NotificationsProps) { + super(parent, id); + this.bucket = props.bucket; + } + + /** + * Adds a notification subscription for this bucket. + * If this is the first notification, a BucketNotification resource is added to the stack. + * + * @param event The type of event + * @param target The target construct + * @param filters A set of S3 key filters + */ + public addNotification(event: EventType, target: IBucketNotificationDestination, ...filters: NotificationKeyFilter[]) { + this.createResourceOnce(); + + // resolve target. this also provides an opportunity for the target to e.g. update + // policies to allow this notification to happen. + const targetProps = target.asBucketNotificationDestination(this.bucket); + const commonConfig: CommonConfiguration = { + Events: [ event ], + Filter: renderFilters(filters), + }; + + // based on the target type, add the the correct configurations array + switch (targetProps.type) { + case BucketNotificationDestinationType.Lambda: + this.lambdaNotifications.push({ ...commonConfig, LambdaFunctionArn: targetProps.arn }); + break; + + case BucketNotificationDestinationType.Queue: + this.queueNotifications.push({ ...commonConfig, QueueArn: targetProps.arn }); + break; + + case BucketNotificationDestinationType.Topic: + this.topicNotifications.push({ ...commonConfig, TopicArn: targetProps.arn }); + break; + + default: + throw new Error('Unsupported notification target type:' + BucketNotificationDestinationType[targetProps.type]); + } + } + + private renderNotificationConfiguration(): NotificationConfiguration { + return { + LambdaFunctionConfigurations: this.lambdaNotifications.length > 0 ? this.lambdaNotifications : undefined, + QueueConfigurations: this.queueNotifications.length > 0 ? this.queueNotifications : undefined, + TopicConfigurations: this.topicNotifications.length > 0 ? this.topicNotifications : undefined + }; + } + + /** + * Defines the bucket notifications resources in the stack only once. + * This is called lazily as we add notifications, so that if notifications are not added, + * there is no notifications resource. + */ + private createResourceOnce() { + if (this.customResourceCreated) { + return; + } + + const handlerArn = NotificationsResourceHandler.singleton(this); + new cdk.Resource(this, 'Resource', { + type: 'Custom::S3BucketNotifications', + properties: { + ServiceToken: handlerArn, + BucketName: this.bucket.bucketName, + NotificationConfiguration: new cdk.Token(() => this.renderNotificationConfiguration()) + } + }); + + this.customResourceCreated = true; + } +} + +function renderFilters(filters?: NotificationKeyFilter[]): Filter | undefined { + if (!filters || filters.length === 0) { + return undefined; + } + + const renderedRules = new Array(); + + for (const rule of filters) { + if (!rule.suffix && !rule.prefix) { + throw new Error('NotificationKeyFilter must specify `prefix` and/or `suffix`'); + } + + if (rule.suffix) { + renderedRules.push({ Name: 'suffix', Value: rule.suffix }); + } + + if (rule.prefix) { + renderedRules.push({ Name: 'prefix', Value: rule.prefix }); + } + } + + return { + Key: { + FilterRules: renderedRules + } + }; +} + +interface NotificationConfiguration { + LambdaFunctionConfigurations?: LambdaFunctionConfiguration[]; + QueueConfigurations?: QueueConfiguration[]; + TopicConfigurations?: TopicConfiguration[]; +} + +interface CommonConfiguration { + Id?: string; + Events: EventType[]; + Filter?: Filter +} + +interface LambdaFunctionConfiguration extends CommonConfiguration { + LambdaFunctionArn: cdk.Arn; +} + +interface QueueConfiguration extends CommonConfiguration { + QueueArn: cdk.Arn; +} + +interface TopicConfiguration extends CommonConfiguration { + TopicArn: cdk.Arn; +} + +interface FilterRule { + Name: 'prefix' | 'suffix'; + Value: string; +} + +interface Filter { + Key: { FilterRules: FilterRule[] } +} diff --git a/packages/@aws-cdk/aws-s3/test/integ.notifications.expected.json b/packages/@aws-cdk/aws-s3/test/integ.notifications.expected.json new file mode 100644 index 0000000000000..5a06b33722f90 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/integ.notifications.expected.json @@ -0,0 +1,273 @@ +{ + "Resources": { + "Bucket83908E77": { + "Type": "AWS::S3::Bucket" + }, + "BucketNotifications8F2E257D": { + "Type": "Custom::S3BucketNotifications", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket83908E77" + }, + "NotificationConfiguration": { + "TopicConfigurations": [ + { + "Events": [ + "s3:ObjectCreated:Put" + ], + "TopicArn": { + "Ref": "TopicBFC7AF6E" + } + }, + { + "Events": [ + "s3:ObjectRemoved:*" + ], + "Filter": { + "Key": { + "FilterRules": [ + { + "Name": "prefix", + "Value": "home/myusername/" + } + ] + } + }, + "TopicArn": { + "Ref": "Topic3DEAE47A7" + } + } + ] + } + } + }, + "TopicBFC7AF6E": { + "Type": "AWS::SNS::Topic" + }, + "TopicPolicy7C94FB28": { + "Type": "AWS::SNS::TopicPolicy", + "Properties": { + "Topics": [ + { + "Ref": "TopicBFC7AF6E" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": "sns:Publish", + "Condition": { + "ArnLike": { + "aws:SourceArn": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "TopicBFC7AF6E" + }, + "Sid": "sid0" + } + ], + "Version": "2012-10-17" + } + } + }, + "Topic3DEAE47A7": { + "Type": "AWS::SNS::Topic" + }, + "Topic3Policy9C00F2FA": { + "Type": "AWS::SNS::TopicPolicy", + "Properties": { + "Topics": [ + { + "Ref": "Topic3DEAE47A7" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": "sns:Publish", + "Condition": { + "ArnLike": { + "aws:SourceArn": { + "Fn::GetAtt": [ + "Bucket83908E77", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "Topic3DEAE47A7" + }, + "Sid": "sid0" + }, + { + "Action": "sns:Publish", + "Condition": { + "ArnLike": { + "aws:SourceArn": { + "Fn::GetAtt": [ + "Bucket25524B414", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "Topic3DEAE47A7" + }, + "Sid": "sid1" + } + ], + "Version": "2012-10-17" + } + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn", + ":", + { + "Ref": "AWS::Partition" + }, + ":", + "iam", + ":", + "", + ":", + "aws", + ":", + "policy", + "/", + "service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:PutBucketNotification", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36", + "Roles": [ + { + "Ref": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + } + ] + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Description": "AWS CloudFormation handler for \"Custom::S3BucketNotifications\" resources (@aws-cdk/aws-s3)", + "Code": { + "ZipFile": "exports.handler = (event, context) => {\n const s3 = new (require('aws-sdk').S3)();\n const https = require(\"https\");\n const url = require(\"url\");\n log(JSON.stringify(event, undefined, 2));\n const props = event.ResourceProperties;\n if (event.RequestType === 'Delete') {\n props.NotificationConfiguration = {}; // this is how you clean out notifications\n }\n const req = {\n Bucket: props.BucketName,\n NotificationConfiguration: props.NotificationConfiguration\n };\n return s3.putBucketNotificationConfiguration(req, (err, data) => {\n log({ err, data });\n if (err) {\n return submitResponse(\"FAILED\", err.message + `\\nMore information in CloudWatch Log Stream: ${context.logStreamName}`);\n }\n else {\n return submitResponse(\"SUCCESS\");\n }\n });\n function log(obj) {\n console.error(event.RequestId, event.StackId, event.LogicalResourceId, obj);\n }\n // tslint:disable-next-line:max-line-length\n // adapted from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html#cfn-lambda-function-code-cfnresponsemodule\n // to allow sending an error messge as a reason.\n function submitResponse(responseStatus, reason) {\n const responseBody = JSON.stringify({\n Status: responseStatus,\n Reason: reason || \"See the details in CloudWatch Log Stream: \" + context.logStreamName,\n PhysicalResourceId: context.logStreamName,\n StackId: event.StackId,\n RequestId: event.RequestId,\n LogicalResourceId: event.LogicalResourceId,\n NoEcho: false,\n });\n log({ responseBody });\n const parsedUrl = url.parse(event.ResponseURL);\n const options = {\n hostname: parsedUrl.hostname,\n port: 443,\n path: parsedUrl.path,\n method: \"PUT\",\n headers: {\n \"content-type\": \"\",\n \"content-length\": responseBody.length\n }\n };\n const request = https.request(options, (r) => {\n log({ statusCode: r.statusCode, statusMessage: r.statusMessage });\n context.done();\n });\n request.on(\"error\", (error) => {\n log({ sendError: error });\n context.done();\n });\n request.write(responseBody);\n request.end();\n }\n};" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC", + "Arn" + ] + }, + "Runtime": "nodejs8.10", + "Timeout": 300 + } + }, + "Bucket25524B414": { + "Type": "AWS::S3::Bucket" + }, + "Bucket2NotificationsD9BA2A77": { + "Type": "Custom::S3BucketNotifications", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "Bucket25524B414" + }, + "NotificationConfiguration": { + "TopicConfigurations": [ + { + "Events": [ + "s3:ObjectRemoved:*" + ], + "Filter": { + "Key": { + "FilterRules": [ + { + "Name": "prefix", + "Value": "foo" + }, + { + "Name": "suffix", + "Value": "foo/bar" + } + ] + } + }, + "TopicArn": { + "Ref": "Topic3DEAE47A7" + } + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3/test/integ.notifications.ts b/packages/@aws-cdk/aws-s3/test/integ.notifications.ts new file mode 100644 index 0000000000000..c6e5977247361 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/integ.notifications.ts @@ -0,0 +1,20 @@ +import cdk = require('@aws-cdk/cdk'); +import { Stack } from '@aws-cdk/cdk'; +import s3 = require('../lib'); +import { Topic } from './notification-dests'; + +const app = new cdk.App(process.argv); + +const stack = new Stack(app, 'test-3'); + +const bucket = new s3.Bucket(stack, 'Bucket'); +const topic = new Topic(stack, 'Topic'); +const topic3 = new Topic(stack, 'Topic3'); + +bucket.onEvent(s3.EventType.ObjectCreatedPut, topic); +bucket.onEvent(s3.EventType.ObjectRemoved, topic3, { prefix: 'home/myusername/' }); + +const bucket2 = new s3.Bucket(stack, 'Bucket2'); +bucket2.onObjectRemoved(topic3, { prefix: 'foo' }, { suffix: 'foo/bar' }); + +process.stdout.write(app.run()); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-s3/test/notification-dests.ts b/packages/@aws-cdk/aws-s3/test/notification-dests.ts new file mode 100644 index 0000000000000..afdf24035bc4c --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/notification-dests.ts @@ -0,0 +1,47 @@ +import cdk = require('@aws-cdk/cdk'); +import s3 = require('../lib'); + +/** + * Since we can't take a dependency on @aws-cdk/sns, this is a simple wrapper + * for AWS::SNS::Topic which implements IBucketNotificationDestination. + */ +export class Topic extends cdk.Construct implements s3.IBucketNotificationDestination { + public readonly topicArn: cdk.Arn; + private readonly policy = new cdk.PolicyDocument(); + private readonly notifyingBucketPaths = new Set(); + + constructor(parent: cdk.Construct, id: string) { + super(parent, id); + + const resource = new cdk.Resource(this, 'Resource', { type: 'AWS::SNS::Topic' }); + + new cdk.Resource(this, 'Policy', { + type: 'AWS::SNS::TopicPolicy', + properties: { + Topics: [ resource.ref ], + PolicyDocument: this.policy + } + }); + + this.topicArn = resource.ref; + } + + public asBucketNotificationDestination(bucket: s3.Bucket): s3.BucketNotificationDestinationProps { + + // add permission to each source bucket + if (!this.notifyingBucketPaths.has(bucket.path)) { + this.policy.addStatement(new cdk.PolicyStatement() + .describe(`sid${this.policy.statementCount}`) + .addServicePrincipal('s3.amazonaws.com') + .addAction('sns:Publish') + .addResource(this.topicArn) + .addCondition('ArnLike', { "aws:SourceArn": bucket.bucketArn })); + this.notifyingBucketPaths.add(bucket.path); + } + + return { + arn: this.topicArn, + type: s3.BucketNotificationDestinationType.Topic + }; + } +} diff --git a/packages/@aws-cdk/aws-s3/test/test.notifications.ts b/packages/@aws-cdk/aws-s3/test/test.notifications.ts new file mode 100644 index 0000000000000..adbd7e7fef925 --- /dev/null +++ b/packages/@aws-cdk/aws-s3/test/test.notifications.ts @@ -0,0 +1,272 @@ +import { expect, haveResource } from '@aws-cdk/assert'; +import cdk = require('@aws-cdk/cdk'); +import { Test } from 'nodeunit'; +import s3 = require('../lib'); +import { Topic } from './notification-dests'; + +// tslint:disable:object-literal-key-quotes +// tslint:disable:max-line-length + +export = { + 'bucket without notifications'(test: Test) { + const stack = new cdk.Stack(); + + new s3.Bucket(stack, 'MyBucket'); + + expect(stack).toMatch({ + "Resources": { + "MyBucketF68F3FF0": { + "Type": "AWS::S3::Bucket" + } + } + }); + + test.done(); + }, + + 'when notification are added, a custom resource is provisioned + a lambda handler for it'(test: Test) { + const stack = new cdk.Stack(); + + const bucket = new s3.Bucket(stack, 'MyBucket'); + + const topic = new Topic(stack, 'MyTopic'); + + bucket.onEvent(s3.EventType.ObjectCreated, topic); + + expect(stack).to(haveResource('AWS::S3::Bucket')); + expect(stack).to(haveResource('AWS::Lambda::Function', { Description: 'AWS CloudFormation handler for "Custom::S3BucketNotifications" resources (@aws-cdk/aws-s3)' })); + expect(stack).to(haveResource('Custom::S3BucketNotifications')); + + test.done(); + }, + + 'bucketNotificationTarget is not called during synthesis'(test: Test) { + const stack = new cdk.Stack(); + + // notice the order here - topic is defined before bucket + // but this shouldn't impact the fact that the topic policy includes + // the bucket information + const topic = new Topic(stack, 'Topic'); + const bucket = new s3.Bucket(stack, 'MyBucket'); + + bucket.onObjectCreated(topic); + + expect(stack).to(haveResource('AWS::SNS::TopicPolicy', { + "Topics": [ + { + "Ref": "TopicBFC7AF6E" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": "sns:Publish", + "Condition": { + "ArnLike": { + "aws:SourceArn": { + "Fn::GetAtt": [ + "MyBucketF68F3FF0", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "TopicBFC7AF6E" + }, + "Sid": "sid0" + } + ], + "Version": "2012-10-17" + } + })); + + test.done(); + }, + + 'subscription types'(test: Test) { + const stack = new cdk.Stack(); + + const bucket = new s3.Bucket(stack, 'TestBucket'); + + const queueTarget: s3.IBucketNotificationDestination = { + asBucketNotificationDestination: _ => ({ + type: s3.BucketNotificationDestinationType.Queue, + arn: new cdk.Arn('arn:aws:sqs:...') + }) + }; + + const lambdaTarget: s3.IBucketNotificationDestination = { + asBucketNotificationDestination: _ => ({ + type: s3.BucketNotificationDestinationType.Lambda, + arn: new cdk.Arn('arn:aws:lambda:...') + }) + }; + + const topicTarget: s3.IBucketNotificationDestination = { + asBucketNotificationDestination: _ => ({ + type: s3.BucketNotificationDestinationType.Topic, + arn: new cdk.Arn('arn:aws:sns:...') + }) + }; + + bucket.onEvent(s3.EventType.ObjectCreated, queueTarget); + bucket.onEvent(s3.EventType.ObjectCreated, lambdaTarget); + bucket.onObjectRemoved(topicTarget, { prefix: 'prefix' }); + + expect(stack).to(haveResource('Custom::S3BucketNotifications', { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestBucket560B80BC" + }, + "NotificationConfiguration": { + "LambdaFunctionConfigurations": [ + { + "Events": [ + "s3:ObjectCreated:*" + ], + "LambdaFunctionArn": "arn:aws:lambda:..." + } + ], + "QueueConfigurations": [ + { + "Events": [ + "s3:ObjectCreated:*" + ], + "QueueArn": "arn:aws:sqs:..." + } + ], + "TopicConfigurations": [ + { + "Events": [ + "s3:ObjectRemoved:*" + ], + "TopicArn": "arn:aws:sns:...", + "Filter": { + "Key": { + "FilterRules": [ + { + "Name": "prefix", + "Value": "prefix" + } + ] + } + } + } + ] + } + })); + + test.done(); + }, + + 'multiple subscriptions of the same type'(test: Test) { + const stack = new cdk.Stack(); + + const bucket = new s3.Bucket(stack, 'TestBucket'); + + bucket.onEvent(s3.EventType.ObjectRemovedDelete, { + asBucketNotificationDestination: _ => ({ + type: s3.BucketNotificationDestinationType.Queue, + arn: new cdk.Arn('arn:aws:sqs:...:queue1') + }) + }); + + bucket.onEvent(s3.EventType.ObjectRemovedDelete, { + asBucketNotificationDestination: _ => ({ + type: s3.BucketNotificationDestinationType.Queue, + arn: new cdk.Arn('arn:aws:sqs:...:queue2') + }) + }); + + expect(stack).to(haveResource('Custom::S3BucketNotifications', { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestBucket560B80BC" + }, + "NotificationConfiguration": { + "QueueConfigurations": [ + { + "Events": [ + "s3:ObjectRemoved:Delete" + ], + "QueueArn": "arn:aws:sqs:...:queue1" + }, + { + "Events": [ + "s3:ObjectRemoved:Delete" + ], + "QueueArn": "arn:aws:sqs:...:queue2" + } + ] + } + })); + + test.done(); + }, + + 'prefix/suffix filters'(test: Test) { + const stack = new cdk.Stack(); + + const bucket = new s3.Bucket(stack, 'TestBucket'); + + const bucketNotificationTarget = { + type: s3.BucketNotificationDestinationType.Queue, + arn: new cdk.Arn('arn:aws:sqs:...') + }; + + bucket.onEvent(s3.EventType.ObjectRemovedDelete, { asBucketNotificationDestination: _ => bucketNotificationTarget }, { prefix: 'images/', suffix: '.jpg' }); + + expect(stack).to(haveResource('Custom::S3BucketNotifications', { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "TestBucket560B80BC" + }, + "NotificationConfiguration": { + "QueueConfigurations": [ + { + "Events": [ + "s3:ObjectRemoved:Delete" + ], + "Filter": { + "Key": { + "FilterRules": [ + { + "Name": "suffix", + "Value": ".jpg" + }, + { + "Name": "prefix", + "Value": "images/" + } + ] + } + }, + "QueueArn": "arn:aws:sqs:..." + } + ] + } + })); + + test.done(); + } +};