From e5ab49ea560ad2845eedafc191aef49c9314f18d Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 22 Jan 2020 12:19:42 -0500 Subject: [PATCH 01/33] feat(lambda-event-sources): add failure handling parameters to lambda event source parameters Closes #5236 --- .../aws-lambda-event-sources/lib/stream.ts | 48 ++- .../aws-lambda-event-sources/package.json | 10 +- .../test/test.dynamo.ts | 319 ++++++++++++++++++ .../aws-lambda/lib/event-source-mapping.ts | 60 ++++ packages/@aws-cdk/aws-lambda/package.json | 8 +- 5 files changed, 440 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index baf06993e74b8..112c079bf7fd9 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -1,5 +1,5 @@ import * as lambda from '@aws-cdk/aws-lambda'; -import {Duration} from '@aws-cdk/core'; +import { Duration, IResolvable } from '@aws-cdk/core'; /** * The set of properties for event sources that follow the streaming model, @@ -21,6 +21,47 @@ export interface StreamEventSourceProps { */ readonly batchSize?: number; + /** + * If the function returns an error, split the batch in two and retry. + * + * @default false + */ + readonly bisectBatchOnFunctionError?: boolean; + + /** + * An Amazon SQS queue or Amazon SNS topic destination for discarded records. + * + * @default discarded records are ignored + */ + readonly destinationConfig?: lambda.CfnEventSourceMapping.DestinationConfigProperty | IResolvable | undefined; + + /** + * The maximum age of a record that Lambda sends to a function for processing. + * Valid Range: + * * Minimum value of 60 + * * Maximum value of 604800 + * + * @default 604800 + */ + readonly maximumRecordAgeInSeconds?: number; + + /** + * Maximum number of retry attempts + * + * @default todoNumber + */ + readonly maximumRetryAttempts?: number; + + /** + * The number of batches to process from each shard concurrently. + * Valid Range: + * * Minimum value of 1 + * * Maximum value of 10 + * + * @default 1 + */ + readonly parallelizationFactor?: number; + /** * Where to begin consuming the stream. */ @@ -48,8 +89,13 @@ export abstract class StreamEventSource implements lambda.IEventSource { return { ...options, batchSize: this.props.batchSize || 100, + bisectBatchOnFunctionError: this.props.bisectBatchOnFunctionError, startingPosition: this.props.startingPosition, maxBatchingWindow: this.props.maxBatchingWindow, + maximumRecordAgeInSeconds: this.props.maximumRecordAgeInSeconds, + maximumRetryAttempts: this.props.maximumRetryAttempts, + parallelizationFactor: this.props.parallelizationFactor, + destinationConfig: this.props.destinationConfig }; } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 42a2374119501..9a9fb62cd5b24 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -109,7 +109,13 @@ "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps", "docs-public-apis:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps", "props-default-doc:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps.filters", - "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps" + "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps", + "duration-prop-type:@aws-cdk/aws-lambda-event-sources.DynamoEventSourceProps.maximumRecordAgeInSeconds", + "duration-prop-type:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps.maximumRecordAgeInSeconds", + "duration-prop-name:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps.maximumRecordAgeInSeconds", + "duration-prop-type:@aws-cdk/aws-lambda-event-sources.StreamEventSourceProps.maximumRecordAgeInSeconds", + "duration-prop-name:@aws-cdk/aws-lambda-event-sources.StreamEventSourceProps.maximumRecordAgeInSeconds", + "duration-prop-name:@aws-cdk/aws-lambda-event-sources.DynamoEventSourceProps.maximumRecordAgeInSeconds" ] } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 68b12725af0e3..044b0e7e73f59 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -231,4 +231,323 @@ export = { test.done(); }, + 'specific maximumRetryAttempts'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // WHEN + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRetryAttempts: 10, + startingPosition: lambda.StartingPosition.LATEST + })); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + "EventSourceArn": { + "Fn::GetAtt": [ + "TD925BC7E", + "StreamArn" + ] + }, + "FunctionName": { + "Ref": "Fn9270CBC0" + }, + "MaximumRetryAttempts": 10, + "StartingPosition": "LATEST" + })); + + test.done(); + }, + + 'fails if maximumRetryAttempts < 0'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRetryAttempts: -1, + startingPosition: lambda.StartingPosition.LATEST + })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got -1/); + + test.done(); + }, + + 'fails if maximumRetryAttempts > 10000'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRetryAttempts: 10001, + startingPosition: lambda.StartingPosition.LATEST + })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got 10001/); + + test.done(); + }, + + 'specific bisectBatchOnFunctionError'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // WHEN + fn.addEventSource(new sources.DynamoEventSource(table, { + bisectBatchOnFunctionError: true, + startingPosition: lambda.StartingPosition.LATEST + })); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + "EventSourceArn": { + "Fn::GetAtt": [ + "TD925BC7E", + "StreamArn" + ] + }, + "FunctionName": { + "Ref": "Fn9270CBC0" + }, + "BisectBatchOnFunctionError": true, + "StartingPosition": "LATEST" + })); + + test.done(); + }, + + 'specific parallelizationFactor'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // WHEN + fn.addEventSource(new sources.DynamoEventSource(table, { + parallelizationFactor: 5, + startingPosition: lambda.StartingPosition.LATEST + })); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + "EventSourceArn": { + "Fn::GetAtt": [ + "TD925BC7E", + "StreamArn" + ] + }, + "FunctionName": { + "Ref": "Fn9270CBC0" + }, + "ParallelizationFactor": 5, + "StartingPosition": "LATEST" + })); + + test.done(); + }, + + 'fails if parallelizationFactor < 1'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + parallelizationFactor: 0, + startingPosition: lambda.StartingPosition.LATEST + })), /parallelizationFactor must be between 1 and 10 inclusive, got 0/); + + test.done(); + }, + + 'fails if parallelizationFactor > 10'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + parallelizationFactor: 11, + startingPosition: lambda.StartingPosition.LATEST + })), /parallelizationFactor must be between 1 and 10 inclusive, got 11/); + + test.done(); + }, + + 'specific maximumRecordAgeInSeconds'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // WHEN + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRecordAgeInSeconds: 100, + startingPosition: lambda.StartingPosition.LATEST + })); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + "EventSourceArn": { + "Fn::GetAtt": [ + "TD925BC7E", + "StreamArn" + ] + }, + "FunctionName": { + "Ref": "Fn9270CBC0" + }, + "MaximumRecordAgeInSeconds": 100, + "StartingPosition": "LATEST" + })); + + test.done(); + }, + + 'fails if maximumRecordAgeInSeconds < 60'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRecordAgeInSeconds: 59, + startingPosition: lambda.StartingPosition.LATEST + })), /maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got 59/); + + test.done(); + }, + + 'fails if maximumRecordAgeInSeconds > 604800'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // THEN + test.throws(() => + fn.addEventSource(new sources.DynamoEventSource(table, { + maximumRecordAgeInSeconds: 604801, + startingPosition: lambda.StartingPosition.LATEST + })), /maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got 604801/); + + test.done(); + }, + + 'specific destinationConfig'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const table = new dynamodb.Table(stack, 'T', { + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING + }, + stream: dynamodb.StreamViewType.NEW_IMAGE + }); + + // WHEN + fn.addEventSource(new sources.DynamoEventSource(table, { + destinationConfig: { + onFailure: { + destination: "DLQ" + } + }, + startingPosition: lambda.StartingPosition.LATEST + })); + + // THEN + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + "EventSourceArn": { + "Fn::GetAtt": [ + "TD925BC7E", + "StreamArn" + ] + }, + "FunctionName": { + "Ref": "Fn9270CBC0" + }, + "DestinationConfig": { + "OnFailure": { + "Destination": "DLQ" + } + }, + "StartingPosition": "LATEST" + })); + + test.done(); + }, }; diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index a30a277af3f31..69222af9ff384 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -21,6 +21,20 @@ export interface EventSourceMappingOptions { */ readonly batchSize?: number; + /** + * If the function returns an error, split the batch in two and retry. + * + * @default false + */ + readonly bisectBatchOnFunctionError?: boolean; + + /** + * An Amazon SQS queue or Amazon SNS topic destination for discarded records. + * + * @default discard records are ignored + */ + readonly destinationConfig?: CfnEventSourceMapping.DestinationConfigProperty | cdk.IResolvable | undefined; + /** * Set to false to disable the event source upon creation. * @@ -45,6 +59,35 @@ export interface EventSourceMappingOptions { * @default Duration.seconds(0) */ readonly maxBatchingWindow?: cdk.Duration; + + /** + * The maximum age of a record that Lambda sends to a function for processing. + * Valid Range: + * * Minimum value of 60 + * * Maximum value of 604800 + * + * @default 604800 + */ + readonly maximumRecordAgeInSeconds?: number; + + /** + * The maximum number of times to retry when the function returns an error. + * + * Valid Range: Minimum value of 0. Maximum value of 10000. + * + * @default 10000 + */ + readonly maximumRetryAttempts?: number; + + /** + * The number of batches to process from each shard concurrently. + * Valid Range: + * * Minimum value of 1 + * * Maximum value of 10 + * + * @default 1 + */ + readonly parallelizationFactor?: number; } export interface EventSourceMappingProps extends EventSourceMappingOptions { @@ -74,13 +117,30 @@ export class EventSourceMapping extends cdk.Resource { throw new Error(`maxBatchingWindow cannot be over 300 seconds, got ${props.maxBatchingWindow.toSeconds()}`); } + if (props.maximumRecordAgeInSeconds && (props.maximumRecordAgeInSeconds < 60 || props.maximumRecordAgeInSeconds > 604800)) { + throw new Error(`maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got ${props.maximumRecordAgeInSeconds}`); + } + + if (props.maximumRetryAttempts && (props.maximumRetryAttempts < 0 || props.maximumRetryAttempts > 10000)) { + throw new Error(`maximumRetryAttempts must be between 0 and 10000 inclusive, got ${props.maximumRetryAttempts}`); + } + + if ((props.parallelizationFactor || props.parallelizationFactor === 0) && (props.parallelizationFactor < 1 || props.parallelizationFactor > 10)) { + throw new Error(`parallelizationFactor must be between 1 and 10 inclusive, got ${props.parallelizationFactor}`); + } + new CfnEventSourceMapping(this, 'Resource', { batchSize: props.batchSize, + bisectBatchOnFunctionError: props.bisectBatchOnFunctionError, + destinationConfig: props.destinationConfig, enabled: props.enabled, eventSourceArn: props.eventSourceArn, functionName: props.target.functionName, startingPosition: props.startingPosition, maximumBatchingWindowInSeconds: props.maxBatchingWindow && props.maxBatchingWindow.toSeconds(), + maximumRecordAgeInSeconds: props.maximumRecordAgeInSeconds, + maximumRetryAttempts: props.maximumRetryAttempts, + parallelizationFactor: props.parallelizationFactor }); } } diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 3c757c06b0873..063fe1facf21b 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -182,8 +182,12 @@ "props-default-doc:@aws-cdk/aws-lambda.Permission.sourceArn", "docs-public-apis:@aws-cdk/aws-lambda.ResourceBindOptions", "docs-public-apis:@aws-cdk/aws-lambda.VersionAttributes", - "props-physical-name:@aws-cdk/aws-lambda.EventInvokeConfigProps" + "props-physical-name:@aws-cdk/aws-lambda.EventInvokeConfigProps", + "duration-prop-type:@aws-cdk/aws-lambda.EventSourceMappingOptions.maximumRecordAgeInSeconds", + "duration-prop-name:@aws-cdk/aws-lambda.EventSourceMappingOptions.maximumRecordAgeInSeconds", + "duration-prop-type:@aws-cdk/aws-lambda.EventSourceMappingProps.maximumRecordAgeInSeconds", + "duration-prop-name:@aws-cdk/aws-lambda.EventSourceMappingProps.maximumRecordAgeInSeconds" ] }, "stability": "stable" -} \ No newline at end of file +} From c271545c2b8b542a72391e8b187d94bbe4da122d Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 27 Jan 2020 08:20:53 -0500 Subject: [PATCH 02/33] Standardize default documentation on maximumRetryAttempts --- packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts | 5 ++++- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 112c079bf7fd9..1e18cb1b8fc11 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -47,8 +47,11 @@ export interface StreamEventSourceProps { /** * Maximum number of retry attempts + * Valid Range: + * * Minimum value of 0 + * * Maximum value of 10000 * - * @default todoNumber + * @default 10000 */ readonly maximumRetryAttempts?: number; diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 69222af9ff384..d8f88ee9a8ca1 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -31,7 +31,7 @@ export interface EventSourceMappingOptions { /** * An Amazon SQS queue or Amazon SNS topic destination for discarded records. * - * @default discard records are ignored + * @default discarded records are ignored */ readonly destinationConfig?: CfnEventSourceMapping.DestinationConfigProperty | cdk.IResolvable | undefined; @@ -73,7 +73,9 @@ export interface EventSourceMappingOptions { /** * The maximum number of times to retry when the function returns an error. * - * Valid Range: Minimum value of 0. Maximum value of 10000. + * Valid Range: + * * Minimum value of 0 + * * Maximum value of 10000 * * @default 10000 */ From ebf5404270c4758d546cfa25883d6ee7d2be483a Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 27 Jan 2020 09:26:52 -0500 Subject: [PATCH 03/33] Refactor maximumRecordAgeInSeconds to be a duration --- .../@aws-cdk/aws-lambda-event-sources/lib/stream.ts | 4 ++-- .../aws-lambda-event-sources/test/test.dynamo.ts | 10 +++++----- .../@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 1e18cb1b8fc11..69738a18457b4 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -43,7 +43,7 @@ export interface StreamEventSourceProps { * * @default 604800 */ - readonly maximumRecordAgeInSeconds?: number; + readonly maximumRecordAge?: Duration; /** * Maximum number of retry attempts @@ -95,7 +95,7 @@ export abstract class StreamEventSource implements lambda.IEventSource { bisectBatchOnFunctionError: this.props.bisectBatchOnFunctionError, startingPosition: this.props.startingPosition, maxBatchingWindow: this.props.maxBatchingWindow, - maximumRecordAgeInSeconds: this.props.maximumRecordAgeInSeconds, + maximumRecordAge: this.props.maximumRecordAge, maximumRetryAttempts: this.props.maximumRetryAttempts, parallelizationFactor: this.props.parallelizationFactor, destinationConfig: this.props.destinationConfig diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 044b0e7e73f59..d4ec5717f9c2a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -441,7 +441,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAgeInSeconds: 100, + maximumRecordAge: cdk.Duration.seconds(100), startingPosition: lambda.StartingPosition.LATEST })); @@ -478,9 +478,9 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAgeInSeconds: 59, + maximumRecordAge: cdk.Duration.seconds(59), startingPosition: lambda.StartingPosition.LATEST - })), /maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got 59/); + })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 59/); test.done(); }, @@ -500,9 +500,9 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAgeInSeconds: 604801, + maximumRecordAge: cdk.Duration.seconds(604801), startingPosition: lambda.StartingPosition.LATEST - })), /maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got 604801/); + })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); test.done(); }, diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index d8f88ee9a8ca1..32f8d569e07e9 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -68,12 +68,12 @@ export interface EventSourceMappingOptions { * * @default 604800 */ - readonly maximumRecordAgeInSeconds?: number; + readonly maximumRecordAge?: cdk.Duration; /** * The maximum number of times to retry when the function returns an error. * - * Valid Range: + * Valid Range: * * Minimum value of 0 * * Maximum value of 10000 * @@ -119,8 +119,8 @@ export class EventSourceMapping extends cdk.Resource { throw new Error(`maxBatchingWindow cannot be over 300 seconds, got ${props.maxBatchingWindow.toSeconds()}`); } - if (props.maximumRecordAgeInSeconds && (props.maximumRecordAgeInSeconds < 60 || props.maximumRecordAgeInSeconds > 604800)) { - throw new Error(`maximumRecordAgeInSeconds must be between 60 and 604800 inclusive, got ${props.maximumRecordAgeInSeconds}`); + if (props.maximumRecordAge && (props.maximumRecordAge.toSeconds() < 60 || props.maximumRecordAge.toSeconds() > 604800)) { + throw new Error(`maximumRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maximumRecordAge.toSeconds()}`); } if (props.maximumRetryAttempts && (props.maximumRetryAttempts < 0 || props.maximumRetryAttempts > 10000)) { @@ -140,7 +140,7 @@ export class EventSourceMapping extends cdk.Resource { functionName: props.target.functionName, startingPosition: props.startingPosition, maximumBatchingWindowInSeconds: props.maxBatchingWindow && props.maxBatchingWindow.toSeconds(), - maximumRecordAgeInSeconds: props.maximumRecordAgeInSeconds, + maximumRecordAgeInSeconds: props.maximumRecordAge && props.maximumRecordAge.toSeconds(), maximumRetryAttempts: props.maximumRetryAttempts, parallelizationFactor: props.parallelizationFactor }); From 3690df164935d5acf4ae07b25417ff0c582e3b78 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Tue, 28 Jan 2020 12:35:48 -0500 Subject: [PATCH 04/33] Refactor away from using lambda.generated for destination config --- .../aws-lambda-event-sources/lib/stream.ts | 6 +++--- .../aws-lambda-event-sources/package.json | 12 ++++-------- .../test/test.dynamo.ts | 17 +++++++++++------ .../aws-lambda/lib/event-source-mapping.ts | 9 +++++++-- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 69738a18457b4..69c6eea57e9e8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -1,5 +1,5 @@ import * as lambda from '@aws-cdk/aws-lambda'; -import { Duration, IResolvable } from '@aws-cdk/core'; +import { Duration } from '@aws-cdk/core'; /** * The set of properties for event sources that follow the streaming model, @@ -33,7 +33,7 @@ export interface StreamEventSourceProps { * * @default discarded records are ignored */ - readonly destinationConfig?: lambda.CfnEventSourceMapping.DestinationConfigProperty | IResolvable | undefined; + readonly destinationOnFailure?: lambda.IDestination; /** * The maximum age of a record that Lambda sends to a function for processing. @@ -98,7 +98,7 @@ export abstract class StreamEventSource implements lambda.IEventSource { maximumRecordAge: this.props.maximumRecordAge, maximumRetryAttempts: this.props.maximumRetryAttempts, parallelizationFactor: this.props.parallelizationFactor, - destinationConfig: this.props.destinationConfig + destinationOnFailure: this.props.destinationOnFailure }; } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index cd96eba0129d5..6d08bc06a3565 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -72,6 +72,7 @@ "@aws-cdk/aws-iam": "1.22.0", "@aws-cdk/aws-kinesis": "1.22.0", "@aws-cdk/aws-lambda": "1.22.0", + "@aws-cdk/aws-lambda-destinations": "1.22.0", "@aws-cdk/aws-s3": "1.22.0", "@aws-cdk/aws-s3-notifications": "1.22.0", "@aws-cdk/aws-sns": "1.22.0", @@ -92,7 +93,8 @@ "@aws-cdk/aws-sns": "1.22.0", "@aws-cdk/aws-sns-subscriptions": "1.22.0", "@aws-cdk/aws-sqs": "1.22.0", - "@aws-cdk/core": "1.22.0" + "@aws-cdk/core": "1.22.0", + "@aws-cdk/aws-lambda-destinations": "1.22.0" }, "engines": { "node": ">= 10.3.0" @@ -109,13 +111,7 @@ "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps", "docs-public-apis:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps", "props-default-doc:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps.filters", - "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps", - "duration-prop-type:@aws-cdk/aws-lambda-event-sources.DynamoEventSourceProps.maximumRecordAgeInSeconds", - "duration-prop-type:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps.maximumRecordAgeInSeconds", - "duration-prop-name:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps.maximumRecordAgeInSeconds", - "duration-prop-type:@aws-cdk/aws-lambda-event-sources.StreamEventSourceProps.maximumRecordAgeInSeconds", - "duration-prop-name:@aws-cdk/aws-lambda-event-sources.StreamEventSourceProps.maximumRecordAgeInSeconds", - "duration-prop-name:@aws-cdk/aws-lambda-event-sources.DynamoEventSourceProps.maximumRecordAgeInSeconds" + "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps" ] } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index d4ec5717f9c2a..b5ceaafb44b6d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -1,6 +1,8 @@ import { expect, haveResource } from '@aws-cdk/assert'; import * as dynamodb from '@aws-cdk/aws-dynamodb'; import * as lambda from '@aws-cdk/aws-lambda'; +import * as destinations from '@aws-cdk/aws-lambda-destinations'; +import * as sqs from '@aws-cdk/aws-sqs'; import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; import * as sources from '../lib'; @@ -511,6 +513,7 @@ export = { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); + const queue = new sqs.Queue(stack, 'Queue'); const table = new dynamodb.Table(stack, 'T', { partitionKey: { name: 'id', @@ -521,11 +524,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - destinationConfig: { - onFailure: { - destination: "DLQ" - } - }, + destinationOnFailure: new destinations.SqsDestination(queue), startingPosition: lambda.StartingPosition.LATEST })); @@ -542,7 +541,13 @@ export = { }, "DestinationConfig": { "OnFailure": { - "Destination": "DLQ" + "Destination": { + "Fn::GetAtt": [ + "Queue4A7E3555", + "Arn" + ] + } + } }, "StartingPosition": "LATEST" diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 32f8d569e07e9..3eb16ba0293f4 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,4 +1,5 @@ import * as cdk from '@aws-cdk/core'; +import { DestinationType, IDestination } from './destination'; import { IFunction } from './function-base'; import { CfnEventSourceMapping } from './lambda.generated'; @@ -33,7 +34,7 @@ export interface EventSourceMappingOptions { * * @default discarded records are ignored */ - readonly destinationConfig?: CfnEventSourceMapping.DestinationConfigProperty | cdk.IResolvable | undefined; + readonly destinationOnFailure?: IDestination; /** * Set to false to disable the event source upon creation. @@ -134,7 +135,11 @@ export class EventSourceMapping extends cdk.Resource { new CfnEventSourceMapping(this, 'Resource', { batchSize: props.batchSize, bisectBatchOnFunctionError: props.bisectBatchOnFunctionError, - destinationConfig: props.destinationConfig, + destinationConfig: props.destinationOnFailure + ? { + onFailure: props.destinationOnFailure.bind(this, props.target, { type: DestinationType.FAILURE }) + } + : undefined, enabled: props.enabled, eventSourceArn: props.eventSourceArn, functionName: props.target.functionName, From 469783183342dd62d225c1653913126a90b5cc9a Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Tue, 28 Jan 2020 12:38:05 -0500 Subject: [PATCH 05/33] Remove unnecessary linter exceptions --- packages/@aws-cdk/aws-lambda/package.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 97bf1e0db015a..7007754cadaa8 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -182,11 +182,7 @@ "props-default-doc:@aws-cdk/aws-lambda.Permission.sourceArn", "docs-public-apis:@aws-cdk/aws-lambda.ResourceBindOptions", "docs-public-apis:@aws-cdk/aws-lambda.VersionAttributes", - "props-physical-name:@aws-cdk/aws-lambda.EventInvokeConfigProps", - "duration-prop-type:@aws-cdk/aws-lambda.EventSourceMappingOptions.maximumRecordAgeInSeconds", - "duration-prop-name:@aws-cdk/aws-lambda.EventSourceMappingOptions.maximumRecordAgeInSeconds", - "duration-prop-type:@aws-cdk/aws-lambda.EventSourceMappingProps.maximumRecordAgeInSeconds", - "duration-prop-name:@aws-cdk/aws-lambda.EventSourceMappingProps.maximumRecordAgeInSeconds" + "props-physical-name:@aws-cdk/aws-lambda.EventInvokeConfigProps" ] }, "stability": "stable" From 95eb54b432d18686d155db110ac325d3a290381b Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Thu, 30 Jan 2020 08:31:44 -0500 Subject: [PATCH 06/33] Simplified comments, variable names, and conditionals --- .../aws-lambda-event-sources/lib/stream.ts | 22 +++++----- .../test/test.dynamo.ts | 16 +++---- .../aws-lambda/lib/event-source-mapping.ts | 44 ++++++++++--------- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 69c6eea57e9e8..0b43f2d727e38 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -26,24 +26,24 @@ export interface StreamEventSourceProps { * * @default false */ - readonly bisectBatchOnFunctionError?: boolean; + readonly bisectBatchOnError?: boolean; /** * An Amazon SQS queue or Amazon SNS topic destination for discarded records. * * @default discarded records are ignored */ - readonly destinationOnFailure?: lambda.IDestination; + readonly onFailure?: lambda.IDestination; /** * The maximum age of a record that Lambda sends to a function for processing. * Valid Range: - * * Minimum value of 60 - * * Maximum value of 604800 + * * Minimum value of 60 seconds + * * Maximum value of 7 days * - * @default 604800 + * @default Duration.days(7) */ - readonly maximumRecordAge?: Duration; + readonly maxRecordAge?: Duration; /** * Maximum number of retry attempts @@ -53,7 +53,7 @@ export interface StreamEventSourceProps { * * @default 10000 */ - readonly maximumRetryAttempts?: number; + readonly retryAttempts?: number; /** * The number of batches to process from each shard concurrently. @@ -92,13 +92,13 @@ export abstract class StreamEventSource implements lambda.IEventSource { return { ...options, batchSize: this.props.batchSize || 100, - bisectBatchOnFunctionError: this.props.bisectBatchOnFunctionError, + bisectBatchOnError: this.props.bisectBatchOnError, startingPosition: this.props.startingPosition, maxBatchingWindow: this.props.maxBatchingWindow, - maximumRecordAge: this.props.maximumRecordAge, - maximumRetryAttempts: this.props.maximumRetryAttempts, + maxRecordAge: this.props.maxRecordAge, + retryAttempts: this.props.retryAttempts, parallelizationFactor: this.props.parallelizationFactor, - destinationOnFailure: this.props.destinationOnFailure + onFailure: this.props.onFailure }; } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index b5ceaafb44b6d..aa8635c6238fb 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -247,7 +247,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRetryAttempts: 10, + retryAttempts: 10, startingPosition: lambda.StartingPosition.LATEST })); @@ -284,7 +284,7 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRetryAttempts: -1, + retryAttempts: -1, startingPosition: lambda.StartingPosition.LATEST })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got -1/); @@ -306,7 +306,7 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRetryAttempts: 10001, + retryAttempts: 10001, startingPosition: lambda.StartingPosition.LATEST })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got 10001/); @@ -327,7 +327,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - bisectBatchOnFunctionError: true, + bisectBatchOnError: true, startingPosition: lambda.StartingPosition.LATEST })); @@ -443,7 +443,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAge: cdk.Duration.seconds(100), + maxRecordAge: cdk.Duration.seconds(100), startingPosition: lambda.StartingPosition.LATEST })); @@ -480,7 +480,7 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAge: cdk.Duration.seconds(59), + maxRecordAge: cdk.Duration.seconds(59), startingPosition: lambda.StartingPosition.LATEST })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 59/); @@ -502,7 +502,7 @@ export = { // THEN test.throws(() => fn.addEventSource(new sources.DynamoEventSource(table, { - maximumRecordAge: cdk.Duration.seconds(604801), + maxRecordAge: cdk.Duration.seconds(604801), startingPosition: lambda.StartingPosition.LATEST })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); @@ -524,7 +524,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - destinationOnFailure: new destinations.SqsDestination(queue), + onFailure: new destinations.SqsDestination(queue), startingPosition: lambda.StartingPosition.LATEST })); diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 3eb16ba0293f4..0a8d5df6a36fd 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -27,14 +27,14 @@ export interface EventSourceMappingOptions { * * @default false */ - readonly bisectBatchOnFunctionError?: boolean; + readonly bisectBatchOnError?: boolean; /** * An Amazon SQS queue or Amazon SNS topic destination for discarded records. * * @default discarded records are ignored */ - readonly destinationOnFailure?: IDestination; + readonly onFailure?: IDestination; /** * Set to false to disable the event source upon creation. @@ -64,12 +64,12 @@ export interface EventSourceMappingOptions { /** * The maximum age of a record that Lambda sends to a function for processing. * Valid Range: - * * Minimum value of 60 - * * Maximum value of 604800 + * * Minimum value of 60 seconds + * * Maximum value of 7 days * - * @default 604800 + * @default Duration.days(7) */ - readonly maximumRecordAge?: cdk.Duration; + readonly maxRecordAge?: cdk.Duration; /** * The maximum number of times to retry when the function returns an error. @@ -80,7 +80,7 @@ export interface EventSourceMappingOptions { * * @default 10000 */ - readonly maximumRetryAttempts?: number; + readonly retryAttempts?: number; /** * The number of batches to process from each shard concurrently. @@ -120,33 +120,37 @@ export class EventSourceMapping extends cdk.Resource { throw new Error(`maxBatchingWindow cannot be over 300 seconds, got ${props.maxBatchingWindow.toSeconds()}`); } - if (props.maximumRecordAge && (props.maximumRecordAge.toSeconds() < 60 || props.maximumRecordAge.toSeconds() > 604800)) { - throw new Error(`maximumRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maximumRecordAge.toSeconds()}`); + if (props.maxRecordAge && (props.maxRecordAge.toSeconds() < 60 || props.maxRecordAge.toDays({integral: false}) > 7)) { + throw new Error(`maximumRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maxRecordAge.toSeconds()}`); } - if (props.maximumRetryAttempts && (props.maximumRetryAttempts < 0 || props.maximumRetryAttempts > 10000)) { - throw new Error(`maximumRetryAttempts must be between 0 and 10000 inclusive, got ${props.maximumRetryAttempts}`); + if (props.retryAttempts && (props.retryAttempts < 0 || props.retryAttempts > 10000)) { + throw new Error(`maximumRetryAttempts must be between 0 and 10000 inclusive, got ${props.retryAttempts}`); } if ((props.parallelizationFactor || props.parallelizationFactor === 0) && (props.parallelizationFactor < 1 || props.parallelizationFactor > 10)) { throw new Error(`parallelizationFactor must be between 1 and 10 inclusive, got ${props.parallelizationFactor}`); } + let onFailure; + + if (props.onFailure) { + onFailure = { + onFailure: props.onFailure.bind(this, props.target, { type: DestinationType.FAILURE }) + }; + } + new CfnEventSourceMapping(this, 'Resource', { batchSize: props.batchSize, - bisectBatchOnFunctionError: props.bisectBatchOnFunctionError, - destinationConfig: props.destinationOnFailure - ? { - onFailure: props.destinationOnFailure.bind(this, props.target, { type: DestinationType.FAILURE }) - } - : undefined, + bisectBatchOnFunctionError: props.bisectBatchOnError, + destinationConfig: onFailure, enabled: props.enabled, eventSourceArn: props.eventSourceArn, functionName: props.target.functionName, startingPosition: props.startingPosition, - maximumBatchingWindowInSeconds: props.maxBatchingWindow && props.maxBatchingWindow.toSeconds(), - maximumRecordAgeInSeconds: props.maximumRecordAge && props.maximumRecordAge.toSeconds(), - maximumRetryAttempts: props.maximumRetryAttempts, + maximumBatchingWindowInSeconds: props.maxBatchingWindow?.toSeconds(), + maximumRecordAgeInSeconds: props.maxRecordAge?.toSeconds(), + maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor }); } From 7662d58200c2c0dab351a9d6b2dc88cf18ea4986 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Thu, 30 Jan 2020 09:15:54 -0500 Subject: [PATCH 07/33] Update README with new parameters --- .../aws-lambda-event-sources/README.md | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 515949dc8da3c..7e29d5c937b75 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -116,11 +116,18 @@ To process events with a Lambda function, first create or update a DynamoDB tabl and add it to your Lambda function. The following parameters will impact Amazon DynamoDB's polling behavior: * __batchSize__: Determines how many records are buffered before invoking your lambda function - could impact your function's memory usage (if too high) and ability to keep up with incoming data velocity (if too low). +* __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. +* __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of possibly delaying processing. +* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. +* __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here +* __parallelizationFactor__: The number of batches to concurrently process on each shard. +* __retryAttempts__: The maximum number of times a record should be retried in the event of failure. * __startingPosition__: Will determine where to being consumption, either at the most recent ('LATEST') record or the oldest record ('TRIM_HORIZON'). 'TRIM_HORIZON' will ensure you process all available data, while 'LATEST' will ignore all reocrds that arrived prior to attaching the event source. ```ts import dynamodb = require('@aws-cdk/aws-dynamodb'); import lambda = require('@aws-cdk/aws-lambda'); +import sqs = require('@aws-cdk/aws-sqs'); import { DynamoEventSource } from '@aws-cdk/aws-lambda-event-sources'; const table = new dynamodb.Table(..., { @@ -128,9 +135,15 @@ const table = new dynamodb.Table(..., { stream: dynamodb.StreamViewType.NEW_IMAGE // make sure stream is configured }); +const deadLetterQueue = new sqs.Queue(this, 'deadLetterQueue'); + const function = new lambda.Function(...); function.addEventSource(new DynamoEventSource(table, { - startingPosition: lambda.StartingPosition.TRIM_HORIZON + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + batchSize: 5, + bisectBatchOnError: true, + onFailure: deadLetterQueue, + retryAttempts: 10 })); ``` @@ -146,6 +159,12 @@ event source parameters. The following parameters will impact Amazon Kinesis's p behavior: * __batchSize__: Determines how many records are buffered before invoking your lambnda function - could impact your function's memory usage (if too high) and ability to keep up with incoming data velocity (if too low). +* __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. +* __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of possibly delaying processing. +* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. +* __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here +* __parallelizationFactor__: The number of batches to concurrently process on each shard. +* __retryAttempts__: The maximum number of times a record should be retried in the event of failure. * __startingPosition__: Will determine where to being consumption, either at the most recent ('LATEST') record or the oldest record ('TRIM_HORIZON'). 'TRIM_HORIZON' will ensure you process all available data, while 'LATEST' will ignore all reocrds that arrived prior to attaching the event source. ```ts From dbafdcd575ae19efd2ce08234b6574b0e2999567 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 08:55:28 -0500 Subject: [PATCH 08/33] Update packages/@aws-cdk/aws-lambda-event-sources/README.md Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 7e29d5c937b75..63e08d37c36b4 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -117,7 +117,7 @@ and add it to your Lambda function. The following parameters will impact Amazon * __batchSize__: Determines how many records are buffered before invoking your lambda function - could impact your function's memory usage (if too high) and ability to keep up with incoming data velocity (if too low). * __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. -* __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of possibly delaying processing. +* __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of delayed processing. * __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. * __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here * __parallelizationFactor__: The number of batches to concurrently process on each shard. From 9a51afd884b6ca96a3da371e2c73637faa1e90ec Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 08:55:44 -0500 Subject: [PATCH 09/33] Update packages/@aws-cdk/aws-lambda-event-sources/README.md Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 63e08d37c36b4..be682d8829f0f 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -119,7 +119,7 @@ and add it to your Lambda function. The following parameters will impact Amazon * __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. * __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of delayed processing. * __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. -* __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here +* __onFailure__: In the event a record fails after all retries or if the record age has exceeded the configured value, the record will be sent to SQS queue or SNS topic that is specified here * __parallelizationFactor__: The number of batches to concurrently process on each shard. * __retryAttempts__: The maximum number of times a record should be retried in the event of failure. * __startingPosition__: Will determine where to being consumption, either at the most recent ('LATEST') record or the oldest record ('TRIM_HORIZON'). 'TRIM_HORIZON' will ensure you process all available data, while 'LATEST' will ignore all reocrds that arrived prior to attaching the event source. From daff2047cf13c9deb8b704ec2d58b39b088b0ad0 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 08:56:04 -0500 Subject: [PATCH 10/33] Update packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index aa8635c6238fb..87362ea215327 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -429,7 +429,7 @@ export = { test.done(); }, - 'specific maximumRecordAgeInSeconds'(test: Test) { + 'specific maxRecordAge'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); From e13f488df0528c251632109a78ee4d1797629142 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 08:56:20 -0500 Subject: [PATCH 11/33] Update packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 87362ea215327..c30edf2ac0a16 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -465,7 +465,7 @@ export = { test.done(); }, - 'fails if maximumRecordAgeInSeconds < 60'(test: Test) { + 'fails if maxRecordAge < 60 seconds'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); From a4fccec77712a28d83f02ccbfb1324f580056e40 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 08:56:45 -0500 Subject: [PATCH 12/33] Update packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index c30edf2ac0a16..6c2376f7b70c7 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -487,7 +487,7 @@ export = { test.done(); }, - 'fails if maximumRecordAgeInSeconds > 604800'(test: Test) { + 'fails if maxRecordAge > 7 days'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); From 8223345cf89654f2d1e00701865b37a00e6f0404 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 09:54:07 -0500 Subject: [PATCH 13/33] Update README with links to lambda event links --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index be682d8829f0f..8d0e8a0b26a9a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -110,7 +110,7 @@ CloudWatch. ### DynamoDB Streams You can write Lambda functions to process change events from a DynamoDB Table. An event is emitted to a DynamoDB stream (if configured) whenever a write (Put, Delete, Update) -operation is performed against the table. See [Using AWS Lambda with Amazon DynamoDB](https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html) for more information. +operation is performed against the table. See [Using AWS Lambda with Amazon DynamoDB](https://docs.aws.amazon.com/lambda/latest/dg/with-ddb.html) for more information about configuring Lambda function event sources with DynamoDB. To process events with a Lambda function, first create or update a DynamoDB table and enable a `stream` specification. Then, create a `DynamoEventSource` and add it to your Lambda function. The following parameters will impact Amazon DynamoDB's polling behavior: @@ -118,7 +118,7 @@ and add it to your Lambda function. The following parameters will impact Amazon * __batchSize__: Determines how many records are buffered before invoking your lambda function - could impact your function's memory usage (if too high) and ability to keep up with incoming data velocity (if too low). * __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. * __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of delayed processing. -* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. +* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. Records that exceed the max age will be treated as failures. * __onFailure__: In the event a record fails after all retries or if the record age has exceeded the configured value, the record will be sent to SQS queue or SNS topic that is specified here * __parallelizationFactor__: The number of batches to concurrently process on each shard. * __retryAttempts__: The maximum number of times a record should be retried in the event of failure. @@ -149,9 +149,9 @@ function.addEventSource(new DynamoEventSource(table, { ### Kinesis -You can write Lambda functions to process streaming data in Amazon Kinesis Streams. For more information about Amazon SQS, see [Amazon Kinesis -Service](https://aws.amazon.com/kinesis/data-streams/). To view a sample event, -see [Amazon SQS Event](https://docs.aws.amazon.com/lambda/latest/dg/eventsources.html#eventsources-kinesis-streams). +You can write Lambda functions to process streaming data in Amazon Kinesis Streams. For more information about Amazon Kinesis, see [Amazon Kinesis +Service](https://aws.amazon.com/kinesis/data-streams/). To learn more about configuring Lambda function event sources with kinesis and view a sample event, +see [Amazon Kinesis Event](https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html). To set up Amazon Kinesis as an event source for AWS Lambda, you first create or update an Amazon Kinesis stream and select custom values for the @@ -161,7 +161,7 @@ behavior: * __batchSize__: Determines how many records are buffered before invoking your lambnda function - could impact your function's memory usage (if too high) and ability to keep up with incoming data velocity (if too low). * __bisectBatchOnError__: If a batch encounters an error, this will cause the batch to be split in two and have each new smaller batch retried, allowing the records in error to be isolated. * __maxBatchingWindow__: The maximum amount of time to gather records before invoking the lambda. This increases the likelihood of a full batch at the cost of possibly delaying processing. -* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. +* __maxRecordAge__: The maximum age of a record that will be sent to the function for processing. Records that exceed the max age will be treated as failures. * __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here * __parallelizationFactor__: The number of batches to concurrently process on each shard. * __retryAttempts__: The maximum number of times a record should be retried in the event of failure. From 3cae74c72e9662e358cf9b2e59cec4f4a6548b15 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Wed, 5 Feb 2020 10:16:32 -0500 Subject: [PATCH 14/33] Use better variable names --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 0a8d5df6a36fd..c37fc976ad982 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -132,10 +132,10 @@ export class EventSourceMapping extends cdk.Resource { throw new Error(`parallelizationFactor must be between 1 and 10 inclusive, got ${props.parallelizationFactor}`); } - let onFailure; + let destinationConfig; if (props.onFailure) { - onFailure = { + destinationConfig = { onFailure: props.onFailure.bind(this, props.target, { type: DestinationType.FAILURE }) }; } @@ -143,7 +143,7 @@ export class EventSourceMapping extends cdk.Resource { new CfnEventSourceMapping(this, 'Resource', { batchSize: props.batchSize, bisectBatchOnFunctionError: props.bisectBatchOnError, - destinationConfig: onFailure, + destinationConfig, enabled: props.enabled, eventSourceArn: props.eventSourceArn, functionName: props.target.functionName, From 61918bbd58ef50623b418e0a3608cf074c158ba4 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Thu, 13 Feb 2020 10:49:00 -0500 Subject: [PATCH 15/33] Add integration test and refactor away from IDestination --- .../aws-lambda-event-sources/lib/dlq.ts | 19 ++ .../aws-lambda-event-sources/lib/index.ts | 1 + .../aws-lambda-event-sources/lib/sns-dlq.ts | 16 ++ .../aws-lambda-event-sources/lib/sqs-dlq.ts | 16 ++ .../aws-lambda-event-sources/lib/stream.ts | 3 +- .../package-lock.json | 55 ++++++ .../aws-lambda-event-sources/package.json | 4 +- .../test/integ.kinesiswithdlq.expected.json | 183 ++++++++++++++++++ .../test/integ.kinesiswithdlq.ts | 65 +++++++ .../test/test.dynamo.ts | 4 +- .../aws-lambda/lib/event-source-mapping.ts | 6 +- 11 files changed, 363 insertions(+), 9 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/package-lock.json create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts new file mode 100644 index 0000000000000..a0d88906baf61 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts @@ -0,0 +1,19 @@ +/** + * A destination configuration + */ +export interface DLQDestinationConfig { + /** + * The Amazon Resource Name (ARN) of the destination resource + */ + readonly destination: string; + } + +/** + * A DLQ for an event source + */ +export interface IEventSourceDLQ { + /** + * Returns the DLQ destination config of the DLQ + */ + bind(): DLQDestinationConfig; +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts index de08ac31022e8..fb4fbc0b8df92 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts @@ -1,5 +1,6 @@ export * from './api'; export * from './dynamodb'; +export * from './dlq'; export * from './kinesis'; export * from './s3'; export * from './sns'; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts new file mode 100644 index 0000000000000..86ab4ee432f43 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -0,0 +1,16 @@ +import * as sns from '@aws-cdk/aws-sns'; +import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; + +export class SnsDLQ implements IEventSourceDLQ { + constructor(private readonly topic: sns.ITopic) { + } + + /** + * Returns a destination configuration for the DLQ + */ + public bind(): DLQDestinationConfig { + return { + destination: this.topic.topicArn + }; + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts new file mode 100644 index 0000000000000..0e4151b474d7b --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -0,0 +1,16 @@ +import * as sqs from '@aws-cdk/aws-sqs'; +import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; + +export class SqsDLQ implements IEventSourceDLQ { + constructor(private readonly queue: sqs.IQueue) { + } + + /** + * Returns a destination configuration for the DLQ + */ + public bind(): DLQDestinationConfig { + return { + destination: this.queue.queueArn + }; + } +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 0b43f2d727e38..44cbd3a884748 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -1,5 +1,6 @@ import * as lambda from '@aws-cdk/aws-lambda'; import { Duration } from '@aws-cdk/core'; +import { IEventSourceDLQ } from './dlq'; /** * The set of properties for event sources that follow the streaming model, @@ -33,7 +34,7 @@ export interface StreamEventSourceProps { * * @default discarded records are ignored */ - readonly onFailure?: lambda.IDestination; + readonly onFailure?: IEventSourceDLQ; /** * The maximum age of a record that Lambda sends to a function for processing. diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json b/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json new file mode 100644 index 0000000000000..f1ecacc5d537b --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json @@ -0,0 +1,55 @@ +{ + "name": "@aws-cdk/aws-lambda-event-sources", + "version": "1.22.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@aws-cdk/aws-kms": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-kms/-/aws-kms-1.23.0.tgz", + "integrity": "sha512-OPNoTefBuHY1RsWZlFTYs460o+iyVeWdOfjBPhlLATAja9yIjYhRH/XEbkhrNt/G/mxwqTJ61nLPWla85b8PCQ==", + "requires": { + "@aws-cdk/aws-iam": "1.23.0", + "@aws-cdk/core": "1.23.0" + }, + "dependencies": { + "@aws-cdk/aws-iam": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-iam/-/aws-iam-1.23.0.tgz", + "integrity": "sha512-BRaPLnZGcd8nxuIiBynLLbuqN+2KfEnQIdsC0uqjDVpTDygQB9ic1ocmf0jugsZ1W4hS1oY8EaZY9v/TX3rQgQ==", + "requires": { + "@aws-cdk/core": "1.23.0", + "@aws-cdk/region-info": "1.23.0" + } + }, + "@aws-cdk/core": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/core/-/core-1.23.0.tgz", + "integrity": "sha512-Fip5/A6uYM3EsZKZknElTO0KM5M/6k1P0ZZgzda7MGmHXegw6MoHzM/zsAMIvpUAIdJlpko+1HgqtcOShRyKYg==", + "requires": { + "@aws-cdk/cx-api": "1.23.0" + } + } + } + }, + "@aws-cdk/cx-api": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cx-api/-/cx-api-1.23.0.tgz", + "integrity": "sha512-xvGxAvBRgNgMhHwpSZ6XKL7p8mllGksglbJ2tp/ZVXpcUMqmKcOU0jvt/tto8VZShS24nO8nkAQmoL/PFlc44w==", + "requires": { + "semver": "^7.1.2" + }, + "dependencies": { + "semver": { + "version": "7.1.2", + "bundled": true + } + } + }, + "@aws-cdk/region-info": { + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/region-info/-/region-info-1.23.0.tgz", + "integrity": "sha512-oJbvjsBHySi6p5c4kW3tjTEgb9eV7zfOW27wIKToRD57e3plbbjJluq3HVumyBxEEdSid4EWM1dnN0bQW2213Q==" + } + } +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 6d08bc06a3565..bf507fd8002ef 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -72,7 +72,6 @@ "@aws-cdk/aws-iam": "1.22.0", "@aws-cdk/aws-kinesis": "1.22.0", "@aws-cdk/aws-lambda": "1.22.0", - "@aws-cdk/aws-lambda-destinations": "1.22.0", "@aws-cdk/aws-s3": "1.22.0", "@aws-cdk/aws-s3-notifications": "1.22.0", "@aws-cdk/aws-sns": "1.22.0", @@ -93,8 +92,7 @@ "@aws-cdk/aws-sns": "1.22.0", "@aws-cdk/aws-sns-subscriptions": "1.22.0", "@aws-cdk/aws-sqs": "1.22.0", - "@aws-cdk/core": "1.22.0", - "@aws-cdk/aws-lambda-destinations": "1.22.0" + "@aws-cdk/core": "1.22.0" }, "engines": { "node": ">= 10.3.0" diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json new file mode 100644 index 0000000000000..232ba3466b227 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json @@ -0,0 +1,183 @@ +{ + "Resources": { + "FServiceRole3AC82EE1": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "FServiceRoleDefaultPolicy17A19BFA": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:DescribeStream", + "kinesis:GetRecords", + "kinesis:GetShardIterator" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "S509448A1", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "FServiceRoleDefaultPolicy17A19BFA", + "Roles": [ + { + "Ref": "FServiceRole3AC82EE1" + } + ] + } + }, + "FC4345940": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "ZipFile": "exports.handler = async function handler(event) {\n console.log('event:', JSON.stringify(event, undefined, 2));\n throw new Error();\n}" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "FServiceRole3AC82EE1", + "Arn" + ] + }, + "Runtime": "nodejs10.x" + }, + "DependsOn": [ + "FServiceRoleDefaultPolicy17A19BFA", + "FServiceRole3AC82EE1" + ] + }, + "FKinesisEventSourcelambdaeventsourcekinesiswithdlqSD357FCB87EEA8CB4": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": { + "Fn::GetAtt": [ + "S509448A1", + "Arn" + ] + }, + "FunctionName": { + "Ref": "FC4345940" + }, + "BatchSize": 100, + "DestinationConfig": { + "OnFailure": { + "Destination": { + "Fn::GetAtt": [ + "Q63C6E3AB", + "Arn" + ] + } + } + }, + "MaximumRetryAttempts": 2, + "StartingPosition": "TRIM_HORIZON" + } + }, + "S509448A1": { + "Type": "AWS::Kinesis::Stream", + "Properties": { + "ShardCount": 1, + "RetentionPeriodHours": 24 + } + }, + "Q63C6E3AB": { + "Type": "AWS::SQS::Queue" + }, + "FPServiceRoleB1957CC2": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "FP85AE6139": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "ZipFile": { + "Fn::Join": [ + "", + [ + "\nvar AWS = require('aws-sdk');\nexports.handler = event => {\n const kinesis = new AWS.Kinesis();\n kinesis.putRecord({\n Data: 'Hello World',\n PartitionKey: 'Hello Key',\n StreamName: '", + { + "Ref": "S509448A1" + }, + "'\n }, (err, data) => {\n if (err) {\n console.log(`Error: ${err}`);\n }\n })\n}\n " + ] + ] + } + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "FPServiceRoleB1957CC2", + "Arn" + ] + }, + "Runtime": "nodejs10.x" + }, + "DependsOn": [ + "FPServiceRoleB1957CC2" + ] + } + } + } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts new file mode 100644 index 0000000000000..eb500f5b51828 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -0,0 +1,65 @@ +import * as kinesis from '@aws-cdk/aws-kinesis'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as sqs from '@aws-cdk/aws-sqs'; +import { App, Stack } from "@aws-cdk/core"; +import { KinesisEventSource } from '../lib'; +import { SqsDLQ } from '../lib/sqs-dlq'; + +/* + * Stack verification steps: + * * aws lambda invoke --function-name --invocation-type Event --payload '"Event"' response.json + * * Validate two executions of fn lambda + * * Validate one message in dlq + */ + +// tslint:disable:no-console +async function handler(event: any) { + console.log('event:', JSON.stringify(event, undefined, 2)); + throw new Error(); +} + +class KinesisWithDLQTest extends Stack { + constructor(scope: App, id: string) { + super(scope, id); + + const fn = new lambda.Function(this, 'F', { + runtime: lambda.Runtime.NODEJS_10_X, + handler: 'index.handler', + code: lambda.Code.fromInline(`exports.handler = ${handler.toString()}`) + }); + + const stream = new kinesis.Stream(this, 'S'); + + const dlq = new sqs.Queue(this, 'Q'); + + fn.addEventSource(new KinesisEventSource(stream, { + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + onFailure: new SqsDLQ(dlq), + retryAttempts: 2 + })); + + new lambda.Function(this, 'FP', { + runtime: lambda.Runtime.NODEJS_10_X, + handler: 'index.handler', + code: lambda.Code.fromInline(` +var AWS = require('aws-sdk'); +exports.handler = event => { + const kinesis = new AWS.Kinesis(); + kinesis.putRecord({ + Data: 'Hello World', + PartitionKey: 'Hello Key', + StreamName: '${stream.streamName}' + }, (err, data) => { + if (err) { + console.log(\`Error: \${err}\`); + } + }) +} + `) + }); + } +} + +const app = new App(); +new KinesisWithDLQTest(app, 'lambda-event-source-kinesis-with-dlq'); +app.synth(); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 6c2376f7b70c7..9299848d2b0e9 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -1,11 +1,11 @@ import { expect, haveResource } from '@aws-cdk/assert'; import * as dynamodb from '@aws-cdk/aws-dynamodb'; import * as lambda from '@aws-cdk/aws-lambda'; -import * as destinations from '@aws-cdk/aws-lambda-destinations'; import * as sqs from '@aws-cdk/aws-sqs'; import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; import * as sources from '../lib'; +import { SqsDLQ } from '../lib/sqs-dlq'; import { TestFunction } from './test-function'; // tslint:disable:object-literal-key-quotes @@ -524,7 +524,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - onFailure: new destinations.SqsDestination(queue), + onFailure: new SqsDLQ(queue), startingPosition: lambda.StartingPosition.LATEST })); diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index c37fc976ad982..2c3ef181d2c7f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,5 +1,5 @@ import * as cdk from '@aws-cdk/core'; -import { DestinationType, IDestination } from './destination'; +import { IEventSourceDLQ } from '@aws-cdk/aws-lambda-event-sources'; import { IFunction } from './function-base'; import { CfnEventSourceMapping } from './lambda.generated'; @@ -34,7 +34,7 @@ export interface EventSourceMappingOptions { * * @default discarded records are ignored */ - readonly onFailure?: IDestination; + readonly onFailure?: IEventSourceDLQ; /** * Set to false to disable the event source upon creation. @@ -136,7 +136,7 @@ export class EventSourceMapping extends cdk.Resource { if (props.onFailure) { destinationConfig = { - onFailure: props.onFailure.bind(this, props.target, { type: DestinationType.FAILURE }) + onFailure: props.onFailure.bind() }; } From f3b3e2201ff4fdadf32bbc969fe2023c716f80ed Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 17 Feb 2020 11:03:30 -0500 Subject: [PATCH 16/33] Pull up DLQ libraries to remedy circular dependency --- packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts | 1 - packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts | 3 +-- .../aws-lambda-event-sources/test/integ.kinesiswithdlq.ts | 3 +-- .../@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 3 +-- .../{aws-lambda-event-sources => aws-lambda}/lib/dlq.ts | 0 packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 2 +- packages/@aws-cdk/aws-lambda/lib/index.ts | 3 +++ .../{aws-lambda-event-sources => aws-lambda}/lib/sns-dlq.ts | 5 ++++- .../{aws-lambda-event-sources => aws-lambda}/lib/sqs-dlq.ts | 3 +++ packages/@aws-cdk/aws-lambda/package.json | 2 ++ 10 files changed, 16 insertions(+), 9 deletions(-) rename packages/@aws-cdk/{aws-lambda-event-sources => aws-lambda}/lib/dlq.ts (100%) rename packages/@aws-cdk/{aws-lambda-event-sources => aws-lambda}/lib/sns-dlq.ts (89%) rename packages/@aws-cdk/{aws-lambda-event-sources => aws-lambda}/lib/sqs-dlq.ts (90%) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts index fb4fbc0b8df92..de08ac31022e8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts @@ -1,6 +1,5 @@ export * from './api'; export * from './dynamodb'; -export * from './dlq'; export * from './kinesis'; export * from './s3'; export * from './sns'; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 44cbd3a884748..371fd44d0a648 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -1,6 +1,5 @@ import * as lambda from '@aws-cdk/aws-lambda'; import { Duration } from '@aws-cdk/core'; -import { IEventSourceDLQ } from './dlq'; /** * The set of properties for event sources that follow the streaming model, @@ -34,7 +33,7 @@ export interface StreamEventSourceProps { * * @default discarded records are ignored */ - readonly onFailure?: IEventSourceDLQ; + readonly onFailure?: lambda.IEventSourceDLQ; /** * The maximum age of a record that Lambda sends to a function for processing. diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index eb500f5b51828..3b0309128faf9 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -3,7 +3,6 @@ import * as lambda from '@aws-cdk/aws-lambda'; import * as sqs from '@aws-cdk/aws-sqs'; import { App, Stack } from "@aws-cdk/core"; import { KinesisEventSource } from '../lib'; -import { SqsDLQ } from '../lib/sqs-dlq'; /* * Stack verification steps: @@ -34,7 +33,7 @@ class KinesisWithDLQTest extends Stack { fn.addEventSource(new KinesisEventSource(stream, { startingPosition: lambda.StartingPosition.TRIM_HORIZON, - onFailure: new SqsDLQ(dlq), + onFailure: new lambda.SqsDLQ(dlq), retryAttempts: 2 })); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 9299848d2b0e9..8194da98e26bb 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -5,7 +5,6 @@ import * as sqs from '@aws-cdk/aws-sqs'; import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; import * as sources from '../lib'; -import { SqsDLQ } from '../lib/sqs-dlq'; import { TestFunction } from './test-function'; // tslint:disable:object-literal-key-quotes @@ -524,7 +523,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - onFailure: new SqsDLQ(queue), + onFailure: new lambda.SqsDLQ(queue), startingPosition: lambda.StartingPosition.LATEST })); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts similarity index 100% rename from packages/@aws-cdk/aws-lambda-event-sources/lib/dlq.ts rename to packages/@aws-cdk/aws-lambda/lib/dlq.ts diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 2c3ef181d2c7f..43a1a7ffc303a 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,5 +1,5 @@ import * as cdk from '@aws-cdk/core'; -import { IEventSourceDLQ } from '@aws-cdk/aws-lambda-event-sources'; +import { IEventSourceDLQ } from './dlq'; import { IFunction } from './function-base'; import { CfnEventSourceMapping } from './lambda.generated'; diff --git a/packages/@aws-cdk/aws-lambda/lib/index.ts b/packages/@aws-cdk/aws-lambda/lib/index.ts index f0a693da65974..9d8ec25cac2a7 100644 --- a/packages/@aws-cdk/aws-lambda/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/index.ts @@ -1,4 +1,5 @@ export * from './alias'; +export * from './dlq'; export * from './function-base'; export * from './function'; export * from './layers'; @@ -6,7 +7,9 @@ export * from './permission'; export * from './runtime'; export * from './code'; export * from './lambda-version'; +export * from './sns-dlq'; export * from './singleton-lambda'; +export * from './sqs-dlq'; export * from './event-source'; export * from './event-source-mapping'; export * from './destination'; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts similarity index 89% rename from packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts rename to packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts index 86ab4ee432f43..b9471b7898554 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts @@ -1,6 +1,9 @@ import * as sns from '@aws-cdk/aws-sns'; import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; +/** + * An SNS destination configuration + */ export class SnsDLQ implements IEventSourceDLQ { constructor(private readonly topic: sns.ITopic) { } @@ -13,4 +16,4 @@ export class SnsDLQ implements IEventSourceDLQ { destination: this.topic.topicArn }; } -} \ No newline at end of file +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts similarity index 90% rename from packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts rename to packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts index 0e4151b474d7b..40378f09a8448 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts @@ -1,6 +1,9 @@ import * as sqs from '@aws-cdk/aws-sqs'; import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; +/** + * An SQS destination configuration + */ export class SqsDLQ implements IEventSourceDLQ { constructor(private readonly queue: sqs.IQueue) { } diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index dc4af155519dd..f4661a6a1c613 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -90,6 +90,7 @@ "@aws-cdk/aws-logs": "1.23.0", "@aws-cdk/aws-s3": "1.23.0", "@aws-cdk/aws-s3-assets": "1.23.0", + "@aws-cdk/aws-sns": "1.23.0", "@aws-cdk/aws-sqs": "1.23.0", "@aws-cdk/core": "1.23.0", "@aws-cdk/cx-api": "1.23.0" @@ -103,6 +104,7 @@ "@aws-cdk/aws-logs": "1.23.0", "@aws-cdk/aws-s3": "1.23.0", "@aws-cdk/aws-s3-assets": "1.23.0", + "@aws-cdk/aws-sns": "1.23.0", "@aws-cdk/aws-sqs": "1.23.0", "@aws-cdk/core": "1.23.0", "@aws-cdk/cx-api": "1.23.0" From 8d7e516c3cd5f5ca0cad27af8c1fe522c081faaa Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 17 Feb 2020 15:19:40 -0500 Subject: [PATCH 17/33] Clarify documentation on dead letter queue objects --- packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts | 2 +- packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts index b9471b7898554..28daed46f1eff 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts @@ -2,7 +2,7 @@ import * as sns from '@aws-cdk/aws-sns'; import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; /** - * An SNS destination configuration + * An SNS dead letter queue destination configuration */ export class SnsDLQ implements IEventSourceDLQ { constructor(private readonly topic: sns.ITopic) { diff --git a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts index 40378f09a8448..9fc805f48d61b 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts @@ -2,7 +2,7 @@ import * as sqs from '@aws-cdk/aws-sqs'; import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; /** - * An SQS destination configuration + * An SQS dead letter queue destination configuration */ export class SqsDLQ implements IEventSourceDLQ { constructor(private readonly queue: sqs.IQueue) { From 7e34a57b6617171c2ddadc562762fb94db613e01 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 2 Mar 2020 12:51:32 -0500 Subject: [PATCH 18/33] Remove package-lock.json --- .../package-lock.json | 55 ------------------- 1 file changed, 55 deletions(-) delete mode 100644 packages/@aws-cdk/aws-lambda-event-sources/package-lock.json diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json b/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json deleted file mode 100644 index f1ecacc5d537b..0000000000000 --- a/packages/@aws-cdk/aws-lambda-event-sources/package-lock.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "@aws-cdk/aws-lambda-event-sources", - "version": "1.22.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "@aws-cdk/aws-kms": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-kms/-/aws-kms-1.23.0.tgz", - "integrity": "sha512-OPNoTefBuHY1RsWZlFTYs460o+iyVeWdOfjBPhlLATAja9yIjYhRH/XEbkhrNt/G/mxwqTJ61nLPWla85b8PCQ==", - "requires": { - "@aws-cdk/aws-iam": "1.23.0", - "@aws-cdk/core": "1.23.0" - }, - "dependencies": { - "@aws-cdk/aws-iam": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/aws-iam/-/aws-iam-1.23.0.tgz", - "integrity": "sha512-BRaPLnZGcd8nxuIiBynLLbuqN+2KfEnQIdsC0uqjDVpTDygQB9ic1ocmf0jugsZ1W4hS1oY8EaZY9v/TX3rQgQ==", - "requires": { - "@aws-cdk/core": "1.23.0", - "@aws-cdk/region-info": "1.23.0" - } - }, - "@aws-cdk/core": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/core/-/core-1.23.0.tgz", - "integrity": "sha512-Fip5/A6uYM3EsZKZknElTO0KM5M/6k1P0ZZgzda7MGmHXegw6MoHzM/zsAMIvpUAIdJlpko+1HgqtcOShRyKYg==", - "requires": { - "@aws-cdk/cx-api": "1.23.0" - } - } - } - }, - "@aws-cdk/cx-api": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/cx-api/-/cx-api-1.23.0.tgz", - "integrity": "sha512-xvGxAvBRgNgMhHwpSZ6XKL7p8mllGksglbJ2tp/ZVXpcUMqmKcOU0jvt/tto8VZShS24nO8nkAQmoL/PFlc44w==", - "requires": { - "semver": "^7.1.2" - }, - "dependencies": { - "semver": { - "version": "7.1.2", - "bundled": true - } - } - }, - "@aws-cdk/region-info": { - "version": "1.23.0", - "resolved": "https://registry.npmjs.org/@aws-cdk/region-info/-/region-info-1.23.0.tgz", - "integrity": "sha512-oJbvjsBHySi6p5c4kW3tjTEgb9eV7zfOW27wIKToRD57e3plbbjJluq3HVumyBxEEdSid4EWM1dnN0bQW2213Q==" - } - } -} From 7263cc5df37ac58a6d10898f2c15b4de7445f684 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 2 Mar 2020 12:56:18 -0500 Subject: [PATCH 19/33] Standardize case on new classes --- packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts | 2 +- .../aws-lambda-event-sources/test/integ.kinesiswithdlq.ts | 2 +- .../@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 2 +- packages/@aws-cdk/aws-lambda/lib/dlq.ts | 6 +++--- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 4 ++-- packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts | 6 +++--- packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts | 6 +++--- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 371fd44d0a648..cb8e7f51f4c4d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -33,7 +33,7 @@ export interface StreamEventSourceProps { * * @default discarded records are ignored */ - readonly onFailure?: lambda.IEventSourceDLQ; + readonly onFailure?: lambda.IEventSourceDlq; /** * The maximum age of a record that Lambda sends to a function for processing. diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index 3b0309128faf9..b09c2d05238c2 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -33,7 +33,7 @@ class KinesisWithDLQTest extends Stack { fn.addEventSource(new KinesisEventSource(stream, { startingPosition: lambda.StartingPosition.TRIM_HORIZON, - onFailure: new lambda.SqsDLQ(dlq), + onFailure: new lambda.SqsDlq(dlq), retryAttempts: 2 })); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 8194da98e26bb..a69868c33f47d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -523,7 +523,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - onFailure: new lambda.SqsDLQ(queue), + onFailure: new lambda.SqsDlq(queue), startingPosition: lambda.StartingPosition.LATEST })); diff --git a/packages/@aws-cdk/aws-lambda/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts index a0d88906baf61..c96b5b53b3f49 100644 --- a/packages/@aws-cdk/aws-lambda/lib/dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/dlq.ts @@ -1,7 +1,7 @@ /** * A destination configuration */ -export interface DLQDestinationConfig { +export interface DlqDestinationConfig { /** * The Amazon Resource Name (ARN) of the destination resource */ @@ -11,9 +11,9 @@ export interface DLQDestinationConfig { /** * A DLQ for an event source */ -export interface IEventSourceDLQ { +export interface IEventSourceDlq { /** * Returns the DLQ destination config of the DLQ */ - bind(): DLQDestinationConfig; + bind(): DlqDestinationConfig; } diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 43a1a7ffc303a..650d0c38a2bfe 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,5 +1,5 @@ import * as cdk from '@aws-cdk/core'; -import { IEventSourceDLQ } from './dlq'; +import { IEventSourceDlq } from './dlq'; import { IFunction } from './function-base'; import { CfnEventSourceMapping } from './lambda.generated'; @@ -34,7 +34,7 @@ export interface EventSourceMappingOptions { * * @default discarded records are ignored */ - readonly onFailure?: IEventSourceDLQ; + readonly onFailure?: IEventSourceDlq; /** * Set to false to disable the event source upon creation. diff --git a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts index 28daed46f1eff..6d5161fafe41f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts @@ -1,17 +1,17 @@ import * as sns from '@aws-cdk/aws-sns'; -import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; +import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** * An SNS dead letter queue destination configuration */ -export class SnsDLQ implements IEventSourceDLQ { +export class SnsDlq implements IEventSourceDlq { constructor(private readonly topic: sns.ITopic) { } /** * Returns a destination configuration for the DLQ */ - public bind(): DLQDestinationConfig { + public bind(): DlqDestinationConfig { return { destination: this.topic.topicArn }; diff --git a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts index 9fc805f48d61b..3ac3c009ef132 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts @@ -1,17 +1,17 @@ import * as sqs from '@aws-cdk/aws-sqs'; -import { DLQDestinationConfig, IEventSourceDLQ } from "./dlq"; +import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** * An SQS dead letter queue destination configuration */ -export class SqsDLQ implements IEventSourceDLQ { +export class SqsDlq implements IEventSourceDlq { constructor(private readonly queue: sqs.IQueue) { } /** * Returns a destination configuration for the DLQ */ - public bind(): DLQDestinationConfig { + public bind(): DlqDestinationConfig { return { destination: this.queue.queueArn }; From 17eee3842258b43bce4300e03a2b350236e36bd4 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 2 Mar 2020 12:59:14 -0500 Subject: [PATCH 20/33] Clarify documentation --- packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts | 2 +- packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts index 6d5161fafe41f..6347b58bf7c94 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts @@ -2,7 +2,7 @@ import * as sns from '@aws-cdk/aws-sns'; import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** - * An SNS dead letter queue destination configuration + * An SNS dead letter queue destination configuration for a Lambda event source */ export class SnsDlq implements IEventSourceDlq { constructor(private readonly topic: sns.ITopic) { diff --git a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts index 3ac3c009ef132..2b422593d8aa1 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts @@ -2,7 +2,7 @@ import * as sqs from '@aws-cdk/aws-sqs'; import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** - * An SQS dead letter queue destination configuration + * An SQS dead letter queue destination configuration for a Lambda event source */ export class SqsDlq implements IEventSourceDlq { constructor(private readonly queue: sqs.IQueue) { From c44be1329ca2780782410fa2ef40f653063ee1ed Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Thu, 5 Mar 2020 11:31:28 -0500 Subject: [PATCH 21/33] Move DLQ implementations to event sources package --- packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts | 2 ++ .../{aws-lambda => aws-lambda-event-sources}/lib/sns-dlq.ts | 2 +- .../{aws-lambda => aws-lambda-event-sources}/lib/sqs-dlq.ts | 2 +- .../aws-lambda-event-sources/test/integ.kinesiswithdlq.ts | 4 ++-- .../@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 3 +-- packages/@aws-cdk/aws-lambda/lib/index.ts | 2 -- 6 files changed, 7 insertions(+), 8 deletions(-) rename packages/@aws-cdk/{aws-lambda => aws-lambda-event-sources}/lib/sns-dlq.ts (84%) rename packages/@aws-cdk/{aws-lambda => aws-lambda-event-sources}/lib/sqs-dlq.ts (84%) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts index de08ac31022e8..19253a743cae8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts @@ -3,5 +3,7 @@ export * from './dynamodb'; export * from './kinesis'; export * from './s3'; export * from './sns'; +export * from './sns-dlq'; export * from './stream'; export * from './sqs'; +export * from './sqs-dlq'; diff --git a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts similarity index 84% rename from packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts rename to packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts index 6347b58bf7c94..e6010dd483aa0 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -1,5 +1,5 @@ +import { DlqDestinationConfig, IEventSourceDlq } from "@aws-cdk/aws-lambda"; import * as sns from '@aws-cdk/aws-sns'; -import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** * An SNS dead letter queue destination configuration for a Lambda event source diff --git a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts similarity index 84% rename from packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts rename to packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts index 2b422593d8aa1..813515b6ddc42 100644 --- a/packages/@aws-cdk/aws-lambda/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -1,5 +1,5 @@ +import { DlqDestinationConfig, IEventSourceDlq } from "@aws-cdk/aws-lambda"; import * as sqs from '@aws-cdk/aws-sqs'; -import { DlqDestinationConfig, IEventSourceDlq } from "./dlq"; /** * An SQS dead letter queue destination configuration for a Lambda event source diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index b09c2d05238c2..fb9c6cc5b0389 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -2,7 +2,7 @@ import * as kinesis from '@aws-cdk/aws-kinesis'; import * as lambda from '@aws-cdk/aws-lambda'; import * as sqs from '@aws-cdk/aws-sqs'; import { App, Stack } from "@aws-cdk/core"; -import { KinesisEventSource } from '../lib'; +import { KinesisEventSource, SqsDlq } from '../lib'; /* * Stack verification steps: @@ -33,7 +33,7 @@ class KinesisWithDLQTest extends Stack { fn.addEventSource(new KinesisEventSource(stream, { startingPosition: lambda.StartingPosition.TRIM_HORIZON, - onFailure: new lambda.SqsDlq(dlq), + onFailure: new SqsDlq(dlq), retryAttempts: 2 })); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 58ba2c20dd431..67c54a990115a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -333,7 +333,6 @@ export = { 'fails if maximumRetryAttempts > 10000'(test: Test) { // GIVEN - const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); const table = new dynamodb.Table(stack, 'T', { @@ -565,7 +564,7 @@ export = { // WHEN fn.addEventSource(new sources.DynamoEventSource(table, { - onFailure: new lambda.SqsDlq(queue), + onFailure: new sources.SqsDlq(queue), startingPosition: lambda.StartingPosition.LATEST })); diff --git a/packages/@aws-cdk/aws-lambda/lib/index.ts b/packages/@aws-cdk/aws-lambda/lib/index.ts index 9d8ec25cac2a7..b494e924c604a 100644 --- a/packages/@aws-cdk/aws-lambda/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/index.ts @@ -7,9 +7,7 @@ export * from './permission'; export * from './runtime'; export * from './code'; export * from './lambda-version'; -export * from './sns-dlq'; export * from './singleton-lambda'; -export * from './sqs-dlq'; export * from './event-source'; export * from './event-source-mapping'; export * from './destination'; From a4f05fe54ecfde6e2173a12d5992441920888bcf Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Fri, 6 Mar 2020 08:44:23 -0500 Subject: [PATCH 22/33] Extract test lambda from integration test into external file --- .../test/integ.kinesiswithdlq.expected.json | 364 ++++++++++-------- .../integ.kinesiswithdlq.handler/index.ts | 21 + .../test/integ.kinesiswithdlq.ts | 17 +- 3 files changed, 224 insertions(+), 178 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json index 232ba3466b227..039d1b9bf10cd 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json @@ -1,183 +1,221 @@ { - "Resources": { - "FServiceRole3AC82EE1": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ + "Resources": { + "FServiceRole3AC82EE1": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - } - ] - } - }, - "FServiceRoleDefaultPolicy17A19BFA": { - "Type": "AWS::IAM::Policy", - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": [ - "kinesis:DescribeStream", - "kinesis:GetRecords", - "kinesis:GetShardIterator" - ], - "Effect": "Allow", - "Resource": { - "Fn::GetAtt": [ - "S509448A1", - "Arn" - ] - } + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" } - ], - "Version": "2012-10-17" - }, - "PolicyName": "FServiceRoleDefaultPolicy17A19BFA", - "Roles": [ - { - "Ref": "FServiceRole3AC82EE1" } - ] - } - }, - "FC4345940": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "ZipFile": "exports.handler = async function handler(event) {\n console.log('event:', JSON.stringify(event, undefined, 2));\n throw new Error();\n}" - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "FServiceRole3AC82EE1", - "Arn" - ] - }, - "Runtime": "nodejs10.x" + ], + "Version": "2012-10-17" }, - "DependsOn": [ - "FServiceRoleDefaultPolicy17A19BFA", - "FServiceRole3AC82EE1" - ] - }, - "FKinesisEventSourcelambdaeventsourcekinesiswithdlqSD357FCB87EEA8CB4": { - "Type": "AWS::Lambda::EventSourceMapping", - "Properties": { - "EventSourceArn": { - "Fn::GetAtt": [ - "S509448A1", - "Arn" + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] ] - }, - "FunctionName": { - "Ref": "FC4345940" - }, - "BatchSize": 100, - "DestinationConfig": { - "OnFailure": { - "Destination": { + } + ] + } + }, + "FServiceRoleDefaultPolicy17A19BFA": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:DescribeStream", + "kinesis:GetRecords", + "kinesis:GetShardIterator" + ], + "Effect": "Allow", + "Resource": { "Fn::GetAtt": [ - "Q63C6E3AB", + "S509448A1", "Arn" ] } } - }, - "MaximumRetryAttempts": 2, - "StartingPosition": "TRIM_HORIZON" - } - }, - "S509448A1": { - "Type": "AWS::Kinesis::Stream", - "Properties": { - "ShardCount": 1, - "RetentionPeriodHours": 24 - } - }, - "Q63C6E3AB": { - "Type": "AWS::SQS::Queue" - }, - "FPServiceRoleB1957CC2": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - } + ], + "Version": "2012-10-17" + }, + "PolicyName": "FServiceRoleDefaultPolicy17A19BFA", + "Roles": [ + { + "Ref": "FServiceRole3AC82EE1" + } + ] + } + }, + "FC4345940": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "ZipFile": "exports.handler = async function handler(event) {\n console.log('event:', JSON.stringify(event, undefined, 2));\n throw new Error();\n}" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "FServiceRole3AC82EE1", + "Arn" ] - } + }, + "Runtime": "nodejs10.x" }, - "FP85AE6139": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "ZipFile": { - "Fn::Join": [ - "", - [ - "\nvar AWS = require('aws-sdk');\nexports.handler = event => {\n const kinesis = new AWS.Kinesis();\n kinesis.putRecord({\n Data: 'Hello World',\n PartitionKey: 'Hello Key',\n StreamName: '", - { - "Ref": "S509448A1" - }, - "'\n }, (err, data) => {\n if (err) {\n console.log(`Error: ${err}`);\n }\n })\n}\n " - ] + "DependsOn": [ + "FServiceRoleDefaultPolicy17A19BFA", + "FServiceRole3AC82EE1" + ] + }, + "FKinesisEventSourcelambdaeventsourcekinesiswithdlqSD357FCB87EEA8CB4": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": { + "Fn::GetAtt": [ + "S509448A1", + "Arn" + ] + }, + "FunctionName": { + "Ref": "FC4345940" + }, + "BatchSize": 100, + "DestinationConfig": { + "OnFailure": { + "Destination": { + "Fn::GetAtt": [ + "Q63C6E3AB", + "Arn" ] } - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "FPServiceRoleB1957CC2", - "Arn" - ] - }, - "Runtime": "nodejs10.x" + } }, - "DependsOn": [ - "FPServiceRoleB1957CC2" + "MaximumRetryAttempts": 2, + "StartingPosition": "TRIM_HORIZON" + } + }, + "S509448A1": { + "Type": "AWS::Kinesis::Stream", + "Properties": { + "ShardCount": 1, + "RetentionPeriodHours": 24 + } + }, + "Q63C6E3AB": { + "Type": "AWS::SQS::Queue" + }, + "FPServiceRoleB1957CC2": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } ] } + }, + "FP85AE6139": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3BucketF010E518" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C" + } + ] + } + ] + } + ] + ] + } + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "FPServiceRoleB1957CC2", + "Arn" + ] + }, + "Runtime": "nodejs10.x" + }, + "DependsOn": [ + "FPServiceRoleB1957CC2" + ] + } + }, + "Parameters": { + "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3BucketF010E518": { + "Type": "String", + "Description": "S3 bucket for asset \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" + }, + "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C": { + "Type": "String", + "Description": "S3 key for asset version \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" + }, + "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604ArtifactHash5300B2A6": { + "Type": "String", + "Description": "Artifact hash for asset \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" } } +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts new file mode 100644 index 0000000000000..5873db82d1cd8 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts @@ -0,0 +1,21 @@ +/* eslint-disable */ +// tslint:disable:no-console +// tslint:disable:no-var-requires + +const AWS = require('aws-sdk'); + +export const handler = (event: any) => { + const kinesis = new AWS.Kinesis(); + console.log(`event: ${JSON.stringify(event)}`); + kinesis.putRecord({ + Data: 'Hello World', + PartitionKey: 'Hello Key', + StreamName: '${stream.streamName}' + }, (err: any, data: any) => { + if (err) { + console.log(`Error: ${err}`); + } else { + console.log(`Data: ${data}`); + } + }); +}; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index fb9c6cc5b0389..74adc8eb844ed 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -2,6 +2,7 @@ import * as kinesis from '@aws-cdk/aws-kinesis'; import * as lambda from '@aws-cdk/aws-lambda'; import * as sqs from '@aws-cdk/aws-sqs'; import { App, Stack } from "@aws-cdk/core"; +import * as path from 'path'; import { KinesisEventSource, SqsDlq } from '../lib'; /* @@ -40,21 +41,7 @@ class KinesisWithDLQTest extends Stack { new lambda.Function(this, 'FP', { runtime: lambda.Runtime.NODEJS_10_X, handler: 'index.handler', - code: lambda.Code.fromInline(` -var AWS = require('aws-sdk'); -exports.handler = event => { - const kinesis = new AWS.Kinesis(); - kinesis.putRecord({ - Data: 'Hello World', - PartitionKey: 'Hello Key', - StreamName: '${stream.streamName}' - }, (err, data) => { - if (err) { - console.log(\`Error: \${err}\`); - } - }) -} - `) + code: lambda.AssetCode.fromAsset(path.join(__dirname, 'integ.kinesiswithdlq.handler')) }); } } From a8b2fedabd2e2adc606399b2aad8cc40ad75ce5e Mon Sep 17 00:00:00 2001 From: Niranjan Jayakar Date: Mon, 9 Mar 2020 11:30:01 +0000 Subject: [PATCH 23/33] fixup readme --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 6895d187e746c..47d2a7edeb736 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -148,7 +148,7 @@ and add it to your Lambda function. The following parameters will impact Amazon import dynamodb = require('@aws-cdk/aws-dynamodb'); import lambda = require('@aws-cdk/aws-lambda'); import sqs = require('@aws-cdk/aws-sqs'); -import { DynamoEventSource } from '@aws-cdk/aws-lambda-event-sources'; +import { DynamoEventSource, SqsDlq } from '@aws-cdk/aws-lambda-event-sources'; const table = new dynamodb.Table(..., { partitionKey: ..., @@ -162,7 +162,7 @@ function.addEventSource(new DynamoEventSource(table, { startingPosition: lambda.StartingPosition.TRIM_HORIZON, batchSize: 5, bisectBatchOnError: true, - onFailure: deadLetterQueue, + onFailure: new SqsDlq(deadLetterQueue), retryAttempts: 10 })); ``` From a4e59b2f446b283859fdcf223327b3c02dcd1fdc Mon Sep 17 00:00:00 2001 From: Niranjan Jayakar Date: Mon, 9 Mar 2020 11:36:46 +0000 Subject: [PATCH 24/33] 2 space indent --- .../aws-lambda-event-sources/lib/sns-dlq.ts | 20 +++---- .../aws-lambda-event-sources/lib/sqs-dlq.ts | 20 +++---- .../test/integ.kinesiswithdlq.ts | 54 +++++++++---------- .../test/test.dynamo.ts | 1 + packages/@aws-cdk/aws-lambda/lib/dlq.ts | 18 +++---- 5 files changed, 57 insertions(+), 56 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts index e6010dd483aa0..cf2df12be51f7 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -5,15 +5,15 @@ import * as sns from '@aws-cdk/aws-sns'; * An SNS dead letter queue destination configuration for a Lambda event source */ export class SnsDlq implements IEventSourceDlq { - constructor(private readonly topic: sns.ITopic) { - } + constructor(private readonly topic: sns.ITopic) { + } - /** - * Returns a destination configuration for the DLQ - */ - public bind(): DlqDestinationConfig { - return { - destination: this.topic.topicArn - }; - } + /** + * Returns a destination configuration for the DLQ + */ + public bind(): DlqDestinationConfig { + return { + destination: this.topic.topicArn + }; + } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts index 813515b6ddc42..d6d94852ce17d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -5,15 +5,15 @@ import * as sqs from '@aws-cdk/aws-sqs'; * An SQS dead letter queue destination configuration for a Lambda event source */ export class SqsDlq implements IEventSourceDlq { - constructor(private readonly queue: sqs.IQueue) { - } + constructor(private readonly queue: sqs.IQueue) { + } - /** - * Returns a destination configuration for the DLQ - */ - public bind(): DlqDestinationConfig { - return { - destination: this.queue.queueArn - }; - } + /** + * Returns a destination configuration for the DLQ + */ + public bind(): DlqDestinationConfig { + return { + destination: this.queue.queueArn + }; + } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index 74adc8eb844ed..41627dcb31c34 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -14,36 +14,36 @@ import { KinesisEventSource, SqsDlq } from '../lib'; // tslint:disable:no-console async function handler(event: any) { - console.log('event:', JSON.stringify(event, undefined, 2)); - throw new Error(); + console.log('event:', JSON.stringify(event, undefined, 2)); + throw new Error(); } class KinesisWithDLQTest extends Stack { - constructor(scope: App, id: string) { - super(scope, id); - - const fn = new lambda.Function(this, 'F', { - runtime: lambda.Runtime.NODEJS_10_X, - handler: 'index.handler', - code: lambda.Code.fromInline(`exports.handler = ${handler.toString()}`) - }); - - const stream = new kinesis.Stream(this, 'S'); - - const dlq = new sqs.Queue(this, 'Q'); - - fn.addEventSource(new KinesisEventSource(stream, { - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - onFailure: new SqsDlq(dlq), - retryAttempts: 2 - })); - - new lambda.Function(this, 'FP', { - runtime: lambda.Runtime.NODEJS_10_X, - handler: 'index.handler', - code: lambda.AssetCode.fromAsset(path.join(__dirname, 'integ.kinesiswithdlq.handler')) - }); - } + constructor(scope: App, id: string) { + super(scope, id); + + const fn = new lambda.Function(this, 'F', { + runtime: lambda.Runtime.NODEJS_10_X, + handler: 'index.handler', + code: lambda.Code.fromInline(`exports.handler = ${handler.toString()}`) + }); + + const stream = new kinesis.Stream(this, 'S'); + + const dlq = new sqs.Queue(this, 'Q'); + + fn.addEventSource(new KinesisEventSource(stream, { + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + onFailure: new SqsDlq(dlq), + retryAttempts: 2 + })); + + new lambda.Function(this, 'FP', { + runtime: lambda.Runtime.NODEJS_10_X, + handler: 'index.handler', + code: lambda.AssetCode.fromAsset(path.join(__dirname, 'integ.kinesiswithdlq.handler')) + }); + } } const app = new App(); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 67c54a990115a..f34e6e30eed24 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -273,6 +273,7 @@ export = { test.throws(() => eventSource.eventSourceMappingId, /DynamoEventSource is not yet bound to an event source mapping/); test.done(); }, + 'specific maximumRetryAttempts'(test: Test) { // GIVEN const stack = new cdk.Stack(); diff --git a/packages/@aws-cdk/aws-lambda/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts index c96b5b53b3f49..30f973119338e 100644 --- a/packages/@aws-cdk/aws-lambda/lib/dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/dlq.ts @@ -2,18 +2,18 @@ * A destination configuration */ export interface DlqDestinationConfig { - /** - * The Amazon Resource Name (ARN) of the destination resource - */ - readonly destination: string; - } + /** + * The Amazon Resource Name (ARN) of the destination resource + */ + readonly destination: string; +} /** * A DLQ for an event source */ export interface IEventSourceDlq { - /** - * Returns the DLQ destination config of the DLQ - */ - bind(): DlqDestinationConfig; + /** + * Returns the DLQ destination config of the DLQ + */ + bind(): DlqDestinationConfig; } From 01f5ec37b85fd99585c5b1febe489cf6c99b0244 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 10:13:55 -0400 Subject: [PATCH 25/33] Add unit test for event source mapping validations --- .../test/test.dynamo.ts | 14 +- .../aws-lambda/lib/event-source-mapping.ts | 4 +- .../test/test.event-source-mapping.ts | 148 ++++++++++++++++++ 3 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index f34e6e30eed24..007d6dfef0a94 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -274,7 +274,7 @@ export = { test.done(); }, - 'specific maximumRetryAttempts'(test: Test) { + 'specific retryAttempts'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -310,7 +310,7 @@ export = { test.done(); }, - 'fails if maximumRetryAttempts < 0'(test: Test) { + 'fails if retryAttempts < 0'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -327,12 +327,12 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { retryAttempts: -1, startingPosition: lambda.StartingPosition.LATEST - })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got -1/); + })), /retryAttempts must be between 0 and 10000 inclusive, got -1/); test.done(); }, - 'fails if maximumRetryAttempts > 10000'(test: Test) { + 'fails if retryAttempts > 10000'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -349,7 +349,7 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { retryAttempts: 10001, startingPosition: lambda.StartingPosition.LATEST - })), /maximumRetryAttempts must be between 0 and 10000 inclusive, got 10001/); + })), /retryAttempts must be between 0 and 10000 inclusive, got 10001/); test.done(); }, @@ -523,7 +523,7 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { maxRecordAge: cdk.Duration.seconds(59), startingPosition: lambda.StartingPosition.LATEST - })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 59/); + })), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 59/); test.done(); }, @@ -545,7 +545,7 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { maxRecordAge: cdk.Duration.seconds(604801), startingPosition: lambda.StartingPosition.LATEST - })), /maximumRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); + })), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); test.done(); }, diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 676989cacf3b3..a955834c17c24 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -127,11 +127,11 @@ export class EventSourceMapping extends cdk.Resource { } if (props.maxRecordAge && (props.maxRecordAge.toSeconds() < 60 || props.maxRecordAge.toDays({integral: false}) > 7)) { - throw new Error(`maximumRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maxRecordAge.toSeconds()}`); + throw new Error(`maxRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maxRecordAge.toSeconds()}`); } if (props.retryAttempts && (props.retryAttempts < 0 || props.retryAttempts > 10000)) { - throw new Error(`maximumRetryAttempts must be between 0 and 10000 inclusive, got ${props.retryAttempts}`); + throw new Error(`retryAttempts must be between 0 and 10000 inclusive, got ${props.retryAttempts}`); } if ((props.parallelizationFactor || props.parallelizationFactor === 0) && (props.parallelizationFactor < 1 || props.parallelizationFactor > 10)) { diff --git a/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts new file mode 100644 index 0000000000000..db962032e3583 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts @@ -0,0 +1,148 @@ +/* tslint:disable */ +import { Test } from 'nodeunit'; +import { Code, EventSourceMapping, Function, Runtime } from '../lib'; +import * as cdk from '@aws-cdk/core'; + + +export = { + 'throws if maxBatchingWindow > 300 seconds'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxBatchingWindow: cdk.Duration.seconds(301) + }), /maxBatchingWindow cannot be over 300 seconds/); + + test.done(); + }, + 'throws if maxRecordAge is below 60 seconds'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxRecordAge: cdk.Duration.seconds(59) + }), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 59/); + + test.done(); + }, + 'throws if maxRecordAge is over 7 days'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxRecordAge: cdk.Duration.seconds(604801) + }), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); + + test.done(); + }, + 'throws if retryAttempts is negative'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + retryAttempts: -1 + }), /retryAttempts must be between 0 and 10000 inclusive, got -1/); + + test.done(); + }, + 'throws if retryAttempts is over 10000'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + retryAttempts: 10001 + }), /retryAttempts must be between 0 and 10000 inclusive, got 10001/); + + test.done(); + }, + 'throws if parallelizationFactor is below 1'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + parallelizationFactor: 0 + }), /parallelizationFactor must be between 1 and 10 inclusive, got 0/); + + test.done(); + }, + 'throws if parallelizationFactor is over 10'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + parallelizationFactor: 11 + }), /parallelizationFactor must be between 1 and 10 inclusive, got 11/); + + test.done(); + }, +}; From faca3074e0ed7d0a9f8b59e58f855d3b42b03591 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 10:33:13 -0400 Subject: [PATCH 26/33] Add permissions to send messages to DLQs to lambda --- .../aws-lambda-event-sources/lib/sns-dlq.ts | 6 ++++-- .../aws-lambda-event-sources/lib/sqs-dlq.ts | 6 ++++-- .../test/integ.kinesiswithdlq.expected.json | 14 ++++++++++++++ packages/@aws-cdk/aws-lambda/lib/dlq.ts | 4 +++- .../aws-lambda/lib/event-source-mapping.ts | 2 +- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts index cf2df12be51f7..b6fc889e88223 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -1,4 +1,4 @@ -import { DlqDestinationConfig, IEventSourceDlq } from "@aws-cdk/aws-lambda"; +import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; import * as sns from '@aws-cdk/aws-sns'; /** @@ -11,7 +11,9 @@ export class SnsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(): DlqDestinationConfig { + public bind(target: IFunction): DlqDestinationConfig { + this.topic.grantPublish(target); + return { destination: this.topic.topicArn }; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts index d6d94852ce17d..02f2381d90781 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -1,4 +1,4 @@ -import { DlqDestinationConfig, IEventSourceDlq } from "@aws-cdk/aws-lambda"; +import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; import * as sqs from '@aws-cdk/aws-sqs'; /** @@ -11,7 +11,9 @@ export class SqsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(): DlqDestinationConfig { + public bind(target: IFunction): DlqDestinationConfig { + this.queue.grantSendMessages(target); + return { destination: this.queue.queueArn }; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json index 039d1b9bf10cd..195c1546cbdda 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json @@ -36,6 +36,20 @@ "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "Q63C6E3AB", + "Arn" + ] + } + }, { "Action": [ "kinesis:DescribeStream", diff --git a/packages/@aws-cdk/aws-lambda/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts index 30f973119338e..7dd88ee1c2df8 100644 --- a/packages/@aws-cdk/aws-lambda/lib/dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/dlq.ts @@ -1,3 +1,5 @@ +import { IFunction } from './function-base'; + /** * A destination configuration */ @@ -15,5 +17,5 @@ export interface IEventSourceDlq { /** * Returns the DLQ destination config of the DLQ */ - bind(): DlqDestinationConfig; + bind(target: IFunction): DlqDestinationConfig; } diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index a955834c17c24..f0ad8a81a8385 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -142,7 +142,7 @@ export class EventSourceMapping extends cdk.Resource { if (props.onFailure) { destinationConfig = { - onFailure: props.onFailure.bind() + onFailure: props.onFailure.bind(props.target) }; } From 068be7af99d2f225accce10cad8c553f7d3adbdf Mon Sep 17 00:00:00 2001 From: Niranjan Jayakar Date: Mon, 9 Mar 2020 17:21:33 +0000 Subject: [PATCH 27/33] fix integ test and stack verification steps --- .../test/integ.kinesiswithdlq.expected.json | 18 ++++++++++++--- .../integ.kinesiswithdlq.handler/index.ts | 21 ------------------ .../test/integ.kinesiswithdlq.ts | 22 ++++++++----------- 3 files changed, 24 insertions(+), 37 deletions(-) delete mode 100644 packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json index 195c1546cbdda..82fc1a6f4a7c3 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json @@ -79,7 +79,7 @@ "Type": "AWS::Lambda::Function", "Properties": { "Code": { - "ZipFile": "exports.handler = async function handler(event) {\n console.log('event:', JSON.stringify(event, undefined, 2));\n throw new Error();\n}" + "ZipFile": "exports.handler = async function handler(event) {\n // tslint:disable-next-line:no-console\n console.log('event:', JSON.stringify(event, undefined, 2));\n throw new Error();\n}" }, "Handler": "index.handler", "Role": { @@ -118,7 +118,7 @@ } } }, - "MaximumRetryAttempts": 2, + "MaximumRetryAttempts": 0, "StartingPosition": "TRIM_HORIZON" } }, @@ -218,6 +218,18 @@ ] } }, + "Outputs": { + "InputKinesisStreamName": { + "Value": { + "Ref": "S509448A1" + } + }, + "DlqSqsQueueUrl": { + "Value": { + "Ref": "Q63C6E3AB" + } + } + }, "Parameters": { "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3BucketF010E518": { "Type": "String", @@ -232,4 +244,4 @@ "Description": "Artifact hash for asset \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" } } -} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts deleted file mode 100644 index 5873db82d1cd8..0000000000000 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.handler/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* eslint-disable */ -// tslint:disable:no-console -// tslint:disable:no-var-requires - -const AWS = require('aws-sdk'); - -export const handler = (event: any) => { - const kinesis = new AWS.Kinesis(); - console.log(`event: ${JSON.stringify(event)}`); - kinesis.putRecord({ - Data: 'Hello World', - PartitionKey: 'Hello Key', - StreamName: '${stream.streamName}' - }, (err: any, data: any) => { - if (err) { - console.log(`Error: ${err}`); - } else { - console.log(`Data: ${data}`); - } - }); -}; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts index 41627dcb31c34..97c470104c8f3 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.ts @@ -1,19 +1,18 @@ import * as kinesis from '@aws-cdk/aws-kinesis'; import * as lambda from '@aws-cdk/aws-lambda'; import * as sqs from '@aws-cdk/aws-sqs'; -import { App, Stack } from "@aws-cdk/core"; -import * as path from 'path'; +import { App, CfnOutput, Stack } from "@aws-cdk/core"; import { KinesisEventSource, SqsDlq } from '../lib'; /* * Stack verification steps: - * * aws lambda invoke --function-name --invocation-type Event --payload '"Event"' response.json - * * Validate two executions of fn lambda - * * Validate one message in dlq + * * aws kinesis put-record --stream-name --partition-key 123 --data testdata + * * aws sqs receive-message --queue-url --max-number-of-messages 1 --query 'Messages[0].Body' + * The last command should return a string that contains the Lambda function ARN in it. */ -// tslint:disable:no-console async function handler(event: any) { + // tslint:disable-next-line:no-console console.log('event:', JSON.stringify(event, undefined, 2)); throw new Error(); } @@ -27,22 +26,19 @@ class KinesisWithDLQTest extends Stack { handler: 'index.handler', code: lambda.Code.fromInline(`exports.handler = ${handler.toString()}`) }); + new CfnOutput(this, 'FunctionArn', { value: fn.functionArn }); const stream = new kinesis.Stream(this, 'S'); + new CfnOutput(this, 'InputKinesisStreamName', { value: stream.streamName }); const dlq = new sqs.Queue(this, 'Q'); + new CfnOutput(this, 'DlqSqsQueueUrl', { value: dlq.queueUrl }); fn.addEventSource(new KinesisEventSource(stream, { startingPosition: lambda.StartingPosition.TRIM_HORIZON, onFailure: new SqsDlq(dlq), - retryAttempts: 2 + retryAttempts: 0, })); - - new lambda.Function(this, 'FP', { - runtime: lambda.Runtime.NODEJS_10_X, - handler: 'index.handler', - code: lambda.AssetCode.fromAsset(path.join(__dirname, 'integ.kinesiswithdlq.handler')) - }); } } From b4f51a1e969abf888957990fab99cb845ac85189 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 13:59:27 -0400 Subject: [PATCH 28/33] Update packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts Co-Authored-By: Niranjan Jayakar <16217941+nija-at@users.noreply.github.com> --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index f0ad8a81a8385..302f45ab20418 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -127,7 +127,7 @@ export class EventSourceMapping extends cdk.Resource { } if (props.maxRecordAge && (props.maxRecordAge.toSeconds() < 60 || props.maxRecordAge.toDays({integral: false}) > 7)) { - throw new Error(`maxRecordAge must be between 60 and 604800 seconds inclusive, got ${props.maxRecordAge.toSeconds()}`); + throw new Error('maxRecordAge must be between 60 seconds and 7 days inclusive'); } if (props.retryAttempts && (props.retryAttempts < 0 || props.retryAttempts > 10000)) { From fb6957b95f1e77537670657e3953d08d904dfe4d Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 14:06:17 -0400 Subject: [PATCH 29/33] Clean up tests --- packages/@aws-cdk/aws-lambda/package.json | 2 - .../test/test.event-source-mapping.ts | 284 +++++++++--------- 2 files changed, 141 insertions(+), 145 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 2d02ed6db7343..91c1b317dfd5b 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -91,7 +91,6 @@ "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", - "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/cx-api": "0.0.0" }, @@ -104,7 +103,6 @@ "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", - "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/core": "0.0.0", "@aws-cdk/cx-api": "0.0.0" diff --git a/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts index db962032e3583..16e5f41aac1e9 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts @@ -1,148 +1,146 @@ -/* tslint:disable */ +import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; import { Code, EventSourceMapping, Function, Runtime } from '../lib'; -import * as cdk from '@aws-cdk/core'; - export = { - 'throws if maxBatchingWindow > 300 seconds'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - maxBatchingWindow: cdk.Duration.seconds(301) - }), /maxBatchingWindow cannot be over 300 seconds/); - - test.done(); - }, - 'throws if maxRecordAge is below 60 seconds'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - maxRecordAge: cdk.Duration.seconds(59) - }), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 59/); - - test.done(); - }, - 'throws if maxRecordAge is over 7 days'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - maxRecordAge: cdk.Duration.seconds(604801) - }), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); - - test.done(); - }, - 'throws if retryAttempts is negative'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - retryAttempts: -1 - }), /retryAttempts must be between 0 and 10000 inclusive, got -1/); - - test.done(); - }, - 'throws if retryAttempts is over 10000'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - retryAttempts: 10001 - }), /retryAttempts must be between 0 and 10000 inclusive, got 10001/); - - test.done(); - }, - 'throws if parallelizationFactor is below 1'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - parallelizationFactor: 0 - }), /parallelizationFactor must be between 1 and 10 inclusive, got 0/); - - test.done(); - }, - 'throws if parallelizationFactor is over 10'(test: Test) { - const stack = new cdk.Stack(); - const fn = new Function(stack, 'fn', { - handler: 'index.handler', - code: Code.fromInline(`exports.handler = \${handler.toString()}`), - runtime: Runtime.NODEJS_10_X - }); - - test.throws(() => - new EventSourceMapping( - stack, - 'test', - { - target: fn, - eventSourceArn: '', - parallelizationFactor: 11 - }), /parallelizationFactor must be between 1 and 10 inclusive, got 11/); - - test.done(); - }, + 'throws if maxBatchingWindow > 300 seconds'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxBatchingWindow: cdk.Duration.seconds(301) + }), /maxBatchingWindow cannot be over 300 seconds/); + + test.done(); + }, + 'throws if maxRecordAge is below 60 seconds'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxRecordAge: cdk.Duration.seconds(59) + }), /maxRecordAge must be between 60 seconds and 7 days inclusive/); + + test.done(); + }, + 'throws if maxRecordAge is over 7 days'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + maxRecordAge: cdk.Duration.seconds(604801) + }), /maxRecordAge must be between 60 seconds and 7 days inclusive/); + + test.done(); + }, + 'throws if retryAttempts is negative'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + retryAttempts: -1 + }), /retryAttempts must be between 0 and 10000 inclusive, got -1/); + + test.done(); + }, + 'throws if retryAttempts is over 10000'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + retryAttempts: 10001 + }), /retryAttempts must be between 0 and 10000 inclusive, got 10001/); + + test.done(); + }, + 'throws if parallelizationFactor is below 1'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + parallelizationFactor: 0 + }), /parallelizationFactor must be between 1 and 10 inclusive, got 0/); + + test.done(); + }, + 'throws if parallelizationFactor is over 10'(test: Test) { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline(`exports.handler = \${handler.toString()}`), + runtime: Runtime.NODEJS_10_X + }); + + test.throws(() => + new EventSourceMapping( + stack, + 'test', + { + target: fn, + eventSourceArn: '', + parallelizationFactor: 11 + }), /parallelizationFactor must be between 1 and 10 inclusive, got 11/); + + test.done(); + } }; From 43cd63b71c2cee719f63005e834042d87ed1c981 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 14:26:46 -0400 Subject: [PATCH 30/33] Add event source mapping to DLQ bind signature --- packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts | 5 +++-- packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts | 5 +++-- .../@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts | 4 ++-- packages/@aws-cdk/aws-lambda/lib/dlq.ts | 3 ++- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts index b6fc889e88223..7b4727eacffb1 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -1,5 +1,6 @@ import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; import * as sns from '@aws-cdk/aws-sns'; +import { IResource } from "@aws-cdk/core"; /** * An SNS dead letter queue destination configuration for a Lambda event source @@ -11,8 +12,8 @@ export class SnsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(target: IFunction): DlqDestinationConfig { - this.topic.grantPublish(target); + public bind(_target: IResource, targetHandler: IFunction): DlqDestinationConfig { + this.topic.grantPublish(targetHandler); return { destination: this.topic.topicArn diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts index 02f2381d90781..c788715d20fe2 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -1,5 +1,6 @@ import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; import * as sqs from '@aws-cdk/aws-sqs'; +import { IResource } from "@aws-cdk/core"; /** * An SQS dead letter queue destination configuration for a Lambda event source @@ -11,8 +12,8 @@ export class SqsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(target: IFunction): DlqDestinationConfig { - this.queue.grantSendMessages(target); + public bind(_target: IResource, targetHandler: IFunction): DlqDestinationConfig { + this.queue.grantSendMessages(targetHandler); return { destination: this.queue.queueArn diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts index 007d6dfef0a94..9c2de1e436e5c 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.dynamo.ts @@ -523,7 +523,7 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { maxRecordAge: cdk.Duration.seconds(59), startingPosition: lambda.StartingPosition.LATEST - })), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 59/); + })), /maxRecordAge must be between 60 seconds and 7 days inclusive/); test.done(); }, @@ -545,7 +545,7 @@ export = { fn.addEventSource(new sources.DynamoEventSource(table, { maxRecordAge: cdk.Duration.seconds(604801), startingPosition: lambda.StartingPosition.LATEST - })), /maxRecordAge must be between 60 and 604800 seconds inclusive, got 604801/); + })), /maxRecordAge must be between 60 seconds and 7 days inclusive/); test.done(); }, diff --git a/packages/@aws-cdk/aws-lambda/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts index 7dd88ee1c2df8..7f74f8ae393a1 100644 --- a/packages/@aws-cdk/aws-lambda/lib/dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/dlq.ts @@ -1,3 +1,4 @@ +import { IResource } from '@aws-cdk/core'; import { IFunction } from './function-base'; /** @@ -17,5 +18,5 @@ export interface IEventSourceDlq { /** * Returns the DLQ destination config of the DLQ */ - bind(target: IFunction): DlqDestinationConfig; + bind(target: IResource, targetHandler: IFunction): DlqDestinationConfig; } diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 302f45ab20418..99e05e61d345f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -142,7 +142,7 @@ export class EventSourceMapping extends cdk.Resource { if (props.onFailure) { destinationConfig = { - onFailure: props.onFailure.bind(props.target) + onFailure: props.onFailure.bind(this, props.target) }; } From 303aabbcd4d3fee34d20f730177adc61f1e1f1b7 Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Mon, 9 Mar 2020 15:15:28 -0400 Subject: [PATCH 31/33] Fix integration test expected --- .../test/integ.kinesiswithdlq.expected.json | 104 +----------------- 1 file changed, 6 insertions(+), 98 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json index 82fc1a6f4a7c3..5439a5515bbd4 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/integ.kinesiswithdlq.expected.json @@ -131,91 +131,6 @@ }, "Q63C6E3AB": { "Type": "AWS::SQS::Queue" - }, - "FPServiceRoleB1957CC2": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Statement": [ - { - "Action": "sts:AssumeRole", - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - } - } - ], - "Version": "2012-10-17" - }, - "ManagedPolicyArns": [ - { - "Fn::Join": [ - "", - [ - "arn:", - { - "Ref": "AWS::Partition" - }, - ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - ] - ] - } - ] - } - }, - "FP85AE6139": { - "Type": "AWS::Lambda::Function", - "Properties": { - "Code": { - "S3Bucket": { - "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3BucketF010E518" - }, - "S3Key": { - "Fn::Join": [ - "", - [ - { - "Fn::Select": [ - 0, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C" - } - ] - } - ] - }, - { - "Fn::Select": [ - 1, - { - "Fn::Split": [ - "||", - { - "Ref": "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C" - } - ] - } - ] - } - ] - ] - } - }, - "Handler": "index.handler", - "Role": { - "Fn::GetAtt": [ - "FPServiceRoleB1957CC2", - "Arn" - ] - }, - "Runtime": "nodejs10.x" - }, - "DependsOn": [ - "FPServiceRoleB1957CC2" - ] } }, "Outputs": { @@ -228,20 +143,13 @@ "Value": { "Ref": "Q63C6E3AB" } - } + }, + "FunctionArn": { + "Value":{ + "Fn::GetAtt":["FC4345940","Arn"] + } + } }, "Parameters": { - "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3BucketF010E518": { - "Type": "String", - "Description": "S3 bucket for asset \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" - }, - "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604S3VersionKeyB38C926C": { - "Type": "String", - "Description": "S3 key for asset version \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" - }, - "AssetParameters013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604ArtifactHash5300B2A6": { - "Type": "String", - "Description": "Artifact hash for asset \"013549436de3944a6176501ce5de71aef355a25e669d5af45fd69f29785df604\"" - } } } \ No newline at end of file From c3f4dfe09030ec02892908df0c2f44c707504ca0 Mon Sep 17 00:00:00 2001 From: Niranjan Jayakar Date: Tue, 10 Mar 2020 08:44:53 +0000 Subject: [PATCH 32/33] added IEventSourceMapping --- .../aws-lambda/lib/event-source-mapping.ts | 28 +++++++++++++++++-- packages/@aws-cdk/aws-lambda/package.json | 1 - .../test/test.event-source-mapping.ts | 11 +++++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 99e05e61d345f..a32a8ffd9b23b 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -93,6 +93,9 @@ export interface EventSourceMappingOptions { readonly parallelizationFactor?: number; } +/** + * Properties for declaring a new event source mapping. + */ export interface EventSourceMappingProps extends EventSourceMappingOptions { /** * The target AWS Lambda function. @@ -100,6 +103,18 @@ export interface EventSourceMappingProps extends EventSourceMappingOptions { readonly target: IFunction; } +/** + * Represents an event source mapping for a lambda function. + * @see https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventsourcemapping.html + */ +export interface IEventSourceMapping extends cdk.IResource { + /** + * The identifier for this EventSourceMapping + * @attribute + */ + readonly eventSourceMappingId: string; +} + /** * Defines a Lambda EventSourceMapping resource. * @@ -112,11 +127,18 @@ export interface EventSourceMappingProps extends EventSourceMappingOptions { * The `SqsEventSource` class will automatically create the mapping, and will also * modify the Lambda's execution role so it can consume messages from the queue. */ -export class EventSourceMapping extends cdk.Resource { +export class EventSourceMapping extends cdk.Resource implements IEventSourceMapping { + /** - * The identifier for this EventSourceMapping - * @attribute + * Import an event source into this stack from its event source id. */ + public static fromEventSourceMappingId(scope: cdk.Construct, id: string, eventSourceMappingId: string): IEventSourceMapping { + class Import extends cdk.Resource implements IEventSourceMapping { + public readonly eventSourceMappingId = eventSourceMappingId; + } + return new Import(scope, id); + } + public readonly eventSourceMappingId: string; constructor(scope: cdk.Construct, id: string, props: EventSourceMappingProps) { diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 91c1b317dfd5b..8064ad98ffa18 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -164,7 +164,6 @@ "props-default-doc:@aws-cdk/aws-lambda.CodeConfig.inlineCode", "props-default-doc:@aws-cdk/aws-lambda.CodeConfig.s3Location", "docs-public-apis:@aws-cdk/aws-lambda.EventSourceMappingOptions", - "docs-public-apis:@aws-cdk/aws-lambda.EventSourceMappingProps", "props-default-doc:@aws-cdk/aws-lambda.FunctionAttributes.role", "props-default-doc:@aws-cdk/aws-lambda.FunctionAttributes.securityGroup", "props-default-doc:@aws-cdk/aws-lambda.FunctionAttributes.securityGroupId", diff --git a/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts index 16e5f41aac1e9..3de9bbfeb5dab 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.event-source-mapping.ts @@ -142,5 +142,14 @@ export = { }), /parallelizationFactor must be between 1 and 10 inclusive, got 11/); test.done(); - } + }, + + 'import event source mapping'(test: Test) { + const stack = new cdk.Stack(undefined, undefined, { stackName: 'test-stack' }); + const imported = EventSourceMapping.fromEventSourceMappingId(stack, 'imported', '14e0db71-5d35-4eb5-b481-8945cf9d10c2'); + + test.equals(imported.eventSourceMappingId, '14e0db71-5d35-4eb5-b481-8945cf9d10c2'); + test.equals(imported.stack.stackName, 'test-stack'); + test.done(); + }, }; From 2d2e1d7b37fc4828300604407025ffe4492c399f Mon Sep 17 00:00:00 2001 From: Brian Rogers Date: Tue, 10 Mar 2020 07:57:56 -0400 Subject: [PATCH 33/33] Refactor to IEventSourceMapping --- packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts | 5 ++--- packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts | 5 ++--- packages/@aws-cdk/aws-lambda/lib/dlq.ts | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts index 7b4727eacffb1..ddac2c080f6a8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sns-dlq.ts @@ -1,6 +1,5 @@ -import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; +import { DlqDestinationConfig, IEventSourceDlq, IEventSourceMapping, IFunction } from "@aws-cdk/aws-lambda"; import * as sns from '@aws-cdk/aws-sns'; -import { IResource } from "@aws-cdk/core"; /** * An SNS dead letter queue destination configuration for a Lambda event source @@ -12,7 +11,7 @@ export class SnsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(_target: IResource, targetHandler: IFunction): DlqDestinationConfig { + public bind(_target: IEventSourceMapping, targetHandler: IFunction): DlqDestinationConfig { this.topic.grantPublish(targetHandler); return { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts index c788715d20fe2..173c82472271b 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/sqs-dlq.ts @@ -1,6 +1,5 @@ -import { DlqDestinationConfig, IEventSourceDlq, IFunction } from "@aws-cdk/aws-lambda"; +import { DlqDestinationConfig, IEventSourceDlq, IEventSourceMapping, IFunction } from "@aws-cdk/aws-lambda"; import * as sqs from '@aws-cdk/aws-sqs'; -import { IResource } from "@aws-cdk/core"; /** * An SQS dead letter queue destination configuration for a Lambda event source @@ -12,7 +11,7 @@ export class SqsDlq implements IEventSourceDlq { /** * Returns a destination configuration for the DLQ */ - public bind(_target: IResource, targetHandler: IFunction): DlqDestinationConfig { + public bind(_target: IEventSourceMapping, targetHandler: IFunction): DlqDestinationConfig { this.queue.grantSendMessages(targetHandler); return { diff --git a/packages/@aws-cdk/aws-lambda/lib/dlq.ts b/packages/@aws-cdk/aws-lambda/lib/dlq.ts index 7f74f8ae393a1..2f2c52448a90d 100644 --- a/packages/@aws-cdk/aws-lambda/lib/dlq.ts +++ b/packages/@aws-cdk/aws-lambda/lib/dlq.ts @@ -1,4 +1,4 @@ -import { IResource } from '@aws-cdk/core'; +import { IEventSourceMapping } from './event-source-mapping'; import { IFunction } from './function-base'; /** @@ -18,5 +18,5 @@ export interface IEventSourceDlq { /** * Returns the DLQ destination config of the DLQ */ - bind(target: IResource, targetHandler: IFunction): DlqDestinationConfig; + bind(target: IEventSourceMapping, targetHandler: IFunction): DlqDestinationConfig; }