From 40bfee4c7fa11a6b5978a40ec6d8cbd6fd85ed09 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 10:01:40 +0100 Subject: [PATCH 01/55] Add SourceAccessConfigurations to event-source-mapping.ts --- .../aws-lambda/lib/event-source-mapping.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index d44ce1cbea1b4..5255f04df90ea 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -101,6 +101,11 @@ export interface EventSourceMappingOptions { * @default - no topic */ readonly kafkaTopic?: string; + + /** + * The Secrets Manager secret that stores your broker credentials. + */ + readonly sourceAccessConfiguration?: SourceAccessConfiguration } /** @@ -196,6 +201,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, + sourceAccessConfigurations: props.sourceAccessConfiguration !== undefined ? [props.sourceAccessConfiguration] : undefined, }); this.eventSourceMappingId = cfnEventSourceMapping.ref; } @@ -218,3 +224,17 @@ export enum StartingPosition { */ LATEST = 'LATEST', } + +/** + * The Secrets Manager secret that stores your broker credentials. To encrypt the secret, you can use customer or service managed keys. When using a customer managed KMS key, the Lambda execution role requires kms:Decrypt permissions. + */ +export interface SourceAccessConfiguration { + /** + * Set the value to BASIC_AUTH. + */ + type?: string, + /** + * The ARN of the AWS Secrets Manager secret. + */ + uri?: string +} From ad0f9a680786cdba8df04e6be0cc29a3ac6581ec Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 14:00:51 +0100 Subject: [PATCH 02/55] Implement Kafka event sources --- .../aws-lambda-event-sources/lib/index.ts | 1 + .../aws-lambda-event-sources/lib/kafka.ts | 63 +++++++++++++++++++ .../aws-lambda-event-sources/package.json | 2 + .../aws-lambda/lib/event-source-mapping.ts | 34 +++++----- 4 files changed, 83 insertions(+), 17 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts index 19253a743cae8..555137e51afbf 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/index.ts @@ -1,5 +1,6 @@ export * from './api'; export * from './dynamodb'; +export * from './kafka'; export * from './kinesis'; export * from './s3'; export * from './sns'; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts new file mode 100644 index 0000000000000..3178f40b2a94d --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -0,0 +1,63 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; +import { StreamEventSource, StreamEventSourceProps } from './stream'; + +export interface KafkaEventSourceProps extends StreamEventSourceProps { +} + +export class ManagedKafkaEventSource extends StreamEventSource { + constructor( + readonly clusterArn: string, + readonly topic: string, + readonly secret: secretsmanager.ISecret, + props: KafkaEventSourceProps) { + super(props); + } + + public bind(target: lambda.IFunction) { + target.addEventSourceMapping( + `KafkaEventSource:${this.topic}`, + this.enrichMappingOptions({ + eventSourceArn: this.clusterArn, + startingPosition: this.props.startingPosition, + kafkaSecretArn: this.secret.secretArn, + kafkaTopic: this.topic, + }), + ); + + this.secret.grantRead(target); + + target.addToRolePolicy(new iam.PolicyStatement( + { + actions: ['kafka:DescribeCluster', 'kafka:GetBootstrapBrokers', 'kafka:ListScramSecrets'], + resources: [this.clusterArn], + }, + )); + + target.role?.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaMSKExecutionRole')); + } +} + +export class SelfManagedKafkaEventSource extends StreamEventSource { + constructor( + readonly bootstrapServers: string[], + readonly kafkaTopic: string, + readonly secret: secretsmanager.ISecret, + props: KafkaEventSourceProps) { + super(props); + } + + public bind(target: lambda.IFunction) { + target.addEventSourceMapping( + `KafkaEventSource:${this.kafkaTopic}`, + this.enrichMappingOptions({ + kafkaBootstrapServers: this.bootstrapServers, + kafkaTopic: this.kafkaTopic, + startingPosition: this.props.startingPosition, + kafkaSecretArn: this.secret.secretArn, + }), + ); + this.secret.grantRead(target); + } +} diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 20112e1029cbf..6eb8db3da7c26 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -78,6 +78,7 @@ "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-notifications": "0.0.0", + "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/aws-sns-subscriptions": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", @@ -94,6 +95,7 @@ "@aws-cdk/aws-lambda": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-notifications": "0.0.0", + "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/aws-sns-subscriptions": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 5255f04df90ea..cdbc33d735e05 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -8,8 +8,10 @@ export interface EventSourceMappingOptions { /** * The Amazon Resource Name (ARN) of the event source. Any record added to * this stream can invoke the Lambda function. + * + * @default - not set */ - readonly eventSourceArn: string; + readonly eventSourceArn?: string; /** * The largest number of records that AWS Lambda will retrieve from your event @@ -103,9 +105,18 @@ export interface EventSourceMappingOptions { readonly kafkaTopic?: string; /** - * The Secrets Manager secret that stores your broker credentials. + * The Secrets Manager secret ARN that stores your broker credentials. + * + * @default - no configuration */ - readonly sourceAccessConfiguration?: SourceAccessConfiguration + readonly kafkaSecretArn?: string + + /** + * A list of Kafka bootstrap servers to connect to your self managed Kafka cluster + * + * @default - none + */ + readonly kafkaBootstrapServers?: string[] } /** @@ -201,7 +212,9 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, - sourceAccessConfigurations: props.sourceAccessConfiguration !== undefined ? [props.sourceAccessConfiguration] : undefined, + sourceAccessConfigurations: props.kafkaSecretArn !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecretArn }] : undefined, + // eslint-disable-next-line max-len + selfManagedEventSource: props.kafkaBootstrapServers !== undefined ? { endpoints: { kafkaBootstrapServers: props.kafkaBootstrapServers } } : undefined, }); this.eventSourceMappingId = cfnEventSourceMapping.ref; } @@ -225,16 +238,3 @@ export enum StartingPosition { LATEST = 'LATEST', } -/** - * The Secrets Manager secret that stores your broker credentials. To encrypt the secret, you can use customer or service managed keys. When using a customer managed KMS key, the Lambda execution role requires kms:Decrypt permissions. - */ -export interface SourceAccessConfiguration { - /** - * Set the value to BASIC_AUTH. - */ - type?: string, - /** - * The ARN of the AWS Secrets Manager secret. - */ - uri?: string -} From 1f510d64a7f0c6b16d2372d9034c6bd05323afcd Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 14:05:32 +0100 Subject: [PATCH 03/55] Add test for ManagedKafkaEventSource --- .../test/test.kafka.ts | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts new file mode 100644 index 0000000000000..4c3c60e0707f0 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -0,0 +1,84 @@ +import { expect, haveResource } from '@aws-cdk/assert'; +import * as lambda from '@aws-cdk/aws-lambda'; +import { Secret } from '@aws-cdk/aws-secretsmanager'; +import * as cdk from '@aws-cdk/core'; +import { Test } from 'nodeunit'; +import * as sources from '../lib'; +import { TestFunction } from './test-function'; + +/* eslint-disable quote-props */ + +export = { + 'sufficiently complex example'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const clusterArn = 'some-arn'; + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + + // WHEN + fn.addEventSource(new sources.ManagedKafkaEventSource( + clusterArn, + kafkaTopic, + secret, + { + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + })); + + // THEN + // expect(stack).to(haveResource('AWS::IAM::Policy', { + // 'PolicyDocument': { + // 'Statement': [ + // { + // 'Action': 'dynamodb:ListStreams', + // 'Effect': 'Allow', + // 'Resource': '*', + // }, + // { + // 'Action': [ + // 'dynamodb:DescribeStream', + // 'dynamodb:GetRecords', + // 'dynamodb:GetShardIterator', + // ], + // 'Effect': 'Allow', + // 'Resource': { + // 'Fn::GetAtt': [ + // 'TD925BC7E', + // 'StreamArn', + // ], + // }, + // }, + // ], + // 'Version': '2012-10-17', + // }, + // 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', + // 'Roles': [{ + // 'Ref': 'FnServiceRoleB9001A96', + // }], + // })); + + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + 'EventSourceArn': clusterArn, + 'FunctionName': { + 'Ref': 'Fn9270CBC0', + }, + 'BatchSize': 100, + 'StartingPosition': 'TRIM_HORIZON', + 'Topics': [ + kafkaTopic, + ], + 'SourceAccessConfigurations': [ + { + 'Type': 'BASIC_AUTH', + 'URI': { + 'Ref': 'SecretA720EF05', + }, + }, + ], + })); + + test.done(); + }, + +} \ No newline at end of file From af073f974271acfe1ded1e89351a5ba2f087a7d0 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 14:38:13 +0100 Subject: [PATCH 04/55] Enhance test --- .../aws-lambda-event-sources/lib/kafka.ts | 12 +++- .../test/test.kafka.ts | 62 ++++++++++--------- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 3178f40b2a94d..96ee6a364f8d1 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -6,6 +6,9 @@ import { StreamEventSource, StreamEventSourceProps } from './stream'; export interface KafkaEventSourceProps extends StreamEventSourceProps { } +/** + * Use a MSK cluster as a streaming source for AWS Lambda + */ export class ManagedKafkaEventSource extends StreamEventSource { constructor( readonly clusterArn: string, @@ -39,10 +42,13 @@ export class ManagedKafkaEventSource extends StreamEventSource { } } +/** + * Use a self hosted Kafka installation as a streaming source for AWS Lambda. + */ export class SelfManagedKafkaEventSource extends StreamEventSource { constructor( readonly bootstrapServers: string[], - readonly kafkaTopic: string, + readonly topic: string, readonly secret: secretsmanager.ISecret, props: KafkaEventSourceProps) { super(props); @@ -50,10 +56,10 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { public bind(target: lambda.IFunction) { target.addEventSourceMapping( - `KafkaEventSource:${this.kafkaTopic}`, + `KafkaEventSource:${this.topic}`, this.enrichMappingOptions({ kafkaBootstrapServers: this.bootstrapServers, - kafkaTopic: this.kafkaTopic, + kafkaTopic: this.topic, startingPosition: this.props.startingPosition, kafkaSecretArn: this.secret.secretArn, }), diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 4c3c60e0707f0..68d76743219a0 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -27,36 +27,38 @@ export = { })); // THEN - // expect(stack).to(haveResource('AWS::IAM::Policy', { - // 'PolicyDocument': { - // 'Statement': [ - // { - // 'Action': 'dynamodb:ListStreams', - // 'Effect': 'Allow', - // 'Resource': '*', - // }, - // { - // 'Action': [ - // 'dynamodb:DescribeStream', - // 'dynamodb:GetRecords', - // 'dynamodb:GetShardIterator', - // ], - // 'Effect': 'Allow', - // 'Resource': { - // 'Fn::GetAtt': [ - // 'TD925BC7E', - // 'StreamArn', - // ], - // }, - // }, - // ], - // 'Version': '2012-10-17', - // }, - // 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', - // 'Roles': [{ - // 'Ref': 'FnServiceRoleB9001A96', - // }], - // })); + expect(stack).to(haveResource('AWS::IAM::Policy', { + 'PolicyDocument': { + 'Statement': [ + { + 'Action': [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + 'Effect': 'Allow', + 'Resource': { + 'Ref': 'SecretA720EF05', + }, + }, + { + 'Action': [ + 'kafka:DescribeCluster', + 'kafka:GetBootstrapBrokers', + 'kafka:ListScramSecrets', + ], + 'Effect': 'Allow', + 'Resource': 'some-arn', + }, + ], + 'Version': '2012-10-17', + }, + 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', + 'Roles': [ + { + 'Ref': 'FnServiceRoleB9001A96', + }, + ], + })); expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { 'EventSourceArn': clusterArn, From 410ac08651a6ed15a8c5bd4e3e26279c06839931 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 14:49:01 +0100 Subject: [PATCH 05/55] Add test for SelfManagedKafkaEventSource --- .../test/test.kafka.ts | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 68d76743219a0..ccdf13ded76ee 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -9,7 +9,7 @@ import { TestFunction } from './test-function'; /* eslint-disable quote-props */ export = { - 'sufficiently complex example'(test: Test) { + 'MSK: sufficiently complex example'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -83,4 +83,75 @@ export = { test.done(); }, + 'self managed Kafka: sufficiently complex example'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + + // WHEN + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + bootstrapServers, + kafkaTopic, + secret, + { + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + })); + + // THEN + expect(stack).to(haveResource('AWS::IAM::Policy', { + 'PolicyDocument': { + 'Statement': [ + { + 'Action': [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + 'Effect': 'Allow', + 'Resource': { + 'Ref': 'SecretA720EF05', + }, + }, + ], + 'Version': '2012-10-17', + }, + 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', + 'Roles': [ + { + 'Ref': 'FnServiceRoleB9001A96', + }, + ], + })); + + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + 'FunctionName': { + 'Ref': 'Fn9270CBC0', + }, + 'BatchSize': 100, + 'SelfManagedEventSource': { + 'Endpoints': { + 'KafkaBootstrapServers': [ + 'kafka-broker:9092', + ], + }, + }, + 'StartingPosition': 'TRIM_HORIZON', + 'Topics': [ + kafkaTopic, + ], + 'SourceAccessConfigurations': [ + { + 'Type': 'BASIC_AUTH', + 'URI': { + 'Ref': 'SecretA720EF05', + }, + }, + ], + })); + + test.done(); + }, + } \ No newline at end of file From 2d6bcccf084ca9def284a16720bc6f48918cad9d Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 14 Jan 2021 16:30:52 +0100 Subject: [PATCH 06/55] Add section to README.md --- .../aws-lambda-event-sources/README.md | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 5e1578d9aefd4..927fe27b99236 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -207,6 +207,55 @@ myFunction.addEventSource(new KinesisEventSource(stream, { })); ``` +## Kafka + +You can write Lambda functions to process data either from [Amazon MSK](https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html) or a [self managed Kafka](https://docs.aws.amazon.com/lambda/latest/dg/kafka-smaa.html) cluster. + +To set up Amazon MSK as an event source use the following, you also have to set up a secret as described [here](https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html). + +```ts +import * as lambda from '@aws-cdk/aws-lambda'; +import { Secret } from '@aws-cdk/aws-secretmanager'; +import { ManagedKafkaEventSource } from '@aws-cdk/aws-lambda-event-sources'; + +// The ARN of your cluster +const clusterArn = 'arn:aws:kafka:us-east-1:0123456789019:cluster/SalesCluster/abcd1234-abcd-cafe-abab-9876543210ab-4' + +// The Kafka topic you want to subscribe to +const topic = 'some-cool-topic' + +// The secret that allows access to your MSK cluster +// You still have to make sure that it is associated with your cluster as described in the documentation +const secret = new Secret(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + +myFunction.addEventSource(new ManagedKafkaEventSource(clusterArn, topic, secret, { + batchSize: 100, // default + startingPosition: lambda.StartingPosition.TRIM_HORIZON +})); +``` + +To set up a self managed Kafka cluster as an event source use the following, you have to set up a secret as described [here](https://docs.aws.amazon.com/lambda/latest/dg/smaa-permissions.html#smaa-permissions-add-secret): + +```ts +import * as lambda from '@aws-cdk/aws-lambda'; +import { Secret } from '@aws-cdk/aws-secretmanager'; +import { SelfManagedKafkaEventSource } from '@aws-cdk/aws-lambda-event-sources'; + +// The list of Kafka brokers +const bootstrapServers = ['kafka-broker:9092'] + +// The Kafka topic you want to subscribe to +const topic = 'some-cool-topic' + +// The secret that allows access to your self hosted Kafka cluster +const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + +myFunction.addEventSource(new SelfManagedKafkaEventSource(bootstrapServers, topic, secret, { + batchSize: 100, // default + startingPosition: lambda.StartingPosition.TRIM_HORIZON +})); +``` + ## Roadmap Eventually, this module will support all the event sources described under From f742a93c5a4e21f4532c4d4efdc06cbbc187fee7 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Fri, 15 Jan 2021 09:49:12 +0100 Subject: [PATCH 07/55] Add docstrings --- .../@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 14 ++++++++++++++ .../aws-lambda-event-sources/lib/stream.ts | 2 +- .../@aws-cdk/aws-lambda-event-sources/package.json | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 96ee6a364f8d1..9a53dd2b36010 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -10,6 +10,13 @@ export interface KafkaEventSourceProps extends StreamEventSourceProps { * Use a MSK cluster as a streaming source for AWS Lambda */ export class ManagedKafkaEventSource extends StreamEventSource { + /** + * Create an event source for MSK + * @param clusterArn - the ARN of the MSK cluster + * @param topic - the Kafka topic to subscribe to + * @param secret - the secret with the Kafka credentials, see https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html for details + * @param props + */ constructor( readonly clusterArn: string, readonly topic: string, @@ -46,6 +53,13 @@ export class ManagedKafkaEventSource extends StreamEventSource { * Use a self hosted Kafka installation as a streaming source for AWS Lambda. */ export class SelfManagedKafkaEventSource extends StreamEventSource { + /** + * Create an event source for a self managed Kafka cluster + * @param bootstrapServers - list of Kafka brokers + * @param topic - the Kafka topic to subscribe to + * @param secret - the secret with the Kafka credentials, see https://docs.aws.amazon.com/lambda/latest/dg/smaa-permissions.html#smaa-permissions-add-secret + * @param props + */ constructor( readonly bootstrapServers: string[], readonly topic: string, diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index d18eaaf3f947c..96907b97835fc 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -3,7 +3,7 @@ import { Duration } from '@aws-cdk/core'; /** * The set of properties for event sources that follow the streaming model, - * such as, Dynamo and Kinesis. + * such as, Dynamo, Kinesis and Kafka. */ export interface StreamEventSourceProps { /** diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 6eb8db3da7c26..cccb7ae5e5e56 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -117,7 +117,8 @@ "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps", "docs-public-apis:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps", "props-default-doc:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps.filters", - "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps" + "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps", + "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KafkaEventSourceProps" ] }, "awscdkio": { From 3dd9628c14d4dfbd488ef89a457821b1beca4408 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 14:11:07 +0100 Subject: [PATCH 08/55] Apply suggestions from code review for README Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 927fe27b99236..ff37a81ce1c50 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -211,7 +211,8 @@ myFunction.addEventSource(new KinesisEventSource(stream, { You can write Lambda functions to process data either from [Amazon MSK](https://docs.aws.amazon.com/lambda/latest/dg/with-msk.html) or a [self managed Kafka](https://docs.aws.amazon.com/lambda/latest/dg/kafka-smaa.html) cluster. -To set up Amazon MSK as an event source use the following, you also have to set up a secret as described [here](https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html). +The following code sets up Amazon MSK as an event source for a lambda function. Credentials will need to be configured to access the +MSK cluster, as described in [Username/Password authentication](https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html). ```ts import * as lambda from '@aws-cdk/aws-lambda'; @@ -234,7 +235,8 @@ myFunction.addEventSource(new ManagedKafkaEventSource(clusterArn, topic, secret, })); ``` -To set up a self managed Kafka cluster as an event source use the following, you have to set up a secret as described [here](https://docs.aws.amazon.com/lambda/latest/dg/smaa-permissions.html#smaa-permissions-add-secret): +The following code sets up a self managed Kafka cluster as an event source. Username and password based authentication +will need to be set up as described in [Managing access and permissions](https://docs.aws.amazon.com/lambda/latest/dg/smaa-permissions.html#smaa-permissions-add-secret). ```ts import * as lambda from '@aws-cdk/aws-lambda'; From a896cf7d12ab999186445d9685799eaa19b8540f Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 14:24:47 +0100 Subject: [PATCH 09/55] Remove obsolete eslint rule --- packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index ccdf13ded76ee..44ee56c5379ae 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -6,8 +6,6 @@ import { Test } from 'nodeunit'; import * as sources from '../lib'; import { TestFunction } from './test-function'; -/* eslint-disable quote-props */ - export = { 'MSK: sufficiently complex example'(test: Test) { // GIVEN From f94133c7a162b3820fe937cddd5651d33edc96d4 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 14:25:46 +0100 Subject: [PATCH 10/55] Remove obsolete eslint rule - and fix it --- .../test/test.kafka.ts | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 44ee56c5379ae..ed536e647086b 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -26,53 +26,53 @@ export = { // THEN expect(stack).to(haveResource('AWS::IAM::Policy', { - 'PolicyDocument': { - 'Statement': [ + PolicyDocument: { + Statement: [ { - 'Action': [ + Action: [ 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', ], - 'Effect': 'Allow', - 'Resource': { - 'Ref': 'SecretA720EF05', + Effect: 'Allow', + Resource: { + Ref: 'SecretA720EF05', }, }, { - 'Action': [ + Action: [ 'kafka:DescribeCluster', 'kafka:GetBootstrapBrokers', 'kafka:ListScramSecrets', ], - 'Effect': 'Allow', - 'Resource': 'some-arn', + Effect: 'Allow', + Resource: 'some-arn', }, ], - 'Version': '2012-10-17', + Version: '2012-10-17', }, - 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', - 'Roles': [ + PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', + Roles: [ { - 'Ref': 'FnServiceRoleB9001A96', + Ref: 'FnServiceRoleB9001A96', }, ], })); expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - 'EventSourceArn': clusterArn, - 'FunctionName': { - 'Ref': 'Fn9270CBC0', + EventSourceArn: clusterArn, + FunctionName: { + Ref: 'Fn9270CBC0', }, - 'BatchSize': 100, - 'StartingPosition': 'TRIM_HORIZON', - 'Topics': [ + BatchSize: 100, + StartingPosition: 'TRIM_HORIZON', + Topics: [ kafkaTopic, ], - 'SourceAccessConfigurations': [ + SourceAccessConfigurations: [ { - 'Type': 'BASIC_AUTH', - 'URI': { - 'Ref': 'SecretA720EF05', + Type: 'BASIC_AUTH', + URI: { + Ref: 'SecretA720EF05', }, }, ], @@ -100,50 +100,50 @@ export = { // THEN expect(stack).to(haveResource('AWS::IAM::Policy', { - 'PolicyDocument': { - 'Statement': [ + PolicyDocument: { + Statement: [ { - 'Action': [ + Action: [ 'secretsmanager:GetSecretValue', 'secretsmanager:DescribeSecret', ], - 'Effect': 'Allow', - 'Resource': { - 'Ref': 'SecretA720EF05', + Effect: 'Allow', + Resource: { + Ref: 'SecretA720EF05', }, }, ], - 'Version': '2012-10-17', + Version: '2012-10-17', }, - 'PolicyName': 'FnServiceRoleDefaultPolicyC6A839BF', - 'Roles': [ + PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', + Roles: [ { - 'Ref': 'FnServiceRoleB9001A96', + Ref: 'FnServiceRoleB9001A96', }, ], })); expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - 'FunctionName': { - 'Ref': 'Fn9270CBC0', + FunctionName: { + Ref: 'Fn9270CBC0', }, - 'BatchSize': 100, - 'SelfManagedEventSource': { - 'Endpoints': { - 'KafkaBootstrapServers': [ + BatchSize: 100, + SelfManagedEventSource: { + Endpoints: { + KafkaBootstrapServers: [ 'kafka-broker:9092', ], }, }, - 'StartingPosition': 'TRIM_HORIZON', - 'Topics': [ + StartingPosition: 'TRIM_HORIZON', + Topics: [ kafkaTopic, ], - 'SourceAccessConfigurations': [ + SourceAccessConfigurations: [ { - 'Type': 'BASIC_AUTH', - 'URI': { - 'Ref': 'SecretA720EF05', + Type: 'BASIC_AUTH', + URI: { + Ref: 'SecretA720EF05', }, }, ], From c967923dbee9c125731b795244aebe85d3647181 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 14:29:04 +0100 Subject: [PATCH 11/55] Document default value --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index cdbc33d735e05..2d4f3426ad3b3 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -9,7 +9,7 @@ export interface EventSourceMappingOptions { * The Amazon Resource Name (ARN) of the event source. Any record added to * this stream can invoke the Lambda function. * - * @default - not set + * @default - not set if using a self managed Kafka cluster */ readonly eventSourceArn?: string; From 4ee85c8c4ef225cf83b9507b457f5b39de2eb30b Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 14:49:19 +0100 Subject: [PATCH 12/55] Rename to kafkaSecret and change type --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 4 ++-- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 5 +++-- packages/@aws-cdk/aws-lambda/package.json | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 9a53dd2b36010..5e4f6d96d12fe 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -31,7 +31,7 @@ export class ManagedKafkaEventSource extends StreamEventSource { this.enrichMappingOptions({ eventSourceArn: this.clusterArn, startingPosition: this.props.startingPosition, - kafkaSecretArn: this.secret.secretArn, + kafkaSecret: this.secret, kafkaTopic: this.topic, }), ); @@ -75,7 +75,7 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { kafkaBootstrapServers: this.bootstrapServers, kafkaTopic: this.topic, startingPosition: this.props.startingPosition, - kafkaSecretArn: this.secret.secretArn, + kafkaSecret: this.secret, }), ); this.secret.grantRead(target); diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 2d4f3426ad3b3..7ad5b7da4cab6 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,3 +1,4 @@ +import { ISecret } from '@aws-cdk/aws-secretsmanager'; import * as cdk from '@aws-cdk/core'; import { Construct } from 'constructs'; import { IEventSourceDlq } from './dlq'; @@ -109,7 +110,7 @@ export interface EventSourceMappingOptions { * * @default - no configuration */ - readonly kafkaSecretArn?: string + readonly kafkaSecret?: ISecret /** * A list of Kafka bootstrap servers to connect to your self managed Kafka cluster @@ -212,7 +213,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, - sourceAccessConfigurations: props.kafkaSecretArn !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecretArn }] : undefined, + sourceAccessConfigurations: props.kafkaSecret !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecret.secretArn }] : undefined, // eslint-disable-next-line max-len selfManagedEventSource: props.kafkaBootstrapServers !== undefined ? { endpoints: { kafkaBootstrapServers: props.kafkaBootstrapServers } } : undefined, }); diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 6eab5cd11b870..2bb7ec3cf329b 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -99,6 +99,7 @@ "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", + "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-signer": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/core": "0.0.0", @@ -120,6 +121,7 @@ "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", + "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-signer": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/core": "0.0.0", From f372c2f68b820ae944eaf4c74d5540bd585ea627 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 15:25:32 +0100 Subject: [PATCH 13/55] Revert "Rename to kafkaSecret and change type" This reverts commit c90691fff38906854a3ea515ce50f8188801b08e. --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 4 ++-- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 5e4f6d96d12fe..9a53dd2b36010 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -31,7 +31,7 @@ export class ManagedKafkaEventSource extends StreamEventSource { this.enrichMappingOptions({ eventSourceArn: this.clusterArn, startingPosition: this.props.startingPosition, - kafkaSecret: this.secret, + kafkaSecretArn: this.secret.secretArn, kafkaTopic: this.topic, }), ); @@ -75,7 +75,7 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { kafkaBootstrapServers: this.bootstrapServers, kafkaTopic: this.topic, startingPosition: this.props.startingPosition, - kafkaSecret: this.secret, + kafkaSecretArn: this.secret.secretArn, }), ); this.secret.grantRead(target); diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 7ad5b7da4cab6..2d4f3426ad3b3 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -1,4 +1,3 @@ -import { ISecret } from '@aws-cdk/aws-secretsmanager'; import * as cdk from '@aws-cdk/core'; import { Construct } from 'constructs'; import { IEventSourceDlq } from './dlq'; @@ -110,7 +109,7 @@ export interface EventSourceMappingOptions { * * @default - no configuration */ - readonly kafkaSecret?: ISecret + readonly kafkaSecretArn?: string /** * A list of Kafka bootstrap servers to connect to your self managed Kafka cluster @@ -213,7 +212,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, - sourceAccessConfigurations: props.kafkaSecret !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecret.secretArn }] : undefined, + sourceAccessConfigurations: props.kafkaSecretArn !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecretArn }] : undefined, // eslint-disable-next-line max-len selfManagedEventSource: props.kafkaBootstrapServers !== undefined ? { endpoints: { kafkaBootstrapServers: props.kafkaBootstrapServers } } : undefined, }); From ac6a393dc0cc23f94fdf18daf7fcbcdc0cb6fcaa Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 16:24:23 +0100 Subject: [PATCH 14/55] Refactor to use props --- .../aws-lambda-event-sources/README.md | 10 ++- .../aws-lambda-event-sources/lib/dynamodb.ts | 2 +- .../aws-lambda-event-sources/lib/kafka.ts | 83 +++++++++++-------- .../aws-lambda-event-sources/lib/kinesis.ts | 2 +- .../aws-lambda-event-sources/lib/stream.ts | 4 +- .../test/test.kafka.ts | 12 +-- 6 files changed, 65 insertions(+), 48 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index ff37a81ce1c50..f72928b77f3c8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -229,7 +229,10 @@ const topic = 'some-cool-topic' // You still have to make sure that it is associated with your cluster as described in the documentation const secret = new Secret(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); -myFunction.addEventSource(new ManagedKafkaEventSource(clusterArn, topic, secret, { +myFunction.addEventSource(new ManagedKafkaEventSource({ + clusterArn: clusterArn, + topic: topic, + secret: secret, batchSize: 100, // default startingPosition: lambda.StartingPosition.TRIM_HORIZON })); @@ -252,7 +255,10 @@ const topic = 'some-cool-topic' // The secret that allows access to your self hosted Kafka cluster const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); -myFunction.addEventSource(new SelfManagedKafkaEventSource(bootstrapServers, topic, secret, { +myFunction.addEventSource(new SelfManagedKafkaEventSource({ + bootstrapServers: bootstrapServers, + topic: topic, + secret: secret, batchSize: 100, // default startingPosition: lambda.StartingPosition.TRIM_HORIZON })); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts index f316ba69cf9ed..29024d03142c8 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts @@ -9,7 +9,7 @@ export interface DynamoEventSourceProps extends StreamEventSourceProps { /** * Use an Amazon DynamoDB stream as an event source for AWS Lambda. */ -export class DynamoEventSource extends StreamEventSource { +export class DynamoEventSource extends StreamEventSource { private _eventSourceMappingId?: string = undefined; constructor(private readonly table: dynamodb.ITable, props: DynamoEventSourceProps) { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 9a53dd2b36010..509d27678bede 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -3,45 +3,66 @@ import * as lambda from '@aws-cdk/aws-lambda'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; import { StreamEventSource, StreamEventSourceProps } from './stream'; +/** + * Properties for a Kafka event source + */ export interface KafkaEventSourceProps extends StreamEventSourceProps { + /** + * the Kafka topic to subscribe to + */ + readonly topic: string, + /** + * the secret with the Kafka credentials, see https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html for details + */ + readonly secret: secretsmanager.ISecret } /** - * Use a MSK cluster as a streaming source for AWS Lambda + * Properties for a MSK event source + */ +export interface ManagedKafkaEventSourceProps extends KafkaEventSourceProps { + /** + * the ARN of the MSK cluster + */ + readonly clusterArn: string +} + +/** + * Properties for a self managed Kafka cluster event source */ -export class ManagedKafkaEventSource extends StreamEventSource { +export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps { /** - * Create an event source for MSK - * @param clusterArn - the ARN of the MSK cluster - * @param topic - the Kafka topic to subscribe to - * @param secret - the secret with the Kafka credentials, see https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html for details - * @param props + * list of Kafka brokers */ - constructor( - readonly clusterArn: string, - readonly topic: string, - readonly secret: secretsmanager.ISecret, - props: KafkaEventSourceProps) { + readonly bootstrapServers: string[] +} + +/** + * Use a MSK cluster as a streaming source for AWS Lambda + */ +export class ManagedKafkaEventSource extends StreamEventSource { + + constructor(props: ManagedKafkaEventSourceProps) { super(props); } public bind(target: lambda.IFunction) { target.addEventSourceMapping( - `KafkaEventSource:${this.topic}`, + `KafkaEventSource:${this.props.topic}`, this.enrichMappingOptions({ - eventSourceArn: this.clusterArn, + eventSourceArn: this.props.clusterArn, startingPosition: this.props.startingPosition, - kafkaSecretArn: this.secret.secretArn, - kafkaTopic: this.topic, + kafkaSecretArn: this.props.secret.secretArn, + kafkaTopic: this.props.topic, }), ); - this.secret.grantRead(target); + this.props.secret.grantRead(target); target.addToRolePolicy(new iam.PolicyStatement( { actions: ['kafka:DescribeCluster', 'kafka:GetBootstrapBrokers', 'kafka:ListScramSecrets'], - resources: [this.clusterArn], + resources: [this.props.clusterArn], }, )); @@ -52,32 +73,22 @@ export class ManagedKafkaEventSource extends StreamEventSource { /** * Use a self hosted Kafka installation as a streaming source for AWS Lambda. */ -export class SelfManagedKafkaEventSource extends StreamEventSource { - /** - * Create an event source for a self managed Kafka cluster - * @param bootstrapServers - list of Kafka brokers - * @param topic - the Kafka topic to subscribe to - * @param secret - the secret with the Kafka credentials, see https://docs.aws.amazon.com/lambda/latest/dg/smaa-permissions.html#smaa-permissions-add-secret - * @param props - */ - constructor( - readonly bootstrapServers: string[], - readonly topic: string, - readonly secret: secretsmanager.ISecret, - props: KafkaEventSourceProps) { +export class SelfManagedKafkaEventSource extends StreamEventSource { + + constructor(props: SelfManagedKafkaEventSourceProps) { super(props); } public bind(target: lambda.IFunction) { target.addEventSourceMapping( - `KafkaEventSource:${this.topic}`, + `KafkaEventSource:${this.props.topic}`, this.enrichMappingOptions({ - kafkaBootstrapServers: this.bootstrapServers, - kafkaTopic: this.topic, + kafkaBootstrapServers: this.props.bootstrapServers, + kafkaTopic: this.props.topic, startingPosition: this.props.startingPosition, - kafkaSecretArn: this.secret.secretArn, + kafkaSecretArn: this.props.secret.secretArn, }), ); - this.secret.grantRead(target); + this.props.secret.grantRead(target); } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts index f0847429c5a45..9ac96c6efd599 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts @@ -9,7 +9,7 @@ export interface KinesisEventSourceProps extends StreamEventSourceProps { /** * Use an Amazon Kinesis stream as an event source for AWS Lambda. */ -export class KinesisEventSource extends StreamEventSource { +export class KinesisEventSource extends StreamEventSource { private _eventSourceMappingId?: string = undefined; constructor(readonly stream: kinesis.IStream, props: KinesisEventSourceProps) { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 96907b97835fc..955cfd8e641ee 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -89,8 +89,8 @@ export interface StreamEventSourceProps { /** * Use an stream as an event source for AWS Lambda. */ -export abstract class StreamEventSource implements lambda.IEventSource { - protected constructor(protected readonly props: StreamEventSourceProps) { +export abstract class StreamEventSource implements lambda.IEventSource { + protected constructor(protected readonly props: T) { } public abstract bind(_target: lambda.IFunction): void; diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index ed536e647086b..19c46a719d2b7 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -17,10 +17,10 @@ export = { // WHEN fn.addEventSource(new sources.ManagedKafkaEventSource( - clusterArn, - kafkaTopic, - secret, { + clusterArn: clusterArn, + topic: kafkaTopic, + secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, })); @@ -91,10 +91,10 @@ export = { // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( - bootstrapServers, - kafkaTopic, - secret, { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, })); From 08a0ba1326f55009bc2c19c5f8b115912c141691 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Mon, 18 Jan 2021 16:45:33 +0100 Subject: [PATCH 15/55] Remove doc error whitelisting --- packages/@aws-cdk/aws-lambda-event-sources/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index cccb7ae5e5e56..6eb8db3da7c26 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -117,8 +117,7 @@ "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KinesisEventSourceProps", "docs-public-apis:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps", "props-default-doc:@aws-cdk/aws-lambda-event-sources.S3EventSourceProps.filters", - "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps", - "docs-public-apis:@aws-cdk/aws-lambda-event-sources.KafkaEventSourceProps" + "docs-public-apis:@aws-cdk/aws-lambda-event-sources.SqsEventSourceProps" ] }, "awscdkio": { From 979414f129a77521744650e78336e449e5fe2c7e Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 19 Jan 2021 08:54:36 +0100 Subject: [PATCH 16/55] It needs to be SASL_SCRAM_512_AUTH --- packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts | 4 ++-- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 19c46a719d2b7..3f8fd733e3bb3 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -70,7 +70,7 @@ export = { ], SourceAccessConfigurations: [ { - Type: 'BASIC_AUTH', + Type: 'SASL_SCRAM_512_AUTH', URI: { Ref: 'SecretA720EF05', }, @@ -141,7 +141,7 @@ export = { ], SourceAccessConfigurations: [ { - Type: 'BASIC_AUTH', + Type: 'SASL_SCRAM_512_AUTH', URI: { Ref: 'SecretA720EF05', }, diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 2d4f3426ad3b3..cfe2e5aad3742 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -212,7 +212,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, - sourceAccessConfigurations: props.kafkaSecretArn !== undefined ? [{ type: 'BASIC_AUTH', uri: props.kafkaSecretArn }] : undefined, + sourceAccessConfigurations: props.kafkaSecretArn !== undefined ? [{ type: 'SASL_SCRAM_512_AUTH', uri: props.kafkaSecretArn }] : undefined, // eslint-disable-next-line max-len selfManagedEventSource: props.kafkaBootstrapServers !== undefined ? { endpoints: { kafkaBootstrapServers: props.kafkaBootstrapServers } } : undefined, }); From 2e06917f9228ad1d270b6579c4609d7c6711c57a Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 19 Jan 2021 12:06:08 +0100 Subject: [PATCH 17/55] Change EventSourceMappingOptions --- .../aws-lambda-event-sources/lib/kafka.ts | 7 ++-- .../test/test.kafka.ts | 6 +-- .../aws-lambda/lib/event-source-mapping.ts | 40 +++++++++++++++---- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 509d27678bede..9d1b665077857 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -52,7 +52,7 @@ export class ManagedKafkaEventSource extends StreamEventSource Date: Tue, 19 Jan 2021 15:33:14 +0100 Subject: [PATCH 18/55] Add support for self managed Kafka running in VPC --- .../aws-lambda-event-sources/lib/kafka.ts | 28 +++++++++++++++++-- .../aws-lambda-event-sources/package.json | 2 ++ .../test/test.kafka.ts | 13 +++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 9d1b665077857..48912172f1fc2 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -1,3 +1,4 @@ +import { ISecurityGroup, ISubnet } from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; @@ -28,13 +29,28 @@ export interface ManagedKafkaEventSourceProps extends KafkaEventSourceProps { } /** - * Properties for a self managed Kafka cluster event source + * Properties for a self managed Kafka cluster event source. + * If your Kafka cluster is only reachable via VPC make sure to configure it. */ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps { /** * list of Kafka brokers */ readonly bootstrapServers: string[] + + /** + * If your Kafka brokers are only reachable via VPC, provide the subnets here + * + * @default - none + */ + readonly subnets?: ISubnet[], + + /** + * If your Kafka brokers are only reachable via VPC, provide the security group here + * + * @default - none + */ + readonly securityGroup?: ISecurityGroup } /** @@ -80,14 +96,20 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { + sourceAccessConfigurations.push({ type: 'VPC_SUBNET', uri: subnet.subnetId }); + }); + } target.addEventSourceMapping( `KafkaEventSource:${this.props.topic}`, this.enrichMappingOptions({ selfManagedEventSource: { endpoints: { kafkaBootstrapServers: this.props.bootstrapServers } }, kafkaTopic: this.props.topic, startingPosition: this.props.startingPosition, - // TODO: make auth type configurable, add vpc config - sourceAccessConfigurations: [{ type: 'SASL_SCRAM_512_AUTH', uri: this.props.secret.secretArn }], + sourceAccessConfigurations, }), ); this.props.secret.grantRead(target); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 6eb8db3da7c26..7a9ab7ada2120 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -72,6 +72,7 @@ "dependencies": { "@aws-cdk/aws-apigateway": "0.0.0", "@aws-cdk/aws-dynamodb": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-events": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", @@ -89,6 +90,7 @@ "peerDependencies": { "@aws-cdk/aws-apigateway": "0.0.0", "@aws-cdk/aws-dynamodb": "0.0.0", + "@aws-cdk/aws-ec2": "0.0.0", "@aws-cdk/aws-events": "0.0.0", "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 7676647cbfe33..c936aa2bc7da4 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -1,4 +1,5 @@ import { expect, haveResource } from '@aws-cdk/assert'; +import { SecurityGroup, Subnet } from '@aws-cdk/aws-ec2'; import * as lambda from '@aws-cdk/aws-lambda'; import { Secret } from '@aws-cdk/aws-secretsmanager'; import * as cdk from '@aws-cdk/core'; @@ -88,6 +89,8 @@ export = { const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); const bootstrapServers = ['kafka-broker:9092']; + const subnet = Subnet.fromSubnetId(stack, 'Subnet', 'subnet-0011001100'); + const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( @@ -96,6 +99,8 @@ export = { topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, + subnets: [subnet], + securityGroup: sg, })); // THEN @@ -144,6 +149,14 @@ export = { Ref: 'SecretA720EF05', }, }, + { + Type: 'VPC_SECURITY_GROUP', + URI: 'sg-0123456789', + }, + { + Type: 'VPC_SUBNET', + URI: 'subnet-0011001100', + }, ], })); From 036174b14e1dfd08fd20109a8991d440914ee7c9 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 19 Jan 2021 16:03:01 +0100 Subject: [PATCH 19/55] Add test and error handling for VPC stuff --- .../aws-lambda-event-sources/README.md | 2 ++ .../aws-lambda-event-sources/lib/kafka.ts | 3 +++ .../test/test.kafka.ts | 22 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index f72928b77f3c8..236f7230fa023 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -264,6 +264,8 @@ myFunction.addEventSource(new SelfManagedKafkaEventSource({ })); ``` +If your self managed Kafka cluster is only reachable via VPC also configure `subnets` and `securityGroup`. + ## Roadmap Eventually, this module will support all the event sources described under diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 48912172f1fc2..cfaec92696008 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -93,6 +93,9 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + subnets: [subnet], + })); + }, /both subnets and securityGroup must be set/); + + test.done(); + }, + } From f6516366b543f4b92e4c2722118d7865092ffbca Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 19 Jan 2021 16:39:53 +0100 Subject: [PATCH 20/55] Fix error --- .../aws-lambda/lib/event-source-mapping.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index e1bfa62dd590d..2048d5777d17f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -19,14 +19,24 @@ export interface SourceAccessConfiguration { readonly uri: string } +/** + * The endpoints for your self managed event source + */ +export interface Endpoints { + /** + * A list of Kafka bootstrap servers in the format HOST:PORT, e.g. 'kafka-broker01:9096' + */ + kafkaBootstrapServers: string[] +} + /** * The configuration for your self managed event source, currently only Kafka is supported */ export interface SelfManagedEventSource { /** - * A list of server endpoints for your self managed event source + * The endpoints for your self managed event source */ - readonly endpoints: {kafkaBootstrapServers: string[]} + readonly endpoints: Endpoints } export interface EventSourceMappingOptions { From 315d56fc007d157f97c6e52d324c803475a184ab Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 19 Jan 2021 16:41:59 +0100 Subject: [PATCH 21/55] Unique identifier --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index cfaec92696008..110e04f4d0640 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -1,3 +1,4 @@ +import * as crypto from 'crypto'; import { ISecurityGroup, ISubnet } from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; @@ -64,7 +65,7 @@ export class ManagedKafkaEventSource extends StreamEventSource Date: Tue, 19 Jan 2021 17:17:06 +0100 Subject: [PATCH 22/55] Fix lambda error and rename --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 2048d5777d17f..d80ca2dd7cd9f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -22,11 +22,11 @@ export interface SourceAccessConfiguration { /** * The endpoints for your self managed event source */ -export interface Endpoints { +export interface SelfManagedEventSourceEndpoints { /** * A list of Kafka bootstrap servers in the format HOST:PORT, e.g. 'kafka-broker01:9096' */ - kafkaBootstrapServers: string[] + readonly kafkaBootstrapServers: string[] } /** @@ -36,7 +36,7 @@ export interface SelfManagedEventSource { /** * The endpoints for your self managed event source */ - readonly endpoints: Endpoints + readonly endpoints: SelfManagedEventSourceEndpoints } export interface EventSourceMappingOptions { From 9199f49130ba8b2f7d515bff40b532503ca2338a Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 11 Feb 2021 20:10:23 +0100 Subject: [PATCH 23/55] Flatten to kafkaBootstrapServers - Add tests - Better docs --- .../aws-lambda-event-sources/lib/kafka.ts | 2 +- .../aws-lambda/lib/event-source-mapping.ts | 53 ++++++++++--------- .../test/event-source-mapping.test.ts | 42 +++++++++++++++ 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 110e04f4d0640..c64457ccb809e 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -111,7 +111,7 @@ export class SelfManagedKafkaEventSource extends StreamEventSource 300) { throw new Error(`maxBatchingWindow cannot be over 300 seconds, got ${props.maxBatchingWindow.toSeconds()}`); } @@ -234,6 +228,15 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp }; } + let selfManagedEventSource; + if (props.kafkaBootstrapServers) { + selfManagedEventSource = { + endpoints: { + kafkaBootstrapServers: props.kafkaBootstrapServers, + }, + }; + } + const cfnEventSourceMapping = new CfnEventSourceMapping(this, 'Resource', { batchSize: props.batchSize, bisectBatchOnFunctionError: props.bisectBatchOnError, @@ -248,7 +251,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, sourceAccessConfigurations: props.sourceAccessConfigurations, - selfManagedEventSource: props.selfManagedEventSource, + selfManagedEventSource, }); this.eventSourceMappingId = cfnEventSourceMapping.ref; } diff --git a/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts b/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts index be42067f263f1..549433ad6f5f4 100644 --- a/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts +++ b/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts @@ -170,4 +170,46 @@ describe('event source mapping', () => { }], }); }); + + test('throws if neither eventSourceArn nor kafkaBootstrapServers are set', () => { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline('exports.handler = ${handler.toString()}'), + runtime: Runtime.NODEJS_10_X, + }); + + expect(() => new EventSourceMapping(stack, 'test', { + target: fn, + })).toThrow(/Either eventSourceArn or kafkaBootstrapServers must be set/); + }); + + test('throws if both eventSourceArn and kafkaBootstrapServers are set', () => { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline('exports.handler = ${handler.toString()}'), + runtime: Runtime.NODEJS_10_X, + }); + + expect(() => new EventSourceMapping(stack, 'test', { + eventSourceArn: '', + kafkaBootstrapServers: [], + target: fn, + })).toThrow(/eventSourceArn and kafkaBootstrapServers are mutually exclusive/); + }); + + test('throws if both kafkaBootstrapServers is set but empty', () => { + const stack = new cdk.Stack(); + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline('exports.handler = ${handler.toString()}'), + runtime: Runtime.NODEJS_10_X, + }); + + expect(() => new EventSourceMapping(stack, 'test', { + kafkaBootstrapServers: [], + target: fn, + })).toThrow(/kafkaBootStrapServers must not be empty if set/); + }); }); From 265f98f6e0d73f6e7b56d3988350d6795ab99059 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 11 Feb 2021 20:52:49 +0100 Subject: [PATCH 24/55] Introduce SourceAccessConfigurationType --- .../aws-lambda-event-sources/lib/kafka.ts | 12 +++-- .../aws-lambda/lib/event-source-mapping.ts | 44 +++++++++++++++++-- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index c64457ccb809e..90daa34e47d60 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -69,7 +69,7 @@ export class ManagedKafkaEventSource extends StreamEventSource { - sourceAccessConfigurations.push({ type: 'VPC_SUBNET', uri: subnet.subnetId }); + sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: subnet.subnetId }); }); } const idHash = crypto.createHash('md5').update(JSON.stringify(this.props.bootstrapServers)).digest('hex'); diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 5808b60cd9f76..76fa393715051 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -4,15 +4,53 @@ import { IEventSourceDlq } from './dlq'; import { IFunction } from './function-base'; import { CfnEventSourceMapping } from './lambda.generated'; +/** + * The type of authentication protocol or the VPC components for your event source's SourceAccessConfiguration + * @see https://docs.aws.amazon.com/lambda/latest/dg/API_SourceAccessConfiguration.html#SSS-Type-SourceAccessConfiguration-Type + */ +export class SourceAccessConfigurationType { + + /** + * (MQ) The Secrets Manager secret that stores your broker credentials. + */ + public static readonly BASIC_AUTH = new SourceAccessConfigurationType('BASIC_AUTH'); + + /** + * The subnets associated with your VPC. Lambda connects to these subnets to fetch data from your Self-Managed Apache Kafka cluster. + */ + public static readonly VPC_SUBNET = new SourceAccessConfigurationType('VPC_SUBNET'); + + /** + * The VPC security group used to manage access to your Self-Managed Apache Kafka brokers. + */ + public static readonly VPC_SECURITY_GROUP = new SourceAccessConfigurationType('VPC_SECURITY_GROUP'); + + /** + * The Secrets Manager ARN of your secret key used for SASL SCRAM-256 authentication of your Self-Managed Apache Kafka brokers. + */ + public static readonly SASL_SCRAM_256_AUTH = new SourceAccessConfigurationType('SASL_SCRAM_256_AUTH'); + + /** + * The Secrets Manager ARN of your secret key used for SASL SCRAM-512 authentication of your Self-Managed Apache Kafka brokers. + */ + public static readonly SASL_SCRAM_512_AUTH = new SourceAccessConfigurationType('SASL_SCRAM_512_AUTH'); + + /** The key to use in `CfnEventSourceMapping.SourceAccessConfigurationProperty.Type` */ + public readonly type: string; + + private constructor(type: string) { + this.type = type; + } +} + /** * Specific settings like the authentication protocol or the VPC components to secure access to your event source. */ export interface SourceAccessConfiguration { /** * The type of authentication protocol or the VPC components for your event source. For example: "SASL_SCRAM_512_AUTH". - * Valid values are: BASIC_AUTH | VPC_SUBNET | VPC_SECURITY_GROUP | SASL_SCRAM_512_AUTH | SASL_SCRAM_256_AUTH */ - readonly type: string, + readonly type: SourceAccessConfigurationType, /** * The value for your chosen configuration in Type. For example: "URI": "arn:aws:secretsmanager:us-east-1:01234567890:secret:MyBrokerSecretName". */ @@ -250,7 +288,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp maximumRetryAttempts: props.retryAttempts, parallelizationFactor: props.parallelizationFactor, topics: props.kafkaTopic !== undefined ? [props.kafkaTopic] : undefined, - sourceAccessConfigurations: props.sourceAccessConfigurations, + sourceAccessConfigurations: props.sourceAccessConfigurations?.map((o) => {return { type: o.type.type, uri: o.uri };}), selfManagedEventSource, }); this.eventSourceMappingId = cfnEventSourceMapping.ref; From dc8ec24fb444eee8c1f668013b31b649d50c5180 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 11 Feb 2021 21:15:51 +0100 Subject: [PATCH 25/55] Make authenticationMethod configurable --- .../aws-lambda-event-sources/lib/kafka.ts | 15 ++++- .../test/test.kafka.ts | 57 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 90daa34e47d60..99cc19e081976 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -52,6 +52,13 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps * @default - none */ readonly securityGroup?: ISecurityGroup + + /** + * The authentication method for your Kafka cluster + * + * @default - SASL_SCRAM_512_AUTH + */ + readonly authenticationMethod?: 'SASL_SCRAM_512_AUTH' | 'SASL_SCRAM_256_AUTH' } /** @@ -100,7 +107,13 @@ export class SelfManagedKafkaEventSource extends StreamEventSource Date: Thu, 11 Feb 2021 21:20:11 +0100 Subject: [PATCH 26/55] Take care of clusterArn being a token --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 99cc19e081976..7c905a8a36bec 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -72,7 +72,7 @@ export class ManagedKafkaEventSource extends StreamEventSource Date: Thu, 11 Feb 2021 21:26:15 +0100 Subject: [PATCH 27/55] Docs --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 76fa393715051..d588332a6f05a 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -52,7 +52,10 @@ export interface SourceAccessConfiguration { */ readonly type: SourceAccessConfigurationType, /** - * The value for your chosen configuration in Type. For example: "URI": "arn:aws:secretsmanager:us-east-1:01234567890:secret:MyBrokerSecretName". + * The value for your chosen configuration in type. + * For example: "URI": "arn:aws:secretsmanager:us-east-1:01234567890:secret:MyBrokerSecretName". + * The exact string depends on the type. + * @see SourceAccessConfigurationType */ readonly uri: string } From 4278cb9d80924532346b09bab1d656637e8ad6ce Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 11 Feb 2021 21:52:34 +0100 Subject: [PATCH 28/55] Refactor to take vpc and subnet selection --- .../aws-lambda-event-sources/README.md | 41 ++++++++++++++++++- .../aws-lambda-event-sources/lib/kafka.ts | 22 ++++++---- .../test/test.kafka.ts | 37 +++++++++++++---- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 236f7230fa023..3a5e2c4f97085 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -264,7 +264,46 @@ myFunction.addEventSource(new SelfManagedKafkaEventSource({ })); ``` -If your self managed Kafka cluster is only reachable via VPC also configure `subnets` and `securityGroup`. +If your self managed Kafka cluster is only reachable via VPC also configure `vpc` `vpcSubnets` and `securityGroup`. + +```ts +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as lambda from '@aws-cdk/aws-lambda'; +import { Secret } from '@aws-cdk/aws-secretmanager'; +import { SelfManagedKafkaEventSource } from '@aws-cdk/aws-lambda-event-sources'; + +// The list of Kafka brokers +const bootstrapServers = ['kafka-broker:9092'] + +// The Kafka topic you want to subscribe to +const topic = 'some-cool-topic' + +// The secret that allows access to your self hosted Kafka cluster +const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + +// Your VPC: +const vpc = ec2.Vpc.fromLookup(stack, 'VPC', { + isDefault: true, +}); + +// Your VPC subnet selection +const vpcSubnets = { subnetType: ec2.SubnetType.PRIVATE }; + +// Your security group +const sg = ec2.SecurityGroup.fromSecurityGroupId(this, 'SG', 'sg-12345'); + +myFunction.addEventSource(new SelfManagedKafkaEventSource({ + bootstrapServers: bootstrapServers, + topic: topic, + secret: secret, + batchSize: 100, // default + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + vpcSubnets: vpcSubnets, + securityGroup: sg, +})); +``` + ## Roadmap diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 7c905a8a36bec..d1cfbbc3d9a6a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -1,5 +1,5 @@ import * as crypto from 'crypto'; -import { ISecurityGroup, ISubnet } from '@aws-cdk/aws-ec2'; +import { ISecurityGroup, IVpc, SubnetSelection } from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; @@ -40,11 +40,18 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps readonly bootstrapServers: string[] /** - * If your Kafka brokers are only reachable via VPC, provide the subnets here + * If your Kafka brokers are only reachable via VPC provide the VPC here + * + * @default none + */ + readonly vpc?: IVpc; + + /** + * If your Kafka brokers are only reachable via VPC, provide the subnets selection here * * @default - none */ - readonly subnets?: ISubnet[], + readonly vpcSubnets?: SubnetSelection, /** * If your Kafka brokers are only reachable via VPC, provide the security group here @@ -101,7 +108,8 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { - sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: subnet.subnetId }); + this.props.vpc?.selectSubnets(this.props.vpcSubnets).subnetIds.forEach((id) => { + sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: id }); }); } const idHash = crypto.createHash('md5').update(JSON.stringify(this.props.bootstrapServers)).digest('hex'); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index a533eabd66385..85450fc46c0f1 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -1,5 +1,5 @@ import { expect, haveResource } from '@aws-cdk/assert'; -import { SecurityGroup, Subnet } from '@aws-cdk/aws-ec2'; +import { SecurityGroup, SubnetType, Vpc } from '@aws-cdk/aws-ec2'; import * as lambda from '@aws-cdk/aws-lambda'; import { Secret } from '@aws-cdk/aws-secretsmanager'; import * as cdk from '@aws-cdk/core'; @@ -89,8 +89,8 @@ export = { const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); const bootstrapServers = ['kafka-broker:9092']; - const subnet = Subnet.fromSubnetId(stack, 'Subnet', 'subnet-0011001100'); const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); + const vpc = new Vpc(stack, 'Vpc'); // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( @@ -99,7 +99,8 @@ export = { topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, - subnets: [subnet], + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, securityGroup: sg, })); @@ -155,7 +156,15 @@ export = { }, { Type: 'VPC_SUBNET', - URI: 'subnet-0011001100', + URI: { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + }, + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, }, ], })); @@ -169,7 +178,7 @@ export = { const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); const bootstrapServers = ['kafka-broker:9092']; - const subnet = Subnet.fromSubnetId(stack, 'Subnet', 'subnet-0011001100'); + const vpc = new Vpc(stack, 'Vpc'); test.throws(() => { fn.addEventSource(new sources.SelfManagedKafkaEventSource( @@ -178,7 +187,8 @@ export = { topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, - subnets: [subnet], + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, })); }, /both subnets and securityGroup must be set/); @@ -192,8 +202,8 @@ export = { const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); const bootstrapServers = ['kafka-broker:9092']; - const subnet = Subnet.fromSubnetId(stack, 'Subnet', 'subnet-0011001100'); const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); + const vpc = new Vpc(stack, 'Vpc'); // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( @@ -202,7 +212,8 @@ export = { topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, - subnets: [subnet], + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, securityGroup: sg, authenticationMethod: 'SASL_SCRAM_256_AUTH', })); @@ -234,7 +245,15 @@ export = { }, { Type: 'VPC_SUBNET', - URI: 'subnet-0011001100', + URI: { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + }, + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, }, ], })); From a549a6b600e5009d9bcd09c6310b14c402536354 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Fri, 12 Feb 2021 12:12:55 +0100 Subject: [PATCH 29/55] Get rid of generics --- .../aws-lambda-event-sources/lib/dynamodb.ts | 2 +- .../aws-lambda-event-sources/lib/kafka.ts | 46 +++++++++++-------- .../aws-lambda-event-sources/lib/kinesis.ts | 2 +- .../aws-lambda-event-sources/lib/stream.ts | 4 +- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts index 29024d03142c8..f316ba69cf9ed 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/dynamodb.ts @@ -9,7 +9,7 @@ export interface DynamoEventSourceProps extends StreamEventSourceProps { /** * Use an Amazon DynamoDB stream as an event source for AWS Lambda. */ -export class DynamoEventSource extends StreamEventSource { +export class DynamoEventSource extends StreamEventSource { private _eventSourceMappingId?: string = undefined; constructor(private readonly table: dynamodb.ITable, props: DynamoEventSourceProps) { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index d1cfbbc3d9a6a..6a4efa22cfffb 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -71,29 +71,32 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps /** * Use a MSK cluster as a streaming source for AWS Lambda */ -export class ManagedKafkaEventSource extends StreamEventSource { +export class ManagedKafkaEventSource extends StreamEventSource { + // This is to work around JSII inheritance problems + private innerProps: ManagedKafkaEventSourceProps; constructor(props: ManagedKafkaEventSourceProps) { super(props); + this.innerProps = props; } public bind(target: lambda.IFunction) { target.addEventSourceMapping( - `KafkaEventSource:${this.props.clusterArn}${this.props.topic}`, + `KafkaEventSource:${this.innerProps.clusterArn}${this.innerProps.topic}`, this.enrichMappingOptions({ - eventSourceArn: this.props.clusterArn, - startingPosition: this.props.startingPosition, - sourceAccessConfigurations: [{ type: lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH, uri: this.props.secret.secretArn }], - kafkaTopic: this.props.topic, + eventSourceArn: this.innerProps.clusterArn, + startingPosition: this.innerProps.startingPosition, + sourceAccessConfigurations: [{ type: lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH, uri: this.innerProps.secret.secretArn }], + kafkaTopic: this.innerProps.topic, }), ); - this.props.secret.grantRead(target); + this.innerProps.secret.grantRead(target); target.addToRolePolicy(new iam.PolicyStatement( { actions: ['kafka:DescribeCluster', 'kafka:GetBootstrapBrokers', 'kafka:ListScramSecrets'], - resources: [this.props.clusterArn], + resources: [this.innerProps.clusterArn], }, )); @@ -104,7 +107,9 @@ export class ManagedKafkaEventSource extends StreamEventSource { +export class SelfManagedKafkaEventSource extends StreamEventSource { + // This is to work around JSII inheritance problems + private innerProps: SelfManagedKafkaEventSourceProps; constructor(props: SelfManagedKafkaEventSourceProps) { super(props); @@ -112,36 +117,37 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { + this.innerProps.vpc?.selectSubnets(this.innerProps.vpcSubnets).subnetIds.forEach((id) => { sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: id }); }); } - const idHash = crypto.createHash('md5').update(JSON.stringify(this.props.bootstrapServers)).digest('hex'); + const idHash = crypto.createHash('md5').update(JSON.stringify(this.innerProps.bootstrapServers)).digest('hex'); target.addEventSourceMapping( - `KafkaEventSource:${idHash}:${this.props.topic}`, + `KafkaEventSource:${idHash}:${this.innerProps.topic}`, this.enrichMappingOptions({ - kafkaBootstrapServers: this.props.bootstrapServers, - kafkaTopic: this.props.topic, - startingPosition: this.props.startingPosition, + kafkaBootstrapServers: this.innerProps.bootstrapServers, + kafkaTopic: this.innerProps.topic, + startingPosition: this.innerProps.startingPosition, sourceAccessConfigurations, }), ); - this.props.secret.grantRead(target); + this.innerProps.secret.grantRead(target); } } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts index 9ac96c6efd599..f0847429c5a45 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kinesis.ts @@ -9,7 +9,7 @@ export interface KinesisEventSourceProps extends StreamEventSourceProps { /** * Use an Amazon Kinesis stream as an event source for AWS Lambda. */ -export class KinesisEventSource extends StreamEventSource { +export class KinesisEventSource extends StreamEventSource { private _eventSourceMappingId?: string = undefined; constructor(readonly stream: kinesis.IStream, props: KinesisEventSourceProps) { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts index 955cfd8e641ee..96907b97835fc 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/stream.ts @@ -89,8 +89,8 @@ export interface StreamEventSourceProps { /** * Use an stream as an event source for AWS Lambda. */ -export abstract class StreamEventSource implements lambda.IEventSource { - protected constructor(protected readonly props: T) { +export abstract class StreamEventSource implements lambda.IEventSource { + protected constructor(protected readonly props: StreamEventSourceProps) { } public abstract bind(_target: lambda.IFunction): void; From 3cb7bb61433bcb3a3e933f8e2dc0a89c9c92faa3 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:28:10 +0100 Subject: [PATCH 30/55] Update packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index d588332a6f05a..951df979a48b5 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -65,7 +65,7 @@ export interface EventSourceMappingOptions { * The Amazon Resource Name (ARN) of the event source. Any record added to * this stream can invoke the Lambda function. * - * @default - not set if using a self managed Kafka cluster + * @default - not set if using a self managed Kafka cluster, throws an error otherwise */ readonly eventSourceArn?: string; @@ -315,4 +315,3 @@ export enum StartingPosition { */ LATEST = 'LATEST', } - From 80fd8d5a7864a6994e5fdeda56c4c9b8be407a9c Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:28:27 +0100 Subject: [PATCH 31/55] Update packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts Co-authored-by: Niranjan Jayakar --- .../test/test.kafka.ts | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 85450fc46c0f1..db5fc49aa971d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -218,44 +218,15 @@ export = { authenticationMethod: 'SASL_SCRAM_256_AUTH', })); - expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - FunctionName: { - Ref: 'Fn9270CBC0', - }, - BatchSize: 100, - SelfManagedEventSource: { - Endpoints: { - KafkaBootstrapServers: bootstrapServers, - }, - }, - StartingPosition: 'TRIM_HORIZON', - Topics: [ - kafkaTopic, - ], - SourceAccessConfigurations: [ + expect(stack).to(haveResourceLike('AWS::Lambda::EventSourceMapping', { + SourceAccessConfigurations: arrayWith( { Type: 'SASL_SCRAM_256_AUTH', URI: { Ref: 'SecretA720EF05', }, }, - { - Type: 'VPC_SECURITY_GROUP', - URI: 'sg-0123456789', - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet1Subnet536B997A', - }, - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet2Subnet3788AAA1', - }, - }, - ], + ), })); test.done(); From 407190d23d24f3cc1945c83f5016e7aabc971fa3 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:30:27 +0100 Subject: [PATCH 32/55] Update packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 6a4efa22cfffb..09c31d718226a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -35,7 +35,8 @@ export interface ManagedKafkaEventSourceProps extends KafkaEventSourceProps { */ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps { /** - * list of Kafka brokers + * The list of host and port pairs that are the addresses of the Kafka brokers in a "bootstrap" Kafka cluster that + * a Kafka client connects to initially to bootstrap itself. They are in the format `abc.xyz.com:xxxx`. */ readonly bootstrapServers: string[] From 91064e77b91a01c6c09a0129d81bb952396956fc Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:30:51 +0100 Subject: [PATCH 33/55] Update packages/@aws-cdk/aws-lambda-event-sources/README.md Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 3a5e2c4f97085..4681db95fd23c 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -282,15 +282,13 @@ const topic = 'some-cool-topic' const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); // Your VPC: -const vpc = ec2.Vpc.fromLookup(stack, 'VPC', { - isDefault: true, -}); +const vpc = new ec2.Vpc(stack, 'VPC', { ... }); // Your VPC subnet selection -const vpcSubnets = { subnetType: ec2.SubnetType.PRIVATE }; +const vpcSubnets: ec2.SubnetSelection = { subnetType: ec2.SubnetType.PRIVATE }; // Your security group -const sg = ec2.SecurityGroup.fromSecurityGroupId(this, 'SG', 'sg-12345'); +const sg = new ec2.SecurityGroup(this, 'SG', { ... }); myFunction.addEventSource(new SelfManagedKafkaEventSource({ bootstrapServers: bootstrapServers, From fb47c96115b73468e60317ad827d34b5c0a83564 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:31:02 +0100 Subject: [PATCH 34/55] Update packages/@aws-cdk/aws-lambda-event-sources/README.md Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 4681db95fd23c..a5f504db7f63d 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -279,7 +279,7 @@ const bootstrapServers = ['kafka-broker:9092'] const topic = 'some-cool-topic' // The secret that allows access to your self hosted Kafka cluster -const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); +const secret = new Secret(this, 'Secret', { ... }); // Your VPC: const vpc = new ec2.Vpc(stack, 'VPC', { ... }); From ba3ebde3e5699cac976e9f351081c4b27dcda914 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:25:49 +0100 Subject: [PATCH 35/55] Fix indentation --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 951df979a48b5..a3a970add1b6f 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -271,11 +271,7 @@ export class EventSourceMapping extends cdk.Resource implements IEventSourceMapp let selfManagedEventSource; if (props.kafkaBootstrapServers) { - selfManagedEventSource = { - endpoints: { - kafkaBootstrapServers: props.kafkaBootstrapServers, - }, - }; + selfManagedEventSource = { endpoints: { kafkaBootstrapServers: props.kafkaBootstrapServers } }; } const cfnEventSourceMapping = new CfnEventSourceMapping(this, 'Resource', { From 1f47f7772dff0dd1610253ec6f795a0f0fb7d0c0 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 17:39:33 +0100 Subject: [PATCH 36/55] Add static method to create custom variants --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index a3a970add1b6f..3fd4d519efc3d 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -35,6 +35,11 @@ export class SourceAccessConfigurationType { */ public static readonly SASL_SCRAM_512_AUTH = new SourceAccessConfigurationType('SASL_SCRAM_512_AUTH'); + /** A custom source access configuration property */ + public static of(name: string): SourceAccessConfigurationType { + return new SourceAccessConfigurationType(name); + } + /** The key to use in `CfnEventSourceMapping.SourceAccessConfigurationProperty.Type` */ public readonly type: string; From 859c9d9856cd5287d86ccb3c4f83c9af79b429b6 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 21:16:12 +0100 Subject: [PATCH 37/55] Introduce AuthenticationMethod enum --- .../aws-lambda-event-sources/lib/kafka.ts | 28 +++++++++++++++---- .../test/test.kafka.ts | 6 ++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 09c31d718226a..2572e6a2390c4 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -29,6 +29,20 @@ export interface ManagedKafkaEventSourceProps extends KafkaEventSourceProps { readonly clusterArn: string } +/** + * The authentication method to use with SelfManagedKafkaEventSource + */ +export enum AuthenticationMethod { + /** + * SASL_SCRAM_512_AUTH authentication method for your Kafka cluster + */ + SASL_SCRAM_512_AUTH = 'SASL_SCRAM_512_AUTH', + /** + * SASL_SCRAM_256_AUTH authentication method for your Kafka cluster + */ + SASL_SCRAM_256_AUTH = 'SASL_SCRAM_512_AUTH', +} + /** * Properties for a self managed Kafka cluster event source. * If your Kafka cluster is only reachable via VPC make sure to configure it. @@ -66,7 +80,7 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps * * @default - SASL_SCRAM_512_AUTH */ - readonly authenticationMethod?: 'SASL_SCRAM_512_AUTH' | 'SASL_SCRAM_256_AUTH' + readonly authenticationMethod?: AuthenticationMethod } /** @@ -123,10 +137,14 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { public bind(target: lambda.IFunction) { let authenticationMethod; - if (this.innerProps.authenticationMethod == undefined || this.innerProps.authenticationMethod == 'SASL_SCRAM_512_AUTH') { - authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH; - } else { - authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_256_AUTH; + switch (this.innerProps.authenticationMethod) { + case AuthenticationMethod.SASL_SCRAM_256_AUTH: + authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_256_AUTH; + break; + case AuthenticationMethod.SASL_SCRAM_512_AUTH: + default: + authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH; + break; } let sourceAccessConfigurations = [{ type: authenticationMethod, uri: this.innerProps.secret.secretArn }]; if (this.innerProps.vpcSubnets !== undefined && this.innerProps.securityGroup !== undefined) { diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index db5fc49aa971d..9ddd6058d62d7 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -1,4 +1,4 @@ -import { expect, haveResource } from '@aws-cdk/assert'; +import { arrayWith, expect, haveResource } from '@aws-cdk/assert'; import { SecurityGroup, SubnetType, Vpc } from '@aws-cdk/aws-ec2'; import * as lambda from '@aws-cdk/aws-lambda'; import { Secret } from '@aws-cdk/aws-secretsmanager'; @@ -215,10 +215,10 @@ export = { vpc: vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE }, securityGroup: sg, - authenticationMethod: 'SASL_SCRAM_256_AUTH', + authenticationMethod: sources.AuthenticationMethod.SASL_SCRAM_256_AUTH, })); - expect(stack).to(haveResourceLike('AWS::Lambda::EventSourceMapping', { + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { SourceAccessConfigurations: arrayWith( { Type: 'SASL_SCRAM_256_AUTH', From b7d6b1a0f691123a16a542fc9b1dcb35b206b4fa Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 21:26:54 +0100 Subject: [PATCH 38/55] Introduce ICluster/Cluster in aws-msk --- packages/@aws-cdk/aws-msk/README.md | 8 ++++++ packages/@aws-cdk/aws-msk/lib/cluster.ts | 34 ++++++++++++++++++++++++ packages/@aws-cdk/aws-msk/lib/index.ts | 1 + packages/@aws-cdk/aws-msk/package.json | 2 +- 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 packages/@aws-cdk/aws-msk/lib/cluster.ts diff --git a/packages/@aws-cdk/aws-msk/README.md b/packages/@aws-cdk/aws-msk/README.md index 1de05861fc74f..51d93453f1eef 100644 --- a/packages/@aws-cdk/aws-msk/README.md +++ b/packages/@aws-cdk/aws-msk/README.md @@ -9,6 +9,14 @@ > > [CFN Resources]: https://docs.aws.amazon.com/cdk/latest/guide/constructs.html#constructs_lib +![cdk-constructs: Experimental](https://img.shields.io/badge/cdk--constructs-experimental-important.svg?style=for-the-badge) + +> The APIs of higher level constructs in this module are experimental and under active development. +> They are subject to non-backward compatible changes or removal in any future version. These are +> not subject to the [Semantic Versioning](https://semver.org/) model and breaking changes will be +> announced in the release notes. This means that while you may use them, you may need to update +> your source code when upgrading to a newer version of this package. + --- diff --git a/packages/@aws-cdk/aws-msk/lib/cluster.ts b/packages/@aws-cdk/aws-msk/lib/cluster.ts new file mode 100644 index 0000000000000..313bd3de5b107 --- /dev/null +++ b/packages/@aws-cdk/aws-msk/lib/cluster.ts @@ -0,0 +1,34 @@ +import { IResource, Resource } from '@aws-cdk/core'; +import { Construct } from 'constructs'; + +/** + * Represents an MSK cluster + */ +export interface ICluster extends IResource { + /** + * the ARN of the MSK cluster + */ + readonly clusterArn: string; +} + +/** + * An MSK cluster + */ +export class Cluster { + /** + * Creates a Cluster construct that represents an existing MSK cluster. + * @param scope + * @param id + * @param clusterArn + */ + public static fromClusterArn(scope: Construct, id: string, clusterArn: string): ICluster { + class Imported extends Resource implements ICluster { + public readonly clusterArn: string; + constructor() { + super(scope, id); + this.clusterArn = clusterArn; + } + } + return new Imported(); + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-msk/lib/index.ts b/packages/@aws-cdk/aws-msk/lib/index.ts index 3cf150cbd7076..9cc4acdc7d61f 100644 --- a/packages/@aws-cdk/aws-msk/lib/index.ts +++ b/packages/@aws-cdk/aws-msk/lib/index.ts @@ -1,2 +1,3 @@ +export * from './cluster'; // AWS::MSK CloudFormation Resources: export * from './msk.generated'; diff --git a/packages/@aws-cdk/aws-msk/package.json b/packages/@aws-cdk/aws-msk/package.json index 939b66619cda7..efa91da07d39f 100644 --- a/packages/@aws-cdk/aws-msk/package.json +++ b/packages/@aws-cdk/aws-msk/package.json @@ -90,7 +90,7 @@ "node": ">= 10.13.0 <13 || >=13.7.0" }, "stability": "experimental", - "maturity": "cfn-only", + "maturity": "experimental", "awscdkio": { "announce": false }, From 545270acec9f373c06130b7ced93c7c81037d2be Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 21:57:23 +0100 Subject: [PATCH 39/55] Fix error handling when VPC is provided --- .../aws-lambda-event-sources/lib/kafka.ts | 14 ++++++---- .../test/test.kafka.ts | 28 +++++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 2572e6a2390c4..3103e62d26ead 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -64,14 +64,14 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps /** * If your Kafka brokers are only reachable via VPC, provide the subnets selection here * - * @default - none + * @default - none, required if setting vpc */ readonly vpcSubnets?: SubnetSelection, /** * If your Kafka brokers are only reachable via VPC, provide the security group here * - * @default - none + * @default - none, required if setting vpc */ readonly securityGroup?: ISecurityGroup @@ -128,9 +128,13 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { constructor(props: SelfManagedKafkaEventSourceProps) { super(props); - if ((props.securityGroup !== undefined && props.vpcSubnets == undefined) || - (props.securityGroup == undefined && props.vpcSubnets !== undefined )) { - throw new Error('both subnets and securityGroup must be set'); + if (props.vpc) { + if (!props.securityGroup) { + throw new Error('securityGroup must be set when providing vpc'); + } + if (!props.vpcSubnets) { + throw new Error('vpcSubnets must be set when providing vpc'); + } } this.innerProps = props; } diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 9ddd6058d62d7..d9fbc48e481e6 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -172,7 +172,31 @@ export = { test.done(); }, - 'self managed Kafka in VPC: subnet and securityGroup must be set together'(test: Test) { + 'self managed Kafka in VPC: setting vpc requires vpcSubnets to be set'(test: Test) { + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const vpc = new Vpc(stack, 'Vpc'); + + test.throws(() => { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + securityGroup: SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'), + + })); + }, /vpcSubnets must be set/); + + test.done(); + }, + + 'self managed Kafka in VPC: setting vpc requires securityGroup to be set'(test: Test) { const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); const kafkaTopic = 'some-topic'; @@ -190,7 +214,7 @@ export = { vpc: vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE }, })); - }, /both subnets and securityGroup must be set/); + }, /securityGroup must be set/); test.done(); }, From 5719072f91f7c158216084e3b46f22394838a78a Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Feb 2021 22:07:20 +0100 Subject: [PATCH 40/55] Make use of Cluster from aws-msk --- packages/@aws-cdk/aws-lambda-event-sources/README.md | 10 ++++++---- .../@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 11 ++++++----- .../@aws-cdk/aws-lambda-event-sources/package.json | 2 ++ .../aws-lambda-event-sources/test/test.kafka.ts | 9 +++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index a5f504db7f63d..69dd46ad98962 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -216,11 +216,13 @@ MSK cluster, as described in [Username/Password authentication](https://docs.aws ```ts import * as lambda from '@aws-cdk/aws-lambda'; +import * as msk from '@aws-cdk/aws-lambda'; import { Secret } from '@aws-cdk/aws-secretmanager'; import { ManagedKafkaEventSource } from '@aws-cdk/aws-lambda-event-sources'; -// The ARN of your cluster -const clusterArn = 'arn:aws:kafka:us-east-1:0123456789019:cluster/SalesCluster/abcd1234-abcd-cafe-abab-9876543210ab-4' +// Your MSK cluster +const cluster = msk.Cluster.fromClusterArn(this, 'Cluster', + 'arn:aws:kafka:us-east-1:0123456789019:cluster/SalesCluster/abcd1234-abcd-cafe-abab-9876543210ab-4'); // The Kafka topic you want to subscribe to const topic = 'some-cool-topic' @@ -230,7 +232,7 @@ const topic = 'some-cool-topic' const secret = new Secret(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); myFunction.addEventSource(new ManagedKafkaEventSource({ - clusterArn: clusterArn, + cluster: cluster, topic: topic, secret: secret, batchSize: 100, // default @@ -253,7 +255,7 @@ const bootstrapServers = ['kafka-broker:9092'] const topic = 'some-cool-topic' // The secret that allows access to your self hosted Kafka cluster -const secret = Secret.fromSecretAttributes(this, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); +const secret = new Secret(this, 'Secret', { ... }); myFunction.addEventSource(new SelfManagedKafkaEventSource({ bootstrapServers: bootstrapServers, diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 3103e62d26ead..d02730da3de9b 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -2,6 +2,7 @@ import * as crypto from 'crypto'; import { ISecurityGroup, IVpc, SubnetSelection } from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; +import * as msk from '@aws-cdk/aws-msk'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; import { StreamEventSource, StreamEventSourceProps } from './stream'; @@ -24,9 +25,9 @@ export interface KafkaEventSourceProps extends StreamEventSourceProps { */ export interface ManagedKafkaEventSourceProps extends KafkaEventSourceProps { /** - * the ARN of the MSK cluster + * an MSK cluster construct */ - readonly clusterArn: string + readonly cluster: msk.ICluster } /** @@ -97,9 +98,9 @@ export class ManagedKafkaEventSource extends StreamEventSource { public bind(target: lambda.IFunction) { target.addEventSourceMapping( - `KafkaEventSource:${this.innerProps.clusterArn}${this.innerProps.topic}`, + `KafkaEventSource:${this.innerProps.cluster.clusterArn}${this.innerProps.topic}`, this.enrichMappingOptions({ - eventSourceArn: this.innerProps.clusterArn, + eventSourceArn: this.innerProps.cluster.clusterArn, startingPosition: this.innerProps.startingPosition, sourceAccessConfigurations: [{ type: lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH, uri: this.innerProps.secret.secretArn }], kafkaTopic: this.innerProps.topic, @@ -111,7 +112,7 @@ export class ManagedKafkaEventSource extends StreamEventSource { target.addToRolePolicy(new iam.PolicyStatement( { actions: ['kafka:DescribeCluster', 'kafka:GetBootstrapBrokers', 'kafka:ListScramSecrets'], - resources: [this.innerProps.clusterArn], + resources: [this.innerProps.cluster.clusterArn], }, )); diff --git a/packages/@aws-cdk/aws-lambda-event-sources/package.json b/packages/@aws-cdk/aws-lambda-event-sources/package.json index 7a9ab7ada2120..33906c1f5219e 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/package.json +++ b/packages/@aws-cdk/aws-lambda-event-sources/package.json @@ -77,6 +77,7 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-msk": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-notifications": "0.0.0", "@aws-cdk/aws-secretsmanager": "0.0.0", @@ -95,6 +96,7 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kinesis": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-msk": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-notifications": "0.0.0", "@aws-cdk/aws-secretsmanager": "0.0.0", diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index d9fbc48e481e6..255808d98327c 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -1,6 +1,7 @@ import { arrayWith, expect, haveResource } from '@aws-cdk/assert'; import { SecurityGroup, SubnetType, Vpc } from '@aws-cdk/aws-ec2'; import * as lambda from '@aws-cdk/aws-lambda'; +import * as msk from '@aws-cdk/aws-msk'; import { Secret } from '@aws-cdk/aws-secretsmanager'; import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; @@ -12,14 +13,14 @@ export = { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); - const clusterArn = 'some-arn'; + const cluster = msk.Cluster.fromClusterArn(stack, 'Cluster', 'some-arn'); const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); // WHEN fn.addEventSource(new sources.ManagedKafkaEventSource( { - clusterArn: clusterArn, + cluster: cluster, topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, @@ -46,7 +47,7 @@ export = { 'kafka:ListScramSecrets', ], Effect: 'Allow', - Resource: 'some-arn', + Resource: cluster.clusterArn, }, ], Version: '2012-10-17', @@ -60,7 +61,7 @@ export = { })); expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - EventSourceArn: clusterArn, + EventSourceArn: cluster.clusterArn, FunctionName: { Ref: 'Fn9270CBC0', }, From 7f5852fd314a8890a7e3601e2ef8b540adebfb70 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 14:37:49 +0100 Subject: [PATCH 41/55] Update packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index 3fd4d519efc3d..d368db6620629 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -40,7 +40,10 @@ export class SourceAccessConfigurationType { return new SourceAccessConfigurationType(name); } - /** The key to use in `CfnEventSourceMapping.SourceAccessConfigurationProperty.Type` */ + /** + * The key to use in `SourceAccessConfigurationProperty.Type` property in CloudFormation + * @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-eventsourcemapping-sourceaccessconfiguration.html#cfn-lambda-eventsourcemapping-sourceaccessconfiguration-type + */ public readonly type: string; private constructor(type: string) { From f44c398f9203c4dc16d05f5530c9e42c5b600504 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 14:38:25 +0100 Subject: [PATCH 42/55] Update packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index d02730da3de9b..2f9c481f67a14 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -79,7 +79,7 @@ export interface SelfManagedKafkaEventSourceProps extends KafkaEventSourceProps /** * The authentication method for your Kafka cluster * - * @default - SASL_SCRAM_512_AUTH + * @default AuthenticationMethod.SASL_SCRAM_512_AUTH */ readonly authenticationMethod?: AuthenticationMethod } From fd318c8fa13e527731f5d61fa6a9f0e6e0efb4e2 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 14:40:25 +0100 Subject: [PATCH 43/55] Remove verbose docs --- .../aws-lambda-event-sources/README.md | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/README.md b/packages/@aws-cdk/aws-lambda-event-sources/README.md index 69dd46ad98962..eb4f206c8f3c9 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/README.md +++ b/packages/@aws-cdk/aws-lambda-event-sources/README.md @@ -268,43 +268,6 @@ myFunction.addEventSource(new SelfManagedKafkaEventSource({ If your self managed Kafka cluster is only reachable via VPC also configure `vpc` `vpcSubnets` and `securityGroup`. -```ts -import * as ec2 from '@aws-cdk/aws-ec2'; -import * as lambda from '@aws-cdk/aws-lambda'; -import { Secret } from '@aws-cdk/aws-secretmanager'; -import { SelfManagedKafkaEventSource } from '@aws-cdk/aws-lambda-event-sources'; - -// The list of Kafka brokers -const bootstrapServers = ['kafka-broker:9092'] - -// The Kafka topic you want to subscribe to -const topic = 'some-cool-topic' - -// The secret that allows access to your self hosted Kafka cluster -const secret = new Secret(this, 'Secret', { ... }); - -// Your VPC: -const vpc = new ec2.Vpc(stack, 'VPC', { ... }); - -// Your VPC subnet selection -const vpcSubnets: ec2.SubnetSelection = { subnetType: ec2.SubnetType.PRIVATE }; - -// Your security group -const sg = new ec2.SecurityGroup(this, 'SG', { ... }); - -myFunction.addEventSource(new SelfManagedKafkaEventSource({ - bootstrapServers: bootstrapServers, - topic: topic, - secret: secret, - batchSize: 100, // default - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - vpcSubnets: vpcSubnets, - securityGroup: sg, -})); -``` - - ## Roadmap Eventually, this module will support all the event sources described under From 439e0c6d286b1cfa27d6ce3e7a8a88617667c36d Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 14:42:44 +0100 Subject: [PATCH 44/55] Update packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 2f9c481f67a14..9ab61a072ae1a 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -102,6 +102,7 @@ export class ManagedKafkaEventSource extends StreamEventSource { this.enrichMappingOptions({ eventSourceArn: this.innerProps.cluster.clusterArn, startingPosition: this.innerProps.startingPosition, + // From https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html#msk-password-limitations, "Amazon MSK only supports SCRAM-SHA-512 authentication." sourceAccessConfigurations: [{ type: lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH, uri: this.innerProps.secret.secretArn }], kafkaTopic: this.innerProps.topic, }), From c0d60061cb791a20d8c5a575b5bb61b2a4f9c42e Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 14:42:08 +0100 Subject: [PATCH 45/55] Fix typo --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 9ab61a072ae1a..fb0528c0afa19 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -41,7 +41,7 @@ export enum AuthenticationMethod { /** * SASL_SCRAM_256_AUTH authentication method for your Kafka cluster */ - SASL_SCRAM_256_AUTH = 'SASL_SCRAM_512_AUTH', + SASL_SCRAM_256_AUTH = 'SASL_SCRAM_256_AUTH', } /** From 4a18ce73d663fe5eb3c6b910376110922166855f Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:04:00 +0100 Subject: [PATCH 46/55] Apply feedback --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index fb0528c0afa19..5841c52f3ad84 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -4,8 +4,13 @@ import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as msk from '@aws-cdk/aws-msk'; import * as secretsmanager from '@aws-cdk/aws-secretsmanager'; +import { Stack } from '@aws-cdk/core'; import { StreamEventSource, StreamEventSourceProps } from './stream'; +// keep this import separate from other imports to reduce chance for merge conflicts with v2-main +// eslint-disable-next-line no-duplicate-imports, import/order +import { Construct } from '@aws-cdk/core'; + /** * Properties for a Kafka event source */ @@ -142,6 +147,7 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { } public bind(target: lambda.IFunction) { + if (!Construct.isConstruct(target)) { throw new Error('Function is not a construct. Unexpected error.'); } let authenticationMethod; switch (this.innerProps.authenticationMethod) { case AuthenticationMethod.SASL_SCRAM_256_AUTH: @@ -163,7 +169,7 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: id }); }); } - const idHash = crypto.createHash('md5').update(JSON.stringify(this.innerProps.bootstrapServers)).digest('hex'); + const idHash = crypto.createHash('md5').update(Stack.of(target).resolve(this.innerProps.bootstrapServers)).digest('hex'); target.addEventSourceMapping( `KafkaEventSource:${idHash}:${this.innerProps.topic}`, this.enrichMappingOptions({ From b2c91454ac4a1d33a128feef23680d3a827d683d Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:10:16 +0100 Subject: [PATCH 47/55] Extract method --- .../aws-lambda-event-sources/lib/kafka.ts | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 5841c52f3ad84..8ea7f27664515 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -148,17 +148,31 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { public bind(target: lambda.IFunction) { if (!Construct.isConstruct(target)) { throw new Error('Function is not a construct. Unexpected error.'); } - let authenticationMethod; + const idHash = crypto.createHash('md5').update(Stack.of(target).resolve(this.innerProps.bootstrapServers)).digest('hex'); + target.addEventSourceMapping( + `KafkaEventSource:${idHash}:${this.innerProps.topic}`, + this.enrichMappingOptions({ + kafkaBootstrapServers: this.innerProps.bootstrapServers, + kafkaTopic: this.innerProps.topic, + startingPosition: this.innerProps.startingPosition, + sourceAccessConfigurations: this.sourceAccessConfigurations(), + }), + ); + this.innerProps.secret.grantRead(target); + } + + private sourceAccessConfigurations() { + let authType; switch (this.innerProps.authenticationMethod) { case AuthenticationMethod.SASL_SCRAM_256_AUTH: - authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_256_AUTH; + authType = lambda.SourceAccessConfigurationType.SASL_SCRAM_256_AUTH; break; case AuthenticationMethod.SASL_SCRAM_512_AUTH: default: - authenticationMethod = lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH; + authType = lambda.SourceAccessConfigurationType.SASL_SCRAM_512_AUTH; break; } - let sourceAccessConfigurations = [{ type: authenticationMethod, uri: this.innerProps.secret.secretArn }]; + let sourceAccessConfigurations = [{ type: authType, uri: this.innerProps.secret.secretArn }]; if (this.innerProps.vpcSubnets !== undefined && this.innerProps.securityGroup !== undefined) { sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SECURITY_GROUP, @@ -169,16 +183,6 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { sourceAccessConfigurations.push({ type: lambda.SourceAccessConfigurationType.VPC_SUBNET, uri: id }); }); } - const idHash = crypto.createHash('md5').update(Stack.of(target).resolve(this.innerProps.bootstrapServers)).digest('hex'); - target.addEventSourceMapping( - `KafkaEventSource:${idHash}:${this.innerProps.topic}`, - this.enrichMappingOptions({ - kafkaBootstrapServers: this.innerProps.bootstrapServers, - kafkaTopic: this.innerProps.topic, - startingPosition: this.innerProps.startingPosition, - sourceAccessConfigurations, - }), - ); - this.innerProps.secret.grantRead(target); + return sourceAccessConfigurations; } } From 87dcaec5e7e1c258fffafec7fbd0b32e3e742901 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:12:44 +0100 Subject: [PATCH 48/55] Update packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts Co-authored-by: Niranjan Jayakar --- packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts index d368db6620629..239bf58671b7e 100644 --- a/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts +++ b/packages/@aws-cdk/aws-lambda/lib/event-source-mapping.ts @@ -179,6 +179,7 @@ export interface EventSourceMappingOptions { /** * Specific settings like the authentication protocol or the VPC components to secure access to your event source. + * @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-eventsourcemapping-sourceaccessconfiguration.html * * @default - none */ From 8d5d938ff9e2521d58f8d528feb721b04b1a4393 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:18:08 +0100 Subject: [PATCH 49/55] Reorganize --- .../test/test.kafka.ts | 414 +++++++++--------- 1 file changed, 209 insertions(+), 205 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 255808d98327c..1661d09174461 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -9,203 +9,231 @@ import * as sources from '../lib'; import { TestFunction } from './test-function'; export = { - 'MSK: sufficiently complex example'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const cluster = msk.Cluster.fromClusterArn(stack, 'Cluster', 'some-arn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + 'msk': { + 'sufficiently complex example'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const cluster = msk.Cluster.fromClusterArn(stack, 'Cluster', 'some-arn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - // WHEN - fn.addEventSource(new sources.ManagedKafkaEventSource( - { - cluster: cluster, - topic: kafkaTopic, - secret: secret, - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - })); + // WHEN + fn.addEventSource(new sources.ManagedKafkaEventSource( + { + cluster: cluster, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + })); - // THEN - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'secretsmanager:GetSecretValue', - 'secretsmanager:DescribeSecret', - ], - Effect: 'Allow', - Resource: { - Ref: 'SecretA720EF05', + // THEN + expect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + Effect: 'Allow', + Resource: { + Ref: 'SecretA720EF05', + }, }, - }, + { + Action: [ + 'kafka:DescribeCluster', + 'kafka:GetBootstrapBrokers', + 'kafka:ListScramSecrets', + ], + Effect: 'Allow', + Resource: cluster.clusterArn, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', + Roles: [ { - Action: [ - 'kafka:DescribeCluster', - 'kafka:GetBootstrapBrokers', - 'kafka:ListScramSecrets', - ], - Effect: 'Allow', - Resource: cluster.clusterArn, + Ref: 'FnServiceRoleB9001A96', }, ], - Version: '2012-10-17', - }, - PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', - Roles: [ - { - Ref: 'FnServiceRoleB9001A96', - }, - ], - })); + })); - expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - EventSourceArn: cluster.clusterArn, - FunctionName: { - Ref: 'Fn9270CBC0', - }, - BatchSize: 100, - StartingPosition: 'TRIM_HORIZON', - Topics: [ - kafkaTopic, - ], - SourceAccessConfigurations: [ - { - Type: 'SASL_SCRAM_512_AUTH', - URI: { - Ref: 'SecretA720EF05', - }, + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + EventSourceArn: cluster.clusterArn, + FunctionName: { + Ref: 'Fn9270CBC0', }, - ], - })); + BatchSize: 100, + StartingPosition: 'TRIM_HORIZON', + Topics: [ + kafkaTopic, + ], + SourceAccessConfigurations: [ + { + Type: 'SASL_SCRAM_512_AUTH', + URI: { + Ref: 'SecretA720EF05', + }, + }, + ], + })); - test.done(); + test.done(); + }, }, - 'self managed Kafka: sufficiently complex example'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); - const vpc = new Vpc(stack, 'Vpc'); + 'self managed kafka': { + 'sufficiently complex example'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); + const vpc = new Vpc(stack, 'Vpc'); - // WHEN - fn.addEventSource(new sources.SelfManagedKafkaEventSource( - { - bootstrapServers: bootstrapServers, - topic: kafkaTopic, - secret: secret, - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - vpcSubnets: { subnetType: SubnetType.PRIVATE }, - securityGroup: sg, - })); + // WHEN + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, + securityGroup: sg, + })); - // THEN - expect(stack).to(haveResource('AWS::IAM::Policy', { - PolicyDocument: { - Statement: [ - { - Action: [ - 'secretsmanager:GetSecretValue', - 'secretsmanager:DescribeSecret', - ], - Effect: 'Allow', - Resource: { - Ref: 'SecretA720EF05', + // THEN + expect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + Effect: 'Allow', + Resource: { + Ref: 'SecretA720EF05', + }, }, + ], + Version: '2012-10-17', + }, + PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', + Roles: [ + { + Ref: 'FnServiceRoleB9001A96', }, ], - Version: '2012-10-17', - }, - PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', - Roles: [ - { - Ref: 'FnServiceRoleB9001A96', - }, - ], - })); + })); - expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - FunctionName: { - Ref: 'Fn9270CBC0', - }, - BatchSize: 100, - SelfManagedEventSource: { - Endpoints: { - KafkaBootstrapServers: bootstrapServers, + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + FunctionName: { + Ref: 'Fn9270CBC0', }, - }, - StartingPosition: 'TRIM_HORIZON', - Topics: [ - kafkaTopic, - ], - SourceAccessConfigurations: [ - { - Type: 'SASL_SCRAM_512_AUTH', - URI: { - Ref: 'SecretA720EF05', + BatchSize: 100, + SelfManagedEventSource: { + Endpoints: { + KafkaBootstrapServers: bootstrapServers, }, }, - { - Type: 'VPC_SECURITY_GROUP', - URI: 'sg-0123456789', - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet1Subnet536B997A', + StartingPosition: 'TRIM_HORIZON', + Topics: [ + kafkaTopic, + ], + SourceAccessConfigurations: [ + { + Type: 'SASL_SCRAM_512_AUTH', + URI: { + Ref: 'SecretA720EF05', + }, }, - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + { + Type: 'VPC_SECURITY_GROUP', + URI: 'sg-0123456789', }, - }, - ], - })); + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + }, + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, + }, + ], + })); - test.done(); - }, + test.done(); + }, - 'self managed Kafka in VPC: setting vpc requires vpcSubnets to be set'(test: Test) { - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const vpc = new Vpc(stack, 'Vpc'); + 'self managed Kafka in VPC: setting vpc requires vpcSubnets to be set'(test: Test) { + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const vpc = new Vpc(stack, 'Vpc'); - test.throws(() => { - fn.addEventSource(new sources.SelfManagedKafkaEventSource( - { - bootstrapServers: bootstrapServers, - topic: kafkaTopic, - secret: secret, - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - securityGroup: SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'), + test.throws(() => { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + securityGroup: SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'), - })); - }, /vpcSubnets must be set/); + })); + }, /vpcSubnets must be set/); - test.done(); - }, + test.done(); + }, - 'self managed Kafka in VPC: setting vpc requires securityGroup to be set'(test: Test) { - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const vpc = new Vpc(stack, 'Vpc'); + 'self managed Kafka in VPC: setting vpc requires securityGroup to be set'(test: Test) { + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const vpc = new Vpc(stack, 'Vpc'); - test.throws(() => { + test.throws(() => { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, + })); + }, /securityGroup must be set/); + + test.done(); + }, + + 'using SCRAM-SHA-256'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); + const vpc = new Vpc(stack, 'Vpc'); + + // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( { bootstrapServers: bootstrapServers, @@ -214,47 +242,23 @@ export = { startingPosition: lambda.StartingPosition.TRIM_HORIZON, vpc: vpc, vpcSubnets: { subnetType: SubnetType.PRIVATE }, + securityGroup: sg, + authenticationMethod: sources.AuthenticationMethod.SASL_SCRAM_256_AUTH, })); - }, /securityGroup must be set/); - test.done(); - }, - - 'self managed Kafka: using SCRAM-SHA-256'(test: Test) { - // GIVEN - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); - const vpc = new Vpc(stack, 'Vpc'); - - // WHEN - fn.addEventSource(new sources.SelfManagedKafkaEventSource( - { - bootstrapServers: bootstrapServers, - topic: kafkaTopic, - secret: secret, - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - vpcSubnets: { subnetType: SubnetType.PRIVATE }, - securityGroup: sg, - authenticationMethod: sources.AuthenticationMethod.SASL_SCRAM_256_AUTH, - })); - - expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { - SourceAccessConfigurations: arrayWith( - { - Type: 'SASL_SCRAM_256_AUTH', - URI: { - Ref: 'SecretA720EF05', + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + SourceAccessConfigurations: arrayWith( + { + Type: 'SASL_SCRAM_256_AUTH', + URI: { + Ref: 'SecretA720EF05', + }, }, - }, - ), - })); + ), + })); - test.done(); + test.done(); + }, }, } From b6be30a09f13e259485100a2bdf0749b686942d7 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:21:40 +0100 Subject: [PATCH 50/55] rename tests --- .../@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 1661d09174461..1b99912c36605 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -10,7 +10,7 @@ import { TestFunction } from './test-function'; export = { 'msk': { - 'sufficiently complex example'(test: Test) { + 'default'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -86,7 +86,7 @@ export = { }, 'self managed kafka': { - 'sufficiently complex example'(test: Test) { + 'default'(test: Test) { // GIVEN const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); @@ -176,7 +176,7 @@ export = { test.done(); }, - 'self managed Kafka in VPC: setting vpc requires vpcSubnets to be set'(test: Test) { + 'VPC: setting vpc requires vpcSubnets to be set'(test: Test) { const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); const kafkaTopic = 'some-topic'; @@ -200,7 +200,7 @@ export = { test.done(); }, - 'self managed Kafka in VPC: setting vpc requires securityGroup to be set'(test: Test) { + 'VPC: setting vpc requires securityGroup to be set'(test: Test) { const stack = new cdk.Stack(); const fn = new TestFunction(stack, 'Fn'); const kafkaTopic = 'some-topic'; From 14c0876927d49fbff0df330961e920fb7d64ce48 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:32:45 +0100 Subject: [PATCH 51/55] Fix and extract --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 8ea7f27664515..03e6b4e5263e0 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -148,9 +148,8 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { public bind(target: lambda.IFunction) { if (!Construct.isConstruct(target)) { throw new Error('Function is not a construct. Unexpected error.'); } - const idHash = crypto.createHash('md5').update(Stack.of(target).resolve(this.innerProps.bootstrapServers)).digest('hex'); target.addEventSourceMapping( - `KafkaEventSource:${idHash}:${this.innerProps.topic}`, + this.mappingId(target), this.enrichMappingOptions({ kafkaBootstrapServers: this.innerProps.bootstrapServers, kafkaTopic: this.innerProps.topic, @@ -161,6 +160,11 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { this.innerProps.secret.grantRead(target); } + private mappingId(target: lambda.IFunction) { + const idHash = crypto.createHash('md5').update(JSON.stringify(Stack.of(target).resolve(this.innerProps.bootstrapServers))).digest('hex'); + return `KafkaEventSource:${idHash}:${this.innerProps.topic}`; + } + private sourceAccessConfigurations() { let authType; switch (this.innerProps.authenticationMethod) { From 838d43f136612d3603e5120d0f6ea37076e92193 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:34:16 +0100 Subject: [PATCH 52/55] Add more tests --- .../test/event-source-mapping.test.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts b/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts index 549433ad6f5f4..5d833a2d865c6 100644 --- a/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts +++ b/packages/@aws-cdk/aws-lambda/test/event-source-mapping.test.ts @@ -212,4 +212,53 @@ describe('event source mapping', () => { target: fn, })).toThrow(/kafkaBootStrapServers must not be empty if set/); }); + + test('eventSourceArn appears in stack', () => { + const stack = new cdk.Stack(); + const topicNameParam = new cdk.CfnParameter(stack, 'TopicNameParam', { + type: 'String', + }); + + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline('exports.handler = ${handler.toString()}'), + runtime: Runtime.NODEJS_10_X, + }); + + let eventSourceArn = 'some-arn'; + + new EventSourceMapping(stack, 'test', { + target: fn, + eventSourceArn: eventSourceArn, + kafkaTopic: topicNameParam.valueAsString, + }); + + expect(stack).toHaveResourceLike('AWS::Lambda::EventSourceMapping', { + EventSourceArn: eventSourceArn, + }); + }); + + test('kafkaBootstrapServers appears in stack', () => { + const stack = new cdk.Stack(); + const topicNameParam = new cdk.CfnParameter(stack, 'TopicNameParam', { + type: 'String', + }); + + const fn = new Function(stack, 'fn', { + handler: 'index.handler', + code: Code.fromInline('exports.handler = ${handler.toString()}'), + runtime: Runtime.NODEJS_10_X, + }); + + let kafkaBootstrapServers = ['kafka-broker.example.com:9092']; + new EventSourceMapping(stack, 'test', { + target: fn, + kafkaBootstrapServers: kafkaBootstrapServers, + kafkaTopic: topicNameParam.valueAsString, + }); + + expect(stack).toHaveResourceLike('AWS::Lambda::EventSourceMapping', { + SelfManagedEventSource: { Endpoints: { KafkaBootstrapServers: kafkaBootstrapServers } }, + }); + }); }); From a177e0d6119d2dda3c720a149756de6daa2d8206 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Thu, 4 Mar 2021 15:40:51 +0100 Subject: [PATCH 53/55] Remove obsolete dependency --- packages/@aws-cdk/aws-lambda/package.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/package.json b/packages/@aws-cdk/aws-lambda/package.json index 2bb7ec3cf329b..6eab5cd11b870 100644 --- a/packages/@aws-cdk/aws-lambda/package.json +++ b/packages/@aws-cdk/aws-lambda/package.json @@ -99,7 +99,6 @@ "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", - "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-signer": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/core": "0.0.0", @@ -121,7 +120,6 @@ "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-s3-assets": "0.0.0", - "@aws-cdk/aws-secretsmanager": "0.0.0", "@aws-cdk/aws-signer": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/core": "0.0.0", From 6f451cf8e196b7c930675e85428d123f10521652 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Mar 2021 10:49:03 +0100 Subject: [PATCH 54/55] Apply feedback --- packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts index 03e6b4e5263e0..f0782964d93ce 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/lib/kafka.ts @@ -161,7 +161,9 @@ export class SelfManagedKafkaEventSource extends StreamEventSource { } private mappingId(target: lambda.IFunction) { - const idHash = crypto.createHash('md5').update(JSON.stringify(Stack.of(target).resolve(this.innerProps.bootstrapServers))).digest('hex'); + let hash = crypto.createHash('md5'); + hash.update(JSON.stringify(Stack.of(target).resolve(this.innerProps.bootstrapServers))); + const idHash = hash.digest('hex'); return `KafkaEventSource:${idHash}:${this.innerProps.topic}`; } From be2b531acc51ba2b213db9f89cac3da1be7371a0 Mon Sep 17 00:00:00 2001 From: Jan Brauer Date: Tue, 16 Mar 2021 10:49:16 +0100 Subject: [PATCH 55/55] Rearrange tests --- .../test/test.kafka.ts | 176 ++++++++++++------ 1 file changed, 123 insertions(+), 53 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts index 1b99912c36605..9cf1af9e50507 100644 --- a/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts +++ b/packages/@aws-cdk/aws-lambda-event-sources/test/test.kafka.ts @@ -93,8 +93,6 @@ export = { const kafkaTopic = 'some-topic'; const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); const bootstrapServers = ['kafka-broker:9092']; - const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); - const vpc = new Vpc(stack, 'Vpc'); // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( @@ -103,9 +101,6 @@ export = { topic: kafkaTopic, secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - vpcSubnets: { subnetType: SubnetType.PRIVATE }, - securityGroup: sg, })); // THEN @@ -154,37 +149,24 @@ export = { Ref: 'SecretA720EF05', }, }, - { - Type: 'VPC_SECURITY_GROUP', - URI: 'sg-0123456789', - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet1Subnet536B997A', - }, - }, - { - Type: 'VPC_SUBNET', - URI: { - Ref: 'VpcPrivateSubnet2Subnet3788AAA1', - }, - }, ], })); test.done(); }, - 'VPC: setting vpc requires vpcSubnets to be set'(test: Test) { - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const vpc = new Vpc(stack, 'Vpc'); + VPC: { + 'correctly rendered in the stack'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const sg = SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'); + const vpc = new Vpc(stack, 'Vpc'); - test.throws(() => { + // WHEN fn.addEventSource(new sources.SelfManagedKafkaEventSource( { bootstrapServers: bootstrapServers, @@ -192,35 +174,123 @@ export = { secret: secret, startingPosition: lambda.StartingPosition.TRIM_HORIZON, vpc: vpc, - securityGroup: SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'), - + vpcSubnets: { subnetType: SubnetType.PRIVATE }, + securityGroup: sg, })); - }, /vpcSubnets must be set/); - test.done(); - }, + // THEN + expect(stack).to(haveResource('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 'secretsmanager:GetSecretValue', + 'secretsmanager:DescribeSecret', + ], + Effect: 'Allow', + Resource: { + Ref: 'SecretA720EF05', + }, + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'FnServiceRoleDefaultPolicyC6A839BF', + Roles: [ + { + Ref: 'FnServiceRoleB9001A96', + }, + ], + })); - 'VPC: setting vpc requires securityGroup to be set'(test: Test) { - const stack = new cdk.Stack(); - const fn = new TestFunction(stack, 'Fn'); - const kafkaTopic = 'some-topic'; - const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); - const bootstrapServers = ['kafka-broker:9092']; - const vpc = new Vpc(stack, 'Vpc'); + expect(stack).to(haveResource('AWS::Lambda::EventSourceMapping', { + FunctionName: { + Ref: 'Fn9270CBC0', + }, + BatchSize: 100, + SelfManagedEventSource: { + Endpoints: { + KafkaBootstrapServers: bootstrapServers, + }, + }, + StartingPosition: 'TRIM_HORIZON', + Topics: [ + kafkaTopic, + ], + SourceAccessConfigurations: [ + { + Type: 'SASL_SCRAM_512_AUTH', + URI: { + Ref: 'SecretA720EF05', + }, + }, + { + Type: 'VPC_SECURITY_GROUP', + URI: 'sg-0123456789', + }, + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet1Subnet536B997A', + }, + }, + { + Type: 'VPC_SUBNET', + URI: { + Ref: 'VpcPrivateSubnet2Subnet3788AAA1', + }, + }, + ], + })); - test.throws(() => { - fn.addEventSource(new sources.SelfManagedKafkaEventSource( - { - bootstrapServers: bootstrapServers, - topic: kafkaTopic, - secret: secret, - startingPosition: lambda.StartingPosition.TRIM_HORIZON, - vpc: vpc, - vpcSubnets: { subnetType: SubnetType.PRIVATE }, - })); - }, /securityGroup must be set/); + test.done(); + }, + 'setting vpc requires vpcSubnets to be set'(test: Test) { + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const vpc = new Vpc(stack, 'Vpc'); - test.done(); + test.throws(() => { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + securityGroup: SecurityGroup.fromSecurityGroupId(stack, 'SecurityGroup', 'sg-0123456789'), + + })); + }, /vpcSubnets must be set/); + + test.done(); + }, + + 'setting vpc requires securityGroup to be set'(test: Test) { + const stack = new cdk.Stack(); + const fn = new TestFunction(stack, 'Fn'); + const kafkaTopic = 'some-topic'; + const secret = new Secret(stack, 'Secret', { secretName: 'AmazonMSK_KafkaSecret' }); + const bootstrapServers = ['kafka-broker:9092']; + const vpc = new Vpc(stack, 'Vpc'); + + test.throws(() => { + fn.addEventSource(new sources.SelfManagedKafkaEventSource( + { + bootstrapServers: bootstrapServers, + topic: kafkaTopic, + secret: secret, + startingPosition: lambda.StartingPosition.TRIM_HORIZON, + vpc: vpc, + vpcSubnets: { subnetType: SubnetType.PRIVATE }, + })); + }, /securityGroup must be set/); + + test.done(); + }, }, 'using SCRAM-SHA-256'(test: Test) {