From 80c49a56067a7cccd848458fd9be85f73abced35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Tue, 18 Feb 2020 16:29:58 -0800 Subject: [PATCH 1/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- .../lib/base/queue-processing-service-base.ts | 34 +++++++++++++++++-- .../ec2/test.queue-processing-ecs-service.ts | 16 ++++++++- .../test.queue-processing-fargate-service.ts | 16 ++++++++- 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts index 633385da4bbfc..11413be8988ab 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts @@ -2,7 +2,7 @@ import { ScalingInterval } from '@aws-cdk/aws-applicationautoscaling'; import { IVpc } from '@aws-cdk/aws-ec2'; import { AwsLogDriver, BaseService, Cluster, ContainerImage, ICluster, LogDriver, PropagatedTagSource, Secret } from '@aws-cdk/aws-ecs'; import { IQueue, Queue } from '@aws-cdk/aws-sqs'; -import { CfnOutput, Construct, Stack } from '@aws-cdk/core'; +import { CfnOutput, Construct, Duration, Stack } from '@aws-cdk/core'; /** * The properties for the base QueueProcessingEc2Service or QueueProcessingFargateService service. @@ -86,6 +86,14 @@ export interface QueueProcessingServiceBaseProps { */ readonly queue?: IQueue; + /** + * The maximum number of times that a message can be received by consumers. + * When this value is exceeded for a message the message will be automatically sent to the Dead Letter Queue. + * + * @default 5 + */ + readonly maxReceiveCount?: number; + /** * Maximum capacity to scale to. * @@ -143,6 +151,11 @@ export abstract class QueueProcessingServiceBase extends Construct { */ public readonly sqsQueue: IQueue; + /** + * The dead letter queue for the SQS queue that the service will process from + */ + public readonly deadLetterQueue?: IQueue; + /** * The cluster where your service will be deployed */ @@ -191,8 +204,23 @@ export abstract class QueueProcessingServiceBase extends Construct { } this.cluster = props.cluster || this.getDefaultCluster(this, props.vpc); - // Create the SQS queue if one is not provided - this.sqsQueue = props.queue !== undefined ? props.queue : new Queue(this, 'EcsProcessingQueue', {}); + // Create the SQS queue if one is not provided, when creating SQS queue setup DLQ + if (props.queue) { + this.sqsQueue = props.queue; + } else { + this.deadLetterQueue = new Queue(this, "EcsProcessingDeadLetterQueue", { + retentionPeriod: Duration.days(14) + }); + this.sqsQueue = new Queue(this, 'EcsProcessingQueue', { + deadLetterQueue: { + queue: this.deadLetterQueue, + maxReceiveCount: props.maxReceiveCount || 5 + } + }); + + new CfnOutput(this, 'SQSDeadLetterQueue', { value: this.deadLetterQueue.queueName }); + new CfnOutput(this, 'SQSDeadLetterQueueArn', { value: this.deadLetterQueue.queueArn }); + } // Setup autoscaling scaling intervals const defaultScalingSteps = [{ upper: 0, change: -1 }, { lower: 100, change: +1 }, { lower: 500, change: +5 }]; diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts index 781ca75b339fa..123abc4a02b2c 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts @@ -27,7 +27,21 @@ export = { LaunchType: "EC2", })); - expect(stack).to(haveResource("AWS::SQS::Queue")); + expect(stack).to(haveResource("AWS::SQS::Queue", { + RedrivePolicy: { + deadLetterTargetArn: { + "Fn::GetAtt": [ + "ServiceEcsProcessingDeadLetterQueue4A89196E", + "Arn" + ] + }, + maxReceiveCount: 5 + } + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + MessageRetentionPeriod: 1209600 + })); expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { ContainerDefinitions: [ diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts index 248d6cbf57be6..18a4d6c7449ee 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts @@ -27,7 +27,21 @@ export = { LaunchType: "FARGATE", })); - expect(stack).to(haveResource("AWS::SQS::Queue")); + expect(stack).to(haveResource("AWS::SQS::Queue", { + RedrivePolicy: { + deadLetterTargetArn: { + "Fn::GetAtt": [ + "ServiceEcsProcessingDeadLetterQueue4A89196E", + "Arn" + ] + }, + maxReceiveCount: 5 + } + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + MessageRetentionPeriod: 1209600 + })); expect(stack).to(haveResource("AWS::IAM::Policy", { PolicyDocument: { From fd91ae028416b0831c05f80a24ed5e051f528b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Tue, 18 Feb 2020 17:19:45 -0800 Subject: [PATCH 2/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- packages/@aws-cdk/aws-ecs-patterns/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/@aws-cdk/aws-ecs-patterns/README.md b/packages/@aws-cdk/aws-ecs-patterns/README.md index 5da8d9feb1391..69ea71896754c 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/README.md +++ b/packages/@aws-cdk/aws-ecs-patterns/README.md @@ -269,6 +269,8 @@ const queueProcessingFargateService = new QueueProcessingFargateService(stack, ' }); ``` +when queue not provided by user, CDK with create a primary queue and dead letter queue with default redrive policy and attach permission to the task to be able to acces the primary queue. + ## Scheduled Tasks To define a task that runs periodically, instantiate an `ScheduledEc2Task`: From b24ace7b91ea17a32debc816cfd71c595a8760df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Tue, 18 Feb 2020 17:21:55 -0800 Subject: [PATCH 3/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- packages/@aws-cdk/aws-ecs-patterns/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-ecs-patterns/README.md b/packages/@aws-cdk/aws-ecs-patterns/README.md index 69ea71896754c..b6dc359f4eefb 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/README.md +++ b/packages/@aws-cdk/aws-ecs-patterns/README.md @@ -269,7 +269,7 @@ const queueProcessingFargateService = new QueueProcessingFargateService(stack, ' }); ``` -when queue not provided by user, CDK with create a primary queue and dead letter queue with default redrive policy and attach permission to the task to be able to acces the primary queue. +when queue not provided by user, CDK will create a primary queue and a dead letter queue with default redrive policy and attach permission to the task to be able to acces the primary queue. ## Scheduled Tasks From 0f33f7c698ed05d6a4857651db5d0f5746431ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Wed, 19 Feb 2020 11:01:05 -0800 Subject: [PATCH 4/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- packages/@aws-cdk/aws-ecs-patterns/README.md | 2 +- .../lib/base/queue-processing-service-base.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs-patterns/README.md b/packages/@aws-cdk/aws-ecs-patterns/README.md index b6dc359f4eefb..e3f1f95984ed4 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/README.md +++ b/packages/@aws-cdk/aws-ecs-patterns/README.md @@ -269,7 +269,7 @@ const queueProcessingFargateService = new QueueProcessingFargateService(stack, ' }); ``` -when queue not provided by user, CDK will create a primary queue and a dead letter queue with default redrive policy and attach permission to the task to be able to acces the primary queue. +when queue not provided by user, CDK will create a primary queue and a dead letter queue with default redrive policy and attach permission to the task to be able to access the primary queue. ## Scheduled Tasks diff --git a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts index 11413be8988ab..47d1bba3bade8 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts @@ -152,7 +152,7 @@ export abstract class QueueProcessingServiceBase extends Construct { public readonly sqsQueue: IQueue; /** - * The dead letter queue for the SQS queue that the service will process from + * The dead letter queue for the primary SQS queue */ public readonly deadLetterQueue?: IQueue; @@ -204,7 +204,7 @@ export abstract class QueueProcessingServiceBase extends Construct { } this.cluster = props.cluster || this.getDefaultCluster(this, props.vpc); - // Create the SQS queue if one is not provided, when creating SQS queue setup DLQ + // Create the SQS queue and it's corresponding DLQ if one is not provided if (props.queue) { this.sqsQueue = props.queue; } else { From a45cd7f738181fdec9760bd9b149da4a3f5cefcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Wed, 19 Feb 2020 16:12:46 -0800 Subject: [PATCH 5/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- .../lib/base/queue-processing-service-base.ts | 4 ++-- .../test/ec2/test.queue-processing-ecs-service.ts | 2 +- .../test/fargate/test.queue-processing-fargate-service.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts index 47d1bba3bade8..7edb24c18d94b 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts @@ -90,7 +90,7 @@ export interface QueueProcessingServiceBaseProps { * The maximum number of times that a message can be received by consumers. * When this value is exceeded for a message the message will be automatically sent to the Dead Letter Queue. * - * @default 5 + * @default 3 */ readonly maxReceiveCount?: number; @@ -214,7 +214,7 @@ export abstract class QueueProcessingServiceBase extends Construct { this.sqsQueue = new Queue(this, 'EcsProcessingQueue', { deadLetterQueue: { queue: this.deadLetterQueue, - maxReceiveCount: props.maxReceiveCount || 5 + maxReceiveCount: props.maxReceiveCount || 3 } }); diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts index 123abc4a02b2c..5cb55401b66fb 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts @@ -35,7 +35,7 @@ export = { "Arn" ] }, - maxReceiveCount: 5 + maxReceiveCount: 3 } })); diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts index 18a4d6c7449ee..5085572ab8699 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts @@ -35,7 +35,7 @@ export = { "Arn" ] }, - maxReceiveCount: 5 + maxReceiveCount: 3 } })); From 3bddd35d0a6c9e385ff866381704877b769f7cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Thu, 20 Feb 2020 08:55:55 -0800 Subject: [PATCH 6/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- .../lib/base/queue-processing-service-base.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts index 7edb24c18d94b..08738ca788485 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/lib/base/queue-processing-service-base.ts @@ -94,6 +94,13 @@ export interface QueueProcessingServiceBaseProps { */ readonly maxReceiveCount?: number; + /** + * The number of seconds that Dead Letter Queue retains a message. + * + * @default Duration.days(14) + */ + readonly retentionPeriod?: Duration; + /** * Maximum capacity to scale to. * @@ -209,7 +216,7 @@ export abstract class QueueProcessingServiceBase extends Construct { this.sqsQueue = props.queue; } else { this.deadLetterQueue = new Queue(this, "EcsProcessingDeadLetterQueue", { - retentionPeriod: Duration.days(14) + retentionPeriod: props.retentionPeriod || Duration.days(14) }); this.sqsQueue = new Queue(this, 'EcsProcessingQueue', { deadLetterQueue: { From 4c18e4616685b0fdf256d94dc96f6c35513e5d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A6=96=20Barun=20Ray?= Date: Thu, 20 Feb 2020 09:15:39 -0800 Subject: [PATCH 7/7] feat(aws-ecs-patterns): setup associated DLQ when Queue created by CDK for QueueProcessingFargateService --- .../ec2/test.queue-processing-ecs-service.ts | 75 ++++++++++++++ .../test.queue-processing-fargate-service.ts | 97 +++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts index 5cb55401b66fb..3daa6ed266ec9 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/ec2/test.queue-processing-ecs-service.ts @@ -80,6 +80,81 @@ export = { test.done(); }, + 'test ECS queue worker service construct - with optional props for queues'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.Vpc(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecsPatterns.QueueProcessingEc2Service(stack, 'Service', { + cluster, + memoryLimitMiB: 512, + image: ecs.ContainerImage.fromRegistry('test'), + maxReceiveCount: 42, + retentionPeriod: cdk.Duration.days(7) + }); + + // THEN - QueueWorker is of EC2 launch type, an SQS queue is created and all default properties are set. + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 1, + LaunchType: "EC2", + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + RedrivePolicy: { + deadLetterTargetArn: { + "Fn::GetAtt": [ + "ServiceEcsProcessingDeadLetterQueue4A89196E", + "Arn" + ] + }, + maxReceiveCount: 42 + } + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + MessageRetentionPeriod: 604800 + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceEcsProcessingQueueC266885C", + "QueueName" + ] + } + } + ], + LogConfiguration: { + LogDriver: "awslogs", + Options: { + "awslogs-group": { + Ref: "ServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupD52338D1" + }, + "awslogs-stream-prefix": "Service", + "awslogs-region": { + Ref: "AWS::Region" + } + } + }, + Essential: true, + Image: "test", + Memory: 512 + } + ], + Family: "ServiceQueueProcessingTaskDef83DB34F1" + })); + + test.done(); + }, + 'test ECS queue worker service construct - with optional props'(test: Test) { // GIVEN const stack = new cdk.Stack(); diff --git a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts index 5085572ab8699..c06f10b10b7aa 100644 --- a/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts +++ b/packages/@aws-cdk/aws-ecs-patterns/test/fargate/test.queue-processing-fargate-service.ts @@ -102,6 +102,103 @@ export = { test.done(); }, + 'test fargate queue worker service construct - with optional props for queues'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.Vpc(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecsPatterns.QueueProcessingFargateService(stack, 'Service', { + cluster, + memoryLimitMiB: 512, + image: ecs.ContainerImage.fromRegistry('test'), + maxReceiveCount: 42, + retentionPeriod: cdk.Duration.days(7) + }); + + // THEN - QueueWorker is of FARGATE launch type, an SQS queue is created and all default properties are set. + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 1, + LaunchType: "FARGATE", + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + RedrivePolicy: { + deadLetterTargetArn: { + "Fn::GetAtt": [ + "ServiceEcsProcessingDeadLetterQueue4A89196E", + "Arn" + ] + }, + maxReceiveCount: 42 + } + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + MessageRetentionPeriod: 604800 + })); + + expect(stack).to(haveResource("AWS::IAM::Policy", { + PolicyDocument: { + Statement: [ + { + Action: [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + Effect: "Allow", + Resource: { + "Fn::GetAtt": [ + "ServiceEcsProcessingQueueC266885C", + "Arn" + ] + } + } + ], + Version: "2012-10-17" + } + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceEcsProcessingQueueC266885C", + "QueueName" + ] + } + } + ], + LogConfiguration: { + LogDriver: "awslogs", + Options: { + "awslogs-group": { + Ref: "ServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupD52338D1" + }, + "awslogs-stream-prefix": "Service", + "awslogs-region": { + Ref: "AWS::Region" + } + } + }, + Image: "test", + } + ], + Family: "ServiceQueueProcessingTaskDef83DB34F1" + })); + + test.done(); + }, + 'test Fargate queue worker service construct - with optional props'(test: Test) { // GIVEN const stack = new cdk.Stack();