From dcc7737c85e9c189b7a938c6779a1685302cff8c Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Fri, 26 Apr 2019 15:10:16 -0700 Subject: [PATCH 1/6] Define the ECS/Fargate QueueWorkerService construct and fix Step Scaling implementation to add ScalingTargetId and default AdjustmentType --- design/aws-ecs-autoscaling-queue-worker.md | 225 ++++++++++++++++++ .../lib/step-scaling-action.ts | 1 + .../lib/step-scaling-policy.ts | 4 +- .../test/test.step-scaling-policy.ts | 38 +++ .../aws-ecs/lib/ecs-queue-worker-service.ts | 68 ++++++ .../lib/fargate-queue-worker-service.ts | 76 ++++++ packages/@aws-cdk/aws-ecs/lib/index.ts | 4 + .../aws-ecs/lib/queue-worker-service-base.ts | 180 ++++++++++++++ packages/@aws-cdk/aws-ecs/package.json | 2 + .../aws-ecs/test/test.ecs-worker-service.ts | 135 +++++++++++ .../test/test.fargate-worker-service.ts | 133 +++++++++++ 11 files changed, 864 insertions(+), 2 deletions(-) create mode 100644 design/aws-ecs-autoscaling-queue-worker.md create mode 100644 packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts create mode 100644 packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts create mode 100644 packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts create mode 100644 packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts create mode 100644 packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts diff --git a/design/aws-ecs-autoscaling-queue-worker.md b/design/aws-ecs-autoscaling-queue-worker.md new file mode 100644 index 0000000000000..b4121401492fe --- /dev/null +++ b/design/aws-ecs-autoscaling-queue-worker.md @@ -0,0 +1,225 @@ +# AWS ECS - L3 Construct for Autoscaling ECS/Fargate Service that Processes Items in a SQS Queue + +To address issue [#2396](https://github.com/awslabs/aws-cdk/issues/2396), the AWS ECS CDK construct library should provide a way for customers to create a queue worker service (an AWS ECS/Fargate service that processes items from an sqs queue). This would mean adding new ECS CDK constructs `Ec2QueueWorkerService` and `FargateQueryWorkerService`, that would take in the necessary properties required to create a task definition, an SQS queue as well as an ECS/Fargate service and enable autoscaling for the service based on cpu usage and the SQS queue's approximateNumberOfMessagesVisible metric. + +## General approach + +The new `ecs.QueueWorkerServiceBase`, `ecs.Ec2QueueWorkerService` and `ecs.FargateQueueWorkerService` classes will create L3 constructs for: + +* Ec2QueueWorkerService +* FargateQueueWorkerService + +A `QueueWorkerService` will create a task definition with the specified container (on both EC2 and Fargate). An AWS SQS `Queue` will be created and autoscaling of the ECS Service will be dependent on both CPU as well as the SQS queue's `ApproximateNumberOfMessagesVisible` metric. + +The `QueueWorkerService` constructs (for EC2 and Fargate) will use the following existing constructs: + +* Ec2TaskDefinition/FargateTaskDefinition - To create a Task Definition for the container to start +* SQSQueue - The queue that the worker is processing from +* Ec2Service/FargateService - The Service running the container + +## Code changes + +Given the above, we should make the following changes to support queue workers on ECS (for both EC2 and Fargate): +1. Create `QueueWorkerServiceBaseProps` interface and `QueueWorkerServiceBase` construct +2. Create `Ec2QueueWorkerServiceProps` interface and `Ec2QueueWorkerService` construct +3. Create `FargateQueueWorkerServiceProps` interface and `FargateQueueWorkerService` construct + +### Part 1: Create `QueueWorkerServiceBaseProps` interface and `QueueWorkerServiceBase` construct + +The `QueueWorkerServiceBaseProps` interface will contain common properties used to construct both the Ec2QueueWorkerService and the FargateQueueWorkerService: + +```ts +/** + * Properties to define an Query Worker service + */ +export interface QueueWorkerServiceBaseProps { + /** + * Cluster where service will be deployed + */ + readonly cluster: ICluster; + + /** + * The image to start. + */ + readonly image: ContainerImage; + + /** + * The CMD value to pass to the container. A string with commands delimited by commas. + * + * @default none + */ + readonly command?: string; + + /** + * Number of desired copies of running tasks + * + * @default 1 + */ + readonly desiredTaskCount?: number; + + /** + * Flag to indicate whether to enable logging + * + * @default true + */ + readonly enableLogging?: boolean; + + /** + * The environment variables to pass to the container. + * + * @default none + */ + readonly environment?: { [key: string]: string }; + + /** + * A name for the queue. + * + * If specified and this is a FIFO queue, must end in the string '.fifo'. + * + * @default CloudFormation-generated name + */ + readonly queueName?: string; + + /** + * Maximum capacity to scale to. + * + * @default (desiredTaskCount * 2) + */ + readonly maxScalingCapacity?: number + + /** + * The intervals for scaling based on the SQS queue's ApproximateNumberOfMessagesVisible metric. + * + * Maps a range of metric values to a particular scaling behavior. + * https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-scaling-simple-step.html + * + * @default [{ upper: 0, change: -1 },{ lower: 100, change: +1 },{ lower: 500, change: +5 }] + */ + readonly scalingSteps: autoScaling.ScalingInterval[]; +} +``` + +### Part 2: Create `Ec2QueueWorkerServiceProps` interface and `Ec2QueueWorkerService` construct + +The `Ec2QueueWorkerServiceProps` interface will contain properties to construct the Ec2TaskDefinition, SQSQueue and Ec2Service: + +```ts +/** + * Properties to define an ECS service + */ +export interface Ec2QueueWorkerServiceProps { + /** + * The minimum number of CPU units to reserve for the container. + * + * @default none + */ + readonly cpu?: number; + + /** + * The hard limit (in MiB) of memory to present to the container. + * + * If your container attempts to exceed the allocated memory, the container + * is terminated. + * + * At least one of memoryLimitMiB and memoryReservationMiB is required for non-Fargate services. + */ + readonly memoryLimitMiB?: number; + + /** + * The soft limit (in MiB) of memory to reserve for the container. + * + * When system memory is under contention, Docker attempts to keep the + * container memory within the limit. If the container requires more memory, + * it can consume up to the value specified by the Memory property or all of + * the available memory on the container instance—whichever comes first. + * + * At least one of memoryLimitMiB and memoryReservationMiB is required for non-Fargate services. + */ + readonly memoryReservationMiB?: number; +} +``` + +An example use case: +```ts +// Create the vpc and cluster used by the Queue Worker task +const vpc = new ec2.VpcNetwork(stack, 'Vpc', { maxAZs: 1 }); +const cluster = new ecs.Cluster(stack, 'EcsCluster', { vpc }); +cluster.addCapacity('DefaultAutoScalingGroup', { + instanceType: new ec2.InstanceType('t2.micro') +}); + +// Create the Queue Worker task +new Ec2QueueWorkerService(stack, 'EcsQueueWorkerService', { + cluster, + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + desiredTaskCount: 2, + maxScalingCapacity: 5, + memoryReservationMiB: 512, + cpu: 256, + queueName: 'EcsWorkerQueue' +}); +``` + +### Part 3: Create `FargateQueueWorkerServiceProps` interface and `FargateQueueWorkerService` construct + +The `FargateQueueWorkerServiceProps` interface will contain properties to construct the FargateTaskDefinition, SQSQueue and FargateService: + +```ts +/** + * Properties to define an Fargate service + */ +export interface FargateQueueWorkerServiceProps { + /** + * The number of cpu units used by the task. + * Valid values, which determines your range of valid values for the memory parameter: + * 256 (.25 vCPU) - Available memory values: 0.5GB, 1GB, 2GB + * 512 (.5 vCPU) - Available memory values: 1GB, 2GB, 3GB, 4GB + * 1024 (1 vCPU) - Available memory values: 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB + * 2048 (2 vCPU) - Available memory values: Between 4GB and 16GB in 1GB increments + * 4096 (4 vCPU) - Available memory values: Between 8GB and 30GB in 1GB increments + * + * This default is set in the underlying FargateTaskDefinition construct. + * + * @default 256 + */ + readonly cpu?: string; + + /** + * The amount (in MiB) of memory used by the task. + * + * This field is required and you must use one of the following values, which determines your range of valid values + * for the cpu parameter: + * + * 0.5GB, 1GB, 2GB - Available cpu values: 256 (.25 vCPU) + * + * 1GB, 2GB, 3GB, 4GB - Available cpu values: 512 (.5 vCPU) + * + * 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB - Available cpu values: 1024 (1 vCPU) + * + * Between 4GB and 16GB in 1GB increments - Available cpu values: 2048 (2 vCPU) + * + * Between 8GB and 30GB in 1GB increments - Available cpu values: 4096 (4 vCPU) + * + * This default is set in the underlying FargateTaskDefinition construct. + * + * @default 512 + */ + readonly memoryMiB?: string; +} +``` + +An example use case: +```ts +// Create the vpc and cluster used by the Queue Worker task +const vpc = new ec2.VpcNetwork(stack, 'Vpc', { maxAZs: 2 }); +const cluster = new ecs.Cluster(stack, 'FargateCluster', { vpc }); + +// Create the Queue Worker task +new FargateQueueWorkerService(stack, 'FargateQueueWorkerService', { + cluster, + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + desiredTaskCount: 2, + maxScalingCapacity: 5, + queueName: 'FargateWorkerQueue' +}); +``` diff --git a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts index d889522658977..339634e36435b 100644 --- a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts +++ b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts @@ -86,6 +86,7 @@ export class StepScalingAction extends cdk.Construct implements cloudwatch.IAlar const resource = new CfnScalingPolicy(this, 'Resource', { policyName: props.policyName || this.node.uniqueId, policyType: 'StepScaling', + scalingTargetId: props.scalingTarget.scalableTargetId, stepScalingPolicyConfiguration: { adjustmentType: props.adjustmentType, cooldown: props.cooldownSec, diff --git a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-policy.ts b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-policy.ts index 55180dc345aec..80cb19e90e620 100644 --- a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-policy.ts +++ b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-policy.ts @@ -85,7 +85,7 @@ export class StepScalingPolicy extends cdk.Construct { const threshold = intervals[alarms.lowerAlarmIntervalIndex].upper; this.lowerAction = new StepScalingAction(this, 'LowerPolicy', { - adjustmentType: props.adjustmentType, + adjustmentType, cooldownSec: props.cooldownSec, metricAggregationType: aggregationTypeFromMetric(props.metric), minAdjustmentMagnitude: props.minAdjustmentMagnitude, @@ -115,7 +115,7 @@ export class StepScalingPolicy extends cdk.Construct { const threshold = intervals[alarms.upperAlarmIntervalIndex].lower; this.upperAction = new StepScalingAction(this, 'UpperPolicy', { - adjustmentType: props.adjustmentType, + adjustmentType, cooldownSec: props.cooldownSec, metricAggregationType: aggregationTypeFromMetric(props.metric), minAdjustmentMagnitude: props.minAdjustmentMagnitude, diff --git a/packages/@aws-cdk/aws-applicationautoscaling/test/test.step-scaling-policy.ts b/packages/@aws-cdk/aws-applicationautoscaling/test/test.step-scaling-policy.ts index 6e60c3cbd6067..1455fa0e84689 100644 --- a/packages/@aws-cdk/aws-applicationautoscaling/test/test.step-scaling-policy.ts +++ b/packages/@aws-cdk/aws-applicationautoscaling/test/test.step-scaling-policy.ts @@ -1,3 +1,4 @@ +import { expect, haveResource } from '@aws-cdk/assert'; import { SynthUtils } from '@aws-cdk/assert'; import cloudwatch = require('@aws-cdk/aws-cloudwatch'); import cdk = require('@aws-cdk/cdk'); @@ -116,6 +117,43 @@ export = { test.done(); }, + + 'test step scaling on metric'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const target = createScalableTarget(stack); + + // WHEN + target.scaleOnMetric('Tracking', { + metric: new cloudwatch.Metric({ namespace: 'Test', metricName: 'Metric' }), + scalingSteps: [ + { upper: 0, change: -1 }, + { lower: 100, change: +1 }, + { lower: 500, change: +5 } + ] + }); + + // THEN + expect(stack).to(haveResource('AWS::ApplicationAutoScaling::ScalingPolicy', { + PolicyType: "StepScaling", + ScalingTargetId: { + Ref: "Target3191CF44" + }, + StepScalingPolicyConfiguration: { + AdjustmentType: "ChangeInCapacity", + MetricAggregationType: "Average", + StepAdjustments: [ + { + MetricIntervalUpperBound: 0, + ScalingAdjustment: -1 + } + ] + } + + })); + + test.done(); + } }; /** diff --git a/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts b/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts new file mode 100644 index 0000000000000..29c0f40bc8a9f --- /dev/null +++ b/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts @@ -0,0 +1,68 @@ +import cdk = require('@aws-cdk/cdk'); +import { Ec2Service } from './ec2/ec2-service'; +import { Ec2TaskDefinition } from './ec2/ec2-task-definition'; +import { QueueWorkerServiceBase, QueueWorkerServiceBaseProps } from './queue-worker-service-base'; + +/** + * Properties to define an ECS query worker service + */ +export interface Ec2QueueWorkerServiceProps extends QueueWorkerServiceBaseProps { + /** + * The minimum number of CPU units to reserve for the container. + * + * @default none + */ + readonly cpu?: number; + + /** + * The hard limit (in MiB) of memory to present to the container. + * + * If your container attempts to exceed the allocated memory, the container + * is terminated. + * + * At least one of memoryLimitMiB and memoryReservationMiB is required for non-Fargate services. + */ + readonly memoryLimitMiB?: number; + + /** + * The soft limit (in MiB) of memory to reserve for the container. + * + * When system memory is under contention, Docker attempts to keep the + * container memory within the limit. If the container requires more memory, + * it can consume up to the value specified by the Memory property or all of + * the available memory on the container instance—whichever comes first. + * + * At least one of memoryLimitMiB and memoryReservationMiB is required for non-Fargate services. + */ + readonly memoryReservationMiB?: number; +} + +/** + * Class to create an ECS query worker service + */ +export class Ec2QueueWorkerService extends QueueWorkerServiceBase { + constructor(scope: cdk.Construct, id: string, props: Ec2QueueWorkerServiceProps) { + super(scope, id, props); + + // Create a Task Definition for the container to start + const taskDefinition = new Ec2TaskDefinition(this, 'QueueWorkerTaskDef'); + taskDefinition.addContainer('QueueWorkerContainer', { + image: props.image, + memoryLimitMiB: props.memoryLimitMiB, + memoryReservationMiB: props.memoryReservationMiB, + cpu: props.cpu, + command: props.command !== undefined ? cdk.Fn.split(",", props.command) : undefined, + environment: this.environment, + logging: this.logDriver + }); + + // Create an ECS service with the previously defined Task Definition and configure + // autoscaling based on cpu utilization and number of messages visible in the sqs queue. + const ecsService = new Ec2Service(this, 'QueueWorkerService', { + cluster: props.cluster, + desiredCount: this.desiredCount, + taskDefinition + }); + this.configureAutoscalingForService(ecsService); + } +} diff --git a/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts b/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts new file mode 100644 index 0000000000000..4f167dfc866e8 --- /dev/null +++ b/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts @@ -0,0 +1,76 @@ +import cdk = require('@aws-cdk/cdk'); +import { FargateService } from './fargate/fargate-service'; +import { FargateTaskDefinition } from './fargate/fargate-task-definition'; +import { QueueWorkerServiceBase, QueueWorkerServiceBaseProps } from './queue-worker-service-base'; + +/** + * Properties to define a Fargate queue worker service + */ +export interface FargateQueueWorkerServiceProps extends QueueWorkerServiceBaseProps { + /** + * The number of cpu units used by the task. + * Valid values, which determines your range of valid values for the memory parameter: + * 256 (.25 vCPU) - Available memory values: 0.5GB, 1GB, 2GB + * 512 (.5 vCPU) - Available memory values: 1GB, 2GB, 3GB, 4GB + * 1024 (1 vCPU) - Available memory values: 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB + * 2048 (2 vCPU) - Available memory values: Between 4GB and 16GB in 1GB increments + * 4096 (4 vCPU) - Available memory values: Between 8GB and 30GB in 1GB increments + * + * This default is set in the underlying FargateTaskDefinition construct. + * + * @default 256 + */ + readonly cpu?: string; + + /** + * The amount (in MiB) of memory used by the task. + * + * This field is required and you must use one of the following values, which determines your range of valid values + * for the cpu parameter: + * + * 0.5GB, 1GB, 2GB - Available cpu values: 256 (.25 vCPU) + * + * 1GB, 2GB, 3GB, 4GB - Available cpu values: 512 (.5 vCPU) + * + * 2GB, 3GB, 4GB, 5GB, 6GB, 7GB, 8GB - Available cpu values: 1024 (1 vCPU) + * + * Between 4GB and 16GB in 1GB increments - Available cpu values: 2048 (2 vCPU) + * + * Between 8GB and 30GB in 1GB increments - Available cpu values: 4096 (4 vCPU) + * + * This default is set in the underlying FargateTaskDefinition construct. + * + * @default 512 + */ + readonly memoryMiB?: string; +} + +/** + * Class to create a Fargate query worker service + */ +export class FargateQueueWorkerService extends QueueWorkerServiceBase { + constructor(scope: cdk.Construct, id: string, props: FargateQueueWorkerServiceProps) { + super(scope, id, props); + + // Create a Task Definition for the container to start + const taskDefinition = new FargateTaskDefinition(this, 'QueueWorkerTaskDef', { + memoryMiB: props.memoryMiB !== undefined ? props.memoryMiB : '512', + cpu: props.cpu !== undefined ? props.cpu : '256', + }); + taskDefinition.addContainer('QueueWorkerContainer', { + image: props.image, + command: props.command !== undefined ? cdk.Fn.split(",", props.command) : undefined, + environment: this.environment, + logging: this.logDriver + }); + + // Create a Fargate service with the previously defined Task Definition and configure + // autoscaling based on cpu utilization and number of messages visible in the sqs queue. + const fargateService = new FargateService(this, 'FargateQueueWorkerService', { + cluster: props.cluster, + desiredCount: this.desiredCount, + taskDefinition + }); + this.configureAutoscalingForService(fargateService); + } +} diff --git a/packages/@aws-cdk/aws-ecs/lib/index.ts b/packages/@aws-cdk/aws-ecs/lib/index.ts index a2a19ba7712ed..8e578b26c3c67 100644 --- a/packages/@aws-cdk/aws-ecs/lib/index.ts +++ b/packages/@aws-cdk/aws-ecs/lib/index.ts @@ -21,6 +21,10 @@ export * from './load-balanced-fargate-service'; export * from './load-balanced-ecs-service'; export * from './load-balanced-fargate-service-applet'; +export * from './queue-worker-service-base'; +export * from './ecs-queue-worker-service'; +export * from './fargate-queue-worker-service'; + export * from './images/asset-image'; export * from './images/repository'; export * from './images/ecr'; diff --git a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts new file mode 100644 index 0000000000000..79ecf201b8579 --- /dev/null +++ b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts @@ -0,0 +1,180 @@ +import autoscaling = require('@aws-cdk/aws-applicationautoscaling'); +import sqs = require('@aws-cdk/aws-sqs'); +import { Queue } from '@aws-cdk/aws-sqs'; +import cdk = require('@aws-cdk/cdk'); +import { BaseService } from './base/base-service'; +import { ICluster } from './cluster'; +import { ContainerImage } from './container-image'; +import { AwsLogDriver } from './log-drivers/aws-log-driver'; +import { LogDriver } from './log-drivers/log-driver'; + +/** + * Properties to define a Query Worker service + */ +export interface QueueWorkerServiceBaseProps { + /** + * Cluster where service will be deployed + */ + readonly cluster: ICluster; + + /** + * The image to start. + */ + readonly image: ContainerImage; + + /** + * The CMD value to pass to the container. A string with commands delimited by commas. + * + * @default none + */ + readonly command?: string; + + /** + * Number of desired copies of running tasks + * + * @default 1 + */ + readonly desiredTaskCount?: number; + + /** + * Flag to indicate whether to enable logging + * + * @default true + */ + readonly enableLogging?: boolean; + + /** + * The environment variables to pass to the container. + * + * @default none + */ + readonly environment?: { [key: string]: string }; + + /** + * A name for the queue. + * + * If specified and this is a FIFO queue, must end in the string '.fifo'. + * + * @default 'ecs-worker-service-queue' + */ + readonly queueName?: string; + + /** + * Maximum capacity to scale to. + * + * @default (desiredTaskCount * 2) + */ + readonly maxScalingCapacity?: number + + /** + * The intervals for scaling based on the SQS queue's ApproximateNumberOfMessagesVisible metric. + * + * Maps a range of metric values to a particular scaling behavior. + * https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-scaling-simple-step.html + * + * @default [{ upper: 0, change: -1 },{ lower: 100, change: +1 },{ lower: 500, change: +5 }] + */ + readonly scalingSteps?: autoscaling.ScalingInterval[]; +} + +/** + * Base class for a Fargate and ECS queue worker service + */ +export abstract class QueueWorkerServiceBase extends cdk.Construct { + /** + * The sqs queue that the worker service will process from + */ + public readonly sqsQueue: Queue; + + // Properties that have defaults defined. The Queue Worker will handle assigning undefined properties with default + // values so that derived classes do not need to maintain the same logic. + + /** + * Environment variables that will include the queue name + */ + public readonly environment: { [key: string]: string }; + /** + * The minimum number of tasks to run + */ + public readonly desiredCount: number; + /** + * The maximum number of instances for autoscaling to scale up to + */ + public readonly maxCapacity: number; + /** + * The scaling interval for autoscaling based off an SQS Queue size + */ + public readonly scalingSteps: autoscaling.ScalingInterval[]; + /** + * The AwsLogDriverto use for logging if logging is enabled. + */ + public readonly logDriver?: LogDriver; + + constructor(scope: cdk.Construct, id: string, props: QueueWorkerServiceBaseProps) { + super(scope, id); + + // Create the worker sqs queue + const queueName = props.queueName !== undefined ? props.queueName : 'ecs-worker-service-queue'; + this.sqsQueue = new sqs.Queue(this, 'ecs-worker-service-queue', { + queueName + }); + + // Setup AutoScaling scaling intervals + const defaultScalingSteps = [{ upper: 0, change: -1 }, { lower: 100, change: +1 }, { lower: 500, change: +5 }]; + this.scalingSteps = props.scalingSteps !== undefined ? props.scalingSteps : defaultScalingSteps; + + // Create log driver if logging is enabled + const enableLogging = props.enableLogging !== undefined ? props.enableLogging : true; + this.logDriver = enableLogging ? this.createAwsLogDriver(this.node.id) : undefined; + + // Add the queuename to environment variables + this.environment = props.environment !== undefined ? + this.addQueueNameToEnvironment(props.environment) : {QUEUE_NAME: this.sqsQueue.queueName}; + + // Determine the desired task count (minimum) and maximum scaling capacity + this.desiredCount = props.desiredTaskCount || 1; + this.maxCapacity = props.maxScalingCapacity || (2 * this.desiredCount); + + new cdk.CfnOutput(this, 'SQSQueue', { value: this.sqsQueue.queueName }); + new cdk.CfnOutput(this, 'SQSQueueArn', { value: this.sqsQueue.queueArn }); + } + + /** + * Configure autoscaling based off of cpu utilization as well as the number of messages visisble in the sqs queue + * + * @param service the ECS/Fargate service for which to apply the autoscaling rules to + */ + protected configureAutoscalingForService(service: BaseService) { + const scalingTarget = service.autoScaleTaskCount({ maxCapacity: this.maxCapacity, minCapacity: this.desiredCount }); + scalingTarget.scaleOnCpuUtilization('CpuScaling', { + targetUtilizationPercent: 50, + }); + scalingTarget.scaleOnMetric('QueueMessagesVisibleScaling', { + metric: this.sqsQueue.metricApproximateNumberOfMessagesVisible(), + scalingSteps: this.scalingSteps + }); + } + + /** + * Create an AWS Log Driver with the provided streamPrefix + * + * @param prefix the cloudwatch logging prefix + */ + private createAwsLogDriver(prefix: string): AwsLogDriver { + return new AwsLogDriver(this, 'QueueWorkerLogging', { streamPrefix: prefix }); + } + + /** + * Add the sqs queue name to the environment variables to pass to the container + * + * @param env the environment variables to pass to the container + */ + private addQueueNameToEnvironment(env: { [key: string]: string }): any { + const environment: { [key: string]: string } = {}; + environment.QUEUE_NAME = this.sqsQueue.queueName; + for (const [key, value] of Object.entries(env)) { + environment[key] = value; + } + return environment; + } +} diff --git a/packages/@aws-cdk/aws-ecs/package.json b/packages/@aws-cdk/aws-ecs/package.json index 532d1f041f6d4..c715ecf3e6b67 100644 --- a/packages/@aws-cdk/aws-ecs/package.json +++ b/packages/@aws-cdk/aws-ecs/package.json @@ -85,6 +85,7 @@ "@aws-cdk/aws-secretsmanager": "^0.31.0", "@aws-cdk/aws-servicediscovery": "^0.31.0", "@aws-cdk/aws-sns": "^0.31.0", + "@aws-cdk/aws-sqs": "^0.31.0", "@aws-cdk/cdk": "^0.31.0", "@aws-cdk/cx-api": "^0.31.0" }, @@ -108,6 +109,7 @@ "@aws-cdk/aws-secretsmanager": "^0.31.0", "@aws-cdk/aws-servicediscovery": "^0.31.0", "@aws-cdk/aws-sns": "^0.31.0", + "@aws-cdk/aws-sqs": "^0.31.0", "@aws-cdk/cdk": "^0.31.0", "@aws-cdk/cx-api": "^0.31.0" }, diff --git a/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts new file mode 100644 index 0000000000000..7f39ee02dea17 --- /dev/null +++ b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts @@ -0,0 +1,135 @@ +import { expect, haveResource, haveResourceLike } from '@aws-cdk/assert'; +import ec2 = require('@aws-cdk/aws-ec2'); +import cdk = require('@aws-cdk/cdk'); +import { Test } from 'nodeunit'; +import ecs = require('../lib'); + +export = { + 'test ECS queue worker service construct - with only required props'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.VpcNetwork(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecs.Ec2QueueWorkerService(stack, 'Service', { + cluster, + memoryLimitMiB: 512, + image: ecs.ContainerImage.fromRegistry('test') + }); + + // THEN - stack contains a load balancer and a service + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 1, + LaunchType: "EC2", + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + QueueName: "ecs-worker-service-queue" + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceecsworkerservicequeueAE5FF7F2", + "QueueName" + ] + } + } + ], + LogConfiguration: { + LogDriver: "awslogs", + Options: { + "awslogs-group": { + Ref: "ServiceQueueWorkerLoggingLogGroup5E11C73B" + }, + "awslogs-stream-prefix": "Service", + "awslogs-region": { + Ref: "AWS::Region" + } + } + }, + Image: "test", + Memory: 512 + } + ] + })); + + test.done(); + }, + + 'test ECS queue worker service construct - with optional props'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.VpcNetwork(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecs.Ec2QueueWorkerService(stack, 'Service', { + cluster, + memoryLimitMiB: 1024, + image: ecs.ContainerImage.fromRegistry('test'), + command: "-c, 4, amazon.com", + enableLogging: false, + desiredTaskCount: 2, + environment: { + TEST_ENVIRONMENT_VARIABLE1: "test environment variable 1 value", + TEST_ENVIRONMENT_VARIABLE2: "test environment variable 2 value" + }, + queueName: 'ecs-test-sqs-queue', + maxScalingCapacity: 5 + }); + + // THEN - stack contains a load balancer and a service + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 2, + LaunchType: "EC2" + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + QueueName: "ecs-test-sqs-queue" + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Command: [ + "-c", + " 4", + " amazon.com" + ], + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceecsworkerservicequeueAE5FF7F2", + "QueueName" + ] + } + }, + { + Name: "TEST_ENVIRONMENT_VARIABLE1", + Value: "test environment variable 1 value" + }, + { + Name: "TEST_ENVIRONMENT_VARIABLE2", + Value: "test environment variable 2 value" + } + ], + Image: "test", + Memory: 1024 + } + ] + })); + + test.done(); + } +}; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts new file mode 100644 index 0000000000000..ecf9fbf815740 --- /dev/null +++ b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts @@ -0,0 +1,133 @@ +import { expect, haveResource, haveResourceLike } from '@aws-cdk/assert'; +import ec2 = require('@aws-cdk/aws-ec2'); +import cdk = require('@aws-cdk/cdk'); +import { Test } from 'nodeunit'; +import ecs = require('../lib'); + +export = { + 'test fargate queue worker service construct - with only required props'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.VpcNetwork(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecs.FargateQueueWorkerService(stack, 'Service', { + cluster, + memoryMiB: '512', + image: ecs.ContainerImage.fromRegistry('test') + }); + + // THEN - stack contains a load balancer and a service + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 1, + LaunchType: "FARGATE", + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + QueueName: "ecs-worker-service-queue" + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceecsworkerservicequeueAE5FF7F2", + "QueueName" + ] + } + } + ], + LogConfiguration: { + LogDriver: "awslogs", + Options: { + "awslogs-group": { + Ref: "ServiceQueueWorkerLoggingLogGroup5E11C73B" + }, + "awslogs-stream-prefix": "Service", + "awslogs-region": { + Ref: "AWS::Region" + } + } + }, + Image: "test", + } + ] + })); + + test.done(); + }, + + 'test Fargate queue worker service construct - with optional props'(test: Test) { + // GIVEN + const stack = new cdk.Stack(); + const vpc = new ec2.VpcNetwork(stack, 'VPC'); + const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); + cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + + // WHEN + new ecs.FargateQueueWorkerService(stack, 'Service', { + cluster, + memoryMiB: '512', + image: ecs.ContainerImage.fromRegistry('test'), + command: "-c, 4, amazon.com", + enableLogging: false, + desiredTaskCount: 2, + environment: { + TEST_ENVIRONMENT_VARIABLE1: "test environment variable 1 value", + TEST_ENVIRONMENT_VARIABLE2: "test environment variable 2 value" + }, + queueName: 'fargate-test-sqs-queue', + maxScalingCapacity: 5 + }); + + // THEN - stack contains a load balancer and a service + expect(stack).to(haveResource("AWS::ECS::Service", { + DesiredCount: 2, + LaunchType: "FARGATE" + })); + + expect(stack).to(haveResource("AWS::SQS::Queue", { + QueueName: "fargate-test-sqs-queue" + })); + + expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { + ContainerDefinitions: [ + { + Command: [ + "-c", + " 4", + " amazon.com" + ], + Environment: [ + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ServiceecsworkerservicequeueAE5FF7F2", + "QueueName" + ] + } + }, + { + Name: "TEST_ENVIRONMENT_VARIABLE1", + Value: "test environment variable 1 value" + }, + { + Name: "TEST_ENVIRONMENT_VARIABLE2", + Value: "test environment variable 2 value" + } + ], + Image: "test", + } + ] + })); + + test.done(); + } +}; From f56c3022be895455a5ea866f5ebb838bacea7de2 Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Fri, 17 May 2019 10:20:33 -0700 Subject: [PATCH 2/6] Address CR Feedback --- design/aws-ecs-autoscaling-queue-worker.md | 2 +- .../aws-ecs/lib/queue-worker-service-base.ts | 24 +++++++++---------- .../aws-ecs/test/test.ecs-worker-service.ts | 4 ++-- .../test/test.fargate-worker-service.ts | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/design/aws-ecs-autoscaling-queue-worker.md b/design/aws-ecs-autoscaling-queue-worker.md index b4121401492fe..20e8a197c176d 100644 --- a/design/aws-ecs-autoscaling-queue-worker.md +++ b/design/aws-ecs-autoscaling-queue-worker.md @@ -30,7 +30,7 @@ The `QueueWorkerServiceBaseProps` interface will contain common properties used ```ts /** - * Properties to define an Query Worker service + * Properties to define a Query Worker service */ export interface QueueWorkerServiceBaseProps { /** diff --git a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts index 79ecf201b8579..f73becc90506b 100644 --- a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts +++ b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts @@ -55,7 +55,7 @@ export interface QueueWorkerServiceBaseProps { * * If specified and this is a FIFO queue, must end in the string '.fifo'. * - * @default 'ecs-worker-service-queue' + * @default 'EcsWorkerServiceQueue' */ readonly queueName?: string; @@ -82,7 +82,7 @@ export interface QueueWorkerServiceBaseProps { */ export abstract class QueueWorkerServiceBase extends cdk.Construct { /** - * The sqs queue that the worker service will process from + * The SQS queue that the worker service will process from */ public readonly sqsQueue: Queue; @@ -106,20 +106,20 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { */ public readonly scalingSteps: autoscaling.ScalingInterval[]; /** - * The AwsLogDriverto use for logging if logging is enabled. + * The AwsLogDriver to use for logging if logging is enabled. */ public readonly logDriver?: LogDriver; constructor(scope: cdk.Construct, id: string, props: QueueWorkerServiceBaseProps) { super(scope, id); - // Create the worker sqs queue - const queueName = props.queueName !== undefined ? props.queueName : 'ecs-worker-service-queue'; - this.sqsQueue = new sqs.Queue(this, 'ecs-worker-service-queue', { + // Create the worker SQS queue + const queueName = props.queueName !== undefined ? props.queueName : 'EcsWorkerServiceQueue'; + this.sqsQueue = new sqs.Queue(this, 'EcsWorkerServiceQueue', { queueName }); - // Setup AutoScaling scaling intervals + // Setup autoscaling scaling intervals const defaultScalingSteps = [{ upper: 0, change: -1 }, { lower: 100, change: +1 }, { lower: 500, change: +5 }]; this.scalingSteps = props.scalingSteps !== undefined ? props.scalingSteps : defaultScalingSteps; @@ -127,9 +127,9 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { const enableLogging = props.enableLogging !== undefined ? props.enableLogging : true; this.logDriver = enableLogging ? this.createAwsLogDriver(this.node.id) : undefined; - // Add the queuename to environment variables + // Add the queue name to environment variables this.environment = props.environment !== undefined ? - this.addQueueNameToEnvironment(props.environment) : {QUEUE_NAME: this.sqsQueue.queueName}; + this.addQueueNameToEnvironment(props.environment) : { QUEUE_NAME: this.sqsQueue.queueName }; // Determine the desired task count (minimum) and maximum scaling capacity this.desiredCount = props.desiredTaskCount || 1; @@ -140,7 +140,7 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { } /** - * Configure autoscaling based off of cpu utilization as well as the number of messages visisble in the sqs queue + * Configure autoscaling based off of CPU utilization as well as the number of messages visible in the SQS queue * * @param service the ECS/Fargate service for which to apply the autoscaling rules to */ @@ -158,14 +158,14 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { /** * Create an AWS Log Driver with the provided streamPrefix * - * @param prefix the cloudwatch logging prefix + * @param prefix the Cloudwatch logging prefix */ private createAwsLogDriver(prefix: string): AwsLogDriver { return new AwsLogDriver(this, 'QueueWorkerLogging', { streamPrefix: prefix }); } /** - * Add the sqs queue name to the environment variables to pass to the container + * Add the SQS queue name to the environment variables to pass to the container * * @param env the environment variables to pass to the container */ diff --git a/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts index 7f39ee02dea17..2874efffd41c2 100644 --- a/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts @@ -19,7 +19,7 @@ export = { image: ecs.ContainerImage.fromRegistry('test') }); - // THEN - stack contains a load balancer and a service + // THEN - QueueWorker is of EC2 launch type, an SQS queue is created and all default properties are set. expect(stack).to(haveResource("AWS::ECS::Service", { DesiredCount: 1, LaunchType: "EC2", @@ -87,7 +87,7 @@ export = { maxScalingCapacity: 5 }); - // THEN - stack contains a load balancer and a service + // THEN - QueueWorker is of EC2 launch type, an SQS queue is created and all optional properties are set. expect(stack).to(haveResource("AWS::ECS::Service", { DesiredCount: 2, LaunchType: "EC2" diff --git a/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts index ecf9fbf815740..dcb0148f20a35 100644 --- a/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts @@ -19,7 +19,7 @@ export = { image: ecs.ContainerImage.fromRegistry('test') }); - // THEN - stack contains a load balancer and a service + // THEN - QueueWorker is of FARGATE launch type, an SQS queue is created and all default properties are set. expect(stack).to(haveResource("AWS::ECS::Service", { DesiredCount: 1, LaunchType: "FARGATE", @@ -86,7 +86,7 @@ export = { maxScalingCapacity: 5 }); - // THEN - stack contains a load balancer and a service + // THEN - QueueWorker is of FARGATE launch type, an SQS queue is created and all optional properties are set. expect(stack).to(haveResource("AWS::ECS::Service", { DesiredCount: 2, LaunchType: "FARGATE" From 0fb3b69371c9d80544f36e4caee45d452af5cb7a Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Fri, 17 May 2019 10:21:56 -0700 Subject: [PATCH 3/6] Address CR Feedback --- packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts | 6 +++--- .../@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts b/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts index 29c0f40bc8a9f..5d33a54a37c62 100644 --- a/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/lib/ecs-queue-worker-service.ts @@ -4,7 +4,7 @@ import { Ec2TaskDefinition } from './ec2/ec2-task-definition'; import { QueueWorkerServiceBase, QueueWorkerServiceBaseProps } from './queue-worker-service-base'; /** - * Properties to define an ECS query worker service + * Properties to define an Ec2 query worker service */ export interface Ec2QueueWorkerServiceProps extends QueueWorkerServiceBaseProps { /** @@ -38,7 +38,7 @@ export interface Ec2QueueWorkerServiceProps extends QueueWorkerServiceBaseProps } /** - * Class to create an ECS query worker service + * Class to create an Ec2 query worker service */ export class Ec2QueueWorkerService extends QueueWorkerServiceBase { constructor(scope: cdk.Construct, id: string, props: Ec2QueueWorkerServiceProps) { @@ -57,7 +57,7 @@ export class Ec2QueueWorkerService extends QueueWorkerServiceBase { }); // Create an ECS service with the previously defined Task Definition and configure - // autoscaling based on cpu utilization and number of messages visible in the sqs queue. + // autoscaling based on cpu utilization and number of messages visible in the SQS queue. const ecsService = new Ec2Service(this, 'QueueWorkerService', { cluster: props.cluster, desiredCount: this.desiredCount, diff --git a/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts b/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts index 4f167dfc866e8..a726e2b3a37ba 100644 --- a/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/lib/fargate-queue-worker-service.ts @@ -65,7 +65,7 @@ export class FargateQueueWorkerService extends QueueWorkerServiceBase { }); // Create a Fargate service with the previously defined Task Definition and configure - // autoscaling based on cpu utilization and number of messages visible in the sqs queue. + // autoscaling based on cpu utilization and number of messages visible in the SQS queue. const fargateService = new FargateService(this, 'FargateQueueWorkerService', { cluster: props.cluster, desiredCount: this.desiredCount, From 5a2f1465421a325d04c5931820fcbe3fb2d0c57e Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Mon, 20 May 2019 06:33:56 -0700 Subject: [PATCH 4/6] fix(appscaling): fix StepScaling (#2522) Add ScalingTargetId and default AdjustmentType. --- .../aws-applicationautoscaling/lib/step-scaling-action.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts index 339634e36435b..860ac7c1f609b 100644 --- a/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts +++ b/packages/@aws-cdk/aws-applicationautoscaling/lib/step-scaling-action.ts @@ -83,6 +83,9 @@ export class StepScalingAction extends cdk.Construct implements cloudwatch.IAlar constructor(scope: cdk.Construct, id: string, props: StepScalingActionProps) { super(scope, id); + // Cloudformation requires either the ResourceId, ScalableDimension, and ServiceNamespace + // properties, or the ScalingTargetId property, but not both. + // https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-applicationautoscaling-scalingpolicy.html const resource = new CfnScalingPolicy(this, 'Resource', { policyName: props.policyName || this.node.uniqueId, policyType: 'StepScaling', From 516336f1382a51febf6c3e63bc992cc1e72ad23e Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Mon, 20 May 2019 10:53:11 -0700 Subject: [PATCH 5/6] Address CR Feedback --- .../aws-ecs/lib/queue-worker-service-base.ts | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts index f73becc90506b..9a86d8fb0a630 100644 --- a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts +++ b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts @@ -1,6 +1,6 @@ import autoscaling = require('@aws-cdk/aws-applicationautoscaling'); import sqs = require('@aws-cdk/aws-sqs'); -import { Queue } from '@aws-cdk/aws-sqs'; +import { Queue, IQueue } from '@aws-cdk/aws-sqs'; import cdk = require('@aws-cdk/cdk'); import { BaseService } from './base/base-service'; import { ICluster } from './cluster'; @@ -51,13 +51,14 @@ export interface QueueWorkerServiceBaseProps { readonly environment?: { [key: string]: string }; /** - * A name for the queue. + * A queue for which to process items from. * * If specified and this is a FIFO queue, must end in the string '.fifo'. + * @see https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html * - * @default 'EcsWorkerServiceQueue' + * @default 'SQSQueue with CloudFormation-generated name' */ - readonly queueName?: string; + readonly queue?: IQueue; /** * Maximum capacity to scale to. @@ -84,7 +85,7 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { /** * The SQS queue that the worker service will process from */ - public readonly sqsQueue: Queue; + public readonly sqsQueue: IQueue; // Properties that have defaults defined. The Queue Worker will handle assigning undefined properties with default // values so that derived classes do not need to maintain the same logic. @@ -113,11 +114,8 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { constructor(scope: cdk.Construct, id: string, props: QueueWorkerServiceBaseProps) { super(scope, id); - // Create the worker SQS queue - const queueName = props.queueName !== undefined ? props.queueName : 'EcsWorkerServiceQueue'; - this.sqsQueue = new sqs.Queue(this, 'EcsWorkerServiceQueue', { - queueName - }); + // Create the worker SQS queue if one is not provided + this.sqsQueue = props.queue !== undefined ? props.queue : new sqs.Queue(this, 'EcsWorkerServiceQueue', {}); // Setup autoscaling scaling intervals const defaultScalingSteps = [{ upper: 0, change: -1 }, { lower: 100, change: +1 }, { lower: 500, change: +5 }]; @@ -128,8 +126,7 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { this.logDriver = enableLogging ? this.createAwsLogDriver(this.node.id) : undefined; // Add the queue name to environment variables - this.environment = props.environment !== undefined ? - this.addQueueNameToEnvironment(props.environment) : { QUEUE_NAME: this.sqsQueue.queueName }; + this.environment = { ...(props.environment || {}), QUEUE_NAME: this.sqsQueue.queueName }; // Determine the desired task count (minimum) and maximum scaling capacity this.desiredCount = props.desiredTaskCount || 1; @@ -163,18 +160,4 @@ export abstract class QueueWorkerServiceBase extends cdk.Construct { private createAwsLogDriver(prefix: string): AwsLogDriver { return new AwsLogDriver(this, 'QueueWorkerLogging', { streamPrefix: prefix }); } - - /** - * Add the SQS queue name to the environment variables to pass to the container - * - * @param env the environment variables to pass to the container - */ - private addQueueNameToEnvironment(env: { [key: string]: string }): any { - const environment: { [key: string]: string } = {}; - environment.QUEUE_NAME = this.sqsQueue.queueName; - for (const [key, value] of Object.entries(env)) { - environment[key] = value; - } - return environment; - } } From 5e6c82bfe554e79ec3fa494c7998c00402cf3aa1 Mon Sep 17 00:00:00 2001 From: Piradeep Kandasamy Date: Mon, 20 May 2019 14:09:33 -0700 Subject: [PATCH 6/6] Address CR Feedback --- .../aws-ecs/lib/queue-worker-service-base.ts | 2 +- .../aws-ecs/test/test.ecs-worker-service.ts | 28 ++++++++-------- .../test/test.fargate-worker-service.ts | 32 +++++++++---------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts index 9a86d8fb0a630..69f8e73f4730e 100644 --- a/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts +++ b/packages/@aws-cdk/aws-ecs/lib/queue-worker-service-base.ts @@ -1,6 +1,6 @@ import autoscaling = require('@aws-cdk/aws-applicationautoscaling'); import sqs = require('@aws-cdk/aws-sqs'); -import { Queue, IQueue } from '@aws-cdk/aws-sqs'; +import { IQueue } from '@aws-cdk/aws-sqs'; import cdk = require('@aws-cdk/cdk'); import { BaseService } from './base/base-service'; import { ICluster } from './cluster'; diff --git a/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts index 2874efffd41c2..249fb4f1804bf 100644 --- a/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/test/test.ecs-worker-service.ts @@ -1,5 +1,6 @@ import { expect, haveResource, haveResourceLike } from '@aws-cdk/assert'; import ec2 = require('@aws-cdk/aws-ec2'); +import sqs = require('@aws-cdk/aws-sqs'); import cdk = require('@aws-cdk/cdk'); import { Test } from 'nodeunit'; import ecs = require('../lib'); @@ -25,9 +26,7 @@ export = { LaunchType: "EC2", })); - expect(stack).to(haveResource("AWS::SQS::Queue", { - QueueName: "ecs-worker-service-queue" - })); + expect(stack).to(haveResource("AWS::SQS::Queue")); expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { ContainerDefinitions: [ @@ -37,7 +36,7 @@ export = { Name: "QUEUE_NAME", Value: { "Fn::GetAtt": [ - "ServiceecsworkerservicequeueAE5FF7F2", + "ServiceEcsWorkerServiceQueue19BF278C", "QueueName" ] } @@ -70,6 +69,7 @@ export = { const vpc = new ec2.VpcNetwork(stack, 'VPC'); const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + const queue = new sqs.Queue(stack, 'ecs-test-queue', { queueName: 'ecs-test-sqs-queue'}); // WHEN new ecs.Ec2QueueWorkerService(stack, 'Service', { @@ -83,7 +83,7 @@ export = { TEST_ENVIRONMENT_VARIABLE1: "test environment variable 1 value", TEST_ENVIRONMENT_VARIABLE2: "test environment variable 2 value" }, - queueName: 'ecs-test-sqs-queue', + queue, maxScalingCapacity: 5 }); @@ -106,15 +106,6 @@ export = { " amazon.com" ], Environment: [ - { - Name: "QUEUE_NAME", - Value: { - "Fn::GetAtt": [ - "ServiceecsworkerservicequeueAE5FF7F2", - "QueueName" - ] - } - }, { Name: "TEST_ENVIRONMENT_VARIABLE1", Value: "test environment variable 1 value" @@ -122,6 +113,15 @@ export = { { Name: "TEST_ENVIRONMENT_VARIABLE2", Value: "test environment variable 2 value" + }, + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "ecstestqueueD1FDA34B", + "QueueName" + ] + } } ], Image: "test", diff --git a/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts index dcb0148f20a35..44d113a63529e 100644 --- a/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts +++ b/packages/@aws-cdk/aws-ecs/test/test.fargate-worker-service.ts @@ -1,5 +1,6 @@ import { expect, haveResource, haveResourceLike } from '@aws-cdk/assert'; import ec2 = require('@aws-cdk/aws-ec2'); +import sqs = require('@aws-cdk/aws-sqs'); import cdk = require('@aws-cdk/cdk'); import { Test } from 'nodeunit'; import ecs = require('../lib'); @@ -25,9 +26,7 @@ export = { LaunchType: "FARGATE", })); - expect(stack).to(haveResource("AWS::SQS::Queue", { - QueueName: "ecs-worker-service-queue" - })); + expect(stack).to(haveResource("AWS::SQS::Queue")); expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { ContainerDefinitions: [ @@ -37,7 +36,7 @@ export = { Name: "QUEUE_NAME", Value: { "Fn::GetAtt": [ - "ServiceecsworkerservicequeueAE5FF7F2", + "ServiceEcsWorkerServiceQueue19BF278C", "QueueName" ] } @@ -69,6 +68,7 @@ export = { const vpc = new ec2.VpcNetwork(stack, 'VPC'); const cluster = new ecs.Cluster(stack, 'Cluster', { vpc }); cluster.addCapacity('DefaultAutoScalingGroup', { instanceType: new ec2.InstanceType('t2.micro') }); + const queue = new sqs.Queue(stack, 'fargate-test-queue', { queueName: 'fargate-test-sqs-queue'}); // WHEN new ecs.FargateQueueWorkerService(stack, 'Service', { @@ -82,7 +82,7 @@ export = { TEST_ENVIRONMENT_VARIABLE1: "test environment variable 1 value", TEST_ENVIRONMENT_VARIABLE2: "test environment variable 2 value" }, - queueName: 'fargate-test-sqs-queue', + queue, maxScalingCapacity: 5 }); @@ -92,9 +92,7 @@ export = { LaunchType: "FARGATE" })); - expect(stack).to(haveResource("AWS::SQS::Queue", { - QueueName: "fargate-test-sqs-queue" - })); + expect(stack).to(haveResource("AWS::SQS::Queue", { QueueName: 'fargate-test-sqs-queue' })); expect(stack).to(haveResourceLike('AWS::ECS::TaskDefinition', { ContainerDefinitions: [ @@ -105,15 +103,6 @@ export = { " amazon.com" ], Environment: [ - { - Name: "QUEUE_NAME", - Value: { - "Fn::GetAtt": [ - "ServiceecsworkerservicequeueAE5FF7F2", - "QueueName" - ] - } - }, { Name: "TEST_ENVIRONMENT_VARIABLE1", Value: "test environment variable 1 value" @@ -121,6 +110,15 @@ export = { { Name: "TEST_ENVIRONMENT_VARIABLE2", Value: "test environment variable 2 value" + }, + { + Name: "QUEUE_NAME", + Value: { + "Fn::GetAtt": [ + "fargatetestqueue28B43841", + "QueueName" + ] + } } ], Image: "test",