feat(cli): hotswap deployments for ECS Services

packages/aws-cdk/README.md

@@ -363,6 +363,7 @@ Hotswapping is currently supported for the following changes
 
 - Code asset changes of AWS Lambda functions.
 - Definition changes of AWS Step Functions State Machines.
+- Container asset changes of AWS ECS Services.
 
 **⚠ Note #1**: This command deliberately introduces drift in CloudFormation stacks in order to speed up deployments.
 For this reason, only use it for development purposes.

packages/aws-cdk/lib/api/aws-auth/sdk.ts

@@ -29,6 +29,7 @@ export interface ISDK {
   s3(): AWS.S3;
   route53(): AWS.Route53;
   ecr(): AWS.ECR;
+  ecs(): AWS.ECS;
   elbv2(): AWS.ELBv2;
   secretsManager(): AWS.SecretsManager;
   kms(): AWS.KMS;
@@ -117,6 +118,10 @@ export class SDK implements ISDK {
     return this.wrapServiceErrorHandling(new AWS.ECR(this.config));
   }
 
+  public ecs(): AWS.ECS {
+    return this.wrapServiceErrorHandling(new AWS.ECS(this.config));
+  }
+
   public elbv2(): AWS.ELBv2 {
     return this.wrapServiceErrorHandling(new AWS.ELBv2(this.config));
   }

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -258,7 +258,6 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
       if (hotswapDeploymentResult) {
         return hotswapDeploymentResult;
       }
-      // could not short-circuit the deployment, perform a full CFN deploy instead
       print('Could not perform a hotswap deployment, as the stack %s contains non-Asset changes', stackArtifact.displayName);
     } catch (e) {
       if (!(e instanceof CfnEvaluationException)) {
@@ -269,6 +268,7 @@ export async function deployStack(options: DeployStackOptions): Promise<DeploySt
     print('Falling back to doing a full deployment');
   }
 
+  // could not short-circuit the deployment, perform a full CFN deploy instead
   return prepareAndExecuteChangeSet(options, cloudFormationStack, stackArtifact, stackParams, bodyParameter);
 }
 

packages/aws-cdk/lib/api/hotswap-deployments.ts

@@ -4,6 +4,7 @@ import { CloudFormation } from 'aws-sdk';
 import { ISDK, Mode, SdkProvider } from './aws-auth';
 import { DeployStackResult } from './deploy-stack';
 import { ChangeHotswapImpact, ChangeHotswapResult, HotswapOperation, ListStackResources, HotswappableChangeCandidate } from './hotswap/common';
+import { isHotswappableEcsServiceChange } from './hotswap/ecs-services';
 import { EvaluateCloudFormationTemplate } from './hotswap/evaluate-cloudformation-template';
 import { isHotswappableLambdaFunctionChange } from './hotswap/lambda-functions';
 import { isHotswappableStateMachineChange } from './hotswap/stepfunctions-state-machines';
@@ -73,6 +74,7 @@ async function findAllHotswappableChanges(
       promises.push([
         isHotswappableLambdaFunctionChange(logicalId, resourceHotswapEvaluation, evaluateCfnTemplate),
         isHotswappableStateMachineChange(logicalId, resourceHotswapEvaluation, evaluateCfnTemplate),
+        isHotswappableEcsServiceChange(logicalId, resourceHotswapEvaluation, evaluateCfnTemplate),
       ]);
     }
   });

packages/aws-cdk/lib/api/hotswap/ecs-services.ts

@@ -0,0 +1,187 @@
+import * as AWS from 'aws-sdk';
+import { ISDK } from '../aws-auth';
+import { ChangeHotswapImpact, ChangeHotswapResult, establishResourcePhysicalName, HotswapOperation, HotswappableChangeCandidate } from './common';
+import { EvaluateCloudFormationTemplate } from './evaluate-cloudformation-template';
+
+export async function isHotswappableEcsServiceChange(
+  logicalId: string, change: HotswappableChangeCandidate, evaluateCfnTemplate: EvaluateCloudFormationTemplate,
+): Promise<ChangeHotswapResult> {
+  // the only resource change we should allow is an ECS TaskDefinition
+  if (change.newValue.Type !== 'AWS::ECS::TaskDefinition') {
+    return ChangeHotswapImpact.REQUIRES_FULL_DEPLOYMENT;
+  }
+
+  for (const updatedPropName in change.propertyUpdates) {
+    // We only allow a change in the ContainerDefinitions of the TaskDefinition for now -
+    // it contains the image and environment variables, so seems like a safe bet for now.
+    // We might revisit this decision in the future though!
+    if (updatedPropName !== 'ContainerDefinitions') {
+      return ChangeHotswapImpact.REQUIRES_FULL_DEPLOYMENT;
+    }
+    const containerDefinitionsDifference = (change.propertyUpdates)[updatedPropName];
+    if (containerDefinitionsDifference.newValue === undefined) {
+      return ChangeHotswapImpact.REQUIRES_FULL_DEPLOYMENT;
+    }
+  }
+  // at this point, we know the TaskDefinition can be hotswapped
+
+  // find all ECS Services that reference the TaskDefinition that changed
+  const resourcesReferencingTaskDef = evaluateCfnTemplate.findReferencesTo(logicalId);
+  const ecsServiceResourcesReferencingTaskDef = resourcesReferencingTaskDef.filter(r => r.Type === 'AWS::ECS::Service');
+  const ecsServicesReferencingTaskDef = new Array<EcsService>();
+  for (const ecsServiceResource of ecsServiceResourcesReferencingTaskDef) {
+    const serviceArn = await evaluateCfnTemplate.findPhysicalNameFor(ecsServiceResource.LogicalId);
+    if (serviceArn) {
+      ecsServicesReferencingTaskDef.push({ serviceArn });
+    }
+  }
+  if (ecsServicesReferencingTaskDef.length === 0 ||
+      resourcesReferencingTaskDef.length > ecsServicesReferencingTaskDef.length) {
+    // if there are either no resources referencing the TaskDefinition,
+    // or something besides an ECS Service is referencing it,
+    // hotswap is not possible
+    return ChangeHotswapImpact.REQUIRES_FULL_DEPLOYMENT;
+  }
+
+  const taskDefinitionResource = change.newValue.Properties;
+  // first, let's get the name of the family
+  const familyNameOrArn = await establishResourcePhysicalName(logicalId, taskDefinitionResource?.Family, evaluateCfnTemplate);
+  if (!familyNameOrArn) {
+    // if the Family property has not bee provided, and we can't find it in the current Stack,
+    // this means hotswapping is not possible
+    return ChangeHotswapImpact.REQUIRES_FULL_DEPLOYMENT;
+  }
+  // the physical name of the Task Definition in CloudFormation includes its current revision number at the end,
+  // remove it if needed
+  const familyNameOrArnParts = familyNameOrArn.split(':');
+  const family = familyNameOrArnParts.length > 1
+    // familyNameOrArn is actually an ARN, of the format 'arn:aws:ecs:region:account:task-definition/<family-name>:<revision-nr>'
+    // so, take the 6th element, at index 5, and split it on '/'
+    ? familyNameOrArnParts[5].split('/')[1]
+    // otherwise, familyNameOrArn is just the simple name evaluated from the CloudFormation template
+    : familyNameOrArn;
+  // then, let's evaluate the body of the remainder of the TaskDef (without the Family property)
+  const evaluatedTaskDef = {
+    ...await evaluateCfnTemplate.evaluateCfnExpression({
+      ...(taskDefinitionResource ?? {}),
+      Family: undefined,
+    }),
+    Family: family,
+  };
+  return new EcsServiceHotswapOperation(evaluatedTaskDef, ecsServicesReferencingTaskDef);
+}
+
+interface EcsService {
+  readonly serviceArn: string;
+}
+
+class EcsServiceHotswapOperation implements HotswapOperation {
+  constructor(
+    private readonly taskDefinitionResource: any,
+    private readonly servicesReferencingTaskDef: EcsService[],
+  ) {}
+
+  public async apply(sdk: ISDK): Promise<any> {
+    // Step 1 - update the changed TaskDefinition, creating a new TaskDefinition Revision
+    // we need to lowercase the evaluated TaskDef from CloudFormation,
+    // as the AWS SDK uses lowercase property names for these
+    const lowercasedTaskDef = lowerCaseFirstCharacterOfObjectKeys(this.taskDefinitionResource);
+    const registerTaskDefResponse = await sdk.ecs().registerTaskDefinition(lowercasedTaskDef).promise();
+    const taskDefRevArn = registerTaskDefResponse.taskDefinition?.taskDefinitionArn;
+
+    // Step 2 - update the services using that TaskDefinition to point to the new TaskDefinition Revision
+    const servicePerClusterUpdates: { [cluster: string]: Array<{ promise: Promise<any>, ecsService: EcsService }> } = {};
+    for (const ecsService of this.servicesReferencingTaskDef) {
+      const clusterName = ecsService.serviceArn.split('/')[1];
+
+      const existingClusterPromises = servicePerClusterUpdates[clusterName];
+      let clusterPromises: Array<{ promise: Promise<any>, ecsService: EcsService }>;
+      if (existingClusterPromises) {
+        clusterPromises = existingClusterPromises;
+      } else {
+        clusterPromises = [];
+        servicePerClusterUpdates[clusterName] = clusterPromises;
+      }
+
+      clusterPromises.push({
+        promise: sdk.ecs().updateService({
+          service: ecsService.serviceArn,
+          taskDefinition: taskDefRevArn,
+          cluster: clusterName,
+          forceNewDeployment: true,
+          deploymentConfiguration: {
+            minimumHealthyPercent: 0,
+          },
+        }).promise(),
+        ecsService: ecsService,
+      });
+    }
+    await Promise.all(Object.values(servicePerClusterUpdates)
+      .map(clusterUpdates => {
+        return Promise.all(clusterUpdates.map(serviceUpdate => serviceUpdate.promise));
+      }),
+    );
+
+    // Step 3 - wait for the service deployments triggered in Step 2 to finish
+    // configure a custom Waiter
+    (sdk.ecs() as any).api.waiters.deploymentToFinish = {
+      name: 'DeploymentToFinish',
+      operation: 'describeServices',
+      delay: 10,
+      maxAttempts: 60,
+      acceptors: [
+        {
+          matcher: 'pathAny',
+          argument: 'failures[].reason',
+          expected: 'MISSING',
+          state: 'failure',
+        },
+        {
+          matcher: 'pathAny',
+          argument: 'services[].status',
+          expected: 'DRAINING',
+          state: 'failure',
+        },
+        {
+          matcher: 'pathAny',
+          argument: 'services[].status',
+          expected: 'INACTIVE',
+          state: 'failure',
+        },
+        {
+          matcher: 'path',
+          argument: "length(services[].deployments[? status == 'PRIMARY' && runningCount < desiredCount][]) == `0`",
+          expected: true,
+          state: 'success',
+        },
+      ],
+    };
+    // create a custom Waiter that uses the deploymentToFinish configuration added above
+    const deploymentWaiter = new (AWS as any).ResourceWaiter(sdk.ecs(), 'deploymentToFinish');
+    // wait for all of the waiters to finish
+    return Promise.all(Object.entries(servicePerClusterUpdates).map(([clusterName, serviceUpdates]) => {
+      return deploymentWaiter.wait({
+        cluster: clusterName,
+        services: serviceUpdates.map(serviceUpdate => serviceUpdate.ecsService.serviceArn),
+      }).promise();
+    }));
+  }
+}
+
+function lowerCaseFirstCharacterOfObjectKeys(val: any): any {
+  if (val == null || typeof val !== 'object') {
+    return val;
+  }
+  if (Array.isArray(val)) {
+    return val.map(lowerCaseFirstCharacterOfObjectKeys);
+  }
+  const ret: { [k: string]: any; } = {};
+  for (const [k, v] of Object.entries(val)) {
+    ret[lowerCaseFirstCharacter(k)] = lowerCaseFirstCharacterOfObjectKeys(v);
+  }
+  return ret;
+}
+
+function lowerCaseFirstCharacter(str: string): string {
+  return str.length > 0 ? `${str[0].toLowerCase()}${str.substr(1)}` : str;
+}

packages/aws-cdk/lib/api/hotswap/evaluate-cloudformation-template.ts

@@ -4,26 +4,33 @@ import { ListStackResources } from './common';
 
 export class CfnEvaluationException extends Error {}
 
+export interface ResourceDefinition {
+  readonly LogicalId: string;
+  readonly Type: string;
+  readonly Properties: { [p: string]: any };
+}
+
 export interface EvaluateCloudFormationTemplateProps {
   readonly stackArtifact: cxapi.CloudFormationStackArtifact;
   readonly parameters: { [parameterName: string]: string };
   readonly account: string;
   readonly region: string;
   readonly partition: string;
   readonly urlSuffix: string;
-
   readonly listStackResources: ListStackResources;
 }
 
 export class EvaluateCloudFormationTemplate {
   private readonly stackResources: ListStackResources;
+  private readonly template: { [section: string]: { [headings: string]: any } };
   private readonly context: { [k: string]: string };
   private readonly account: string;
   private readonly region: string;
   private readonly partition: string;
 
   constructor(props: EvaluateCloudFormationTemplateProps) {
     this.stackResources = props.listStackResources;
+    this.template = props.stackArtifact.template;
     this.context = {
       'AWS::AccountId': props.account,
       'AWS::Region': props.region,
@@ -41,6 +48,19 @@ export class EvaluateCloudFormationTemplate {
     return stackResources.find(sr => sr.LogicalResourceId === logicalId)?.PhysicalResourceId;
   }
 
+  public findReferencesTo(logicalId: string): Array<ResourceDefinition> {
+    const ret = new Array<ResourceDefinition>();
+    for (const [resourceLogicalId, resourceDef] of Object.entries(this.template?.Resources ?? {})) {
+      if (logicalId !== resourceLogicalId && this.references(logicalId, resourceDef)) {
+        ret.push({
+          ...(resourceDef as any),
+          LogicalId: resourceLogicalId,
+        });
+      }
+    }
+    return ret;
+  }
+
   public async evaluateCfnExpression(cfnExpression: any): Promise<any> {
     const self = this;
     class CfnIntrinsics {
@@ -131,6 +151,26 @@ export class EvaluateCloudFormationTemplate {
     return cfnExpression;
   }
 
+  private references(logicalId: string, templateElement: any): boolean {
+    if (typeof templateElement === 'string') {
+      return logicalId === templateElement;
+    }
+
+    if (templateElement == null) {
+      return false;
+    }
+
+    if (Array.isArray(templateElement)) {
+      return templateElement.some(el => this.references(logicalId, el));
+    }
+
+    if (typeof templateElement === 'object') {
+      return Object.values(templateElement).some(el => this.references(logicalId, el));
+    }
+
+    return false;
+  }
+
   private parseIntrinsic(x: any): Intrinsic | undefined {
     const keys = Object.keys(x);
     if (keys.length === 1 && (keys[0].startsWith('Fn::') || keys[0] === 'Ref')) {