feat(cli): configurable timeout for ECS hotswap operation

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cdk-hotswap-deployment-for-ecs-service-detects-failed-deployment-and-errors.integtest.ts

@@ -10,7 +10,7 @@ integTest(
 
     // WHEN
     const deployOutput = await fixture.cdkDeploy('ecs-hotswap', {
-      options: ['--hotswap'],
+      options: ['--hotswap', '--hotswap-ecs-stabilization-timeout-seconds', '10'],
       modEnv: {
         USE_INVALID_ECS_HOTSWAP_IMAGE: 'true',
       },

packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/sdk.ts

@@ -488,7 +488,7 @@ export interface IECSClient {
   registerTaskDefinition(input: RegisterTaskDefinitionCommandInput): Promise<RegisterTaskDefinitionCommandOutput>;
   updateService(input: UpdateServiceCommandInput): Promise<UpdateServiceCommandOutput>;
   // Waiters
-  waitUntilServicesStable(input: DescribeServicesCommandInput): Promise<WaiterResult>;
+  waitUntilServicesStable(input: DescribeServicesCommandInput, timeoutSeconds?: number): Promise<WaiterResult>;
 }
 
 export interface IElasticLoadBalancingV2Client {
@@ -827,11 +827,11 @@ export class SDK {
       updateService: (input: UpdateServiceCommandInput): Promise<UpdateServiceCommandOutput> =>
         client.send(new UpdateServiceCommand(input)),
       // Waiters
-      waitUntilServicesStable: (input: DescribeServicesCommandInput): Promise<WaiterResult> => {
+      waitUntilServicesStable: (input: DescribeServicesCommandInput, timeoutSeconds?: number): Promise<WaiterResult> => {
         return waitUntilServicesStable(
           {
             client,
-            maxWaitTime: 600,
+            maxWaitTime: timeoutSeconds ?? 600,
             minDelay: 6,
             maxDelay: 6,
           },

packages/@aws-cdk/toolkit-lib/lib/api/hotswap/common.ts

@@ -86,21 +86,27 @@ export class EcsHotswapProperties {
   readonly minimumHealthyPercent?: number;
   // The upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount
   readonly maximumHealthyPercent?: number;
+  // The number of seconds to wait for a single service to reach stable state.
+  readonly stabilizationTimeoutSeconds?: number;
 
-  public constructor (minimumHealthyPercent?: number, maximumHealthyPercent?: number) {
+  public constructor (minimumHealthyPercent?: number, maximumHealthyPercent?: number, stabilizationTimeoutSeconds?: number) {
     if (minimumHealthyPercent !== undefined && minimumHealthyPercent < 0 ) {
       throw new ToolkitError('hotswap-ecs-minimum-healthy-percent can\'t be a negative number');
     }
     if (maximumHealthyPercent !== undefined && maximumHealthyPercent < 0 ) {
       throw new ToolkitError('hotswap-ecs-maximum-healthy-percent can\'t be a negative number');
     }
+    if (stabilizationTimeoutSeconds !== undefined && stabilizationTimeoutSeconds < 0 ) {
+      throw new ToolkitError('hotswap-ecs-stabilization-timeout-seconds can\'t be a negative number');
+    }
     // In order to preserve the current behaviour, when minimumHealthyPercent is not defined, it will be set to the currently default value of 0
     if (minimumHealthyPercent == undefined) {
       this.minimumHealthyPercent = 0;
     } else {
       this.minimumHealthyPercent = minimumHealthyPercent;
     }
     this.maximumHealthyPercent = maximumHealthyPercent;
+    this.stabilizationTimeoutSeconds = stabilizationTimeoutSeconds;
   }
 
   /**

packages/@aws-cdk/toolkit-lib/lib/api/hotswap/ecs-services.ts

@@ -131,6 +131,7 @@ export async function isHotswappableEcsServiceChange(
         let ecsHotswapProperties = hotswapPropertyOverrides.ecsHotswapProperties;
         let minimumHealthyPercent = ecsHotswapProperties?.minimumHealthyPercent;
         let maximumHealthyPercent = ecsHotswapProperties?.maximumHealthyPercent;
+        let stabilizationTimeoutSeconds = ecsHotswapProperties?.stabilizationTimeoutSeconds;
 
         // Step 2 - update the services using that TaskDefinition to point to the new TaskDefinition Revision
         // Forcing New Deployment and setting Minimum Healthy Percent to 0.
@@ -153,7 +154,7 @@ export async function isHotswappableEcsServiceChange(
             await sdk.ecs().waitUntilServicesStable({
               cluster: update.service?.clusterArn,
               services: [service.serviceArn],
-            });
+            }, stabilizationTimeoutSeconds);
           }),
         );
       },

packages/@aws-cdk/toolkit-lib/test/api/hotswap/ecs-services-hotswap-deployments.test.ts

@@ -724,5 +724,97 @@ describe.each([
       },
       forceNewDeployment: true,
     });
+    expect(mockECSClient).toHaveReceivedCommandWith(DescribeServicesCommand, {
+      cluster: 'arn:aws:ecs:region:account:service/my-cluster',
+      services: ['arn:aws:ecs:region:account:service/my-cluster/my-service'],
+    });
+  });
+});
+
+test.each([
+  // default case
+  [101, undefined],
+  [2, 10],
+  [11, 60],
+])('DesribeService is called %p times when timeout is %p', async (describeAttempts: number, timeoutSeconds?: number) => {
+  setup.setCurrentCfnStackTemplate({
+    Resources: {
+      TaskDef: {
+        Type: 'AWS::ECS::TaskDefinition',
+        Properties: {
+          Family: 'my-task-def',
+          ContainerDefinitions: [
+            { Image: 'image1' },
+          ],
+        },
+      },
+      Service: {
+        Type: 'AWS::ECS::Service',
+        Properties: {
+          TaskDefinition: { Ref: 'TaskDef' },
+        },
+      },
+    },
+  });
+  setup.pushStackResourceSummaries(
+    setup.stackSummaryOf('Service', 'AWS::ECS::Service',
+      'arn:aws:ecs:region:account:service/my-cluster/my-service'),
+  );
+  mockECSClient.on(RegisterTaskDefinitionCommand).resolves({
+    taskDefinition: {
+      taskDefinitionArn: 'arn:aws:ecs:region:account:task-definition/my-task-def:3',
+    },
+  });
+  const cdkStackArtifact = setup.cdkStackArtifactOf({
+    template: {
+      Resources: {
+        TaskDef: {
+          Type: 'AWS::ECS::TaskDefinition',
+          Properties: {
+            Family: 'my-task-def',
+            ContainerDefinitions: [
+              { Image: 'image2' },
+            ],
+          },
+        },
+        Service: {
+          Type: 'AWS::ECS::Service',
+          Properties: {
+            TaskDefinition: { Ref: 'TaskDef' },
+          },
+        },
+      },
+    },
+  });
+
+  // WHEN
+  let ecsHotswapProperties = new EcsHotswapProperties(undefined, undefined, timeoutSeconds);
+  // mock the client such that the service never becomes stable using desiredCount > runningCount
+  mockECSClient.on(DescribeServicesCommand).resolves({
+    services: [
+      {
+        serviceArn: 'arn:aws:ecs:region:account:service/my-cluster/my-service',
+        taskDefinition: 'arn:aws:ecs:region:account:task-definition/my-task-def:3',
+        desiredCount: 1,
+        runningCount: 0,
+      },
+    ],
+  });
+
+  jest.useFakeTimers();
+  jest.spyOn(global, 'setTimeout').mockImplementation((callback, ms) => {
+    callback();
+    jest.advanceTimersByTime(ms ?? 0);
+    return {} as NodeJS.Timeout;
   });
+
+  await expect(hotswapMockSdkProvider.tryHotswapDeployment(
+    HotswapMode.HOTSWAP_ONLY,
+    cdkStackArtifact,
+    {},
+    new HotswapPropertyOverrides(ecsHotswapProperties),
+  )).rejects.toThrow('Resource is not in the expected state due to waiter status');
+
+  // THEN
+  expect(mockECSClient).toHaveReceivedCommandTimes(DescribeServicesCommand, describeAttempts);
 });

packages/aws-cdk/README.md

@@ -505,19 +505,34 @@ Hotswapping is currently supported for the following changes
 - VTL mapping template changes for AppSync Resolvers and Functions.
 - Schema changes for AppSync GraphQL Apis.
 
-You can optionally configure the behavior of your hotswap deployments in `cdk.json`. Currently you can only configure ECS hotswap behavior:
+You can optionally configure the behavior of your hotswap deployments. Currently you can only configure ECS hotswap behavior:
+
+| Property                       | Description                          | Default     |
+|--------------------------------|--------------------------------------|-------------|
+| minimumHealthyPercent          | Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount  | **REPLICA:** 100, **DAEMON:** 0 |
+| maximumHealthyPercent          | Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount    | **REPLICA:** 200, **DAEMON:**: N/A |
+| stabilizationTimeoutSeconds    | Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount    | 600 |
+
+##### cdk.json
 
 ```json
 {
 "hotswap": {
     "ecs": {
       "minimumHealthyPercent": 100,
-      "maximumHealthyPercent": 250
+      "maximumHealthyPercent": 250,
+      "stabilizationTimeoutSeconds": 300,
     }
   }
 }
 ```
 
+##### cli arguments
+
+```console
+cdk deploy --hotswap --hotswap-ecs-minimum-healthy-percent 100 --hotswap-ecs-maximum-healthy-percent 250 --hotswap-ecs-stabilization-timeout-seconds 300
+```
+
 **⚠ Note #1**: This command deliberately introduces drift in CloudFormation stacks in order to speed up deployments.
 For this reason, only use it for development purposes.
 **Never use this flag for your production deployments**!

packages/aws-cdk/lib/cli/cdk-toolkit.ts

@@ -9,8 +9,8 @@ import * as uuid from 'uuid';
 import { CliIoHost } from './io-host';
 import type { Configuration } from './user-configuration';
 import { PROJECT_CONFIG } from './user-configuration';
+import type { ToolkitAction } from '../../../@aws-cdk/toolkit-lib';
 import { StackSelectionStrategy, ToolkitError } from '../../../@aws-cdk/toolkit-lib';
-import type { ToolkitAction } from '../../../@aws-cdk/toolkit-lib/lib/api';
 import { asIoHelper } from '../../../@aws-cdk/toolkit-lib/lib/api/io/private';
 import { PermissionChangeType } from '../../../@aws-cdk/toolkit-lib/lib/payloads';
 import type { ToolkitOptions } from '../../../@aws-cdk/toolkit-lib/lib/toolkit';
@@ -390,6 +390,7 @@ export class CdkToolkit {
     hotswapPropertyOverrides.ecsHotswapProperties = new EcsHotswapProperties(
       hotswapPropertiesFromSettings.ecs?.minimumHealthyPercent,
       hotswapPropertiesFromSettings.ecs?.maximumHealthyPercent,
+      hotswapPropertiesFromSettings.ecs?.stabilizationTimeoutSeconds,
     );
 
     const stacks = stackCollection.stackArtifacts;

packages/aws-cdk/lib/cli/cli-config.ts

@@ -158,6 +158,18 @@ export async function makeConfig(): Promise<CliConfig> {
               'and falls back to a full deployment if that is not possible. ' +
               'Do not use this in production environments',
           },
+          'hotswap-ecs-minimum-healthy-percent': {
+            type: 'string',
+            desc: 'Lower limit on the number of your service\'s tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount',
+          },
+          'hotswap-ecs-maximum-healthy-percent': {
+            type: 'string',
+            desc: 'Upper limit on the number of your service\'s tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount',
+          },
+          'hotswap-ecs-stabilization-timeout-seconds': {
+            type: 'string',
+            desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
+          },
           'watch': {
             type: 'boolean',
             desc: 'Continuously observe the project files, ' +
@@ -275,6 +287,18 @@ export async function makeConfig(): Promise<CliConfig> {
               'which skips CloudFormation and updates the resources directly, ' +
               'and falls back to a full deployment if that is not possible.',
           },
+          'hotswap-ecs-minimum-healthy-percent': {
+            type: 'string',
+            desc: 'Lower limit on the number of your service\'s tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount',
+          },
+          'hotswap-ecs-maximum-healthy-percent': {
+            type: 'string',
+            desc: 'Upper limit on the number of your service\'s tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount',
+          },
+          'hotswap-ecs-stabilization-timeout-seconds': {
+            type: 'string',
+            desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
+          },
           'logs': {
             type: 'boolean',
             default: true,

packages/aws-cdk/lib/cli/convert-to-user-input.ts

@@ -114,6 +114,9 @@ export function convertYargsToUserInput(args: any): UserInput {
         rollback: args.rollback,
         hotswap: args.hotswap,
         hotswapFallback: args.hotswapFallback,
+        hotswapEcsMinimumHealthyPercent: args.hotswapEcsMinimumHealthyPercent,
+        hotswapEcsMaximumHealthyPercent: args.hotswapEcsMaximumHealthyPercent,
+        hotswapEcsStabilizationTimeoutSeconds: args.hotswapEcsStabilizationTimeoutSeconds,
         watch: args.watch,
         logs: args.logs,
         concurrency: args.concurrency,
@@ -159,6 +162,9 @@ export function convertYargsToUserInput(args: any): UserInput {
         rollback: args.rollback,
         hotswap: args.hotswap,
         hotswapFallback: args.hotswapFallback,
+        hotswapEcsMinimumHealthyPercent: args.hotswapEcsMinimumHealthyPercent,
+        hotswapEcsMaximumHealthyPercent: args.hotswapEcsMaximumHealthyPercent,
+        hotswapEcsStabilizationTimeoutSeconds: args.hotswapEcsStabilizationTimeoutSeconds,
         logs: args.logs,
         concurrency: args.concurrency,
         STACKS: args.STACKS,
@@ -356,6 +362,9 @@ export function convertConfigToUserInput(config: any): UserInput {
     rollback: config.deploy?.rollback,
     hotswap: config.deploy?.hotswap,
     hotswapFallback: config.deploy?.hotswapFallback,
+    hotswapEcsMinimumHealthyPercent: config.deploy?.hotswapEcsMinimumHealthyPercent,
+    hotswapEcsMaximumHealthyPercent: config.deploy?.hotswapEcsMaximumHealthyPercent,
+    hotswapEcsStabilizationTimeoutSeconds: config.deploy?.hotswapEcsStabilizationTimeoutSeconds,
     watch: config.deploy?.watch,
     logs: config.deploy?.logs,
     concurrency: config.deploy?.concurrency,
@@ -389,6 +398,9 @@ export function convertConfigToUserInput(config: any): UserInput {
     rollback: config.watch?.rollback,
     hotswap: config.watch?.hotswap,
     hotswapFallback: config.watch?.hotswapFallback,
+    hotswapEcsMinimumHealthyPercent: config.watch?.hotswapEcsMinimumHealthyPercent,
+    hotswapEcsMaximumHealthyPercent: config.watch?.hotswapEcsMaximumHealthyPercent,
+    hotswapEcsStabilizationTimeoutSeconds: config.watch?.hotswapEcsStabilizationTimeoutSeconds,
     logs: config.watch?.logs,
     concurrency: config.watch?.concurrency,
   };

packages/aws-cdk/lib/cli/parse-command-line-arguments.ts

@@ -464,6 +464,21 @@ export function parseCommandLineArguments(args: Array<string>): any {
           type: 'boolean',
           desc: "Attempts to perform a 'hotswap' deployment, which skips CloudFormation and updates the resources directly, and falls back to a full deployment if that is not possible. Do not use this in production environments",
         })
+        .option('hotswap-ecs-minimum-healthy-percent', {
+          default: undefined,
+          type: 'string',
+          desc: "Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount",
+        })
+        .option('hotswap-ecs-maximum-healthy-percent', {
+          default: undefined,
+          type: 'string',
+          desc: "Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount",
+        })
+        .option('hotswap-ecs-stabilization-timeout-seconds', {
+          default: undefined,
+          type: 'string',
+          desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
+        })
         .option('watch', {
           default: undefined,
           type: 'boolean',
@@ -628,6 +643,21 @@ export function parseCommandLineArguments(args: Array<string>): any {
           type: 'boolean',
           desc: "Attempts to perform a 'hotswap' deployment, which skips CloudFormation and updates the resources directly, and falls back to a full deployment if that is not possible.",
         })
+        .option('hotswap-ecs-minimum-healthy-percent', {
+          default: undefined,
+          type: 'string',
+          desc: "Lower limit on the number of your service's tasks that must remain in the RUNNING state during a deployment, as a percentage of the desiredCount",
+        })
+        .option('hotswap-ecs-maximum-healthy-percent', {
+          default: undefined,
+          type: 'string',
+          desc: "Upper limit on the number of your service's tasks that are allowed in the RUNNING or PENDING state during a deployment, as a percentage of the desiredCount",
+        })
+        .option('hotswap-ecs-stabilization-timeout-seconds', {
+          default: undefined,
+          type: 'string',
+          desc: 'Number of seconds to wait for a single service to reach stable state, where the desiredCount is equal to the runningCount',
+        })
         .option('logs', {
           default: true,
           type: 'boolean',

packages/aws-cdk/lib/cli/user-configuration.ts

@@ -305,8 +305,9 @@ export function commandLineArgumentsToSettings(argv: Arguments): Settings {
     ignoreNoStacks: argv['ignore-no-stacks'],
     hotswap: {
       ecs: {
-        minimumEcsHealthyPercent: argv.minimumEcsHealthyPercent,
-        maximumEcsHealthyPercent: argv.maximumEcsHealthyPercent,
+        minimumHealthyPercent: argv.hotswapEcsMinimumHealthyPercent,
+        maximumHealthyPercent: argv.hotswapEcsMaximumHealthyPercent,
+        stabilizationTimeoutSeconds: argv.hotswapEcsStabilizationTimeoutSeconds,
       },
     },
     unstable: argv.unstable,