fix(toolkit): CLI tool fails on CloudFormation Throttling

The CDK (particularly, `cdk deploy`) might crash after getting throttled
by CloudFormation, after the default configured 6 retries has been
reached.

This changes the retry configuration of the CloudFormation client (and
only that one) to use a custom backoff function that will allow up to
`100` retries to be made before failing (until it reaches an error that
is either not declared as retryable; or that is not a throttling error);
and will exponentially back-off (with a maximum wait time between two
attempts of 1 minute).

This should allow heavily parallel deployments on the same account and
region to avoid getting killed by a throttle; but will reduce the
responsiveness of the progress UI.

Fixes #5637

Author: RomainMuller

packages/aws-cdk/lib/api/aws-auth/sdk.ts

@@ -42,14 +42,7 @@ export class SDK implements ISDK {
   private readonly config: ConfigurationOptions;
 
   /**
-   * Default retry options for SDK clients
-   *
-   * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be
-   * a little more tenacious than the defaults, and with a little more breathing
-   * room between calls (defaults are {retries=3, base=100}).
-   *
-   * I've left this running in a tight loop for an hour and the throttle errors
-   * haven't escaped the retry mechanism.
+   * Default retry options for SDK clients.
    */
   private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
 
@@ -64,7 +57,43 @@ export class SDK implements ISDK {
   }
 
   public cloudFormation(): AWS.CloudFormation {
-    return wrapServiceErrorHandling(new AWS.CloudFormation(this.config));
+    const defaultMaxRetries = this.retryOptions.maxRetries;
+    const defaultRetryBase = this.retryOptions.retryDelayOptions.base;
+
+    return wrapServiceErrorHandling(new AWS.CloudFormation({
+      ...this.config,
+      maxRetries: 100,
+      retryDelayOptions: {
+        customBackoff: (retryCount: number, err?: Error): number => {
+          const { code, retryable } = (err as AWS.AWSError) ?? {};
+
+          // Note: "retryable" will be "undefined" if it wasn't an AWS.AWSError
+          //       and we still want to retry in this case. Hence the "===false"
+          if (retryable === false) {
+            return -1;
+          }
+
+          const exponential = Math.pow(2, retryCount) * defaultRetryBase;
+          const jitter = Math.random() * defaultRetryBase;
+
+          // We're capping this at 60 seconds.
+          const backoff = Math.min(exponential + jitter, 60_000);
+
+          if (retryCount < defaultMaxRetries || code === 'Throttling') {
+            if (code === 'Throttling' && backoff > 5_000) {
+              // Drop a debug line if we'll be waiting more than 5 seconds before the next attempt
+              debug(`CloudFormation Throttled: ${retryCount} retries so far, will retry in ${backoff} milliseconds`);
+            }
+
+            // Exponential back-off per the default configuration
+            return backoff;
+          }
+
+          // No more retrying
+          return -1;
+        },
+      },
+    }));
   }
 
   public ec2(): AWS.EC2 {
@@ -212,4 +241,4 @@ function allChainedExceptionMessages(e: Error | undefined) {
     e = (e as any).originalError;
   }
   return ret.join(': ');
-}
+}