fix(toolkit): CLI tool fails on CloudFormation Throttling

packages/aws-cdk/lib/api/aws-auth/sdk.ts

@@ -42,29 +42,34 @@ export class SDK implements ISDK {
   private readonly config: ConfigurationOptions;
 
   /**
-   * Default retry options for SDK clients
-   *
-   * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be
-   * a little more tenacious than the defaults, and with a little more breathing
-   * room between calls (defaults are {retries=3, base=100}).
+   * Default retry options for SDK clients.
+   */
+  private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 } };
+
+  /**
+   * The more generous retry policy for CloudFormation, which has a 1 TPM limit on certain APIs,
+   * which are abundantly used for deployment tracking, ...
    *
-   * I've left this running in a tight loop for an hour and the throttle errors
-   * haven't escaped the retry mechanism.
+   * So we're allowing way more retries, but waiting a bit more.
    */
-  private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
+  private readonly cloudFormationRetryOptions = { maxRetries: 10, retryDelayOptions: { base: 1_000 } };
 
   constructor(private readonly credentials: AWS.Credentials, region: string, httpOptions: ConfigurationOptions = {}) {
     this.config = {
       ...httpOptions,
       ...this.retryOptions,
       credentials,
       region,
+      logger: { log: (...messages) => messages.forEach(m => debug('%s', m)) },
     };
     this.currentRegion = region;
   }
 
   public cloudFormation(): AWS.CloudFormation {
-    return wrapServiceErrorHandling(new AWS.CloudFormation(this.config));
+    return wrapServiceErrorHandling(new AWS.CloudFormation({
+      ...this.config,
+      ...this.cloudFormationRetryOptions,
+    }));
   }
 
   public ec2(): AWS.EC2 {
@@ -212,4 +217,4 @@ function allChainedExceptionMessages(e: Error | undefined) {
     e = (e as any).originalError;
   }
   return ret.join(': ');
-}
+}