fix(toolkit): increase number of retries (#2053)

This will help in situations where the default retry behavior is
not enough to cover throttling errors.

Fixes #1647.

packages/aws-cdk/lib/api/bootstrap-environment.ts

@@ -12,7 +12,7 @@ export async function bootstrapEnvironment(environment: Environment, aws: SDK, t
  environment,
  metadata: {},
  template: {
- Description: "The CDK Toolkit Stack. It cas created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
+ Description: "The CDK Toolkit Stack. It was created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
  Resources: {
  StagingBucket: {
  Type: "AWS::S3::Bucket",

packages/aws-cdk/lib/api/util/sdk.ts

@@ -50,6 +50,18 @@ export class SDK {
  private readonly credentialsCache: CredentialsCache;
  private readonly profile?: string;
 
+ /**
+ * Default retry options for SDK clients
+ *
+ * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be
+ * a little more tenacious than the defaults, and with a little more breathing
+ * room between calls (defaults are {retries=3, base=100}).
+ *
+ * I've left this running in a tight loop for an hour and the throttle errors
+ * haven't escaped the retry mechanism.
+ */
+ private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
+
  constructor(options: SDKOptions = {}) {
  this.profile = options.profile;
 
@@ -78,41 +90,47 @@ export class SDK {
 
  public async cloudFormation(environment: Environment, mode: Mode): Promise<AWS.CloudFormation> {
  return new AWS.CloudFormation({
+ ...this.retryOptions,
  region: environment.region,
  credentials: await this.credentialsCache.get(environment.account, mode)
  });
  }
 
  public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.EC2> {
  return new AWS.EC2({
+ ...this.retryOptions,
  region,
  credentials: await this.credentialsCache.get(awsAccountId, mode)
  });
  }
 
  public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.SSM> {
  return new AWS.SSM({
+ ...this.retryOptions,
  region,
  credentials: await this.credentialsCache.get(awsAccountId, mode)
  });
  }
 
  public async s3(environment: Environment, mode: Mode): Promise<AWS.S3> {
  return new AWS.S3({
+ ...this.retryOptions,
  region: environment.region,
  credentials: await this.credentialsCache.get(environment.account, mode)
  });
  }
 
  public async route53(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.Route53> {
  return new AWS.Route53({
+ ...this.retryOptions,
  region,
  credentials: await this.credentialsCache.get(awsAccountId, mode),
  });
  }
 
  public async ecr(environment: Environment, mode: Mode): Promise<AWS.ECR> {
  return new AWS.ECR({
+ ...this.retryOptions,
  region: environment.region,
  credentials: await this.credentialsCache.get(environment.account, mode)
  });