From 133dc98450186f19c4a56b13e1a09352e794f90e Mon Sep 17 00:00:00 2001
From: Rico Huijbers <rix0rrr@gmail.com>
Date: Tue, 19 Mar 2019 15:10:12 +0100
Subject: [PATCH] fix(toolkit): increase number of retries (#2053)

This will help in situations where the default retry behavior is
not enough to cover throttling errors.

Fixes #1647.
---
 .../aws-cdk/lib/api/bootstrap-environment.ts   |  2 +-
 packages/aws-cdk/lib/api/util/sdk.ts           | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/aws-cdk/lib/api/bootstrap-environment.ts b/packages/aws-cdk/lib/api/bootstrap-environment.ts
index 6503c5870aa02..122899919e97a 100644
--- a/packages/aws-cdk/lib/api/bootstrap-environment.ts
+++ b/packages/aws-cdk/lib/api/bootstrap-environment.ts
@@ -12,7 +12,7 @@ export async function bootstrapEnvironment(environment: Environment, aws: SDK, t
     environment,
     metadata: {},
     template: {
-      Description: "The CDK Toolkit Stack. It cas created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
+      Description: "The CDK Toolkit Stack. It was created by `cdk bootstrap` and manages resources necessary for managing your Cloud Applications with AWS CDK.",
       Resources: {
         StagingBucket: {
           Type: "AWS::S3::Bucket",
diff --git a/packages/aws-cdk/lib/api/util/sdk.ts b/packages/aws-cdk/lib/api/util/sdk.ts
index 639667dd2de16..59ce0adf02b7f 100644
--- a/packages/aws-cdk/lib/api/util/sdk.ts
+++ b/packages/aws-cdk/lib/api/util/sdk.ts
@@ -50,6 +50,18 @@ export class SDK {
   private readonly credentialsCache: CredentialsCache;
   private readonly profile?: string;
 
+  /**
+   * Default retry options for SDK clients
+   *
+   * Biggest bottleneck is CloudFormation, with a 1tps call rate. We want to be
+   * a little more tenacious than the defaults, and with a little more breathing
+   * room between calls (defaults are {retries=3, base=100}).
+   *
+   * I've left this running in a tight loop for an hour and the throttle errors
+   * haven't escaped the retry mechanism.
+   */
+  private readonly retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
+
   constructor(options: SDKOptions = {}) {
     this.profile = options.profile;
 
@@ -78,6 +90,7 @@ export class SDK {
 
   public async cloudFormation(environment: Environment, mode: Mode): Promise<AWS.CloudFormation> {
     return new AWS.CloudFormation({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });
@@ -85,6 +98,7 @@ export class SDK {
 
   public async ec2(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.EC2> {
     return new AWS.EC2({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode)
     });
@@ -92,6 +106,7 @@ export class SDK {
 
   public async ssm(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.SSM> {
     return new AWS.SSM({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode)
     });
@@ -99,6 +114,7 @@ export class SDK {
 
   public async s3(environment: Environment, mode: Mode): Promise<AWS.S3> {
     return new AWS.S3({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });
@@ -106,6 +122,7 @@ export class SDK {
 
   public async route53(awsAccountId: string | undefined, region: string | undefined, mode: Mode): Promise<AWS.Route53> {
     return new AWS.Route53({
+      ...this.retryOptions,
       region,
       credentials: await this.credentialsCache.get(awsAccountId, mode),
     });
@@ -113,6 +130,7 @@ export class SDK {
 
   public async ecr(environment: Environment, mode: Mode): Promise<AWS.ECR> {
     return new AWS.ECR({
+      ...this.retryOptions,
       region: environment.region,
       credentials: await this.credentialsCache.get(environment.account, mode)
     });