feat(rds): support rolling instance updates to reduce downtime (aws#2…

…0054)

Support defining the instance update behaviour of RDS instances. This allows to switch between bulk (all instances at once) and rolling updates (one instance after another). While bulk updates are faster, they have a higher risk for longer downtimes as all instances might be simultaneously unreachable due to the update. Rolling updates take longer but ensure that all but one instance are not updated and thus downtimes are limited to the (at most two) changes of the primary instance.

We keep the current behaviour, namely a bulk update, as default.

This implementation follows proposal A by  @hixi-hyi  in issue aws#10595.

Fixes  aws#10595

packages/@aws-cdk/aws-rds/README.md

@@ -63,9 +63,30 @@ new rds.DatabaseClusterFromSnapshot(this, 'Database', {
 });
 ```
 
+### Updating the database instances in a cluster
+
+Database cluster instances may be updated in bulk or on a rolling basis. 
+
+An update to all instances in a cluster may cause significant downtime. To reduce the downtime, set the `instanceUpdateBehavior` property in `DatabaseClusterBaseProps` to `InstanceUpdateBehavior.ROLLING`. This adds a dependency between each instance so the update is performed on only one instance at a time.
+
+Use `InstanceUpdateBehavior.BULK` to update all instances at once.
+
+```ts
+declare const vpc: ec2.Vpc;
+const cluster = new rds.DatabaseCluster(this, 'Database', {
+  engine: rds.DatabaseClusterEngine.auroraMysql({ version: rds.AuroraMysqlEngineVersion.VER_3_01_0 }),
+  instances: 2,
+  instanceProps: {
+    instanceType: ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.SMALL),
+    vpc,
+  },
+  instanceUpdateBehaviour: rds.InstanceUpdateBehaviour.ROLLING, // Optional - defaults to rds.InstanceUpdateBehaviour.BULK
+});
+```
+
 ## Starting an instance database
 
-To set up a instance database, define a `DatabaseInstance`. You must
+To set up an instance database, define a `DatabaseInstance`. You must
 always launch a database in a VPC. Use the `vpcSubnets` attribute to control whether
 your instances will be launched privately or publicly:
 

packages/@aws-cdk/aws-rds/lib/cluster.ts

@@ -41,6 +41,13 @@ interface DatabaseClusterBaseProps {
    */
   readonly instanceProps: InstanceProps;
 
+  /**
+   * The ordering of updates for instances
+   *
+   * @default InstanceUpdateBehaviour.BULK
+   */
+  readonly instanceUpdateBehaviour?: InstanceUpdateBehaviour;
+
   /**
    * The number of seconds to set a cluster's target backtrack window to.
    * This feature is only supported by the Aurora MySQL database engine and
@@ -275,6 +282,25 @@ interface DatabaseClusterBaseProps {
   readonly copyTagsToSnapshot?: boolean;
 }
 
+/**
+ * The orchestration of updates of multiple instances
+ */
+export enum InstanceUpdateBehaviour {
+  /**
+   * In a bulk update, all instances of the cluster are updated at the same time.
+   * This results in a faster update procedure.
+   * During the update, however, all instances might be unavailable at the same time and thus a downtime might occur.
+   */
+  BULK = 'BULK',
+
+  /**
+   * In a rolling update, one instance after another is updated.
+   * This results in at most one instance being unavailable during the update.
+   * If your cluster consists of more than 1 instance, the downtime periods are limited to the time a primary switch needs.
+   */
+  ROLLING = 'ROLLING'
+}
+
 /**
  * A new or imported clustered database.
  */
@@ -805,6 +831,7 @@ interface InstanceConfig {
  */
 function createInstances(cluster: DatabaseClusterNew, props: DatabaseClusterBaseProps, subnetGroup: ISubnetGroup): InstanceConfig {
   const instanceCount = props.instances != null ? props.instances : 2;
+  const instanceUpdateBehaviour = props.instanceUpdateBehaviour ?? InstanceUpdateBehaviour.BULK;
   if (Token.isUnresolved(instanceCount)) {
     throw new Error('The number of instances an RDS Cluster consists of cannot be provided as a deploy-time only value!');
   }
@@ -852,6 +879,8 @@ function createInstances(cluster: DatabaseClusterNew, props: DatabaseClusterBase
   );
   const instanceParameterGroupConfig = instanceParameterGroup?.bindToInstance({});
 
+  const instances: CfnDBInstance[] = [];
+
   for (let i = 0; i < instanceCount; i++) {
     const instanceIndex = i + 1;
     const instanceIdentifier = props.instanceIdentifierBase != null ? `${props.instanceIdentifierBase}${instanceIndex}` :
@@ -895,6 +924,14 @@ function createInstances(cluster: DatabaseClusterNew, props: DatabaseClusterBase
 
     instanceIdentifiers.push(instance.ref);
     instanceEndpoints.push(new Endpoint(instance.attrEndpointAddress, portAttribute));
+    instances.push(instance);
+  }
+
+  // Adding dependencies here to ensure that the instances are updated one after the other.
+  if (instanceUpdateBehaviour === InstanceUpdateBehaviour.ROLLING) {
+    for (let i = 1; i < instanceCount; i++) {
+      instances[i].node.addDependency(instances[i-1]);
+    }
   }
 
   return { instanceEndpoints, instanceIdentifiers };

packages/@aws-cdk/aws-rds/package.json

@@ -86,6 +86,7 @@
     "@aws-cdk/cdk-build-tools": "0.0.0",
     "@aws-cdk/custom-resources": "0.0.0",
     "@aws-cdk/integ-runner": "0.0.0",
+    "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/cfn2ts": "0.0.0",
     "@aws-cdk/cx-api": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",

packages/@aws-cdk/aws-rds/test/cluster.test.ts

@@ -10,7 +10,7 @@ import * as cxapi from '@aws-cdk/cx-api';
 import {
   AuroraEngineVersion, AuroraMysqlEngineVersion, AuroraPostgresEngineVersion, CfnDBCluster, Credentials, DatabaseCluster,
   DatabaseClusterEngine, DatabaseClusterFromSnapshot, ParameterGroup, PerformanceInsightRetention, SubnetGroup, DatabaseSecret,
-  DatabaseInstanceEngine, SqlServerEngineVersion, SnapshotCredentials,
+  DatabaseInstanceEngine, SqlServerEngineVersion, SnapshotCredentials, InstanceUpdateBehaviour,
 } from '../lib';
 
 describe('cluster', () => {
@@ -122,6 +122,36 @@ describe('cluster', () => {
     });
   });
 
+  test('can create a cluster with ROLLING instance update behaviour', () => {
+    // GIVEN
+    const stack = testStack();
+    const vpc = new ec2.Vpc(stack, 'VPC');
+
+    // WHEN
+    new DatabaseCluster(stack, 'Database', {
+      engine: DatabaseClusterEngine.AURORA,
+      instances: 5,
+      instanceProps: {
+        vpc,
+      },
+      instanceUpdateBehaviour: InstanceUpdateBehaviour.ROLLING,
+    });
+
+    // THEN
+    const instanceResources = Template.fromStack(stack).findResources('AWS::RDS::DBInstance');
+    const instances = Object.keys(instanceResources);
+    const instanceDependencies = Object.values(instanceResources)
+      .map(properties => (properties.DependsOn as string[]).filter(dependency => instances.includes(dependency)));
+    // check that there are only required dependencies to form a chain of dependant instances
+    for (const dependencies of instanceDependencies) {
+      expect(dependencies.length).toBeLessThanOrEqual(1);
+    }
+    // check that all but one instance are a dependency of another instance
+    const dependantInstances = instanceDependencies.flat();
+    expect(dependantInstances).toHaveLength(instances.length - 1);
+    expect(instances.filter(it => !dependantInstances.includes(it))).toHaveLength(1);
+  });
+
   test('can create a cluster with imported vpc and security group', () => {
     // GIVEN
     const stack = testStack();

packages/@aws-cdk/aws-rds/test/integ.rolling-instance-updates.ts

@@ -0,0 +1,45 @@
+import * as ec2 from '@aws-cdk/aws-ec2';
+import * as cdk from '@aws-cdk/core';
+import * as integTests from '@aws-cdk/integ-tests';
+import * as constructs from 'constructs';
+import * as rds from '../lib';
+
+interface RollingInstanceUpdateTestStackProps extends cdk.StackProps {
+  instanceUpdateBehaviour: rds.InstanceUpdateBehaviour;
+}
+
+class RollingInstanceUpdateTestStack extends cdk.Stack {
+  constructor(scope: constructs.Construct, id: string, props: RollingInstanceUpdateTestStackProps) {
+    super(scope, id, props);
+    const vpc = new ec2.Vpc(this, 'Vpc', {
+      maxAzs: 2,
+    });
+
+    new rds.DatabaseCluster(this, 'DatabaseCluster', {
+      engine: rds.DatabaseClusterEngine.AURORA,
+      instances: 3,
+      instanceProps: {
+        instanceType: ec2.InstanceType.of(ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.SMALL),
+        vpc,
+      },
+      removalPolicy: cdk.RemovalPolicy.DESTROY,
+      instanceUpdateBehaviour: props.instanceUpdateBehaviour,
+    });
+  }
+}
+
+
+// Beginning of the test suite
+const app = new cdk.App();
+new integTests.IntegTest(app, 'InstanceUpdateBehaviorTests', {
+  testCases: [
+    new RollingInstanceUpdateTestStack(app, 'BulkUpdate', {
+      instanceUpdateBehaviour: rds.InstanceUpdateBehaviour.BULK,
+    }),
+    new RollingInstanceUpdateTestStack(app, 'RollingUpdate', {
+      instanceUpdateBehaviour: rds.InstanceUpdateBehaviour.ROLLING,
+    }),
+  ],
+});
+
+app.synth();