fix(aws-cdk): continue after exceptions in stack monitor (#791)

If exceptions occur in the stack monitor, a new invocation wouldn't be
scheduled. We're now catching the exception and continuing. This will
not fix #787, but it will reduce the frequency and impact of it
occurring.

Also fixes a race condition where the stack monitor might print more
lines after it has already been stopped and the CDK final output has
already been printed.

packages/aws-cdk/lib/api/deploy-stack.ts

@@ -68,7 +68,7 @@ export async function deployStack(stack: cxapi.SynthesizedStack,
  const monitor = quiet ? undefined : new StackActivityMonitor(cfn, deployName, stack.metadata, changeSetDescription.Changes.length).start();
  debug('Execution of changeset %s on stack %s has started; waiting for the update to complete...', changeSetName, deployName);
  await waitForStack(cfn, deployName);
- if (monitor) { monitor.stop(); }
+ if (monitor) { await monitor.stop(); }
  debug('Stack %s has completed updating', deployName);
  return { noOp: false, outputs: await getStackOutputs(cfn, deployName), stackArn: changeSet.StackId! };
 }
@@ -127,7 +127,7 @@ export async function destroyStack(stack: cxapi.StackInfo, sdk: SDK, deployName?
  const monitor = quiet ? undefined : new StackActivityMonitor(cfn, deployName).start();
  await cfn.deleteStack({ StackName: deployName }).promise().catch(e => { throw e; });
  const destroyedStack = await waitForStack(cfn, deployName, false);
- if (monitor) { monitor.stop(); }
+ if (monitor) { await monitor.stop(); }
  if (destroyedStack && destroyedStack.StackStatus !== 'DELETE_COMPLETE') {
  const status = StackStatus.fromStackDescription(destroyedStack);
  throw new Error(`Failed to destroy ${deployName}: ${status}`);

packages/aws-cdk/lib/api/util/cloudformation/stack-activity-monitor.ts

@@ -2,6 +2,7 @@ import cxapi = require('@aws-cdk/cx-api');
 import aws = require('aws-sdk');
 import colors = require('colors/safe');
 import util = require('util');
+import { error } from '../../../logging';
 
 interface StackActivity {
  readonly event: aws.CloudFormation.StackEvent;
@@ -59,6 +60,11 @@ export class StackActivityMonitor {
  */
  private lastPrintTime = Date.now();
 
+ /**
+ * Set to the activity of reading the current events
+ */
+ private readPromise?: Promise<AWS.CloudFormation.StackEvent[]>;
+
  constructor(private readonly cfn: aws.CloudFormation,
  private readonly stackName: string,
  private readonly metadata?: cxapi.StackMetadata,
@@ -80,11 +86,17 @@ export class StackActivityMonitor {
  return this;
  }
 
- public stop() {
+ public async stop() {
  this.active = false;
  if (this.tickTimer) {
  clearTimeout(this.tickTimer);
  }
+
+ if (this.readPromise) {
+ // We're currently reading events, wait for it to finish and print them before continuing.
+ await this.readPromise;
+ this.flushEvents();
+ }
  }
 
  private scheduleNextTick() {
@@ -99,8 +111,18 @@ export class StackActivityMonitor {
  return;
  }
 
- await this.readEvents();
- this.flushEvents();
+ try {
+ this.readPromise = this.readEvents();
+ await this.readPromise;
+ this.readPromise = undefined;
+
+ // We might have been stop()ped while the network call was in progress.
+ if (!this.active) { return; }
+
+ this.flushEvents();
+ } catch (e) {
+ error("Error occurred while monitoring stack: %s", e);
+ }
  this.scheduleNextTick();
  }
 
@@ -220,7 +242,7 @@ export class StackActivityMonitor {
  return colors.reset;
  }
 
- private async readEvents(nextToken?: string) {
+ private async readEvents(nextToken?: string): Promise<AWS.CloudFormation.StackEvent[]> {
  const output = await this.cfn.describeStackEvents({ StackName: this.stackName, NextToken: nextToken }).promise()
  .catch( e => {
  if (e.code === 'ValidationError' && e.message === `Stack [${this.stackName}] does not exist`) {