fix(cli): cdk synth too eager with validation in Pipelines (aws#15147)

In aws#14613, we introduced validation so that `cdk synth` would guard
against incomplete context lookups. In the past, stacks with missing
context would have successfully completed synthesis, propagated down
the pipeline and caused confusing error messages during deployment.
The new behavior was to error out early in case there was missing
context.

This broke people who had resorted to resolving context in the pipeline
using multiple `synth` commands in `for`-loop: this used to work because
the `synths` would be incomplete but silently succeed, but with the new
validation the very first `cdk synth` would start failing and the `for`
loop would never complete.

This PR adds a `--no-validation` flag to `cdk synth` to stop the
additional validation, so the `for` loop can complete successfully.

The same behavior can be controlled with an environment variable,
by setting `CDK_VALIDATION=false`.

Fixes aws#15130.

packages/aws-cdk/bin/cdk.ts

@@ -70,6 +70,7 @@ async function parseCommandLineArguments() {
     )
     .command(['synthesize [STACKS..]', 'synth [STACKS..]'], 'Synthesizes and prints the CloudFormation template for this stack', yargs => yargs
       .option('exclusively', { type: 'boolean', alias: 'e', desc: 'Only synthesize requested stacks, don\'t include dependencies' })
+      .option('validation', { type: 'boolean', desc: 'After synthesis, validate stacks with the "validateOnSynth" attribute set (can also be controlled with CDK_VALIDATION)', default: true })
       .option('quiet', { type: 'boolean', alias: 'q', desc: 'Do not output CloudFormation Template to stdout', default: false }))
     .command('bootstrap [ENVIRONMENTS..]', 'Deploys the CDK toolkit stack into an AWS environment', yargs => yargs
       .option('bootstrap-bucket-name', { type: 'string', alias: ['b', 'toolkit-bucket-name'], desc: 'The name of the CDK toolkit bucket; bucket will be created and must not exist', default: undefined })
@@ -328,9 +329,9 @@ async function initCommandLine() {
       case 'synthesize':
       case 'synth':
         if (args.exclusively) {
-          return cli.synth(args.STACKS, args.exclusively, args.quiet);
+          return cli.synth(args.STACKS, args.exclusively, args.quiet, args.validation);
         } else {
-          return cli.synth(args.STACKS, true, args.quiet);
+          return cli.synth(args.STACKS, true, args.quiet, args.validation);
         }
 
 

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -296,8 +296,8 @@ export class CdkToolkit {
    * OUTPUT: If more than one stack ends up being selected, an output directory
    * should be supplied, where the templates will be written.
    */
-  public async synth(stackNames: string[], exclusively: boolean, quiet: boolean): Promise<any> {
-    const stacks = await this.selectStacksForDiff(stackNames, exclusively);
+  public async synth(stackNames: string[], exclusively: boolean, quiet: boolean, autoValidate?: boolean): Promise<any> {
+    const stacks = await this.selectStacksForDiff(stackNames, exclusively, autoValidate);
 
     // if we have a single stack, print it to STDOUT
     if (stacks.stackCount === 1) {
@@ -392,7 +392,7 @@ export class CdkToolkit {
     return stacks;
   }
 
-  private async selectStacksForDiff(stackNames: string[], exclusively?: boolean) {
+  private async selectStacksForDiff(stackNames: string[], exclusively?: boolean, autoValidate?: boolean) {
     const assembly = await this.assembly();
 
     const selectedForDiff = await assembly.selectStacks({ patterns: stackNames }, {
@@ -401,9 +401,11 @@ export class CdkToolkit {
     });
 
     const allStacks = await this.selectStacksForList([]);
-    const flaggedStacks = allStacks.filter(art => art.validateOnSynth ?? false);
+    const autoValidateStacks = autoValidate
+      ? allStacks.filter(art => art.validateOnSynth ?? false)
+      : new StackCollection(assembly, []);
 
-    await this.validateStacks(selectedForDiff.concat(flaggedStacks));
+    await this.validateStacks(selectedForDiff.concat(autoValidateStacks));
 
     return selectedForDiff;
   }

packages/aws-cdk/test/cdk-toolkit.test.ts

@@ -178,22 +178,32 @@ describe('synth', () => {
     process.env.STACKS_TO_VALIDATE = undefined;
   });
 
-  test('stack has error and is flagged for validation', async() => {
-    cloudExecutable = new MockCloudExecutable({
-      stacks: [
-        MockStack.MOCK_STACK_A,
-        MockStack.MOCK_STACK_B,
-      ],
-      nestedAssemblies: [{
+  describe('stack with error and flagged for validation', () => {
+    beforeEach(() => {
+      cloudExecutable = new MockCloudExecutable({
         stacks: [
-          { properties: { validateOnSynth: true }, ...MockStack.MOCK_STACK_WITH_ERROR },
+          MockStack.MOCK_STACK_A,
+          MockStack.MOCK_STACK_B,
         ],
-      }],
+        nestedAssemblies: [{
+          stacks: [
+            { properties: { validateOnSynth: true }, ...MockStack.MOCK_STACK_WITH_ERROR },
+          ],
+        }],
+      });
     });
 
-    const toolkit = defaultToolkitSetup();
+    test('causes synth to fail if autoValidate=true', async() => {
+      const toolkit = defaultToolkitSetup();
+      const autoValidate = true;
+      await expect(toolkit.synth([], false, true, autoValidate)).rejects.toBeDefined();
+    });
 
-    await expect(toolkit.synth([], false, true)).rejects.toBeDefined();
+    test('causes synth to succeed if autoValidate=false', async() => {
+      const toolkit = defaultToolkitSetup();
+      const autoValidate = false;
+      await expect(toolkit.synth([], false, true, autoValidate)).resolves.toBeUndefined();
+    });
   });
 
   test('stack has error and was explicitly selected', async() => {

packages/aws-cdk/test/integ/cli/app/app.js

@@ -8,6 +8,7 @@ const lambda = require('@aws-cdk/aws-lambda');
 const docker = require('@aws-cdk/aws-ecr-assets');
 const core = require('@aws-cdk/core')
 const { StackWithNestedStack, StackWithNestedStackUsingParameters } = require('./nested-stack');
+const { Annotations } = require('@aws-cdk/core');
 
 const stackPrefix = process.env.STACK_NAME_PREFIX;
 if (!stackPrefix) {
@@ -136,7 +137,22 @@ class ProvidingStack extends cdk.Stack {
   }
 }
 
+class StackWithError extends cdk.Stack {
+  constructor(parent, id, props) {
+    super(parent, id, props);
 
+    this.topic = new sns.Topic(this, 'BogusTopic'); // Some filler
+    Annotations.of(this).addError('This is an error');
+  }
+}
+
+class StageWithError extends cdk.Stage {
+  constructor(parent, id, props) {
+    super(parent, id, props);
+
+    new StackWithError(this, 'Stack');
+  }
+}
 
 class ConsumingStack extends cdk.Stack {
   constructor(parent, id, props) {
@@ -335,6 +351,11 @@ switch (stackSet) {
     });
     break;
 
+  case 'stage-with-errors':
+    const stage = new StageWithError(app, `${stackPrefix}-stage-with-errors`);
+    stage.synth({ validateOnSynthesis: true });
+    break;
+
   default:
     throw new Error(`Unrecognized INTEG_STACK_SET: '${stackSet}'`);
 }

packages/aws-cdk/test/integ/cli/cli.integtest.ts

@@ -536,6 +536,25 @@ integTest('cdk ls', withDefaultFixture(async (fixture) => {
   }
 }));
 
+integTest('synthing a stage with errors leads to failure', withDefaultFixture(async (fixture) => {
+  const output = await fixture.cdk(['synth'], {
+    allowErrExit: true,
+    modEnv: {
+      INTEG_STACK_SET: 'stage-with-errors',
+    },
+  });
+
+  expect(output).toContain('This is an error');
+}));
+
+integTest('synthing a stage with errors can be suppressed', withDefaultFixture(async (fixture) => {
+  await fixture.cdk(['synth', '--no-validation'], {
+    modEnv: {
+      INTEG_STACK_SET: 'stage-with-errors',
+    },
+  });
+}));
+
 integTest('deploy stack without resource', withDefaultFixture(async (fixture) => {
   // Deploy the stack without resources
   await fixture.cdkDeploy('conditional-resource', { modEnv: { NO_RESOURCE: 'TRUE' } });