feat(core): template validation after synthesis (#23951)

Integrate policy as code tools into CDK synthesis via a plugin mechanism. Immediately after synthesis, the framework invokes all the registered plugins, collect the results and, if there are any violations, show a report to the user.

Application developers register plugins to a `Stage`:

```ts
const app = new App({
  validationPlugins: [
	new SomePolicyAgentPlugin(),
	new AnotherPolicyAgentPugin(),
  ]
});
```

Plugin authors must implement the `IPolicyValidationPlugin` interface. Hypothetical example of a CloudFormation Guard plugin:

```ts
export class CfnGuardValidator implements IPolicyValidationPlugin {
  public readonly name = 'cfn-guard-validator';
  constructor() {}

  validate(context: IPolicyValidationContext): PolicyValidationPluginReport {
    // execute the cfn-guard cli and get the JSON response from the tool
    const cliResultJson = executeCfnGuardCli();

    // parse the results and return the violations format
    // that the framework expects
    const violations = parseGuardResults(cliResultJson);

    // construct the report and return it to the framework
    // this is a vastly over simplified example that is only
    // meant to show the structure of the report that is returned
    return {
      success: false,
      violations: [{
        ruleName: violations.ruleName,
        recommendation: violations.recommendation,
        fix: violations.fix,
        violatingResources: [{
          resourceName: violations.resourceName,
          locations: violations.locations,
          templatePath: violations.templatePath,
        }],
      }],
    };
  }
}
```

Co-authored-by: corymhall <43035978+corymhall@users.noreply.github.com>

package.json

@@ -83,8 +83,6 @@
       "@aws-cdk/assertions-alpha/fs-extra/**",
       "@aws-cdk/assertions/fs-extra",
       "@aws-cdk/assertions/fs-extra/**",
-      "@aws-cdk/aws-iot-actions-alpha/case",
-      "@aws-cdk/aws-iot-actions-alpha/case/**",
       "@aws-cdk/aws-codebuild/yaml",
       "@aws-cdk/aws-codebuild/yaml/**",
       "@aws-cdk/aws-codepipeline-actions/case",
@@ -99,6 +97,8 @@
       "@aws-cdk/aws-eks/yaml/**",
       "@aws-cdk/aws-events-targets/aws-sdk",
       "@aws-cdk/aws-events-targets/aws-sdk/**",
+      "@aws-cdk/aws-iot-actions-alpha/case",
+      "@aws-cdk/aws-iot-actions-alpha/case/**",
       "@aws-cdk/aws-iot-actions/case",
       "@aws-cdk/aws-iot-actions/case/**",
       "@aws-cdk/aws-s3-deployment/case",
@@ -117,6 +117,8 @@
       "@aws-cdk/core/ignore/**",
       "@aws-cdk/core/minimatch",
       "@aws-cdk/core/minimatch/**",
+      "@aws-cdk/core/table",
+      "@aws-cdk/core/table/**",
       "@aws-cdk/cx-api/semver",
       "@aws-cdk/cx-api/semver/**",
       "@aws-cdk/pipelines/aws-sdk",

packages/@aws-cdk/core/README.md

@@ -1302,4 +1302,118 @@ permissions boundary attached.
 
 For more details see the [Permissions Boundary](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_iam-readme.html#permissions-boundaries) section in the IAM guide.
 
+## Policy Validation
+
+If you or your organization use (or would like to use) any policy validation tool, such as
+[CloudFormation
+Guard](https://docs.aws.amazon.com/cfn-guard/latest/ug/what-is-guard.html) or
+[OPA](https://www.openpolicyagent.org/), to define constraints on your
+CloudFormation template, you can incorporate them into the CDK application.
+By using the appropriate plugin, you can make the CDK application check the
+generated CloudFormation templates against your policies immediately after
+synthesis. If there are any violations, the synthesis will fail and a report
+will be printed to the console or to a file (see below).
+
+> **Note**
+> This feature is considered experimental, and both the plugin API and the
+> format of the validation report are subject to change in the future.
+
+### For application developers
+
+To use one or more validation plugins in your application, use the
+`policyValidationBeta1` property of `Stage`:
+
+```ts
+// globally for the entire app (an app is a stage)
+const app = new App({
+  policyValidationBeta1: [
+    // These hypothetical classes implement IValidationPlugin:
+    new ThirdPartyPluginX(), 
+    new ThirdPartyPluginY(),
+  ],
+});
+
+// only apply to a particular stage
+const prodStage = new Stage(app, 'ProdStage', {
+  policyValidationBeta1: [...],
+});
+```
+
+Immediately after synthesis, all plugins registered this way will be invoked to
+validate all the templates generated in the scope you defined. In particular, if
+you register the templates in the `App` object, all templates will be subject to
+validation.
+
+> **Warning**
+> Other than modifying the cloud assembly, plugins can do anything that your CDK
+> application can. They can read data from the filesystem, access the network
+> etc. It's your responsibility as the consumer of a plugin to verify that it is
+> secure to use.
+
+By default, the report will be printed in a human readable format. If you want a
+report in JSON format, enable it using the `@aws-cdk/core:validationReportJson` 
+context passing it directly to the application:
+
+```ts
+const app = new App({ 
+  context: { '@aws-cdk/core:validationReportJson': true }, 
+});
+```
+
+Alternatively, you can set this context key-value pair using the `cdk.json` or
+`cdk.context.json` files in your project directory (see
+[Runtime context](https://docs.aws.amazon.com/cdk/v2/guide/context.html)).
+
+If you choose the JSON format, the CDK will print the policy validation report
+to a file called `policy-validation-report.json` in the cloud assembly
+directory. For the default, human-readable format, the report will be printed to
+the standard output.
+
+### For plugin authors
+
+The communication protocol between the CDK core module and your policy tool is
+defined by the `IValidationPluginBeta1` interface. To create a new plugin you must
+write a class that implements this interface. There are two things you need to
+implement: the plugin name (by overriding the `name` property), and the
+`validate()` method.
+
+The framework will call `validate()`, passing an `IValidationContextBeta1` object.
+The location of the templates to be validated is given by `templatePaths`. The
+plugin should return an instance of `ValidationPluginReportBeta1`. This object
+represents the report that the user wil receive at the end of the synthesis.
+
+```ts
+validate(context: ValidationContextBeta1): ValidationReportBeta1 {
+  // First read the templates using context.templatePaths...
+
+  // ...then perform the validation, and then compose and return the report.
+  // Using hard-coded values here for better clarity:
+  return {
+    success: false,
+    violations: [{
+      ruleName: 'CKV_AWS_117',
+      recommendation: 'Ensure that AWS Lambda function is configured inside a VPC',
+      fix: 'https://docs.bridgecrew.io/docs/ensure-that-aws-lambda-function-is-configured-inside-a-vpc-1',
+      violatingResources: [{
+        resourceName: 'MyFunction3BAA72D1',
+        templatePath: '/home/johndoe/myapp/cdk.out/MyService.template.json',
+        locations: 'Properties/VpcConfig',
+      }],
+    }],
+  };
+}
+```
+
+Note that plugins are not allowed to modify anything in the cloud assembly. Any
+attempt to do so will result in synthesis failure.
+
+If your plugin depends on an external tool, keep in mind that some developers may
+not have that tool installed in their workstations yet. To minimize friction, we
+highly recommend that you provide some installation script along with your
+plugin package, to automate the whole process. Better yet, run that script as
+part of the installation of your package. With `npm`, for example, you can run
+add it to the `postinstall`
+[script](https://docs.npmjs.com/cli/v9/using-npm/scripts) in the `package.json`
+file.
+
 <!--END CORE DOCUMENTATION-->

packages/@aws-cdk/core/lib/app.ts

@@ -5,6 +5,7 @@ import { PRIVATE_CONTEXT_DEFAULT_STACK_SYNTHESIZER } from './private/private-con
 import { addCustomSynthesis, ICustomSynthesis } from './private/synthesis';
 import { IReusableStackSynthesizer } from './stack-synthesizers';
 import { Stage } from './stage';
+import { IPolicyValidationPluginBeta1 } from './validation/validation';
 
 const APP_SYMBOL = Symbol.for('@aws-cdk/core.App');
 
@@ -118,6 +119,13 @@ export interface AppProps {
    * @default - A `DefaultStackSynthesizer` with default settings
    */
   readonly defaultStackSynthesizer?: IReusableStackSynthesizer;
+
+  /**
+   * Validation plugins to run after synthesis
+   *
+   * @default - no validation plugins
+   */
+  readonly policyValidationBeta1?: IPolicyValidationPluginBeta1[];
 }
 
 /**
@@ -159,6 +167,7 @@ export class App extends Stage {
   constructor(props: AppProps = {}) {
     super(undefined as any, '', {
       outdir: props.outdir ?? process.env[cxapi.OUTDIR_ENV],
+      policyValidationBeta1: props.policyValidationBeta1,
     });
 
     Object.defineProperty(this, APP_SYMBOL, { value: true });

packages/@aws-cdk/core/lib/index.ts

@@ -64,6 +64,8 @@ export * from './cloudformation.generated';
 export * from './feature-flags';
 export * from './permissions-boundary';
 
+export * from './validation';
+
 // WARNING: Should not be exported, but currently is because of a bug. See the
 // class description for more information.
 export * from './private/intrinsic';

packages/@aws-cdk/core/lib/private/runtime-info.ts

@@ -1,4 +1,5 @@
 import { IConstruct } from 'constructs';
+import { App } from '../app';
 import { Stack } from '../stack';
 import { Stage } from '../stage';
 
@@ -38,6 +39,34 @@ export function constructInfoFromConstruct(construct: IConstruct): ConstructInfo
   return undefined;
 }
 
+/**
+ * Add analytics data for any validation plugins that are used.
+ * Since validation plugins are not constructs we have to handle them
+ * as a special case
+ */
+function addValidationPluginInfo(stack: Stack, allConstructInfos: ConstructInfo[]): void {
+  let stage = Stage.of(stack);
+  let done = false;
+  do {
+    if (App.isApp(stage)) {
+      done = true;
+    }
+    if (stage) {
+      allConstructInfos.push(...stage.policyValidationBeta1.map(
+        plugin => {
+          return {
+            // the fqn can be in the format of `package.module.construct`
+            // those get pulled out into separate fields
+            fqn: `policyValidation.${plugin.name}`,
+            version: plugin.version ?? '0.0.0',
+          };
+        },
+      ));
+      stage = Stage.of(stage);
+    }
+  } while (!done && stage);
+}
+
 /**
  * For a given stack, walks the tree and finds the runtime info for all constructs within the tree.
  * Returns the unique list of construct info present in the stack,
@@ -57,6 +86,8 @@ export function constructInfoFromStack(stack: Stack): ConstructInfo[] {
     version: getJsiiAgentVersion(),
   });
 
+  addValidationPluginInfo(stack, allConstructInfos);
+
   // Filter out duplicate values
   const uniqKeys = new Set();
   return allConstructInfos.filter(construct => {

packages/@aws-cdk/core/lib/private/synthesis.ts

@@ -1,4 +1,8 @@
+import { createHash } from 'crypto';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as cxapi from '@aws-cdk/cx-api';
+import { CloudAssembly } from '@aws-cdk/cx-api';
 import { IConstruct } from 'constructs';
 import { MetadataResource } from './metadata-resource';
 import { prepareApp } from './prepare-app';
@@ -9,6 +13,12 @@ import { Aspects, IAspect } from '../aspect';
 import { Stack } from '../stack';
 import { ISynthesisSession } from '../stack-synthesizers/types';
 import { Stage, StageSynthesisOptions } from '../stage';
+import { IPolicyValidationPluginBeta1 } from '../validation';
+import { ConstructTree } from '../validation/private/construct-tree';
+import { PolicyValidationReportFormatter, NamedValidationPluginReport } from '../validation/private/report';
+
+const POLICY_VALIDATION_FILE_PATH = 'policy-validation-report.json';
+const VALIDATION_REPORT_JSON_CONTEXT = '@aws-cdk/core:validationReportJson';
 
 /**
  * Options for `synthesize()`
@@ -49,7 +59,115 @@ export function synthesize(root: IConstruct, options: SynthesisOptions = { }): c
   // stacks to add themselves to the synthesized cloud assembly.
   synthesizeTree(root, builder, options.validateOnSynthesis);
 
-  return builder.buildAssembly();
+  const assembly = builder.buildAssembly();
+
+  invokeValidationPlugins(root, builder.outdir, assembly);
+
+  return assembly;
+}
+
+/**
+ * Find all the assemblies in the app, including all levels of nested assemblies
+ * and return a map where the assemblyId is the key
+ */
+function getAssemblies(root: App, rootAssembly: CloudAssembly): Map<string, CloudAssembly> {
+  const assemblies = new Map<string, CloudAssembly>();
+  assemblies.set(root.artifactId, rootAssembly);
+  visitAssemblies(root, 'pre', construct => {
+    const stage = construct as Stage;
+    if (stage.parentStage && assemblies.has(stage.parentStage.artifactId)) {
+      assemblies.set(
+        stage.artifactId,
+        assemblies.get(stage.parentStage.artifactId)!.getNestedAssembly(stage.artifactId),
+      );
+    }
+  });
+  return assemblies;
+}
+
+/**
+ * Invoke validation plugins for all stages in an App.
+ */
+function invokeValidationPlugins(root: IConstruct, outdir: string, assembly: CloudAssembly) {
+  if (!App.isApp(root)) return;
+  const hash = computeChecksumOfFolder(outdir);
+  const assemblies = getAssemblies(root, assembly);
+  const templatePathsByPlugin: Map<IPolicyValidationPluginBeta1, string[]> = new Map();
+  visitAssemblies(root, 'post', construct => {
+    if (Stage.isStage(construct)) {
+      for (const plugin of construct.policyValidationBeta1) {
+        if (!templatePathsByPlugin.has(plugin)) {
+          templatePathsByPlugin.set(plugin, []);
+        }
+        let assemblyToUse = assemblies.get(construct.artifactId);
+        if (!assemblyToUse) throw new Error(`Validation failed, cannot find cloud assembly for stage ${construct.stageName}`);
+        templatePathsByPlugin.get(plugin)!.push(...assemblyToUse.stacksRecursively.map(stack => stack.templateFullPath));
+      }
+    }
+  });
+
+  const reports: NamedValidationPluginReport[] = [];
+  if (templatePathsByPlugin.size > 0) {
+    // eslint-disable-next-line no-console
+    console.log('Performing Policy Validations\n');
+  }
+  for (const [plugin, paths] of templatePathsByPlugin.entries()) {
+    try {
+      const report = plugin.validate({ templatePaths: paths });
+      reports.push({ ...report, pluginName: plugin.name });
+    } catch (e: any) {
+      reports.push({
+        success: false,
+        pluginName: plugin.name,
+        pluginVersion: plugin.version,
+        violations: [],
+        metadata: {
+          error: `Validation plugin '${plugin.name}' failed: ${e.message}`,
+        },
+      });
+    }
+    if (computeChecksumOfFolder(outdir) !== hash) {
+      throw new Error(`Illegal operation: validation plugin '${plugin.name}' modified the cloud assembly`);
+    }
+  }
+
+  if (reports.length > 0) {
+    const tree = new ConstructTree(root);
+    const formatter = new PolicyValidationReportFormatter(tree);
+    const formatJson = root.node.tryGetContext(VALIDATION_REPORT_JSON_CONTEXT) ?? false;
+    const output = formatJson
+      ? formatter.formatJson(reports)
+      : formatter.formatPrettyPrinted(reports);
+
+    if (formatJson) {
+      fs.writeFileSync(path.join(assembly.directory, POLICY_VALIDATION_FILE_PATH), JSON.stringify(output, undefined, 2));
+    } else {
+      // eslint-disable-next-line no-console
+      console.error(output);
+    }
+    const failed = reports.some(r => !r.success);
+    if (failed) {
+      throw new Error('Validation failed. See the validation report above for details');
+    } else {
+      // eslint-disable-next-line no-console
+      console.log('Policy Validation Successful!');
+    }
+  }
+}
+
+function computeChecksumOfFolder(folder: string): string {
+  const hash = createHash('sha256');
+  const files = fs.readdirSync(folder, { withFileTypes: true });
+
+  for (const file of files) {
+    const fullPath = path.join(folder, file.name);
+    if (file.isDirectory()) {
+      hash.update(computeChecksumOfFolder(fullPath));
+    } else if (file.isFile()) {
+      hash.update(fs.readFileSync(fullPath));
+    }
+  }
+  return hash.digest().toString('hex');
 }
 
 const CUSTOM_SYNTHESIS_SYM = Symbol.for('@aws-cdk/core:customSynthesis');
@@ -232,6 +350,24 @@ function validateTree(root: IConstruct) {
   }
 }
 
+/**
+ * Visit the given construct tree in either pre or post order, only looking at Assemblies
+ */
+function visitAssemblies(root: IConstruct, order: 'pre' | 'post', cb: (x: IConstruct) => void) {
+  if (order === 'pre') {
+    cb(root);
+  }
+
+  for (const child of root.node.children) {
+    if (!Stage.isStage(child)) { continue; }
+    visitAssemblies(child, order, cb);
+  }
+
+  if (order === 'post') {
+    cb(root);
+  }
+}
+
 /**
  * Visit the given construct tree in either pre or post order, stopping at Assemblies
  */