fix(cli): build assets before deploying any stacks (#21513)

Changes the CDK CLI to build assets before deploying any stacks. This allows the CDK CLI to catch docker build errors, such as from rate limiting, before any stacks are deployed. Moving asset builds this early prevents these build failures from interrupting multi-stack deployments part way through. 

Fixes #21511

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/aws-cdk/lib/api/cloudformation-deployments.ts

@@ -2,7 +2,7 @@ import * as cxapi from '@aws-cdk/cx-api';
 import { AssetManifest } from 'cdk-assets';
 import { Tag } from '../cdk-toolkit';
 import { debug, warning } from '../logging';
-import { publishAssets } from '../util/asset-publishing';
+import { buildAssets, publishAssets } from '../util/asset-publishing';
 import { Mode } from './aws-auth/credentials';
 import { ISDK } from './aws-auth/sdk';
 import { SdkProvider } from './aws-auth/sdk-provider';
@@ -236,6 +236,43 @@ export interface DeployStackOptions {
    * @default - Use the stored template
    */
   readonly overrideTemplate?: any;
+
+  /**
+   * Whether to build assets before publishing.
+   *
+   * @default true To remain backward compatible.
+   */
+  readonly buildAssets?: boolean;
+}
+
+export interface BuildStackAssetsOptions {
+  /**
+   * Stack with assets to build.
+   */
+  readonly stack: cxapi.CloudFormationStackArtifact;
+
+  /**
+   * Name of the toolkit stack, if not the default name.
+   *
+   * @default 'CDKToolkit'
+   */
+  readonly toolkitStackName?: string;
+
+  /**
+   * Execution role for the building.
+   *
+   * @default - Current role
+   */
+  readonly roleArn?: string;
+}
+
+interface PublishStackAssetsOptions {
+  /**
+   * Whether to build assets before publishing.
+   *
+   * @default true To remain backward compatible.
+   */
+  readonly buildAssets?: boolean;
 }
 
 export interface DestroyStackOptions {
@@ -340,7 +377,9 @@ export class CloudFormationDeployments {
 
     // Publish any assets before doing the actual deploy (do not publish any assets on import operation)
     if (options.resourcesToImport === undefined) {
-      await this.publishStackAssets(options.stack, toolkitInfo);
+      await this.publishStackAssets(options.stack, toolkitInfo, {
+        buildAssets: options.buildAssets ?? true,
+      });
     }
 
     // Do a verification of the bootstrap stack version
@@ -451,10 +490,32 @@ export class CloudFormationDeployments {
     };
   }
 
+  /**
+   * Build a stack's assets.
+   */
+  public async buildStackAssets(options: BuildStackAssetsOptions) {
+    const { stackSdk, resolvedEnvironment } = await this.prepareSdkFor(options.stack, options.roleArn);
+    const toolkitInfo = await ToolkitInfo.lookup(resolvedEnvironment, stackSdk, options.toolkitStackName);
+
+    const stackEnv = await this.sdkProvider.resolveEnvironment(options.stack.environment);
+    const assetArtifacts = options.stack.dependencies.filter(cxapi.AssetManifestArtifact.isAssetManifestArtifact);
+
+    for (const assetArtifact of assetArtifacts) {
+      await this.validateBootstrapStackVersion(
+        options.stack.stackName,
+        assetArtifact.requiresBootstrapStackVersion,
+        assetArtifact.bootstrapStackVersionSsmParameter,
+        toolkitInfo);
+
+      const manifest = AssetManifest.fromFile(assetArtifact.file);
+      await buildAssets(manifest, this.sdkProvider, stackEnv);
+    }
+  }
+
   /**
    * Publish all asset manifests that are referenced by the given stack
    */
-  private async publishStackAssets(stack: cxapi.CloudFormationStackArtifact, toolkitInfo: ToolkitInfo) {
+  private async publishStackAssets(stack: cxapi.CloudFormationStackArtifact, toolkitInfo: ToolkitInfo, options: PublishStackAssetsOptions = {}) {
     const stackEnv = await this.sdkProvider.resolveEnvironment(stack.environment);
     const assetArtifacts = stack.dependencies.filter(cxapi.AssetManifestArtifact.isAssetManifestArtifact);
 
@@ -466,7 +527,9 @@ export class CloudFormationDeployments {
         toolkitInfo);
 
       const manifest = AssetManifest.fromFile(assetArtifact.file);
-      await publishAssets(manifest, this.sdkProvider, stackEnv);
+      await publishAssets(manifest, this.sdkProvider, stackEnv, {
+        buildAssets: options.buildAssets ?? true,
+      });
     }
   }
 

packages/aws-cdk/lib/build.ts

@@ -0,0 +1,23 @@
+import * as cxapi from '@aws-cdk/cx-api';
+
+type Options = {
+  buildStackAssets: (stack: cxapi.CloudFormationStackArtifact) => Promise<void>;
+};
+
+export async function buildAllStackAssets(stacks: cxapi.CloudFormationStackArtifact[], options: Options): Promise<void> {
+  const { buildStackAssets } = options;
+
+  const buildingErrors: Error[] = [];
+
+  for (const stack of stacks) {
+    try {
+      await buildStackAssets(stack);
+    } catch (err) {
+      buildingErrors.push(err);
+    }
+  }
+
+  if (buildingErrors.length) {
+    throw Error(`Building Assets Failed: ${buildingErrors.join(', ')}`);
+  }
+}

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -14,6 +14,7 @@ import { CloudExecutable } from './api/cxapp/cloud-executable';
 import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs';
 import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor';
 import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
+import { buildAllStackAssets } from './build';
 import { printSecurityDiff, printStackDiff, RequireApproval } from './diff';
 import { ResourceImporter } from './import';
 import { data, debug, error, highlight, print, success, warning } from './logging';
@@ -166,6 +167,28 @@ export class CdkToolkit {
     const stackOutputs: { [key: string]: any } = { };
     const outputsFile = options.outputsFile;
 
+    const buildStackAssets = async (stack: cxapi.CloudFormationStackArtifact) => {
+      // Check whether the stack has an asset manifest before trying to build and publish.
+      if (!stack.dependencies.some(cxapi.AssetManifestArtifact.isAssetManifestArtifact)) {
+        return;
+      }
+
+      print('%s: building assets...\n', chalk.bold(stack.displayName));
+      await this.props.cloudFormation.buildStackAssets({
+        stack,
+        roleArn: options.roleArn,
+        toolkitStackName: options.toolkitStackName,
+      });
+      print('\n%s: assets built\n', chalk.bold(stack.displayName));
+    };
+
+    try {
+      await buildAllStackAssets(stacks.stackArtifacts, { buildStackAssets });
+    } catch (e) {
+      error('\n ❌ Building assets failed: %s', e);
+      throw e;
+    }
+
     for (const stack of stacks.stackArtifacts) {
       if (stacks.stackCount !== 1) { highlight(stack.displayName); }
       if (!stack.environment) {
@@ -234,6 +257,7 @@ export class CdkToolkit {
           rollback: options.rollback,
           hotswap: options.hotswap,
           extraUserAgent: options.extraUserAgent,
+          buildAssets: false,
         });
 
         const message = result.noOp

packages/aws-cdk/lib/util/asset-publishing.ts

@@ -11,6 +11,13 @@ export interface PublishAssetsOptions {
    * Print progress at 'debug' level
    */
   readonly quiet?: boolean;
+
+  /**
+   * Whether to build assets before publishing.
+   *
+   * @default true To remain backward compatible.
+   */
+  readonly buildAssets?: boolean;
 }
 
 /**
@@ -30,21 +37,64 @@ export async function publishAssets(
     targetEnv.region === undefined ||
     targetEnv.account === cxapi.UNKNOWN_REGION
   ) {
-    throw new Error(`Asset publishing requires resolved account and region, got ${JSON.stringify( targetEnv)}`);
+    throw new Error(`Asset publishing requires resolved account and region, got ${JSON.stringify(targetEnv)}`);
   }
 
   const publisher = new cdk_assets.AssetPublishing(manifest, {
     aws: new PublishingAws(sdk, targetEnv),
     progressListener: new PublishingProgressListener(options.quiet ?? false),
     throwOnError: false,
     publishInParallel: true,
+    buildAssets: options.buildAssets ?? true,
+    publishAssets: true,
   });
   await publisher.publish();
   if (publisher.hasFailures) {
     throw new Error('Failed to publish one or more assets. See the error messages above for more information.');
   }
 }
 
+export interface BuildAssetsOptions {
+  /**
+   * Print progress at 'debug' level
+   */
+  readonly quiet?: boolean;
+}
+
+/**
+ * Use cdk-assets to build all assets in the given manifest.
+ */
+export async function buildAssets(
+  manifest: cdk_assets.AssetManifest,
+  sdk: SdkProvider,
+  targetEnv: cxapi.Environment,
+  options: BuildAssetsOptions = {},
+) {
+  // This shouldn't really happen (it's a programming error), but we don't have
+  // the types here to guide us. Do an runtime validation to be super super sure.
+  if (
+    targetEnv.account === undefined ||
+    targetEnv.account === cxapi.UNKNOWN_ACCOUNT ||
+    targetEnv.region === undefined ||
+    targetEnv.account === cxapi.UNKNOWN_REGION
+  ) {
+    throw new Error(`Asset building requires resolved account and region, got ${JSON.stringify(targetEnv)}`);
+  }
+
+  const publisher = new cdk_assets.AssetPublishing(manifest, {
+    aws: new PublishingAws(sdk, targetEnv),
+    progressListener: new PublishingProgressListener(options.quiet ?? false),
+    throwOnError: false,
+    publishInParallel: true,
+    buildAssets: true,
+    publishAssets: false,
+  });
+  await publisher.publish();
+  if (publisher.hasFailures) {
+    throw new Error('Failed to build one or more assets. See the error messages above for more information.');
+  }
+}
+
 class PublishingAws implements cdk_assets.IAws {
   private sdkCache: Map<String, ISDK> = new Map();