Merge branch 'main' into mrgrain/feat/snapshot-nested-stacks

package.json

@@ -18,7 +18,7 @@
   "devDependencies": {
     "@types/prettier": "2.6.0",
     "@yarnpkg/lockfile": "^1.1.0",
-    "cdk-generate-synthetic-examples": "^0.1.57",
+    "cdk-generate-synthetic-examples": "^0.1.64",
     "conventional-changelog-cli": "^2.2.2",
     "fs-extra": "^9.1.0",
     "graceful-fs": "^4.2.10",

packages/@aws-cdk/aws-codepipeline-actions/package.json

@@ -87,7 +87,7 @@
     "@aws-cdk/cx-api": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
     "@types/jest": "^27.5.2",
-    "@types/lodash": "^4.14.188",
+    "@types/lodash": "^4.14.189",
     "jest": "^27.5.1",
     "lodash": "^4.17.21"
   },

packages/@aws-cdk/aws-eks/package.json

@@ -93,9 +93,9 @@
     "@types/sinon": "^9.0.11",
     "@types/yaml": "1.9.6",
     "aws-sdk": "^2.1211.0",
-    "cdk8s": "^2.5.39",
+    "cdk8s": "^2.5.46",
     "cdk8s-plus-21": "^2.0.0-beta.12",
-    "cdk8s-plus-23": "2.0.21",
+    "cdk8s-plus-23": "2.0.28",
     "jest": "^27.5.1",
     "sinon": "^9.2.4"
   },

packages/@aws-cdk/aws-lambda-event-sources/README.md

@@ -196,7 +196,7 @@ behavior:
 * __onFailure__: In the event a record fails and consumes all retries, the record will be sent to SQS queue or SNS topic that is specified here
 * __parallelizationFactor__: The number of batches to concurrently process on each shard.
 * __retryAttempts__: The maximum number of times a record should be retried in the event of failure.
-* __startingPosition__: Will determine where to being consumption. 'LATEST' will start at the most recent record and ignore all records that arrived prior to attaching the event source, 'TRIM_HORIZON' will start at the oldest record and ensure you process all available data, while 'AT_TIMESTAMP' will start reading records from a specified time stamp. Note that 'AT_TIMESTAMP' is only supported for Amazon Kinesis streams.
+* __startingPosition__: Will determine where to begin consumption. 'LATEST' will start at the most recent record and ignore all records that arrived prior to attaching the event source, 'TRIM_HORIZON' will start at the oldest record and ensure you process all available data, while 'AT_TIMESTAMP' will start reading records from a specified time stamp. Note that 'AT_TIMESTAMP' is only supported for Amazon Kinesis streams.
 * __startingPositionTimestamp__: The time stamp from which to start reading. Used in conjunction with __startingPosition__ when set to 'AT_TIMESTAMP'.
 * __tumblingWindow__: The duration in seconds of a processing window when using streams.
 * __enabled__: If the DynamoDB Streams event source mapping should be enabled. The default is true.

packages/@aws-cdk/aws-lambda-nodejs/package.json

@@ -79,7 +79,7 @@
     "@aws-cdk/pkglint": "0.0.0",
     "@types/jest": "^27.5.2",
     "delay": "5.0.0",
-    "esbuild": "^0.15.13"
+    "esbuild": "^0.15.14"
   },
   "dependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",

packages/@aws-cdk/aws-lambda/package.json

@@ -93,7 +93,7 @@
     "@aws-cdk/pkglint": "0.0.0",
     "@types/aws-lambda": "^8.10.108",
     "@types/jest": "^27.5.2",
-    "@types/lodash": "^4.14.188",
+    "@types/lodash": "^4.14.189",
     "jest": "^27.5.1",
     "lodash": "^4.17.21"
   },

packages/@aws-cdk/core/package.json

@@ -187,7 +187,7 @@
     "@types/aws-lambda": "^8.10.108",
     "@types/fs-extra": "^8.1.2",
     "@types/jest": "^27.5.2",
-    "@types/lodash": "^4.14.188",
+    "@types/lodash": "^4.14.189",
     "@types/minimatch": "^3.0.5",
     "aws-sdk": "^2.928.0",
     "@types/node": "^14.18.33",

packages/@aws-cdk/integ-runner/THIRD_PARTY_LICENSES

@@ -1,6 +1,6 @@
 The @aws-cdk/integ-runner package includes the following third-party software/licensing:
 
-** ajv@8.11.0 - https://www.npmjs.com/package/ajv/v/8.11.0 | MIT
+** ajv@8.11.2 - https://www.npmjs.com/package/ajv/v/8.11.2 | MIT
 The MIT License (MIT)
 
 Copyright (c) 2015-2021 Evgeny Poberezkin
@@ -156,7 +156,7 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH RE
 
 ----------------
 
-** aws-sdk@2.1250.0 - https://www.npmjs.com/package/aws-sdk/v/2.1250.0 | Apache-2.0
+** aws-sdk@2.1255.0 - https://www.npmjs.com/package/aws-sdk/v/2.1255.0 | Apache-2.0
 AWS SDK for JavaScript
 Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 

packages/@aws-cdk/lambda-layer-awscli/package.json

@@ -84,14 +84,14 @@
   "dependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
     "@aws-cdk/core": "0.0.0",
-    "@aws-cdk/asset-awscli-v1": "^2.2.2",
+    "@aws-cdk/asset-awscli-v1": "^2.2.9",
     "constructs": "^10.0.0"
   },
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
     "@aws-cdk/core": "0.0.0",
-    "@aws-cdk/asset-awscli-v1": "^2.2.2",
+    "@aws-cdk/asset-awscli-v1": "^2.2.9",
     "constructs": "^10.0.0"
   },
   "engines": {

packages/@aws-cdk/lambda-layer-node-proxy-agent/package.json

@@ -84,14 +84,14 @@
   "dependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
     "@aws-cdk/core": "0.0.0",
-    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.8",
+    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.15",
     "constructs": "^10.0.0"
   },
   "homepage": "https://github.com/aws/aws-cdk",
   "peerDependencies": {
     "@aws-cdk/aws-lambda": "0.0.0",
     "@aws-cdk/core": "0.0.0",
-    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.8",
+    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.15",
     "constructs": "^10.0.0"
   },
   "engines": {

packages/@aws-cdk/pipelines/README.md

@@ -217,7 +217,7 @@ const originalPipeline = new pipelines.CdkPipeline(this, 'Pipeline', {
 This section of the documentation describes the AWS CodePipeline engine,
 which comes with this library. If you want to use a different deployment
 engine, read the section
-[Using a different deployment engine](#using-a-different-deployment-engine)below.
+[Using a different deployment engine](#using-a-different-deployment-engine) below.
 
 ### Synth and sources
 
@@ -1685,7 +1685,7 @@ cannot be remedied by CDK at this point. They are reproduced here for completene
   console will assume all links are relative to the current account. You will
   not be able to use the pipeline console to click through to a CloudFormation
   stack in a different account.
-- **If a change set failed to apply the pipeline must restarted**: if a change
+- **If a change set failed to apply the pipeline must be restarted**: if a change
   set failed to apply, it cannot be retried. The pipeline must be restarted from
   the top by clicking **Release Change**.
 - **A stack that failed to create must be deleted manually**: if a stack

packages/aws-cdk-lib/package.json

@@ -114,8 +114,8 @@
     "punycode": "^2.1.1",
     "semver": "^7.3.8",
     "yaml": "1.10.2",
-    "@aws-cdk/asset-awscli-v1": "^2.2.2",
-    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.8",
+    "@aws-cdk/asset-awscli-v1": "^2.2.9",
+    "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.15",
     "@aws-cdk/asset-kubectl-v20": "^2.1.1"
   },
   "devDependencies": {
@@ -374,7 +374,7 @@
     "@types/fs-extra": "^8.1.2",
     "@types/node": "^14.18.33",
     "constructs": "^10.0.0",
-    "esbuild": "^0.15.13",
+    "esbuild": "^0.15.14",
     "fs-extra": "^9.1.0",
     "ts-node": "^9.1.1",
     "typescript": "~3.8.3"

packages/aws-cdk/THIRD_PARTY_LICENSES

@@ -67,7 +67,7 @@ THE SOFTWARE.
 
 ----------------
 
-** ajv@8.11.0 - https://www.npmjs.com/package/ajv/v/8.11.0 | MIT
+** ajv@8.11.2 - https://www.npmjs.com/package/ajv/v/8.11.2 | MIT
 The MIT License (MIT)
 
 Copyright (c) 2015-2021 Evgeny Poberezkin
@@ -268,7 +268,7 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH RE
 
 ----------------
 
-** aws-sdk@2.1250.0 - https://www.npmjs.com/package/aws-sdk/v/2.1250.0 | Apache-2.0
+** aws-sdk@2.1255.0 - https://www.npmjs.com/package/aws-sdk/v/2.1255.0 | Apache-2.0
 AWS SDK for JavaScript
 Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 

packages/aws-cdk/jest.config.js

@@ -7,4 +7,7 @@ module.exports = {
             branches: 45,
         },
     },
+
+    // We have many tests here that commonly time out
+    testTimeout: 30_000,
 };

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -69,6 +69,24 @@ export interface CdkToolkitProps {
   sdkProvider: SdkProvider;
 }
 
+/**
+ * When to build assets
+ */
+export enum AssetBuildTime {
+  /**
+   * Build all assets before deploying the first stack
+   *
+   * This is intended for expensive Docker image builds; so that if the Docker image build
+   * fails, no stacks are unnecessarily deployed (with the attendant wait time).
+   */
+  ALL_BEFORE_DEPLOY,
+
+  /**
+   * Build assets just-in-time, before publishing
+   */
+  JUST_IN_TIME,
+};
+
 /**
  * Toolkit logic
  *
@@ -167,17 +185,22 @@ export class CdkToolkit {
     }
 
     const stacks = stackCollection.stackArtifacts;
+    const assetBuildTime = options.assetBuildTime ?? AssetBuildTime.ALL_BEFORE_DEPLOY;
+
 
     const stackOutputs: { [key: string]: any } = { };
     const outputsFile = options.outputsFile;
 
-    try {
-      await buildAllStackAssets(stackCollection.stackArtifacts, {
-        buildStackAssets: (a) => this.buildAllAssetsForSingleStack(a, options),
-      });
-    } catch (e) {
-      error('\n ❌ Building assets failed: %s', e);
-      throw e;
+    if (assetBuildTime === AssetBuildTime.ALL_BEFORE_DEPLOY) {
+      // Prebuild all assets
+      try {
+        await buildAllStackAssets(stackCollection.stackArtifacts, {
+          buildStackAssets: (a) => this.buildAllAssetsForSingleStack(a, options),
+        });
+      } catch (e) {
+        error('\n ❌ Building assets failed: %s', e);
+        throw e;
+      }
     }
 
     const deployStack = async (stack: cxapi.CloudFormationStackArtifact) => {
@@ -257,7 +280,7 @@ export class CdkToolkit {
           rollback: options.rollback,
           hotswap: options.hotswap,
           extraUserAgent: options.extraUserAgent,
-          buildAssets: false,
+          buildAssets: assetBuildTime !== AssetBuildTime.ALL_BEFORE_DEPLOY,
           assetParallelism: options.assetParallelism,
         });
 
@@ -1042,6 +1065,15 @@ export interface DeployOptions extends CfnDeployOptions, WatchOptions {
    * @default true
    */
   readonly assetParallelism?: boolean;
+
+  /**
+   * When to build assets
+   *
+   * The default is the Docker-friendly default.
+   *
+   * @default AssetBuildTime.ALL_BEFORE_DEPLOY
+   */
+  readonly assetBuildTime?: AssetBuildTime;
 }
 
 export interface ImportOptions extends CfnDeployOptions {
@@ -1136,4 +1168,4 @@ function roundPercentage(num: number): number {
  */
 function millisecondsToSeconds(num: number): number {
   return num / 1000;
-}
+}

packages/aws-cdk/lib/cli.ts

@@ -13,7 +13,7 @@ import { execProgram } from '../lib/api/cxapp/exec';
 import { PluginHost } from '../lib/api/plugin';
 import { ToolkitInfo } from '../lib/api/toolkit-info';
 import { StackActivityProgress } from '../lib/api/util/cloudformation/stack-activity-monitor';
-import { CdkToolkit } from '../lib/cdk-toolkit';
+import { CdkToolkit, AssetBuildTime } from '../lib/cdk-toolkit';
 import { realHandler as context } from '../lib/commands/context';
 import { realHandler as docs } from '../lib/commands/docs';
 import { realHandler as doctor } from '../lib/commands/doctor';
@@ -157,7 +157,8 @@ async function parseCommandLineArguments() {
           "Only in effect if specified alongside the '--watch' option",
       })
       .option('concurrency', { type: 'number', desc: 'Maximum number of simultaneous deployments (dependency permitting) to execute.', default: 1, requiresArg: true })
-      .option('asset-parallelism', { type: 'boolean', desc: 'Whether to build/publish assets in parallel' }),
+      .option('asset-parallelism', { type: 'boolean', desc: 'Whether to build/publish assets in parallel' })
+      .option('asset-prebuild', { type: 'boolean', desc: 'Whether to build all assets before deploying the first stack (useful for failing Docker builds)', default: true }),
     )
     .command('import [STACK]', 'Import existing resource(s) into the given STACK', (yargs: Argv) => yargs
       .option('execute', { type: 'boolean', desc: 'Whether to execute ChangeSet (--no-execute will NOT execute the ChangeSet)', default: true })
@@ -521,6 +522,7 @@ async function initCommandLine() {
           traceLogs: args.logs,
           concurrency: args.concurrency,
           assetParallelism: configuration.settings.get(['assetParallelism']),
+          assetBuildTime: configuration.settings.get(['assetPrebuild']) ? AssetBuildTime.ALL_BEFORE_DEPLOY : AssetBuildTime.JUST_IN_TIME,
         });
 
       case 'import':

packages/aws-cdk/lib/settings.ts

@@ -290,6 +290,7 @@ export class Settings {
       rollback: argv.rollback,
       notices: argv.notices,
       assetParallelism: argv['asset-parallelism'],
+      assetPrebuild: argv['asset-prebuild'],
     });
   }
 

packages/aws-cdk/test/cdk-toolkit.test.ts

@@ -62,7 +62,7 @@ import { Bootstrapper } from '../lib/api/bootstrap';
 import { CloudFormationDeployments, DeployStackOptions, DestroyStackOptions } from '../lib/api/cloudformation-deployments';
 import { DeployStackResult } from '../lib/api/deploy-stack';
 import { Template } from '../lib/api/util/cloudformation';
-import { CdkToolkit, Tag } from '../lib/cdk-toolkit';
+import { CdkToolkit, Tag, AssetBuildTime } from '../lib/cdk-toolkit';
 import { RequireApproval } from '../lib/diff';
 import { flatten } from '../lib/util';
 import { instanceMockFrom, MockCloudExecutable, TestStackArtifact, withMocked } from './util';
@@ -589,6 +589,32 @@ describe('deploy', () => {
         }));
       });
     });
+
+    test('can disable asset prebuild', async () => {
+      // GIVEN
+      cloudExecutable = new MockCloudExecutable({
+        stacks: [MockStack.MOCK_STACK_WITH_ASSET],
+      });
+      const fakeCloudFormation = new FakeCloudFormation({});
+
+      const toolkit = new CdkToolkit({
+        cloudExecutable,
+        configuration: cloudExecutable.configuration,
+        sdkProvider: cloudExecutable.sdkProvider,
+        cloudFormation: fakeCloudFormation,
+      });
+
+      // WHEN
+      // Not the best test but following this through to the asset publishing library fails
+      await withMocked(fakeCloudFormation, 'buildStackAssets', async (mockBuildStackAssets) => {
+        await toolkit.deploy({
+          selector: { patterns: ['Test-Stack-Asset'] },
+          assetBuildTime: AssetBuildTime.JUST_IN_TIME,
+        });
+
+        expect(mockBuildStackAssets).not.toHaveBeenCalled();
+      });
+    });
   });
 });