From f983fbb474ecd6727b0c5a35333718cc55d78bf1 Mon Sep 17 00:00:00 2001 From: Gerald McAlister Date: Sun, 28 Feb 2021 00:48:37 -0800 Subject: [PATCH 1/3] fix(lambda-nodejs): paths with spaces break esbuild (#13312) Problem: Paths with spaces break ESBuild on Windows. Solution: Add double quotes around the input paths for the ESBuild command. Testing: Updated unit tests and confirmed in my own package this fix works. Closes #13311 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../aws-lambda-nodejs/lib/bundling.ts | 4 +-- .../aws-lambda-nodejs/test/bundling.test.ts | 32 ++++--------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 5051fc9012ada..536ca1ea7646a 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -140,10 +140,10 @@ export class Bundling implements cdk.BundlingOptions { const esbuildCommand: string = [ npx, 'esbuild', - '--bundle', pathJoin(inputDir, this.relativeEntryPath).replace(/(\s+)/g, '\\$1'), + '--bundle', `"${pathJoin(inputDir, this.relativeEntryPath)}"`, `--target=${this.props.target ?? toTarget(this.props.runtime)}`, '--platform=node', - `--outfile=${pathJoin(outputDir, 'index.js')}`, + `--outfile="${pathJoin(outputDir, 'index.js')}"`, ...this.props.minify ? ['--minify'] : [], ...this.props.sourceMap ? ['--sourcemap'] : [], ...this.externals.map(external => `--external:${external}`), diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index 2f96c10ff78ed..382baafee54a0 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -53,7 +53,7 @@ test('esbuild bundling in Docker', () => { }, command: [ 'bash', '-c', - 'npx esbuild --bundle /asset-input/lib/handler.ts --target=node12 --platform=node --outfile=/asset-output/index.js --external:aws-sdk --loader:.png=dataurl', + 'npx esbuild --bundle "/asset-input/lib/handler.ts" --target=node12 --platform=node --outfile="/asset-output/index.js" --external:aws-sdk --loader:.png=dataurl', ], workingDirectory: '/', }), @@ -74,7 +74,7 @@ test('esbuild bundling with handler named index.ts', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'npx esbuild --bundle /asset-input/lib/index.ts --target=node12 --platform=node --outfile=/asset-output/index.js --external:aws-sdk', + 'npx esbuild --bundle "/asset-input/lib/index.ts" --target=node12 --platform=node --outfile="/asset-output/index.js" --external:aws-sdk', ], }), }); @@ -94,7 +94,7 @@ test('esbuild bundling with tsx handler', () => { bundling: expect.objectContaining({ command: [ 'bash', '-c', - 'npx esbuild --bundle /asset-input/lib/handler.tsx --target=node12 --platform=node --outfile=/asset-output/index.js --external:aws-sdk', + 'npx esbuild --bundle "/asset-input/lib/handler.tsx" --target=node12 --platform=node --outfile="/asset-output/index.js" --external:aws-sdk', ], }), }); @@ -139,7 +139,7 @@ test('esbuild bundling with externals and dependencies', () => { command: [ 'bash', '-c', [ - 'npx esbuild --bundle /asset-input/test/bundling.test.js --target=node12 --platform=node --outfile=/asset-output/index.js --external:abc --external:delay', + 'npx esbuild --bundle "/asset-input/test/bundling.test.js" --target=node12 --platform=node --outfile="/asset-output/index.js" --external:abc --external:delay', `echo \'{\"dependencies\":{\"delay\":\"${delayVersion}\"}}\' > /asset-output/package.json`, 'cp /asset-input/package-lock.json /asset-output/package-lock.json', 'cd /asset-output', @@ -181,8 +181,8 @@ test('esbuild bundling with esbuild options', () => { command: [ 'bash', '-c', [ - 'npx esbuild --bundle /asset-input/lib/handler.ts', - '--target=es2020 --platform=node --outfile=/asset-output/index.js', + 'npx esbuild --bundle "/asset-input/lib/handler.ts"', + '--target=es2020 --platform=node --outfile="/asset-output/index.js"', '--minify --sourcemap --external:aws-sdk --loader:.png=dataurl', '--define:DEBUG=true --define:process.env.KEY="VALUE"', '--log-level=silent --keep-names --tsconfig=/asset-input/lib/custom-tsconfig.ts', @@ -334,23 +334,3 @@ test('with command hooks', () => { }), }); }); - -test('escapes spaces in path', () => { - Bundling.bundle({ - entry: '/project/lib/my cool lambda/handler.ts', - depsLockFilePath, - runtime: Runtime.NODEJS_12_X, - forceDockerBundling: true, - }); - - // Correctly bundles with esbuild - expect(Code.fromAsset).toHaveBeenCalledWith(path.dirname(depsLockFilePath), { - assetHashType: AssetHashType.OUTPUT, - bundling: expect.objectContaining({ - command: [ - 'bash', '-c', - expect.stringContaining('lib/my\\ cool\\ lambda/handler.ts'), - ], - }), - }); -}); From 6de533faa9290b3b87e4a779eb6589efce300ed3 Mon Sep 17 00:00:00 2001 From: Shonn Lyga Date: Sun, 28 Feb 2021 01:22:31 -0800 Subject: [PATCH 2/3] chore: minor tweaks to DESIGN_GUIDELINES.md (#13073) Was reading the Design Guidelines before contributing, and figured I'd improve it as I read through it ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- DESIGN_GUIDELINES.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/DESIGN_GUIDELINES.md b/DESIGN_GUIDELINES.md index e6cebe9f434c6..20fd5be016870 100644 --- a/DESIGN_GUIDELINES.md +++ b/DESIGN_GUIDELINES.md @@ -14,7 +14,7 @@ APIs and verifies that the APIs adhere to the guidelines. When a guideline is backed by a linter rule, the rule name will be referenced like this: _[awslint:resource-class-is-construct]_. -For the purpose of this document we will use "Foo" to denote the official name +For the purpose of this document, we will use "Foo" to denote the official name of the resource as defined in the AWS CloudFormation resource specification (i.e. "Bucket", "Queue", "Topic", etc). This notation allows deriving names from the official name. For example, `FooProps` would be `BucketProps`, `TopicProps`, @@ -98,8 +98,8 @@ or abstractions. However, you will notice that some sections explicitly call out guidelines that apply only to AWS resources (and in many cases enforced/implemented by the **Resource** base class). -AWS services are modeled around the concept of *resources*. Service normally -expose through their APIs one or more resources, which can be provisioned +AWS services are modeled around the concept of *resources*. Services normally +expose one or more resources through their APIs, which can be provisioned through the APIs control plane or through AWS CloudFormation. Every resource available in the AWS platform will have a corresponding resource @@ -397,12 +397,12 @@ For example, prefer “readCapacity” versus “readCapacityUnits”. We prefer the terminology used by the official AWS service documentation over new terminology, even if you think it's not ideal. It helps users diagnose issues and map the mental model of the construct to the service APIs, -documentation and examples. For example don't be tempted to change SQS's +documentation and examples. For example, don't be tempted to change SQS's **dataKeyReusePeriod** with **keyRotation** because it will be hard for people to diagnose problems. They won't be able to just search for “sqs dataKeyReuse” and find topics on it. -> We can relax this guidelines when this is about generic terms (like +> We can relax this guideline when this is about generic terms (like `httpStatus` instead of `statusCode`). The important semantics to preserve are for *service features*: I wouldn't want to rename "lambda layers" to "lambda dependencies" just because it makes more sense because then users won't be @@ -697,8 +697,8 @@ _[awslint:from-signature]_: #### “from” Methods Resource constructs should export static “from” methods for importing unowned -resources given one more of its physical attributes such as ARN, name, etc. All -constructs should have at least one "fromXxx" method _[awslint:from-method]_: +resources given one or more of its physical attributes such as ARN, name, etc. All +constructs should have at least one `fromXxx` method _[awslint:from-method]_: ```ts static fromFooArn(scope: Construct, id: string, bucketArn: string): IFoo; @@ -870,7 +870,7 @@ vpcSubnetSelection?: ec2.SubnetSelection; ### Grants -Grants are one of the most powerful concept in the AWS Construct Library. They +Grants are one of the most powerful concepts in the AWS Construct Library. They offer a higher level, intent-based, API for managing IAM permissions for AWS resources. @@ -974,7 +974,7 @@ class Function extends Resource implements IFunction { ### Events -Many AWS resource emit events to the CloudWatch event bus. Such resources should +Many AWS resources emit events to the CloudWatch event bus. Such resources should have a set of “onXxx” methods available on their construct interface _[awslint:events-in-interface]_. From 6eca979f65542f3e44461588d8220e8c0bf76a6e Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Sun, 28 Feb 2021 10:56:25 +0100 Subject: [PATCH 3/3] feat(core): customize bundling output packaging (#13152) Redo of #13076 after #13131. The fix is [`7b3d829` (#13152)](https://github.com/aws/aws-cdk/pull/13152/commits/7b3d829b3db6aacbc0372855128c3e5a8187d78d). If the bundling output contains a single archive file (zip or jar), upload it as-is to S3 without zipping it. Allow to customize this behavior with `bundling.outputType`: * `NOT_ARCHIVED`: The bundling output will always be zipped and uploaded to S3. * `ARCHIVED`: The bundling output will not be zipped. Bundling will fail if the bundling output doesn't contain a single archive file. * `AUTO_DISCOVER`: If the bundling output contains a single archive file (zip or jar) it will not be zipped. Otherwise it will be zipped. This is the default. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- allowed-breaking-changes.txt | 7 - packages/@aws-cdk/aws-s3-assets/README.md | 21 +++ packages/@aws-cdk/aws-s3-assets/lib/asset.ts | 30 +--- packages/@aws-cdk/core/lib/asset-staging.ts | 132 ++++++++++++-- packages/@aws-cdk/core/lib/bundling.ts | 35 ++++ .../@aws-cdk/core/test/archive/archive.zip | 0 packages/@aws-cdk/core/test/docker-stub.sh | 15 +- packages/@aws-cdk/core/test/staging.test.ts | 168 +++++++++++++++++- 8 files changed, 360 insertions(+), 48 deletions(-) create mode 100644 packages/@aws-cdk/core/test/archive/archive.zip diff --git a/allowed-breaking-changes.txt b/allowed-breaking-changes.txt index 964bb4d5c7712..2ca2ca5b6067f 100644 --- a/allowed-breaking-changes.txt +++ b/allowed-breaking-changes.txt @@ -56,10 +56,3 @@ incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.addVolume # We made properties optional and it's really fine but our differ doesn't think so. weakened:@aws-cdk/cloud-assembly-schema.DockerImageSource weakened:@aws-cdk/cloud-assembly-schema.FileSource - -# https://github.com/aws/aws-cdk/pull/13145 -removed:@aws-cdk/core.AssetStaging.isArchive -removed:@aws-cdk/core.AssetStaging.packaging -removed:@aws-cdk/core.BundlingOutput -removed:@aws-cdk/core.BundlingOptions.outputType - diff --git a/packages/@aws-cdk/aws-s3-assets/README.md b/packages/@aws-cdk/aws-s3-assets/README.md index aab4c46d9c44d..f2583b7c10a24 100644 --- a/packages/@aws-cdk/aws-s3-assets/README.md +++ b/packages/@aws-cdk/aws-s3-assets/README.md @@ -124,6 +124,27 @@ new assets.Asset(this, 'BundledAsset', { Although optional, it's recommended to provide a local bundling method which can greatly improve performance. +If the bundling output contains a single archive file (zip or jar) it will be +uploaded to S3 as-is and will not be zipped. Otherwise the contents of the +output directory will be zipped and the zip file will be uploaded to S3. This +is the default behavior for `bundling.outputType` (`BundlingOutput.AUTO_DISCOVER`). + +Use `BundlingOutput.NOT_ARCHIVED` if the bundling output must always be zipped: + +```ts +const asset = new assets.Asset(this, 'BundledAsset', { + path: '/path/to/asset', + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: ['command-that-produces-an-archive.sh'], + outputType: BundlingOutput.NOT_ARCHIVED, // Bundling output will be zipped even though it produces a single archive file. + }, +}); +``` + +Use `BundlingOutput.ARCHIVED` if the bundling output contains a single archive file and +you don't want it to be zippped. + ## CloudFormation Resource Metadata > NOTE: This section is relevant for authors of AWS Resource Constructs. diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 938778d1381f4..510834a61c634 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -1,4 +1,3 @@ -import * as fs from 'fs'; import * as path from 'path'; import * as assets from '@aws-cdk/assets'; import * as iam from '@aws-cdk/aws-iam'; @@ -13,8 +12,6 @@ import { toSymlinkFollow } from './compat'; // eslint-disable-next-line no-duplicate-imports, import/order import { Construct as CoreConstruct } from '@aws-cdk/core'; -const ARCHIVE_EXTENSIONS = ['.zip', '.jar']; - export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions { /** * A list of principals that should be able to read this asset from S3. @@ -139,17 +136,12 @@ export class Asset extends CoreConstruct implements cdk.IAsset { this.assetPath = staging.relativeStagedPath(stack); - const packaging = determinePackaging(staging.sourcePath); - - this.isFile = packaging === cdk.FileAssetPackaging.FILE; + this.isFile = staging.packaging === cdk.FileAssetPackaging.FILE; - // sets isZipArchive based on the type of packaging and file extension - this.isZipArchive = packaging === cdk.FileAssetPackaging.ZIP_DIRECTORY - ? true - : ARCHIVE_EXTENSIONS.some(ext => staging.sourcePath.toLowerCase().endsWith(ext)); + this.isZipArchive = staging.isArchive; const location = stack.synthesizer.addFileAsset({ - packaging, + packaging: staging.packaging, sourceHash: this.sourceHash, fileName: this.assetPath, }); @@ -210,19 +202,3 @@ export class Asset extends CoreConstruct implements cdk.IAsset { this.bucket.grantRead(grantee); } } - -function determinePackaging(assetPath: string): cdk.FileAssetPackaging { - if (!fs.existsSync(assetPath)) { - throw new Error(`Cannot find asset at ${assetPath}`); - } - - if (fs.statSync(assetPath).isDirectory()) { - return cdk.FileAssetPackaging.ZIP_DIRECTORY; - } - - if (fs.statSync(assetPath).isFile()) { - return cdk.FileAssetPackaging.FILE; - } - - throw new Error(`Asset ${assetPath} is expected to be either a directory or a regular file`); -} diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 66c65e3d14864..6a34bd9b4b1ac 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,8 +5,8 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import * as minimatch from 'minimatch'; -import { AssetHashType, AssetOptions } from './assets'; -import { BundlingOptions } from './bundling'; +import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; +import { BundlingOptions, BundlingOutput } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { Names } from './names'; import { Cache } from './private/cache'; @@ -17,6 +17,8 @@ import { Stage } from './stage'; // eslint-disable-next-line import { Construct as CoreConstruct } from './construct-compat'; +const ARCHIVE_EXTENSIONS = ['.zip', '.jar']; + /** * A previously staged asset */ @@ -30,6 +32,16 @@ interface StagedAsset { * The hash we used previously */ readonly assetHash: string; + + /** + * The packaging of the asset + */ + readonly packaging: FileAssetPackaging, + + /** + * Whether this asset is an archive + */ + readonly isArchive: boolean; } /** @@ -124,6 +136,16 @@ export class AssetStaging extends CoreConstruct { */ public readonly assetHash: string; + /** + * How this asset should be packaged. + */ + public readonly packaging: FileAssetPackaging; + + /** + * Whether this asset is an archive (zip or jar). + */ + public readonly isArchive: boolean; + private readonly fingerprintOptions: FingerprintOptions; private readonly hashType: AssetHashType; @@ -138,12 +160,20 @@ export class AssetStaging extends CoreConstruct { private readonly cacheKey: string; + private readonly sourceStats: fs.Stats; + constructor(scope: Construct, id: string, props: AssetStagingProps) { super(scope, id); this.sourcePath = path.resolve(props.sourcePath); this.fingerprintOptions = props; + if (!fs.existsSync(this.sourcePath)) { + throw new Error(`Cannot find asset at ${this.sourcePath}`); + } + + this.sourceStats = fs.statSync(this.sourcePath); + const outdir = Stage.of(this)?.assetOutdir; if (!outdir) { throw new Error('unable to determine cloud assembly asset output directory. Assets must be defined indirectly within a "Stage" or an "App" scope'); @@ -192,6 +222,8 @@ export class AssetStaging extends CoreConstruct { this.stagedPath = staged.stagedPath; this.absoluteStagedPath = staged.stagedPath; this.assetHash = staged.assetHash; + this.packaging = staged.packaging; + this.isArchive = staged.isArchive; } /** @@ -248,8 +280,18 @@ export class AssetStaging extends CoreConstruct { ? this.sourcePath : path.resolve(this.assetOutdir, renderAssetFilename(assetHash, path.extname(this.sourcePath))); + if (!this.sourceStats.isDirectory() && !this.sourceStats.isFile()) { + throw new Error(`Asset ${this.sourcePath} is expected to be either a directory or a regular file`); + } + this.stageAsset(this.sourcePath, stagedPath, 'copy'); - return { assetHash, stagedPath }; + + return { + assetHash, + stagedPath, + packaging: this.sourceStats.isDirectory() ? FileAssetPackaging.ZIP_DIRECTORY : FileAssetPackaging.FILE, + isArchive: this.sourceStats.isDirectory() || ARCHIVE_EXTENSIONS.includes(path.extname(this.sourcePath).toLowerCase()), + }; } /** @@ -258,6 +300,10 @@ export class AssetStaging extends CoreConstruct { * Optionally skip, in which case we pretend we did something but we don't really. */ private stageByBundling(bundling: BundlingOptions, skip: boolean): StagedAsset { + if (!this.sourceStats.isDirectory()) { + throw new Error(`Asset ${this.sourcePath} is expected to be a directory when bundling`); + } + if (skip) { // We should have bundled, but didn't to save time. Still pretend to have a hash. // If the asset uses OUTPUT or BUNDLE, we use a CUSTOM hash to avoid fingerprinting @@ -270,6 +316,8 @@ export class AssetStaging extends CoreConstruct { return { assetHash: this.calculateHash(hashType, bundling), stagedPath: this.sourcePath, + packaging: FileAssetPackaging.ZIP_DIRECTORY, + isArchive: true, }; } @@ -281,12 +329,21 @@ export class AssetStaging extends CoreConstruct { const bundleDir = this.determineBundleDir(this.assetOutdir, assetHash); this.bundle(bundling, bundleDir); - // Calculate assetHash afterwards if we still must - assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundleDir); - const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash)); + // Check bundling output content and determine if we will need to archive + const bundlingOutputType = bundling.outputType ?? BundlingOutput.AUTO_DISCOVER; + const bundledAsset = determineBundledAsset(bundleDir, bundlingOutputType); - this.stageAsset(bundleDir, stagedPath, 'move'); - return { assetHash, stagedPath }; + // Calculate assetHash afterwards if we still must + assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundledAsset.path); + const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash, bundledAsset.extension)); + + this.stageAsset(bundledAsset.path, stagedPath, 'move'); + return { + assetHash, + stagedPath, + packaging: bundledAsset.packaging, + isArchive: true, // bundling always produces an archive + }; } /** @@ -320,10 +377,9 @@ export class AssetStaging extends CoreConstruct { } // Copy file/directory to staging directory - const stat = fs.statSync(sourcePath); - if (stat.isFile()) { + if (this.sourceStats.isFile()) { fs.copyFileSync(sourcePath, targetPath); - } else if (stat.isDirectory()) { + } else if (this.sourceStats.isDirectory()) { fs.mkdirSync(targetPath); FileSystem.copyDirectory(sourcePath, targetPath, this.fingerprintOptions); } else { @@ -502,3 +558,57 @@ function sortObject(object: { [key: string]: any }): { [key: string]: any } { } return ret; } + +/** + * Returns the single archive file of a directory or undefined + */ +function singleArchiveFile(directory: string): string | undefined { + if (!fs.existsSync(directory)) { + throw new Error(`Directory ${directory} does not exist.`); + } + + if (!fs.statSync(directory).isDirectory()) { + throw new Error(`${directory} is not a directory.`); + } + + const content = fs.readdirSync(directory); + if (content.length === 1) { + const file = path.join(directory, content[0]); + const extension = path.extname(content[0]).toLowerCase(); + if (fs.statSync(file).isFile() && ARCHIVE_EXTENSIONS.includes(extension)) { + return file; + } + } + + return undefined; +} + +interface BundledAsset { + path: string, + packaging: FileAssetPackaging, + extension?: string +} + +/** + * Returns the bundled asset to use based on the content of the bundle directory + * and the type of output. + */ +function determineBundledAsset(bundleDir: string, outputType: BundlingOutput): BundledAsset { + const archiveFile = singleArchiveFile(bundleDir); + + // auto-discover means that if there is an archive file, we take it as the + // bundle, otherwise, we will archive here. + if (outputType === BundlingOutput.AUTO_DISCOVER) { + outputType = archiveFile ? BundlingOutput.ARCHIVED : BundlingOutput.NOT_ARCHIVED; + } + + switch (outputType) { + case BundlingOutput.NOT_ARCHIVED: + return { path: bundleDir, packaging: FileAssetPackaging.ZIP_DIRECTORY }; + case BundlingOutput.ARCHIVED: + if (!archiveFile) { + throw new Error('Bundling output directory is expected to include only a single .zip or .jar file when `output` is set to `ARCHIVED`'); + } + return { path: archiveFile, packaging: FileAssetPackaging.FILE, extension: path.extname(archiveFile) }; + } +} diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 0179b07d2f8f3..213b6bbaba281 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -80,6 +80,41 @@ export interface BundlingOptions { * @experimental */ readonly local?: ILocalBundling; + + /** + * The type of output that this bundling operation is producing. + * + * @default BundlingOutput.AUTO_DISCOVER + * + * @experimental + */ + readonly outputType?: BundlingOutput; +} + +/** + * The type of output that a bundling operation is producing. + * + * @experimental + */ +export enum BundlingOutput { + /** + * The bundling output directory includes a single .zip or .jar file which + * will be used as the final bundle. If the output directory does not + * include exactly a single archive, bundling will fail. + */ + ARCHIVED = 'archived', + + /** + * The bundling output directory contains one or more files which will be + * archived and uploaded as a .zip file to S3. + */ + NOT_ARCHIVED = 'not-archived', + + /** + * If the bundling output directory contains a single archive file (zip or jar) + * it will be used as the bundle output as-is. Otherwise all the files in the bundling output directory will be zipped. + */ + AUTO_DISCOVER = 'auto-discover', } /** diff --git a/packages/@aws-cdk/core/test/archive/archive.zip b/packages/@aws-cdk/core/test/archive/archive.zip new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/core/test/docker-stub.sh b/packages/@aws-cdk/core/test/docker-stub.sh index fe48e93d4a207..94f806f69a120 100755 --- a/packages/@aws-cdk/core/test/docker-stub.sh +++ b/packages/@aws-cdk/core/test/docker-stub.sh @@ -24,5 +24,18 @@ if echo "$@" | grep "DOCKER_STUB_SUCCESS"; then exit 0 fi -echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS" +if echo "$@" | grep "DOCKER_STUB_MULTIPLE_FILES"; then + outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1) + touch ${outdir}/test1.txt + touch ${outdir}/test2.txt + exit 0 +fi + +if echo "$@" | grep "DOCKER_STUB_SINGLE_ARCHIVE"; then + outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1) + touch ${outdir}/test.zip + exit 0 +fi + +echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS,DOCKER_STUB_MULTIPLE_FILES,DOCKER_SINGLE_ARCHIVE" exit 1 diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 347c5fcea3b63..ee87780a0957e 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1,10 +1,11 @@ import * as os from 'os'; import * as path from 'path'; +import { FileAssetPackaging } from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import { nodeunitShim, Test } from 'nodeunit-shim'; import * as sinon from 'sinon'; -import { App, AssetHashType, AssetStaging, BundlingDockerImage, BundlingOptions, FileSystem, Stack, Stage } from '../lib'; +import { App, AssetHashType, AssetStaging, BundlingDockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage } from '../lib'; const STUB_INPUT_FILE = '/tmp/docker-stub.input'; const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; @@ -12,7 +13,9 @@ const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; enum DockerStubCommand { SUCCESS = 'DOCKER_STUB_SUCCESS', FAIL = 'DOCKER_STUB_FAIL', - SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT' + SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT', + MULTIPLE_FILES = 'DOCKER_STUB_MULTIPLE_FILES', + SINGLE_ARCHIVE = 'DOCKER_STUB_SINGLE_ARCHIVE', } const FIXTURE_TEST1_DIR = path.join(__dirname, 'fs', 'fixtures', 'test1'); @@ -50,6 +53,84 @@ nodeunitShim({ test.deepEqual(staging.sourcePath, sourcePath); test.deepEqual(path.basename(staging.stagedPath), 'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); test.deepEqual(path.basename(staging.relativeStagedPath(stack)), 'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); + test.deepEqual(staging.packaging, FileAssetPackaging.ZIP_DIRECTORY); + test.deepEqual(staging.isArchive, true); + test.done(); + }, + + 'staging of an archive file correctly sets packaging and isArchive'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + // WHEN + const staging = new AssetStaging(stack, 's1', { sourcePath }); + + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, true); + test.done(); + }, + + 'asset packaging type is correct when staging is skipped because of memory cache'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + // WHEN + const staging1 = new AssetStaging(stack, 's1', { sourcePath }); + const staging2 = new AssetStaging(stack, 's2', { sourcePath }); + + test.deepEqual(staging1.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging1.isArchive, true); + test.deepEqual(staging2.packaging, staging1.packaging); + test.deepEqual(staging2.isArchive, staging1.isArchive); + test.done(); + }, + + 'asset packaging type is correct when staging is skipped because of disk cache'(test: Test) { + // GIVEN + const TEST_OUTDIR = path.join(__dirname, 'cdk.out'); + if (fs.existsSync(TEST_OUTDIR)) { + fs.removeSync(TEST_OUTDIR); + } + + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + const app1 = new App({ outdir: TEST_OUTDIR }); + const stack1 = new Stack(app1, 'Stack'); + + const app2 = new App({ outdir: TEST_OUTDIR }); // same OUTDIR + const stack2 = new Stack(app2, 'stack'); + + // WHEN + const staging1 = new AssetStaging(stack1, 'Asset', { sourcePath }); + + // Now clear asset hash cache to show that during the second staging + // even though the asset is already available on disk it will correctly + // be considered as a FileAssetPackaging.FILE. + AssetStaging.clearAssetHashCache(); + + const staging2 = new AssetStaging(stack2, 'Asset', { sourcePath }); + + // THEN + test.deepEqual(staging1.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging1.isArchive, true); + test.deepEqual(staging2.packaging, staging1.packaging); + test.deepEqual(staging2.isArchive, staging1.isArchive); + + test.done(); + }, + + 'staging of a non-archive file correctly sets packaging and isArchive'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = __filename; + + // WHEN + const staging = new AssetStaging(stack, 's1', { sourcePath }); + + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, false); test.done(); }, @@ -785,6 +866,89 @@ nodeunitShim({ ); test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f'); // hash of MyStack/Asset + test.done(); + }, + + 'bundling that produces a single archive file is autodiscovered'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const staging = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SINGLE_ARCHIVE], + }, + }); + + // THEN + const assembly = app.synth(); + test.deepEqual(fs.readdirSync(assembly.directory), [ + 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48', // this is the bundle dir but it's empty + 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48.zip', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + test.equal(fs.readdirSync(path.join(assembly.directory, 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48')).length, 0); // empty bundle dir + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, true); + + test.done(); + }, + + 'bundling that produces a single archive file with NOT_ARCHIVED'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const staging = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SINGLE_ARCHIVE], + outputType: BundlingOutput.NOT_ARCHIVED, + }, + }); + + // THEN + const assembly = app.synth(); + test.deepEqual(fs.readdirSync(assembly.directory), [ + 'asset.86ec07746e1d859290cfd8b9c648e581555649c75f51f741f11e22cab6775abc', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + test.deepEqual(staging.packaging, FileAssetPackaging.ZIP_DIRECTORY); + test.deepEqual(staging.isArchive, true); + + test.done(); + }, + + 'throws with ARCHIVED and bundling that does not produce a single archive file'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.MULTIPLE_FILES], + outputType: BundlingOutput.ARCHIVED, + }, + }), /Bundling output directory is expected to include only a single .zip or .jar file when `output` is set to `ARCHIVED`/); + + test.done(); }, });