diff --git a/allowed-breaking-changes.txt b/allowed-breaking-changes.txt index 964bb4d5c7712..2ca2ca5b6067f 100644 --- a/allowed-breaking-changes.txt +++ b/allowed-breaking-changes.txt @@ -56,10 +56,3 @@ incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.addVolume # We made properties optional and it's really fine but our differ doesn't think so. weakened:@aws-cdk/cloud-assembly-schema.DockerImageSource weakened:@aws-cdk/cloud-assembly-schema.FileSource - -# https://github.com/aws/aws-cdk/pull/13145 -removed:@aws-cdk/core.AssetStaging.isArchive -removed:@aws-cdk/core.AssetStaging.packaging -removed:@aws-cdk/core.BundlingOutput -removed:@aws-cdk/core.BundlingOptions.outputType - diff --git a/packages/@aws-cdk/aws-s3-assets/README.md b/packages/@aws-cdk/aws-s3-assets/README.md index aab4c46d9c44d..f2583b7c10a24 100644 --- a/packages/@aws-cdk/aws-s3-assets/README.md +++ b/packages/@aws-cdk/aws-s3-assets/README.md @@ -124,6 +124,27 @@ new assets.Asset(this, 'BundledAsset', { Although optional, it's recommended to provide a local bundling method which can greatly improve performance. +If the bundling output contains a single archive file (zip or jar) it will be +uploaded to S3 as-is and will not be zipped. Otherwise the contents of the +output directory will be zipped and the zip file will be uploaded to S3. This +is the default behavior for `bundling.outputType` (`BundlingOutput.AUTO_DISCOVER`). + +Use `BundlingOutput.NOT_ARCHIVED` if the bundling output must always be zipped: + +```ts +const asset = new assets.Asset(this, 'BundledAsset', { + path: '/path/to/asset', + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: ['command-that-produces-an-archive.sh'], + outputType: BundlingOutput.NOT_ARCHIVED, // Bundling output will be zipped even though it produces a single archive file. + }, +}); +``` + +Use `BundlingOutput.ARCHIVED` if the bundling output contains a single archive file and +you don't want it to be zippped. + ## CloudFormation Resource Metadata > NOTE: This section is relevant for authors of AWS Resource Constructs. diff --git a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts index 938778d1381f4..510834a61c634 100644 --- a/packages/@aws-cdk/aws-s3-assets/lib/asset.ts +++ b/packages/@aws-cdk/aws-s3-assets/lib/asset.ts @@ -1,4 +1,3 @@ -import * as fs from 'fs'; import * as path from 'path'; import * as assets from '@aws-cdk/assets'; import * as iam from '@aws-cdk/aws-iam'; @@ -13,8 +12,6 @@ import { toSymlinkFollow } from './compat'; // eslint-disable-next-line no-duplicate-imports, import/order import { Construct as CoreConstruct } from '@aws-cdk/core'; -const ARCHIVE_EXTENSIONS = ['.zip', '.jar']; - export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions { /** * A list of principals that should be able to read this asset from S3. @@ -139,17 +136,12 @@ export class Asset extends CoreConstruct implements cdk.IAsset { this.assetPath = staging.relativeStagedPath(stack); - const packaging = determinePackaging(staging.sourcePath); - - this.isFile = packaging === cdk.FileAssetPackaging.FILE; + this.isFile = staging.packaging === cdk.FileAssetPackaging.FILE; - // sets isZipArchive based on the type of packaging and file extension - this.isZipArchive = packaging === cdk.FileAssetPackaging.ZIP_DIRECTORY - ? true - : ARCHIVE_EXTENSIONS.some(ext => staging.sourcePath.toLowerCase().endsWith(ext)); + this.isZipArchive = staging.isArchive; const location = stack.synthesizer.addFileAsset({ - packaging, + packaging: staging.packaging, sourceHash: this.sourceHash, fileName: this.assetPath, }); @@ -210,19 +202,3 @@ export class Asset extends CoreConstruct implements cdk.IAsset { this.bucket.grantRead(grantee); } } - -function determinePackaging(assetPath: string): cdk.FileAssetPackaging { - if (!fs.existsSync(assetPath)) { - throw new Error(`Cannot find asset at ${assetPath}`); - } - - if (fs.statSync(assetPath).isDirectory()) { - return cdk.FileAssetPackaging.ZIP_DIRECTORY; - } - - if (fs.statSync(assetPath).isFile()) { - return cdk.FileAssetPackaging.FILE; - } - - throw new Error(`Asset ${assetPath} is expected to be either a directory or a regular file`); -} diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 66c65e3d14864..6a34bd9b4b1ac 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,8 +5,8 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import * as minimatch from 'minimatch'; -import { AssetHashType, AssetOptions } from './assets'; -import { BundlingOptions } from './bundling'; +import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; +import { BundlingOptions, BundlingOutput } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { Names } from './names'; import { Cache } from './private/cache'; @@ -17,6 +17,8 @@ import { Stage } from './stage'; // eslint-disable-next-line import { Construct as CoreConstruct } from './construct-compat'; +const ARCHIVE_EXTENSIONS = ['.zip', '.jar']; + /** * A previously staged asset */ @@ -30,6 +32,16 @@ interface StagedAsset { * The hash we used previously */ readonly assetHash: string; + + /** + * The packaging of the asset + */ + readonly packaging: FileAssetPackaging, + + /** + * Whether this asset is an archive + */ + readonly isArchive: boolean; } /** @@ -124,6 +136,16 @@ export class AssetStaging extends CoreConstruct { */ public readonly assetHash: string; + /** + * How this asset should be packaged. + */ + public readonly packaging: FileAssetPackaging; + + /** + * Whether this asset is an archive (zip or jar). + */ + public readonly isArchive: boolean; + private readonly fingerprintOptions: FingerprintOptions; private readonly hashType: AssetHashType; @@ -138,12 +160,20 @@ export class AssetStaging extends CoreConstruct { private readonly cacheKey: string; + private readonly sourceStats: fs.Stats; + constructor(scope: Construct, id: string, props: AssetStagingProps) { super(scope, id); this.sourcePath = path.resolve(props.sourcePath); this.fingerprintOptions = props; + if (!fs.existsSync(this.sourcePath)) { + throw new Error(`Cannot find asset at ${this.sourcePath}`); + } + + this.sourceStats = fs.statSync(this.sourcePath); + const outdir = Stage.of(this)?.assetOutdir; if (!outdir) { throw new Error('unable to determine cloud assembly asset output directory. Assets must be defined indirectly within a "Stage" or an "App" scope'); @@ -192,6 +222,8 @@ export class AssetStaging extends CoreConstruct { this.stagedPath = staged.stagedPath; this.absoluteStagedPath = staged.stagedPath; this.assetHash = staged.assetHash; + this.packaging = staged.packaging; + this.isArchive = staged.isArchive; } /** @@ -248,8 +280,18 @@ export class AssetStaging extends CoreConstruct { ? this.sourcePath : path.resolve(this.assetOutdir, renderAssetFilename(assetHash, path.extname(this.sourcePath))); + if (!this.sourceStats.isDirectory() && !this.sourceStats.isFile()) { + throw new Error(`Asset ${this.sourcePath} is expected to be either a directory or a regular file`); + } + this.stageAsset(this.sourcePath, stagedPath, 'copy'); - return { assetHash, stagedPath }; + + return { + assetHash, + stagedPath, + packaging: this.sourceStats.isDirectory() ? FileAssetPackaging.ZIP_DIRECTORY : FileAssetPackaging.FILE, + isArchive: this.sourceStats.isDirectory() || ARCHIVE_EXTENSIONS.includes(path.extname(this.sourcePath).toLowerCase()), + }; } /** @@ -258,6 +300,10 @@ export class AssetStaging extends CoreConstruct { * Optionally skip, in which case we pretend we did something but we don't really. */ private stageByBundling(bundling: BundlingOptions, skip: boolean): StagedAsset { + if (!this.sourceStats.isDirectory()) { + throw new Error(`Asset ${this.sourcePath} is expected to be a directory when bundling`); + } + if (skip) { // We should have bundled, but didn't to save time. Still pretend to have a hash. // If the asset uses OUTPUT or BUNDLE, we use a CUSTOM hash to avoid fingerprinting @@ -270,6 +316,8 @@ export class AssetStaging extends CoreConstruct { return { assetHash: this.calculateHash(hashType, bundling), stagedPath: this.sourcePath, + packaging: FileAssetPackaging.ZIP_DIRECTORY, + isArchive: true, }; } @@ -281,12 +329,21 @@ export class AssetStaging extends CoreConstruct { const bundleDir = this.determineBundleDir(this.assetOutdir, assetHash); this.bundle(bundling, bundleDir); - // Calculate assetHash afterwards if we still must - assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundleDir); - const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash)); + // Check bundling output content and determine if we will need to archive + const bundlingOutputType = bundling.outputType ?? BundlingOutput.AUTO_DISCOVER; + const bundledAsset = determineBundledAsset(bundleDir, bundlingOutputType); - this.stageAsset(bundleDir, stagedPath, 'move'); - return { assetHash, stagedPath }; + // Calculate assetHash afterwards if we still must + assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundledAsset.path); + const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash, bundledAsset.extension)); + + this.stageAsset(bundledAsset.path, stagedPath, 'move'); + return { + assetHash, + stagedPath, + packaging: bundledAsset.packaging, + isArchive: true, // bundling always produces an archive + }; } /** @@ -320,10 +377,9 @@ export class AssetStaging extends CoreConstruct { } // Copy file/directory to staging directory - const stat = fs.statSync(sourcePath); - if (stat.isFile()) { + if (this.sourceStats.isFile()) { fs.copyFileSync(sourcePath, targetPath); - } else if (stat.isDirectory()) { + } else if (this.sourceStats.isDirectory()) { fs.mkdirSync(targetPath); FileSystem.copyDirectory(sourcePath, targetPath, this.fingerprintOptions); } else { @@ -502,3 +558,57 @@ function sortObject(object: { [key: string]: any }): { [key: string]: any } { } return ret; } + +/** + * Returns the single archive file of a directory or undefined + */ +function singleArchiveFile(directory: string): string | undefined { + if (!fs.existsSync(directory)) { + throw new Error(`Directory ${directory} does not exist.`); + } + + if (!fs.statSync(directory).isDirectory()) { + throw new Error(`${directory} is not a directory.`); + } + + const content = fs.readdirSync(directory); + if (content.length === 1) { + const file = path.join(directory, content[0]); + const extension = path.extname(content[0]).toLowerCase(); + if (fs.statSync(file).isFile() && ARCHIVE_EXTENSIONS.includes(extension)) { + return file; + } + } + + return undefined; +} + +interface BundledAsset { + path: string, + packaging: FileAssetPackaging, + extension?: string +} + +/** + * Returns the bundled asset to use based on the content of the bundle directory + * and the type of output. + */ +function determineBundledAsset(bundleDir: string, outputType: BundlingOutput): BundledAsset { + const archiveFile = singleArchiveFile(bundleDir); + + // auto-discover means that if there is an archive file, we take it as the + // bundle, otherwise, we will archive here. + if (outputType === BundlingOutput.AUTO_DISCOVER) { + outputType = archiveFile ? BundlingOutput.ARCHIVED : BundlingOutput.NOT_ARCHIVED; + } + + switch (outputType) { + case BundlingOutput.NOT_ARCHIVED: + return { path: bundleDir, packaging: FileAssetPackaging.ZIP_DIRECTORY }; + case BundlingOutput.ARCHIVED: + if (!archiveFile) { + throw new Error('Bundling output directory is expected to include only a single .zip or .jar file when `output` is set to `ARCHIVED`'); + } + return { path: archiveFile, packaging: FileAssetPackaging.FILE, extension: path.extname(archiveFile) }; + } +} diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 0179b07d2f8f3..213b6bbaba281 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -80,6 +80,41 @@ export interface BundlingOptions { * @experimental */ readonly local?: ILocalBundling; + + /** + * The type of output that this bundling operation is producing. + * + * @default BundlingOutput.AUTO_DISCOVER + * + * @experimental + */ + readonly outputType?: BundlingOutput; +} + +/** + * The type of output that a bundling operation is producing. + * + * @experimental + */ +export enum BundlingOutput { + /** + * The bundling output directory includes a single .zip or .jar file which + * will be used as the final bundle. If the output directory does not + * include exactly a single archive, bundling will fail. + */ + ARCHIVED = 'archived', + + /** + * The bundling output directory contains one or more files which will be + * archived and uploaded as a .zip file to S3. + */ + NOT_ARCHIVED = 'not-archived', + + /** + * If the bundling output directory contains a single archive file (zip or jar) + * it will be used as the bundle output as-is. Otherwise all the files in the bundling output directory will be zipped. + */ + AUTO_DISCOVER = 'auto-discover', } /** diff --git a/packages/@aws-cdk/core/test/archive/archive.zip b/packages/@aws-cdk/core/test/archive/archive.zip new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/core/test/docker-stub.sh b/packages/@aws-cdk/core/test/docker-stub.sh index fe48e93d4a207..94f806f69a120 100755 --- a/packages/@aws-cdk/core/test/docker-stub.sh +++ b/packages/@aws-cdk/core/test/docker-stub.sh @@ -24,5 +24,18 @@ if echo "$@" | grep "DOCKER_STUB_SUCCESS"; then exit 0 fi -echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS" +if echo "$@" | grep "DOCKER_STUB_MULTIPLE_FILES"; then + outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1) + touch ${outdir}/test1.txt + touch ${outdir}/test2.txt + exit 0 +fi + +if echo "$@" | grep "DOCKER_STUB_SINGLE_ARCHIVE"; then + outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1) + touch ${outdir}/test.zip + exit 0 +fi + +echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS,DOCKER_STUB_MULTIPLE_FILES,DOCKER_SINGLE_ARCHIVE" exit 1 diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 347c5fcea3b63..ee87780a0957e 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1,10 +1,11 @@ import * as os from 'os'; import * as path from 'path'; +import { FileAssetPackaging } from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import { nodeunitShim, Test } from 'nodeunit-shim'; import * as sinon from 'sinon'; -import { App, AssetHashType, AssetStaging, BundlingDockerImage, BundlingOptions, FileSystem, Stack, Stage } from '../lib'; +import { App, AssetHashType, AssetStaging, BundlingDockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage } from '../lib'; const STUB_INPUT_FILE = '/tmp/docker-stub.input'; const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; @@ -12,7 +13,9 @@ const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; enum DockerStubCommand { SUCCESS = 'DOCKER_STUB_SUCCESS', FAIL = 'DOCKER_STUB_FAIL', - SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT' + SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT', + MULTIPLE_FILES = 'DOCKER_STUB_MULTIPLE_FILES', + SINGLE_ARCHIVE = 'DOCKER_STUB_SINGLE_ARCHIVE', } const FIXTURE_TEST1_DIR = path.join(__dirname, 'fs', 'fixtures', 'test1'); @@ -50,6 +53,84 @@ nodeunitShim({ test.deepEqual(staging.sourcePath, sourcePath); test.deepEqual(path.basename(staging.stagedPath), 'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); test.deepEqual(path.basename(staging.relativeStagedPath(stack)), 'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00'); + test.deepEqual(staging.packaging, FileAssetPackaging.ZIP_DIRECTORY); + test.deepEqual(staging.isArchive, true); + test.done(); + }, + + 'staging of an archive file correctly sets packaging and isArchive'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + // WHEN + const staging = new AssetStaging(stack, 's1', { sourcePath }); + + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, true); + test.done(); + }, + + 'asset packaging type is correct when staging is skipped because of memory cache'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + // WHEN + const staging1 = new AssetStaging(stack, 's1', { sourcePath }); + const staging2 = new AssetStaging(stack, 's2', { sourcePath }); + + test.deepEqual(staging1.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging1.isArchive, true); + test.deepEqual(staging2.packaging, staging1.packaging); + test.deepEqual(staging2.isArchive, staging1.isArchive); + test.done(); + }, + + 'asset packaging type is correct when staging is skipped because of disk cache'(test: Test) { + // GIVEN + const TEST_OUTDIR = path.join(__dirname, 'cdk.out'); + if (fs.existsSync(TEST_OUTDIR)) { + fs.removeSync(TEST_OUTDIR); + } + + const sourcePath = path.join(__dirname, 'archive', 'archive.zip'); + + const app1 = new App({ outdir: TEST_OUTDIR }); + const stack1 = new Stack(app1, 'Stack'); + + const app2 = new App({ outdir: TEST_OUTDIR }); // same OUTDIR + const stack2 = new Stack(app2, 'stack'); + + // WHEN + const staging1 = new AssetStaging(stack1, 'Asset', { sourcePath }); + + // Now clear asset hash cache to show that during the second staging + // even though the asset is already available on disk it will correctly + // be considered as a FileAssetPackaging.FILE. + AssetStaging.clearAssetHashCache(); + + const staging2 = new AssetStaging(stack2, 'Asset', { sourcePath }); + + // THEN + test.deepEqual(staging1.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging1.isArchive, true); + test.deepEqual(staging2.packaging, staging1.packaging); + test.deepEqual(staging2.isArchive, staging1.isArchive); + + test.done(); + }, + + 'staging of a non-archive file correctly sets packaging and isArchive'(test: Test) { + // GIVEN + const stack = new Stack(); + const sourcePath = __filename; + + // WHEN + const staging = new AssetStaging(stack, 's1', { sourcePath }); + + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, false); test.done(); }, @@ -785,6 +866,89 @@ nodeunitShim({ ); test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f'); // hash of MyStack/Asset + test.done(); + }, + + 'bundling that produces a single archive file is autodiscovered'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const staging = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SINGLE_ARCHIVE], + }, + }); + + // THEN + const assembly = app.synth(); + test.deepEqual(fs.readdirSync(assembly.directory), [ + 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48', // this is the bundle dir but it's empty + 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48.zip', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + test.equal(fs.readdirSync(path.join(assembly.directory, 'asset.f43148c61174f444925231b5849b468f21e93b5d1469cd07c53625ffd039ef48')).length, 0); // empty bundle dir + test.deepEqual(staging.packaging, FileAssetPackaging.FILE); + test.deepEqual(staging.isArchive, true); + + test.done(); + }, + + 'bundling that produces a single archive file with NOT_ARCHIVED'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const staging = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.SINGLE_ARCHIVE], + outputType: BundlingOutput.NOT_ARCHIVED, + }, + }); + + // THEN + const assembly = app.synth(); + test.deepEqual(fs.readdirSync(assembly.directory), [ + 'asset.86ec07746e1d859290cfd8b9c648e581555649c75f51f741f11e22cab6775abc', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + test.deepEqual(staging.packaging, FileAssetPackaging.ZIP_DIRECTORY); + test.deepEqual(staging.isArchive, true); + + test.done(); + }, + + 'throws with ARCHIVED and bundling that does not produce a single archive file'(test: Test) { + // GIVEN + const app = new App(); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + test.throws(() => new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: BundlingDockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.MULTIPLE_FILES], + outputType: BundlingOutput.ARCHIVED, + }, + }), /Bundling output directory is expected to include only a single .zip or .jar file when `output` is set to `ARCHIVED`/); + + test.done(); }, });