feat: stage assets under .cdk.assets

To ensure that assets are available for the toolchain to deploy after the CDK app
exists, the CLI will, by default, request that the app will stage the assets under
the `.cdk.assets` directory (relative to working directory).

The CDK will then *copy* all assets from their source locations to this staging
directory and will refer to the staging location as the asset path. Assets will
be stored using their content fingerprint (md5 hash) so they will never be copied
twice unless they change.

Fixes #1716

TODO:
- [ ] docker assets
- [ ] toolkit support
- [ ] toolkit test

packages/@aws-cdk/assets/lib/asset.ts

@@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk');
 import cxapi = require('@aws-cdk/cx-api');
 import fs = require('fs');
 import path = require('path');
+import { copyDirectory, fingerprint } from './fs';
 
 /**
  * Defines the way an asset is packaged before it is uploaded to S3.
@@ -61,7 +62,10 @@ export class Asset extends cdk.Construct {
   public readonly s3Url: string;
 
   /**
-   * Resolved full-path location of this asset.
+   * The path to the asset (stringinfied token).
+   *
+   * If asset staging is disabled, this will just be the original path.
+   * If asset staging is enabled it will be the staged path.
    */
   public readonly assetPath: string;
 
@@ -81,19 +85,55 @@ export class Asset extends cdk.Construct {
    */
   private readonly s3Prefix: string;
 
+  /**
+   * Type of asset packaging (file/zip-dir);.
+   */
+  private readonly packaging: AssetPackaging;
+
+  /**
+   * The path of the asset as it was referenced by the user.
+   */
+  private readonly sourcePath: string;
+
+  /**
+   * If this is defined, assets will be staged (copied) under this directory.
+   * Otherwise, they will just be consumed from their original location.
+   *
+   * This is controlled by the context key 'aws:cdk:assets-staging-dir' and
+   * enabled by the CLI by default in order to ensure that when the CDK app
+   * exists, all assets are available for deployment. Otherwise, if an app
+   * references assets in temporary locations, those will not be available when
+   * it exists (see https://github.com/awslabs/aws-cdk/issues/1716).
+   */
+  private readonly stagingDir?: string;
+
+  /**
+   * The asset path after "prepare" is called.
+   *
+   * If staging is disabled, this will just be the original path.
+   * If staging is enabled it will be the staged path.
+   */
+  private _preparedAssetPath?: string;
+
   constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) {
     super(scope, id);
 
+    this.packaging = props.packaging;
+
+    this.stagingDir = this.node.getContext(cxapi.ASSETS_STAGING_DIR_CONTEXT);
+
     // resolve full path
-    this.assetPath = path.resolve(props.path);
+    this.sourcePath = path.resolve(props.path);
 
     // sets isZipArchive based on the type of packaging and file extension
     const allowedExtensions: string[] = ['.jar', '.zip'];
     this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory
       ? true
-      : allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext));
+      : allowedExtensions.some(ext => this.sourcePath.toLowerCase().endsWith(ext));
+
+    validateAssetOnDisk(this.sourcePath, props.packaging);
 
-    validateAssetOnDisk(this.assetPath, props.packaging);
+    this.assetPath = new cdk.Token(() => this._preparedAssetPath).toString();
 
     // add parameters for s3 bucket and s3 key. those will be set by
     // the toolkit or by CI/CD when the stack is deployed and will include
@@ -178,6 +218,43 @@ export class Asset extends cdk.Construct {
     // when deploying a new version.
     this.bucket.grantRead(grantee, `${this.s3Prefix}*`);
   }
+
+  public prepare() {
+    const stagingDir = this.stagingDir;
+    if (!stagingDir) {
+      this._preparedAssetPath = this.sourcePath;
+      return;
+    }
+
+    if (!fs.existsSync(stagingDir)) {
+      fs.mkdirSync(stagingDir);
+    }
+
+    const hash = fingerprint(this.sourcePath);
+    const targetPath = path.join(stagingDir, hash + path.extname(this.sourcePath));
+
+    this._preparedAssetPath = targetPath;
+
+    // nothing to do if we already have the asset there
+    if (fs.existsSync(targetPath)) {
+      return;
+    }
+
+    // copy file/directory to .assets
+    switch (this.packaging) {
+      case AssetPackaging.File:
+        fs.copyFileSync(this.sourcePath, targetPath);
+        break;
+
+      case AssetPackaging.ZipDirectory:
+        fs.mkdirSync(targetPath);
+        copyDirectory(this.sourcePath, targetPath);
+        break;
+
+      default:
+        throw new Error(`Unknown asset packaging type: ${this.packaging}`);
+    }
+  }
 }
 
 export interface FileAssetProps {

packages/@aws-cdk/assets/lib/fs/copy.ts

@@ -0,0 +1,89 @@
+import fs = require('fs');
+import minimatch = require('minimatch');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+export interface CopyOptions {
+  /**
+   * @default External only follows symlinks that are external to the source directory
+   */
+  follow?: FollowMode;
+
+  /**
+   * glob patterns to exclude from the copy.
+   */
+  exclude?: string[];
+}
+
+export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const exclude = options.exclude || [];
+
+  rootDir = rootDir || srcDir;
+
+  if (!fs.statSync(srcDir).isDirectory()) {
+    throw new Error(`${srcDir} is not a directory`);
+  }
+
+  const files = fs.readdirSync(srcDir);
+  for (const file of files) {
+    const sourceFilePath = path.join(srcDir, file);
+
+    if (shouldExclude(path.relative(rootDir, sourceFilePath))) {
+      continue;
+    }
+
+    const destFilePath = path.join(destDir, file);
+
+    let stat: fs.Stats | undefined = follow === FollowMode.Always
+      ? fs.statSync(sourceFilePath)
+      : fs.lstatSync(sourceFilePath);
+
+    if (stat && stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(sourceFilePath);
+
+      // determine if this is an external link (i.e. the target's absolute path
+      // is outside of the root directory).
+      const targetPath = path.normalize(path.resolve(srcDir, target));
+      const rootPath = path.normalize(rootDir);
+      const external = !targetPath.startsWith(rootPath);
+
+      if (follow === FollowMode.External && external) {
+        stat = fs.statSync(sourceFilePath);
+      } else {
+        fs.symlinkSync(target, destFilePath);
+        stat = undefined;
+      }
+    }
+
+    if (stat && stat.isDirectory()) {
+      fs.mkdirSync(destFilePath);
+      copyDirectory(sourceFilePath, destFilePath, options, rootDir);
+      stat = undefined;
+    }
+
+    if (stat && stat.isFile()) {
+      fs.copyFileSync(sourceFilePath, destFilePath);
+      stat = undefined;
+    }
+  }
+
+  function shouldExclude(filePath: string): boolean {
+    let excludeOutput = false;
+
+    for (const pattern of exclude) {
+      const negate = pattern.startsWith('!');
+      const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true });
+
+      if (!negate && match) {
+        excludeOutput = true;
+      }
+
+      if (negate && match) {
+        excludeOutput = false;
+      }
+    }
+
+    return excludeOutput;
+  }
+}

packages/@aws-cdk/assets/lib/fs/fingerprint.ts

@@ -0,0 +1,86 @@
+import crypto = require('crypto');
+import fs = require('fs');
+import path = require('path');
+import { FollowMode } from './follow-mode';
+
+const BUFFER_SIZE = 8 * 1024;
+
+export interface FingerprintOptions {
+  /**
+   * Extra information to encode into the fingerprint (e.g. build instructions
+   * and other inputs)
+   */
+  extra?: string;
+
+  /**
+   * List of exclude patterns (see `CopyOptions`)
+   * @default include all files
+   */
+  exclude?: string[];
+
+  /**
+   * What to do when we encounter symlinks.
+   * @default External only follows symlinks that are external to the source
+   * directory
+   */
+  follow?: FollowMode;
+}
+
+/**
+ * Produces fingerprint based on the contents of a single file or an entire directory tree.
+ *
+ * The fingerprint will also include:
+ * 1. An extra string if defined in `options.extra`.
+ * 2. The set of exclude patterns, if defined in `options.exclude`
+ * 3. The symlink follow mode value.
+ *
+ * @param fileOrDirectory The directory or file to fingerprint
+ * @param options Fingerprinting options
+ */
+export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) {
+  const follow = options.follow !== undefined ? options.follow : FollowMode.External;
+  const hash = crypto.createHash('md5');
+  addToHash(fileOrDirectory);
+
+  hash.update(`==follow==${follow}==\n\n`);
+
+  if (options.extra) {
+    hash.update(`==extra==${options.extra}==\n\n`);
+  }
+
+  for (const ex of options.exclude || []) {
+    hash.update(`==exclude==${ex}==\n\n`);
+  }
+
+  return hash.digest('hex');
+
+  function addToHash(pathToAdd: string) {
+    hash.update('==\n');
+    const relativePath = path.relative(fileOrDirectory, pathToAdd);
+    hash.update(relativePath + '\n');
+    hash.update('~~~~~~~~~~~~~~~~~~\n');
+    const stat = fs.statSync(pathToAdd);
+
+    if (stat.isSymbolicLink()) {
+      const target = fs.readlinkSync(pathToAdd);
+      hash.update(target);
+    } else if (stat.isDirectory()) {
+      for (const file of fs.readdirSync(pathToAdd)) {
+        addToHash(path.join(pathToAdd, file));
+      }
+    } else {
+      const file = fs.openSync(pathToAdd, 'r');
+      const buffer = Buffer.alloc(BUFFER_SIZE);
+
+      try {
+        let bytesRead;
+        do {
+          bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null);
+          hash.update(buffer.slice(0, bytesRead));
+        } while (bytesRead === BUFFER_SIZE);
+      } finally {
+        fs.closeSync(file);
+      }
+    }
+  }
+}

packages/@aws-cdk/assets/lib/fs/follow-mode.ts

@@ -0,0 +1,29 @@
+export enum FollowMode {
+  /**
+   * Never follow symlinks.
+   */
+  Never = 'never',
+
+  /**
+   * Materialize all symlinks, whether they are internal or external to the source directory.
+   */
+  Always = 'always',
+
+  /**
+   * Only follows symlinks that are external to the source directory.
+   */
+  External = 'external',
+
+  // ----------------- TODO::::::::::::::::::::::::::::::::::::::::::::
+  /**
+   * Forbids source from having any symlinks pointing outside of the source
+   * tree.
+   *
+   * This is the safest mode of operation as it ensures that copy operations
+   * won't materialize files from the user's file system. Internal symlinks are
+   * not followed.
+   *
+   * If the copy operation runs into an external symlink, it will fail.
+   */
+  BlockExternal = 'internal-only',
+}

packages/@aws-cdk/assets/lib/fs/index.ts

@@ -0,0 +1,3 @@
+export * from './fingerprint';
+export * from './follow-mode';
+export * from './copy';