fix(toolkit): ensure asset zips are consistently produced (#2931)

Zip files were not consistent across deploys resulting in unnecessary S3 uploads and stack updates.

Ensure consistency by appending files in series (guarantees file ordering in the zip) and reseting
dates (guarantees same hash for same content).

Closes #1997, Closes #2759

packages/aws-cdk/lib/archive.ts

@@ -1,26 +1,39 @@
 import archiver = require('archiver');
 import crypto = require('crypto');
 import fs = require('fs-extra');
+import glob = require('glob');
+import path = require('path');
 
 export function zipDirectory(directory: string, outputFile: string): Promise<void> {
  return new Promise((ok, fail) => {
- const output = fs.createWriteStream(outputFile);
- const archive = archiver('zip');
  // The below options are needed to support following symlinks when building zip files:
- // -  nodir: This will prevent symlinks themselves from being copied into the zip.
+ // - nodir: This will prevent symlinks themselves from being copied into the zip.
  // - follow: This will follow symlinks and copy the files within.
  const globOptions = {
  dot: true,
  nodir: true,
  follow: true,
- cwd: directory
+ cwd: directory,
  };
- archive.glob('**', globOptions);
- archive.pipe(output);
- archive.finalize();
+ const files = glob.sync('**', globOptions); // The output here is already sorted
+
+ const output = fs.createWriteStream(outputFile);
 
+ const archive = archiver('zip');
  archive.on('warning', fail);
  archive.on('error', fail);
+ archive.pipe(output);
+
+ files.forEach(file => { // Append files serially to ensure file order
+ archive.append(fs.createReadStream(path.join(directory, file)), {
+ name: file,
+ date: new Date('1980-01-01T00:00:00.000Z'), // reset dates to get the same hash for the same content
+ });
+ });
+
+ archive.finalize();
+
+ // archive has been finalized and the output file descriptor has closed, resolve promise
  output.once('close', () => ok());
  });
 }

packages/aws-cdk/package.json

@@ -36,6 +36,7 @@
  "devDependencies": {
  "@types/archiver": "^3.0.0",
  "@types/fs-extra": "^7.0.0",
+ "@types/jszip": "^3.1.6",
  "@types/minimatch": "^3.0.3",
  "@types/mockery": "^1.4.29",
  "@types/request": "^2.48.1",
@@ -46,6 +47,7 @@
  "@types/yaml": "^1.0.2",
  "@types/yargs": "^13.0.0",
  "cdk-build-tools": "^0.35.0",
+ "jszip": "^3.2.1",
  "mockery": "^2.1.0",
  "pkglint": "^0.35.0",
  "sinon": "^7.3.2"
@@ -60,6 +62,7 @@
  "colors": "^1.3.3",
  "decamelize": "^3.2.0",
  "fs-extra": "^8.0.1",
+ "glob": "^7.1.4",
  "json-diff": "^0.5.4",
  "minimatch": ">=3.0",
  "promptly": "^3.0.3",

packages/aws-cdk/test/test.archive.ts

@@ -1,5 +1,6 @@
 import { exec as _exec } from 'child_process';
 import fs = require('fs-extra');
+import jszip = require('jszip');
 import { Test } from 'nodeunit';
 import os = require('os');
 import path = require('path');
@@ -24,6 +25,13 @@ export = {
  test.ok(false, `extracted directory ${extractDir} differs from original ${originalDir}`);
  }
 
+ // inspect the zile file to check that dates are reset
+ const zip = await fs.readFile(zipFile);
+ const zipData = await jszip.loadAsync(zip);
+ const dates = Object.values(zipData.files).map(file => file.date.toISOString());
+ test.equal(dates[0], '1980-01-01T00:00:00.000Z', 'Dates are not reset');
+ test.equal(new Set(dates).size, 1, 'Dates are not equal');
+
  await fs.remove(stagingDir);
  await fs.remove(extractDir);
  test.done();

packages/aws-cdk/tsconfig.json

@@ -2,7 +2,7 @@
  "compilerOptions": {
  "target":"ES2018",
  "module": "commonjs",
- "lib": ["es2016", "es2017.object", "es2017.string"],
+ "lib": ["es2016", "es2017.object", "es2017.string", "dom"],
  "declaration": true,
  "strict": true,
  "noImplicitAny": true,