From 91011611feaf36920d48b2e2d7c7f4363e2016ee Mon Sep 17 00:00:00 2001 From: Jonathan Goldwasser Date: Fri, 21 Jun 2019 13:47:32 +0200 Subject: [PATCH] fix(toolkit): ensure asset zips are consistently produced (#2931) Zip files were not consistent across deploys resulting in unnecessary S3 uploads and stack updates. Ensure consistency by appending files in series (guarantees file ordering in the zip) and reseting dates (guarantees same hash for same content). Closes #1997, Closes #2759 --- packages/aws-cdk/lib/archive.ts | 27 +++++++++++---- packages/aws-cdk/package-lock.json | 48 +++++++++++++++++++++++++++ packages/aws-cdk/package.json | 3 ++ packages/aws-cdk/test/test.archive.ts | 8 +++++ packages/aws-cdk/tsconfig.json | 2 +- 5 files changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/aws-cdk/lib/archive.ts b/packages/aws-cdk/lib/archive.ts index c924035bca369..a5994ea035369 100644 --- a/packages/aws-cdk/lib/archive.ts +++ b/packages/aws-cdk/lib/archive.ts @@ -1,26 +1,39 @@ import archiver = require('archiver'); import crypto = require('crypto'); import fs = require('fs-extra'); +import glob = require('glob'); +import path = require('path'); export function zipDirectory(directory: string, outputFile: string): Promise { return new Promise((ok, fail) => { - const output = fs.createWriteStream(outputFile); - const archive = archiver('zip'); // The below options are needed to support following symlinks when building zip files: - // - nodir: This will prevent symlinks themselves from being copied into the zip. + // - nodir: This will prevent symlinks themselves from being copied into the zip. // - follow: This will follow symlinks and copy the files within. const globOptions = { dot: true, nodir: true, follow: true, - cwd: directory + cwd: directory, }; - archive.glob('**', globOptions); - archive.pipe(output); - archive.finalize(); + const files = glob.sync('**', globOptions); // The output here is already sorted + + const output = fs.createWriteStream(outputFile); + const archive = archiver('zip'); archive.on('warning', fail); archive.on('error', fail); + archive.pipe(output); + + files.forEach(file => { // Append files serially to ensure file order + archive.append(fs.createReadStream(path.join(directory, file)), { + name: file, + date: new Date('1980-01-01T00:00:00.000Z'), // reset dates to get the same hash for the same content + }); + }); + + archive.finalize(); + + // archive has been finalized and the output file descriptor has closed, resolve promise output.once('close', () => ok()); }); } diff --git a/packages/aws-cdk/package-lock.json b/packages/aws-cdk/package-lock.json index 38d5104b4d970..5cf170ac1a8ce 100644 --- a/packages/aws-cdk/package-lock.json +++ b/packages/aws-cdk/package-lock.json @@ -107,6 +107,15 @@ "@types/node": "*" } }, + "@types/jszip": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@types/jszip/-/jszip-3.1.6.tgz", + "integrity": "sha512-m8uFcI+O2EupCfbEVQWsBM/4nhbegjOHL7cQgBpM95FeF98kdFJXzy9/8yhx4b3lCRl/gMBhcvyh30Qt3X+XPQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -990,6 +999,12 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", + "dev": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -1098,6 +1113,18 @@ "verror": "1.10.0" } }, + "jszip": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.2.1.tgz", + "integrity": "sha512-iCMBbo4eE5rb1VCpm5qXOAaUiRKRUKiItn8ah2YQQx9qymmSAY98eyQfioChEYcVQLh0zxJ3wS4A0mh90AVPvw==", + "dev": true, + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, "just-extend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", @@ -1129,6 +1156,15 @@ "type-check": "~0.3.2" } }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "requires": { + "immediate": "~3.0.5" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -1388,6 +1424,12 @@ "thunkify": "^2.1.2" } }, + "pako": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.10.tgz", + "integrity": "sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==", + "dev": true + }, "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", @@ -1608,6 +1650,12 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "dev": true + }, "setprototypeof": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", diff --git a/packages/aws-cdk/package.json b/packages/aws-cdk/package.json index 341f535d29d20..864715ba187a0 100644 --- a/packages/aws-cdk/package.json +++ b/packages/aws-cdk/package.json @@ -36,6 +36,7 @@ "devDependencies": { "@types/archiver": "^3.0.0", "@types/fs-extra": "^7.0.0", + "@types/jszip": "^3.1.6", "@types/minimatch": "^3.0.3", "@types/mockery": "^1.4.29", "@types/request": "^2.48.1", @@ -46,6 +47,7 @@ "@types/yaml": "^1.0.2", "@types/yargs": "^13.0.0", "cdk-build-tools": "^0.35.0", + "jszip": "^3.2.1", "mockery": "^2.1.0", "pkglint": "^0.35.0", "sinon": "^7.3.2" @@ -60,6 +62,7 @@ "colors": "^1.3.3", "decamelize": "^3.2.0", "fs-extra": "^8.0.1", + "glob": "^7.1.4", "json-diff": "^0.5.4", "minimatch": ">=3.0", "promptly": "^3.0.3", diff --git a/packages/aws-cdk/test/test.archive.ts b/packages/aws-cdk/test/test.archive.ts index f26b662e79ca2..c709cce749765 100644 --- a/packages/aws-cdk/test/test.archive.ts +++ b/packages/aws-cdk/test/test.archive.ts @@ -1,5 +1,6 @@ import { exec as _exec } from 'child_process'; import fs = require('fs-extra'); +import jszip = require('jszip'); import { Test } from 'nodeunit'; import os = require('os'); import path = require('path'); @@ -24,6 +25,13 @@ export = { test.ok(false, `extracted directory ${extractDir} differs from original ${originalDir}`); } + // inspect the zile file to check that dates are reset + const zip = await fs.readFile(zipFile); + const zipData = await jszip.loadAsync(zip); + const dates = Object.values(zipData.files).map(file => file.date.toISOString()); + test.equal(dates[0], '1980-01-01T00:00:00.000Z', 'Dates are not reset'); + test.equal(new Set(dates).size, 1, 'Dates are not equal'); + await fs.remove(stagingDir); await fs.remove(extractDir); test.done(); diff --git a/packages/aws-cdk/tsconfig.json b/packages/aws-cdk/tsconfig.json index 0c02dab2c6a45..0eac39a6083af 100644 --- a/packages/aws-cdk/tsconfig.json +++ b/packages/aws-cdk/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target":"ES2018", "module": "commonjs", - "lib": ["es2016", "es2017.object", "es2017.string"], + "lib": ["es2016", "es2017.object", "es2017.string", "dom"], "declaration": true, "strict": true, "noImplicitAny": true,