From f0bf75f12ea759c3d2cb6bcb03868f9c3caf906c Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 18 Jul 2018 12:53:21 +0300 Subject: [PATCH] cdk-integ: Improvements (#369) * `--verbose`: emits verbose output from "cdk" * `--no-clean`: skips stack cleanup * README Fixes #40 --- tools/cdk-integ-tools/README.md | 44 ++++++++++++++++--- tools/cdk-integ-tools/bin/cdk-integ-assert.ts | 4 +- tools/cdk-integ-tools/bin/cdk-integ.ts | 30 ++++++++++--- tools/cdk-integ-tools/chmod.bat | 2 - tools/cdk-integ-tools/lib/integ-helpers.ts | 30 ++++++++----- 5 files changed, 81 insertions(+), 29 deletions(-) delete mode 100644 tools/cdk-integ-tools/chmod.bat diff --git a/tools/cdk-integ-tools/README.md b/tools/cdk-integ-tools/README.md index 8b7dd248752ec..9c30292e40aad 100644 --- a/tools/cdk-integ-tools/README.md +++ b/tools/cdk-integ-tools/README.md @@ -1,9 +1,39 @@ -CDK Build Tools -================ +# Integration Test Tools -These scripts wrap the common operations that need to happen -during a CDK build, in a common place so it's easy to change -the build for all packages. +A testing tool for CDK constructs integration testing. -Written in TypeScript instead of shell so that they can work -on Windows with no extra effort. +Integration tests are simple CDK apps under `test/integ.*.js`. Each one defines +a single stack. + +There are two modes of operation: + +1. `cdk-integ`: Executed by developers against their developer account. This + command actually deploys the stack and stores a local copy of the synthesized + CloudFormation template under `.expected.json`. +2. `cdk-integ-assert`: Executed during build (CI/CD). It will only synthesize + the template and then compare the result to the stored copy. If they differ, + the test will fail the build. + +This approach pragmatically ensures that unexpected changes are not introduced +without a developer actually deploying a stack and verifying them. + +## cdk-integ + +Usage: + + cdk-integ [TEST...] [--no-clean] [--verbose] + +Will deploy test stacks from `test/integ.*.js` and store the synthesized output +under `test/integ.*.expected.json`. + +* Optionally, you can specify a list of test `integ.*.js` files (they must be + under `test/`) to execute only a subset of the tests. +* Use `--no-clean` to skip the clean up of the stack. This is useful in case you + wish to manually examine the stack to ensure that the result is what you + expected. +* Use `--verbose` to print verbose output from `cdk` executions. + +## cdk-integ-assert + +No arguments - will synthesize all `test/integ.*.js` apps and compare them to +their `.expected.json` counterparts. diff --git a/tools/cdk-integ-tools/bin/cdk-integ-assert.ts b/tools/cdk-integ-tools/bin/cdk-integ-assert.ts index 6baf359554db7..cd1d16726e3af 100644 --- a/tools/cdk-integ-tools/bin/cdk-integ-assert.ts +++ b/tools/cdk-integ-tools/bin/cdk-integ-assert.ts @@ -6,7 +6,7 @@ import { IntegrationTests, STATIC_TEST_CONTEXT } from '../lib/integ-helpers'; // tslint:disable:no-console async function main() { - const tests = await new IntegrationTests('test').fromCliArgs(process.argv); + const tests = await new IntegrationTests('test').fromCliArgs(); // always assert all tests const failures: string[] = []; for (const test of tests) { @@ -17,7 +17,7 @@ async function main() { } const expected = await test.readExpected(); - const actual = await test.invoke(['--json', 'synth'], true, STATIC_TEST_CONTEXT); + const actual = await test.invoke(['--json', 'synth'], { json: true, context: STATIC_TEST_CONTEXT }); const diff = diffTemplate(expected, actual); diff --git a/tools/cdk-integ-tools/bin/cdk-integ.ts b/tools/cdk-integ-tools/bin/cdk-integ.ts index 04eeb04919887..b6051e450316c 100644 --- a/tools/cdk-integ-tools/bin/cdk-integ.ts +++ b/tools/cdk-integ-tools/bin/cdk-integ.ts @@ -1,27 +1,45 @@ #!/usr/bin/env node // Exercise all integ stacks and if they deploy, update the expected synth files +import yargs = require('yargs'); import { IntegrationTests, STATIC_TEST_CONTEXT } from '../lib/integ-helpers'; // tslint:disable:no-console async function main() { - const tests = await new IntegrationTests('test').fromCliArgs(process.argv); + const argv = yargs + .usage('Usage: cdk-integ [TEST...]') + .option('clean', { type: 'boolean', default: true, desc: 'Skipps stack clean up after test is completed (use --no-clean to negate)' }) + .option('verbose', { type: 'boolean', default: false, alias: 'v', desc: 'Verbose logs' }) + .argv; - for (const test of tests) { + const tests = await new IntegrationTests('test').fromCliArgs(argv._); + for (const test of tests) { console.error(`Trying to deploy ${test.name}`); + // injects "--verbose" to the command line of "cdk" if we are in verbose mode + const makeArgs = (...args: string[]) => !argv.verbose ? args : [ '--verbose', ...args ]; + try { - await test.invoke(['deploy']); // Note: no context, so use default user settings! + await test.invoke(makeArgs('deploy'), { verbose: argv.verbose }); // Note: no context, so use default user settings! console.error(`Success! Writing out reference synth.`); // If this all worked, write the new expectation file - const actual = await test.invoke(['--json', 'synth'], true, STATIC_TEST_CONTEXT); + const actual = await test.invoke(makeArgs('--json', 'synth'), { + json: true, + context: STATIC_TEST_CONTEXT, + verbose: argv.verbose + }); + await test.writeExpected(actual); } finally { - console.error(`Cleaning up.`); - await test.invoke(['destroy', '--force']); + if (argv.clean) { + console.error(`Cleaning up.`); + await test.invoke(['destroy', '--force']); + } else { + console.error('Skipping clean up (--no-clean).'); + } } } } diff --git a/tools/cdk-integ-tools/chmod.bat b/tools/cdk-integ-tools/chmod.bat deleted file mode 100644 index 59ac42c0974a0..0000000000000 --- a/tools/cdk-integ-tools/chmod.bat +++ /dev/null @@ -1,2 +0,0 @@ -@rem Just here so that running 'chmod' doesn't fail on Windows. -@rem Doesn't actually do anything, because it doesn't need to. diff --git a/tools/cdk-integ-tools/lib/integ-helpers.ts b/tools/cdk-integ-tools/lib/integ-helpers.ts index 2a5018677a038..04c0ea35300d1 100644 --- a/tools/cdk-integ-tools/lib/integ-helpers.ts +++ b/tools/cdk-integ-tools/lib/integ-helpers.ts @@ -9,9 +9,9 @@ export class IntegrationTests { constructor(private readonly directory: string) { } - public fromCliArgs(argv: string[]): Promise { - if (argv.length > 2) { - return this.request(argv.slice(2)); + public fromCliArgs(tests?: string[]): Promise { + if (tests && tests.length > 0) { + return this.request(tests); } else { return this.discover(); } @@ -39,18 +39,22 @@ export class IntegrationTest { this.cdkConfigPath = path.join(this.directory, 'cdk.json'); } - public async invoke(args: string[], json?: boolean, context?: any): Promise { + public async invoke(args: string[], options: { json?: boolean, context?: any, verbose?: boolean } = { }): Promise { // Write context to cdk.json, afterwards delete. We need to do this because there is no way // to pass structured context data from the command-line, currently. - if (context) { - await this.writeCdkConfig({ context }); + if (options.context) { + await this.writeCdkConfig({ context: options.context }); } else { this.deleteCdkConfig(); } try { const cdk = require.resolve('aws-cdk/bin/cdk'); - return exec([cdk, '-a', `node ${this.name}`].concat(args), this.directory, json); + return exec([cdk, '-a', `node ${this.name}`].concat(args), { + cwd: this.directory, + json: options.json, + verbose: options.verbose + }); } finally { this.deleteCdkConfig(); } @@ -91,22 +95,24 @@ export const STATIC_TEST_CONTEXT = { /** * Our own execute function which doesn't use shells and strings. */ -function exec(commandLine: string[], cwd?: string, json?: boolean): any { +function exec(commandLine: string[], options: { cwd?: string, json?: boolean, verbose?: boolean} = { }): any { const proc = spawnSync(commandLine[0], commandLine.slice(1), { - stdio: ['ignore', 'pipe', 'pipe'], - cwd + stdio: [ 'ignore', 'pipe', options.verbose ? 'inherit' : 'pipe' ], // inherit STDERR in verbose mode + cwd: options.cwd }); if (proc.error) { throw proc.error; } if (proc.status !== 0) { - process.stderr.write(proc.stderr); + if (process.stderr) { // will be 'null' in verbose mode + process.stderr.write(proc.stderr); + } throw new Error(`Command exited with ${proc.status ? `status ${proc.status}` : `signal ${proc.signal}`}`); } const output = proc.stdout.toString('utf-8').trim(); try { - if (json) { + if (options.json) { if (output.length === 0) { return {}; } return JSON.parse(output);