From c05a9a5d4385d14a837cae3c85c99986702ff136 Mon Sep 17 00:00:00 2001 From: corymhall <43035978+corymhall@users.noreply.github.com> Date: Fri, 8 Apr 2022 19:27:44 +0000 Subject: [PATCH 1/3] feat(integ-runner): integ-runner enhancements This PR adds a couple of enhancements to the `integ-runner` 1. Add `--profiles` option which allows you to provide a list of AWS profiles to use. This will be used in combination with the `--parallel-regions` option to run the integration tests in parallel across multiple accounts + regions. 2. Adds `--from-file` option that allows you to store a list of tests to run (or exclude) in a file, rather than passing test names as arguments. 3. Adds `--exclude` option, which when provided will excluded the tests provided as arguments 4. Adds some additional metrics. Now each test will show the total time in the logs. Also, if `--verbose` option is provided a summary will be printed which will contain metrics for the entire test. ```bash --- Integration test metrics --- Profile mb-dev + Region us-east-2 total time: 349.866 test/integ.bundling.js: 58.75 test/integ.lambda-insights-mapping.js: 61.012 test/integ.assets.lit.js: 61.121 test/integ.lambda.docker.js: 81.936 test/integ.log-retention.js: 87.047 Profile mb-dev + Region us-west-2 total time: 359.775 test/integ.layer-version.lit.js: 72.61 test/integ.autoscaling.lit.js: 82.82 test/integ.current-version.js: 84.982 test/integ.lambda.js: 119.363 Profile mb-dev + Region us-east-1 total time: 378.713 test/integ.assets.file.js: 67.915 test/integ.runtime.inlinecode.js: 68.211 ``` --- packages/@aws-cdk/integ-runner/README.md | 6 +- packages/@aws-cdk/integ-runner/lib/cli.ts | 45 +- .../integ-runner/lib/runner/integ-tests.ts | 77 +++- .../integ-runner/lib/runner/runners.ts | 16 + .../integ-runner/lib/workers/common.ts | 64 ++- .../lib/workers/extract/extract_worker.ts | 130 +++++- .../lib/workers/integ-snapshot-worker.ts | 83 +--- .../lib/workers/integ-test-worker.ts | 165 +++---- packages/@aws-cdk/integ-runner/package.json | 2 + .../test/runner/integration-tests.test.ts | 97 +++- .../integ-runner/test/runner/runners.test.ts | 91 ++++ .../test/workers/integ-worker.test.ts | 414 ++++++++++++++++++ .../test/workers/mock-extract_worker.ts | 10 +- .../test/workers/snapshot-worker.test.ts | 60 +++ .../integ-runner/test/workers/workers.test.ts | 303 ------------- 15 files changed, 1053 insertions(+), 510 deletions(-) create mode 100644 packages/@aws-cdk/integ-runner/test/workers/integ-worker.test.ts create mode 100644 packages/@aws-cdk/integ-runner/test/workers/snapshot-worker.test.ts delete mode 100644 packages/@aws-cdk/integ-runner/test/workers/workers.test.ts diff --git a/packages/@aws-cdk/integ-runner/README.md b/packages/@aws-cdk/integ-runner/README.md index 9fd2b667346ae..68d078c9b33d0 100644 --- a/packages/@aws-cdk/integ-runner/README.md +++ b/packages/@aws-cdk/integ-runner/README.md @@ -54,7 +54,11 @@ to be a self contained CDK app. The runner will execute the following for each f Search for integration tests recursively from this starting directory - `--force` (default=`false`) Rerun integration test even if the test passes -- `--file` +- `--profiles` + List of AWS Profiles to use when running tests in parallel +- `--exclude` (default=`false`) + If this is set to `true` then the list of tests provided will be excluded +- `--from-file` Read the list of tests from this file Example: diff --git a/packages/@aws-cdk/integ-runner/lib/cli.ts b/packages/@aws-cdk/integ-runner/lib/cli.ts index daa23ec687cb6..6a1bd80545e8d 100644 --- a/packages/@aws-cdk/integ-runner/lib/cli.ts +++ b/packages/@aws-cdk/integ-runner/lib/cli.ts @@ -1,10 +1,9 @@ // Exercise all integ stacks and if they deploy, update the expected synth files -import * as os from 'os'; import * as path from 'path'; import * as workerpool from 'workerpool'; import * as logger from './logger'; import { IntegrationTests, IntegTestConfig } from './runner/integ-tests'; -import { runSnapshotTests, runIntegrationTests } from './workers'; +import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics } from './workers'; // https://github.com/yargs/yargs/issues/1929 // https://github.com/evanw/esbuild/issues/1492 @@ -24,13 +23,14 @@ async function main() { .option('parallel', { type: 'boolean', default: false, desc: 'run integration tests in parallel' }) .option('parallel-regions', { type: 'array', desc: 'if --parallel is used then these regions are used to run tests in parallel', nargs: 1, default: [] }) .options('directory', { type: 'string', default: 'test', desc: 'starting directory to discover integration tests' }) + .options('profiles', { type: 'array', desc: 'list of AWS profiles to use. Tests will be run in parallel across each profile+regions', nargs: 1, default: [] }) + .options('max-workers', { type: 'number', desc: 'The max number of workerpool workers to use when running integration tests in parallel', default: 16 }) + .options('exclude', { type: 'boolean', desc: 'All tests should be run, except for the list of tests provided', default: false }) + .options('from-file', { type: 'string', desc: 'Import tests to include or exclude from a file' }) .argv; - // Cap to a reasonable top-level limit to prevent thrash on machines with many, many cores. - const maxWorkers = parseInt(process.env.CDK_INTEG_MAX_WORKER_COUNT ?? '16'); - const N = Math.min(maxWorkers, Math.max(1, Math.ceil(os.cpus().length / 2))); const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), { - maxWorkers: N, + maxWorkers: argv['max-workers'], }); // list of integration tests that will be executed @@ -38,21 +38,31 @@ async function main() { const testsFromArgs: IntegTestConfig[] = []; const parallelRegions = arrayFromYargs(argv['parallel-regions']); const testRegions: string[] = parallelRegions ?? ['us-east-1', 'us-east-2', 'us-west-2']; + const profiles = arrayFromYargs(argv.profiles); const runUpdateOnFailed = argv['update-on-failed'] ?? false; + const fromFile: string | undefined = argv['from-file']; + const exclude: boolean = argv.exclude; let failedSnapshots: IntegTestConfig[] = []; - try { + if (argv['max-workers'] < testRegions.length * (profiles ?? [1]).length) { + logger.warning('You are attempting to run %s tests in parallel, but only have %s workers. Not all of your profiles+regions will be utilized', argv.profiles*argv['parallel-regions'], argv['max-workers']); + } + try { if (argv.list) { const tests = await new IntegrationTests(argv.directory).fromCliArgs(); process.stdout.write(tests.map(t => t.fileName).join('\n') + '\n'); return; } - if (argv._.length === 0) { + if (argv._.length > 0 && fromFile) { + throw new Error('A list of tests cannot be provided if "--from-file" is provided'); + } else if (argv._.length === 0 && !fromFile) { testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs())); + } else if (fromFile) { + testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromFile(fromFile))); } else { - testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs(argv._.map((x: any) => x.toString())))); + testsFromArgs.push(...(await new IntegrationTests(argv.directory).fromCliArgs(argv._.map((x: any) => x.toString()), exclude))); } // If `--force` is not used then first validate the snapshots and gather @@ -65,13 +75,13 @@ async function main() { testsToRun.push(...testsFromArgs); } - // run integration tests if `--update-on-failed` OR `--force` is used if (runUpdateOnFailed || argv.force) { - const success = await runIntegrationTests({ + const { success, metrics } = await runIntegrationTests({ pool, tests: testsToRun, regions: testRegions, + profiles, clean: argv.clean, dryRun: argv['dry-run'], verbose: argv.verbose, @@ -79,6 +89,9 @@ async function main() { if (!success) { throw new Error('Some integration tests failed!'); } + if (argv.verbose) { + printMetrics(metrics); + } if (argv.clean === false) { logger.warning('Not cleaning up stacks since "--no-clean" was used'); @@ -97,6 +110,16 @@ async function main() { } } +function printMetrics(metrics: IntegRunnerMetrics[]): void { + logger.highlight(' --- Integration test metrics ---'); + const sortedMetrics = metrics.sort((a, b) => a.duration - b.duration); + sortedMetrics.forEach(metric => { + logger.print('Profile %s + Region %s total time: %s', metric.profile, metric.region, metric.duration); + const sortedTests = Object.entries(metric.tests).sort((a, b) => a[1] - b[1]); + sortedTests.forEach(test => logger.print(' %s: %s', test[0], test[1])); + }); +} + /** * Translate a Yargs input array to something that makes more sense in a programming language * model (telling the difference between absence and an empty array) diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts b/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts index 780c1d5fef4a0..acb9980d3a126 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts @@ -5,10 +5,33 @@ import * as fs from 'fs-extra'; * Represents a single integration test */ export interface IntegTestConfig { - readonly directory: string; + /** + * The name of the file that contains the + * integration tests. This will be in the format + * of integ.{test-name}.js + */ readonly fileName: string; } +/** + * The list of tests to run can be provided in a file + * instead of as command line arguments. + */ +export interface IntegrationTestFileConfig { + /** + * If this is set to true then the list of tests + * provided will be excluded + * + * @default false + */ + readonly exclude?: boolean; + + /** + * List of tests to include (or exclude if `exclude=true`) + */ + readonly tests: string[]; +} + /** * Discover integration tests */ @@ -17,28 +40,40 @@ export class IntegrationTests { } /** - * Takes an optional list of tests to look for, otherwise - * it will look for all tests from the directory + * Takes a file name of a file that contains a list of test + * to either run or exclude and returns a list of Integration Tests to run */ - public async fromCliArgs(tests?: string[]): Promise { - let allTests = await this.discover(); - const all = allTests.map(x => x.fileName); - let foundAll = true; + public async fromFile(fileName: string): Promise { + const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' })); + const foundTests = await this.discover(); - if (tests && tests.length > 0) { - // Pare down found tests to filter - allTests = allTests.filter(t => { - const parts = path.parse(t.fileName); - return (tests.includes(t.fileName) || tests.includes(parts.base)); - }); + const allTests = this.filterTests(foundTests, file.tests, file.exclude); + return allTests; + } + + private filterTests(discoveredTests: IntegTestConfig[], requestedTests?: string[], exclude?: boolean): IntegTestConfig[] { + if (!requestedTests || requestedTests.length === 0) { + return discoveredTests; + } + const all = discoveredTests.map(x => x.fileName); + let foundAll = true; + // Pare down found tests to filter + const allTests = discoveredTests.filter(t => { + const parts = path.parse(t.fileName); + if (exclude) { + return (!requestedTests.includes(t.fileName) && !requestedTests.includes(parts.base)); + } + return (requestedTests.includes(t.fileName) || requestedTests.includes(parts.base)); + }); + + if (!exclude) { const selectedNames = allTests.map(t => t.fileName); - for (const unmatched of tests.filter(t => !selectedNames.includes(t))) { + for (const unmatched of requestedTests.filter(t => !selectedNames.includes(t))) { process.stderr.write(`No such integ test: ${unmatched}\n`); foundAll = false; } } - if (!foundAll) { process.stderr.write(`Available tests: ${all.join(' ')}\n`); return []; @@ -47,6 +82,18 @@ export class IntegrationTests { return allTests; } + /** + * Takes an optional list of tests to look for, otherwise + * it will look for all tests from the directory + */ + public async fromCliArgs(tests?: string[], exclude?: boolean): Promise { + const discoveredTests = await this.discover(); + + const allTests = this.filterTests(discoveredTests, tests, exclude); + + return allTests; + } + private async discover(): Promise { const files = await this.readTree(); const integs = files.filter(fileName => path.basename(fileName).startsWith('integ.') && path.basename(fileName).endsWith('.js')); diff --git a/packages/@aws-cdk/integ-runner/lib/runner/runners.ts b/packages/@aws-cdk/integ-runner/lib/runner/runners.ts index 787e2a0fbd872..df22eb5afe0e3 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/runners.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/runners.ts @@ -29,6 +29,13 @@ export interface IntegRunnerOptions { */ readonly fileName: string, + /** + * The AWS profile to use when invoking the CDK CLI + * + * @default - no profile is passed, the default profile is used + */ + readonly profile?: string; + /** * Additional environment variables that will be available * to the CDK CLI @@ -120,6 +127,8 @@ export abstract class IntegRunner { */ protected readonly cdkOutDir: string; + protected readonly profile?: string; + constructor(options: IntegRunnerOptions) { const parsed = path.parse(options.fileName); this.directory = parsed.dir; @@ -146,6 +155,7 @@ export abstract class IntegRunner { }); this.cdkOutDir = options.integOutDir ?? `${CDK_OUTDIR_PREFIX}.${testName}`; this.cdkApp = `node ${parsed.base}`; + this.profile = options.profile; if (this.hasSnapshot()) { this.loadManifest(); } @@ -291,6 +301,7 @@ export abstract class IntegRunner { ...this.defaultArgs, all: true, app: this.cdkApp, + profile: this.profile, output: this.cdkOutDir, })).split('\n'); if (stacks.length !== 1) { @@ -299,6 +310,9 @@ export abstract class IntegRunner { ` ${CDK_INTEG_STACK_PRAGMA} STACK ...\n\n` + ` Available stacks: ${stacks.join(' ')} (wildcards are also supported)\n`); } + if (stacks.length === 1 && stacks[0] === '') { + throw new Error(`No stack found for test ${this.testName}`); + } tests.stacks.push(...stacks); } @@ -433,6 +447,7 @@ export class IntegTestRunner extends IntegRunner { if (!options.dryRun) { this.cdk.deploy({ ...this.defaultArgs, + profile: this.profile, stacks: options.testCase.stacks, requireApproval: RequireApproval.NEVER, output: this.cdkOutDir, @@ -460,6 +475,7 @@ export class IntegTestRunner extends IntegRunner { if (clean) { this.cdk.destroy({ ...this.defaultArgs, + profile: this.profile, stacks: options.testCase.stacks, force: true, app: this.cdkApp, diff --git a/packages/@aws-cdk/integ-runner/lib/workers/common.ts b/packages/@aws-cdk/integ-runner/lib/workers/common.ts index b1bd40e6a2312..0a550660f9372 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/common.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/common.ts @@ -2,11 +2,51 @@ import * as chalk from 'chalk'; import * as logger from '../logger'; import { IntegTestConfig } from '../runner/integ-tests'; +/** + * Represents integration tests metrics for a given worker + */ +export interface IntegRunnerMetrics { + /** + * The region the test was run in + */ + readonly region: string; + + /** + * The total duration of the worker. + * This will be the sum of all individual test durations + */ + readonly duration: number; + + /** + * Contains the duration of individual tests that the + * worker executed. + * + * Map of testName to duration. + */ + readonly tests: { [testName: string]: number }; + + /** + * The profile that was used to run the test + * + * @default - default profile + */ + readonly profile?: string; +} + /** * Integration test results */ export interface IntegBatchResponse { - failedTests: IntegTestConfig[]; + /** + * List of failed tests + */ + readonly failedTests: IntegTestConfig[]; + + /** + * List of Integration test metrics. Each entry in the + * list represents metrics from a single worker (account + region). + */ + readonly metrics: IntegRunnerMetrics[]; } /** @@ -91,6 +131,11 @@ export interface Diagnostic { */ readonly message: string; + /** + * The time it took to run the test + */ + readonly duration?: number; + /** * The reason for the diagnostic */ @@ -111,18 +156,25 @@ export function printSummary(total: number, failed: number): void { export function printResults(diagnostic: Diagnostic): void { switch (diagnostic.reason) { case DiagnosticReason.SNAPSHOT_SUCCESS: - logger.success(' %s No Change!', diagnostic.testName); + logger.success(' %s No Change! %s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`)); break; case DiagnosticReason.TEST_SUCCESS: - logger.success(' %s Test Succeeded!', diagnostic.testName); + logger.success(' %s Test Succeeded! %s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`)); break; case DiagnosticReason.NO_SNAPSHOT: - logger.error(' %s - No Snapshot!', diagnostic.testName); + logger.error(' %s - No Snapshot! %s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`)); break; case DiagnosticReason.SNAPSHOT_FAILED: - logger.error(' %s - Snapshot changed!\n%s', diagnostic.testName, diagnostic.message); + logger.error(' %s - Snapshot changed! %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), diagnostic.message); break; case DiagnosticReason.TEST_FAILED: - logger.error(' %s - Failed!\n%s', diagnostic.testName, diagnostic.message); + logger.error(' %s - Failed! %s\n%s', diagnostic.testName, chalk.gray(`${diagnostic.duration}s`), diagnostic.message); } } + +/** + * Flatten a list of lists into a list of elements + */ +export function flatten(xs: T[][]): T[] { + return Array.prototype.concat.apply([], xs); +} diff --git a/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts index f4991f56d1826..4f6c233cae142 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts @@ -1,35 +1,123 @@ import * as workerpool from 'workerpool'; import { IntegTestConfig } from '../../runner/integ-tests'; -import { Diagnostic, IntegBatchResponse } from '../common'; -import { singleThreadedSnapshotRunner } from '../integ-snapshot-worker'; -import { singleThreadedTestRunner, IntegTestBatchRequest } from '../integ-test-worker'; +import { IntegSnapshotRunner, IntegTestRunner } from '../../runner/runners'; +import { DiagnosticReason } from '../common'; +import { IntegTestBatchRequest } from '../integ-test-worker'; /** - * Options for running snapshot tests + * Runs a single integration test batch request. + * If the test does not have an existing snapshot, + * this will first generate a snapshot and then execute + * the integration tests. + * + * If the tests succeed it will then save the snapshot */ -export interface SnapshotBatchRequest { - readonly tests: IntegTestConfig[]; +export function integTestWorker(request: IntegTestBatchRequest): IntegTestConfig[] { + const failures: IntegTestConfig[] = []; + for (const test of request.tests) { + const runner = new IntegTestRunner({ + fileName: test.fileName, + profile: request.profile, + env: { + AWS_REGION: request.region, + }, + }); + const start = Date.now(); + try { + if (!runner.hasSnapshot()) { + runner.generateSnapshot(); + } + + if (!runner.tests || Object.keys(runner.tests).length === 0) { + throw new Error(`No tests defined for ${runner.testName}`); + } + for (const [testName, testCase] of Object.entries(runner.tests)) { + try { + runner.runIntegTestCase({ + testCase: testCase, + clean: request.clean, + dryRun: request.dryRun, + }); + workerpool.workerEmit({ + reason: DiagnosticReason.TEST_SUCCESS, + testName: testName, + message: 'Success', + duration: (Date.now() - start) / 1000, + }); + } catch (e) { + failures.push(test); + workerpool.workerEmit({ + reason: DiagnosticReason.TEST_FAILED, + testName: testName, + message: `Integration test failed: ${e}`, + duration: (Date.now() - start) / 1000, + }); + } + } + } catch (e) { + failures.push(test); + workerpool.workerEmit({ + reason: DiagnosticReason.TEST_FAILED, + testName: test.fileName, + message: `Integration test failed: ${e}`, + duration: (Date.now() - start) / 1000, + }); + } + } + + return failures; } /** - * Snapshot test results + * Runs a single snapshot test batch request. + * For each integration test this will check to see + * if there is an existing snapshot, and if there is will + * check if there are any changes */ -export interface SnapshotBatchResponse { - diagnostics: Diagnostic[]; - failedTests: IntegTestConfig[]; -} - -function integTestBatch(request: IntegTestBatchRequest): IntegBatchResponse { - const result = singleThreadedTestRunner(request); - return result; -} +export function snapshotTestWorker(test: IntegTestConfig): IntegTestConfig[] { + const failedTests = new Array(); + const runner = new IntegSnapshotRunner({ fileName: test.fileName }); + const start = Date.now(); + try { + if (!runner.hasSnapshot()) { + workerpool.workerEmit({ + reason: DiagnosticReason.NO_SNAPSHOT, + testName: runner.testName, + message: 'No Snapshot', + duration: (Date.now() - start) / 1000, + }); + failedTests.push(test); + } else { + const snapshotDiagnostics = runner.testSnapshot(); + if (snapshotDiagnostics.length > 0) { + snapshotDiagnostics.forEach(diagnostic => workerpool.workerEmit({ + ...diagnostic, + duration: (Date.now() - start) / 1000, + })); + failedTests.push(test); + } else { + workerpool.workerEmit({ + reason: DiagnosticReason.SNAPSHOT_SUCCESS, + testName: runner.testName, + message: 'Success', + duration: (Date.now() - start) / 1000, + }); + } + } + } catch (e) { + failedTests.push(test); + workerpool.workerEmit({ + message: e.message, + testName: runner.testName, + reason: DiagnosticReason.SNAPSHOT_FAILED, + duration: (Date.now() - start) / 1000, + }); + } -function snapshotTestBatch(request: SnapshotBatchRequest): IntegBatchResponse { - const result = singleThreadedSnapshotRunner(request.tests); - return result; + return failedTests; } workerpool.worker({ - snapshotTestBatch, - integTestBatch, + snapshotTestWorker, + integTestWorker, }); diff --git a/packages/@aws-cdk/integ-runner/lib/workers/integ-snapshot-worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/integ-snapshot-worker.ts index f2c96cf464972..fc1c7a897693e 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/integ-snapshot-worker.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/integ-snapshot-worker.ts @@ -1,13 +1,15 @@ import * as workerpool from 'workerpool'; import * as logger from '../logger'; import { IntegTestConfig } from '../runner/integ-tests'; -import { IntegSnapshotRunner } from '../runner/runners'; -import { IntegBatchResponse, printSummary, Diagnostic, DiagnosticReason, printResults } from './common'; +import { printSummary, Diagnostic, printResults, flatten } from './common'; /** * Options for running snapshot tests */ export interface SnapshotBatchRequest { + /** + * List of tests to run + */ readonly tests: IntegTestConfig[]; } @@ -15,22 +17,15 @@ export interface SnapshotBatchRequest { * Snapshot test results */ export interface SnapshotBatchResponse { + /** + * Test diagnostics + */ diagnostics: Diagnostic[]; - failedTests: IntegTestConfig[]; -} -/** - * Split a list of snapshot tests into batches that can be run using a workerpool. - */ -function batchTests(tests: IntegTestConfig[]): SnapshotBatchRequest[] { - let batchSize = 3; - const ret: SnapshotBatchRequest[] = []; - for (let i = 0; i < tests.length; i += batchSize) { - ret.push({ - tests: tests.slice(i, i + batchSize), - }); - } - return ret; + /** + * List of failed tests + */ + failedTests: IntegTestConfig[]; } /** @@ -39,66 +34,16 @@ function batchTests(tests: IntegTestConfig[]): SnapshotBatchRequest[] { * Use a workerpool to run the batches in parallel. */ export async function runSnapshotTests(pool: workerpool.WorkerPool, tests: IntegTestConfig[]): Promise { - const testsToRun: IntegTestConfig[] = []; - const requests = batchTests(tests); logger.highlight('\nVerifying integration test snapshots...\n'); - const responses: IntegBatchResponse[] = await Promise.all( - requests.map((request) => pool.exec('snapshotTestBatch', [request], { + const failedTests: IntegTestConfig[][] = await Promise.all( + tests.map((test) => pool.exec('snapshotTestWorker', [test], { on: printResults, })), ); - for (const response of responses) { - testsToRun.push(...response.failedTests); - } + const testsToRun = flatten(failedTests); logger.highlight('\nSnapshot Results: \n'); printSummary(tests.length, testsToRun.length); return testsToRun; } - -/** - * Runs a single snapshot test batch request. - * For each integration test this will check to see - * if there is an existing snapshot, and if there is will - * check if there are any changes - */ -export function singleThreadedSnapshotRunner(tests: IntegTestConfig[]): IntegBatchResponse { - const failedTests = new Array(); - for (const test of tests) { - const runner = new IntegSnapshotRunner({ fileName: test.fileName }); - try { - if (!runner.hasSnapshot()) { - workerpool.workerEmit({ - reason: DiagnosticReason.NO_SNAPSHOT, - testName: runner.testName, - message: 'No Snapshot', - }); - failedTests.push(test); - } else { - const snapshotDiagnostics = runner.testSnapshot(); - if (snapshotDiagnostics.length > 0) { - snapshotDiagnostics.forEach(diagnostic => printResults(diagnostic)); - failedTests.push(test); - } else { - workerpool.workerEmit({ - reason: DiagnosticReason.SNAPSHOT_SUCCESS, - testName: runner.testName, - message: 'Success', - }); - } - } - } catch (e) { - failedTests.push(test); - workerpool.workerEmit({ - message: e.message, - testName: runner.testName, - reason: DiagnosticReason.SNAPSHOT_FAILED, - }); - } - } - - return { - failedTests, - }; -} diff --git a/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts b/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts index 9bda4d3d707fe..1b45d5d480f33 100644 --- a/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts +++ b/packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts @@ -1,8 +1,7 @@ import * as workerpool from 'workerpool'; import * as logger from '../logger'; import { IntegTestConfig } from '../runner/integ-tests'; -import { IntegTestRunner } from '../runner/runners'; -import { printResults, printSummary, IntegBatchResponse, IntegTestOptions, DiagnosticReason } from './common'; +import { printResults, printSummary, IntegBatchResponse, IntegTestOptions, IntegRunnerMetrics, flatten } from './common'; /** * Options for an integration test batch @@ -12,6 +11,11 @@ export interface IntegTestBatchRequest extends IntegTestOptions { * The AWS region to run this batch in */ readonly region: string; + + /** + * The AWS profile to use when running this test + */ + readonly profile?: string; } /** @@ -24,6 +28,12 @@ export interface IntegTestRunOptions extends IntegTestOptions { */ readonly regions: string[]; + /** + * List of AWS profiles. This will be used in conjunction with `regions` + * to run tests in parallel across accounts + regions + */ + readonly profiles?: string[]; + /** * The workerpool to use */ @@ -33,22 +43,63 @@ export interface IntegTestRunOptions extends IntegTestOptions { /** * Run Integration tests. */ -export async function runIntegrationTests(options: IntegTestRunOptions): Promise { +export async function runIntegrationTests(options: IntegTestRunOptions): Promise<{ success: boolean, metrics: IntegRunnerMetrics[] }> { logger.highlight('\nRunning integration tests for failed tests...\n'); - logger.print('Running in parallel across: %s', options.regions.join(', ')); + logger.print( + 'Running in parallel across %sregions: %s', + options.profiles ? `profiles ${options.profiles.join(', ')} and `: '', + options.regions.join(', ')); const totalTests = options.tests.length; - const failedTests: IntegTestConfig[] = []; const responses = await runIntegrationTestsInParallel(options); - for (const response of responses) { - failedTests.push(...response.failedTests); - } logger.highlight('\nTest Results: \n'); - printSummary(totalTests, failedTests.length); - if (failedTests.length > 0) { - return false; + printSummary(totalTests, responses.failedTests.length); + return { + success: responses.failedTests.length === 0, + metrics: responses.metrics, + }; +} + +/** + * Represents a worker for a single account + region + */ +interface AccountWorker { + /** + * The region the worker should run in + */ + readonly region: string; + + /** + * The AWS profile that the worker should use + * This will be passed as the '--profile' option to the CDK CLI + * + * @default - default profile + */ + readonly profile?: string; +} + +/** + * Returns a list of AccountWorkers based on the list of regions and profiles + * given to the CLI. + */ +function getAccountWorkers(regions: string[], profiles?: string[]): AccountWorker[] { + const workers: AccountWorker[] = []; + function pushWorker(profile?: string) { + for (const region of regions) { + workers.push({ + region, + profile, + }); + } } - return true; + if (profiles && profiles.length > 0) { + for (const profile of profiles ?? []) { + pushWorker(profile); + } + } else { + pushWorker(); + } + return workers; } /** @@ -58,18 +109,26 @@ export async function runIntegrationTests(options: IntegTestRunOptions): Promise */ export async function runIntegrationTestsInParallel( options: IntegTestRunOptions, -): Promise { +): Promise { const queue = options.tests; - const results: IntegBatchResponse[] = []; + const results: IntegBatchResponse = { + metrics: [], + failedTests: [], + }; + const accountWorkers: AccountWorker[] = getAccountWorkers(options.regions, options.profiles); - async function runTest(region: string): Promise { + async function runTest(worker: AccountWorker): Promise { + const start = Date.now(); + const tests: { [testName: string]: number } = {}; do { const test = queue.pop(); if (!test) break; - logger.highlight(`Running test ${test.fileName} in ${region}`); - const response: IntegBatchResponse = await options.pool.exec('integTestBatch', [{ - region, + const testStart = Date.now(); + logger.highlight(`Running test ${test.fileName} in ${worker.profile ? worker.profile + '/' : ''}${worker.region}`); + const response: IntegTestConfig[][] = await options.pool.exec('integTestWorker', [{ + region: worker.region, + profile: worker.profile, tests: [test], clean: options.clean, dryRun: options.dryRun, @@ -78,67 +137,21 @@ export async function runIntegrationTestsInParallel( on: printResults, }); - results.push(response); + results.failedTests.push(...flatten(response)); + tests[test.fileName] = (Date.now() - testStart) / 1000; } while (queue.length > 0); + const metrics: IntegRunnerMetrics = { + region: worker.region, + profile: worker.profile, + duration: (Date.now() - start) / 1000, + tests, + }; + if (Object.keys(tests).length > 0) { + results.metrics.push(metrics); + } } - const workers = options.regions.map((region) => runTest(region)); + const workers = accountWorkers.map((worker) => runTest(worker)); await Promise.all(workers); return results; } - -/** - * Runs a single integration test batch request. - * If the test does not have an existing snapshot, - * this will first generate a snapshot and then execute - * the integration tests. - * - * If the tests succeed it will then save the snapshot - */ -export function singleThreadedTestRunner(request: IntegTestBatchRequest): IntegBatchResponse { - const failures: IntegTestConfig[] = []; - for (const test of request.tests) { - const runner = new IntegTestRunner({ - fileName: test.fileName, - env: { - AWS_REGION: request.region, - }, - }); - try { - if (!runner.hasSnapshot()) { - runner.generateSnapshot(); - } - - if (!runner.tests) { - throw new Error(`No tests defined for ${runner.testName}`); - } - for (const [testName, testCase] of Object.entries(runner.tests)) { - try { - runner.runIntegTestCase({ - testCase: testCase, - clean: request.clean, - dryRun: request.dryRun, - }); - workerpool.workerEmit({ - reason: DiagnosticReason.TEST_SUCCESS, - testName: testName, - message: 'Success', - }); - } catch (e) { - failures.push(test); - workerpool.workerEmit({ - reason: DiagnosticReason.TEST_FAILED, - testName: testName, - message: `Integration test failed: ${e}`, - }); - } - } - } catch (e) { - logger.error(`Errors running test cases: ${e}`); - } - } - - return { - failedTests: failures, - }; -} diff --git a/packages/@aws-cdk/integ-runner/package.json b/packages/@aws-cdk/integ-runner/package.json index 20eaf0193740b..4f13a2849fc17 100644 --- a/packages/@aws-cdk/integ-runner/package.json +++ b/packages/@aws-cdk/integ-runner/package.json @@ -53,6 +53,8 @@ "license": "Apache-2.0", "devDependencies": { "@aws-cdk/cdk-build-tools": "0.0.0", + "@types/mock-fs": "^4.13.1", + "mock-fs": "^4.14.0", "@aws-cdk/pkglint": "0.0.0", "@types/fs-extra": "^8.1.2", "@types/jest": "^27.4.1", diff --git a/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts b/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts index d83cd687243dd..153c919cd5cb8 100644 --- a/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts +++ b/packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts @@ -1,12 +1,105 @@ +import * as mockfs from 'mock-fs'; import { IntegrationTests } from '../../lib/runner/integ-tests'; describe('IntegrationTests', () => { - test('from cli args', async () => { - const tests = new IntegrationTests('test'); + const testsFile = '/tmp/foo/bar/does/not/exist/tests.json'; + const testsFileExclude = '/tmp/foo/bar/does/not/exist/tests-exclude.json'; + const tests = new IntegrationTests('test'); + let stderrMock: jest.SpyInstance; + stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); + beforeEach(() => { + mockfs({ + 'test/test-data': { + 'integ.integ-test1.js': 'content', + 'integ.integ-test2.js': 'content', + 'integ.integ-test3.js': 'content', + }, + [testsFileExclude]: JSON.stringify({ + exclude: true, + tests: [ + 'test/test-data/integ.integ-test1.js', + ], + }), + [testsFile]: JSON.stringify({ + tests: [ + 'test/test-data/integ.integ-test1.js', + ], + }), + }); + }); + + afterEach(() => { + mockfs.restore(); + }); + test('from cli args', async () => { const integTests = await tests.fromCliArgs(['test/test-data/integ.integ-test1.js']); expect(integTests.length).toEqual(1); expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/)); }); + + test('from cli args, test not found', async () => { + const integTests = await tests.fromCliArgs(['test/test-data/integ.integ-test16.js']); + + expect(integTests.length).toEqual(0); + expect(stderrMock.mock.calls[0][0]).toContain( + 'No such integ test: test/test-data/integ.integ-test16.js', + ); + expect(stderrMock.mock.calls[1][0]).toContain( + 'Available tests: test/test-data/integ.integ-test1.js test/test-data/integ.integ-test2.js test/test-data/integ.integ-test3.js', + ); + }); + + test('from cli args, exclude', async () => { + const integTests = await tests.fromCliArgs(['test/test-data/integ.integ-test1.js'], true); + + const fileNames = integTests.map(test => test.fileName); + expect(integTests.length).toEqual(2); + expect(fileNames).not.toContain( + 'test/test-data/integ.integ-test1.js', + ); + }); + + test('from file', async () => { + const integTests = await tests.fromFile(testsFile); + + const fileNames = integTests.map(test => test.fileName); + expect(integTests.length).toEqual(1); + expect(fileNames).toContain( + 'test/test-data/integ.integ-test1.js', + ); + }); + + test('from file, test not found', async () => { + mockfs({ + 'test/test-data': { + 'integ.integ-test1.js': 'content', + }, + [testsFile]: JSON.stringify({ + tests: [ + 'test/test-data/integ.integ-test16.js', + ], + }), + }); + const integTests = await tests.fromFile(testsFile); + + expect(integTests.length).toEqual(0); + expect(stderrMock.mock.calls[0][0]).toContain( + 'No such integ test: test/test-data/integ.integ-test16.js', + ); + expect(stderrMock.mock.calls[1][0]).toContain( + 'Available tests: test/test-data/integ.integ-test1.js test/test-data/integ.integ-test2.js test/test-data/integ.integ-test3.js', + ); + }); + + test('from file exclude', async () => { + const integTests = await tests.fromFile(testsFileExclude); + + const fileNames = integTests.map(test => test.fileName); + expect(integTests.length).toEqual(2); + expect(fileNames).not.toContain( + 'test/test-data/integ.integ-test1.js', + ); + }); }); diff --git a/packages/@aws-cdk/integ-runner/test/runner/runners.test.ts b/packages/@aws-cdk/integ-runner/test/runner/runners.test.ts index fcf616e81fbf0..94092fcd38516 100644 --- a/packages/@aws-cdk/integ-runner/test/runner/runners.test.ts +++ b/packages/@aws-cdk/integ-runner/test/runner/runners.test.ts @@ -170,6 +170,39 @@ describe('IntegTest runIntegTests', () => { }); }); + test('with profile', () => { + // WHEN + integTest.runIntegTestCase({ + testCase: { + stacks: ['stack1'], + }, + }); + + // THEN + expect(deployMock).toHaveBeenCalledTimes(1); + expect(destroyMock).toHaveBeenCalledTimes(1); + expect(synthMock).toHaveBeenCalledTimes(0); + expect(deployMock.mock.calls[0][0]).toEqual({ + app: 'node integ.integ-test1.js', + requireApproval: 'never', + pathMetadata: false, + assetMetadata: false, + versionReporting: false, + lookups: false, + stacks: ['stack1'], + output: 'cdk-integ.out.integ-test1', + }); + expect(destroyMock.mock.calls[0][0]).toEqual({ + app: 'node integ.integ-test1.js', + pathMetadata: false, + assetMetadata: false, + versionReporting: false, + force: true, + stacks: ['stack1'], + output: 'cdk-integ.out.integ-test1', + }); + }); + test('with lookups', () => { // WHEN integTest = new IntegTestRunner({ fileName: path.join(__dirname, '../test-data/integ.test-with-snapshot-assets-diff.js') }); @@ -319,3 +352,61 @@ describe('IntegTest no pragma', () => { }); }); }); + +describe('IntegTest runIntegTests with profile', () => { + let integTest: IntegTestRunner; + let deployMock: jest.SpyInstance; + let destroyMock: jest.SpyInstance; + let synthMock: jest.SpyInstance; + // let stderrMock: jest.SpyInstance; + beforeEach(() => { + integTest = new IntegTestRunner({ fileName: 'test/test-data/integ.integ-test1.js', profile: 'test-profile' }); + deployMock = jest.spyOn(integTest.cdk, 'deploy').mockImplementation(); + destroyMock = jest.spyOn(integTest.cdk, 'destroy').mockImplementation(); + synthMock = jest.spyOn(integTest.cdk, 'synthFast').mockImplementation(); + jest.spyOn(integTest.cdk, 'list').mockImplementation(); + jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); + }); + afterEach(() => { + jest.clearAllMocks(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + test('with defaults', () => { + // WHEN + integTest.runIntegTestCase({ + testCase: { + stacks: ['stack1'], + }, + }); + + // THEN + expect(deployMock).toHaveBeenCalledTimes(1); + expect(destroyMock).toHaveBeenCalledTimes(1); + expect(synthMock).toHaveBeenCalledTimes(0); + expect(deployMock.mock.calls[0][0]).toEqual({ + app: 'node integ.integ-test1.js', + requireApproval: 'never', + pathMetadata: false, + assetMetadata: false, + versionReporting: false, + profile: 'test-profile', + lookups: false, + stacks: ['stack1'], + output: 'cdk-integ.out.integ-test1', + }); + expect(destroyMock.mock.calls[0][0]).toEqual({ + app: 'node integ.integ-test1.js', + pathMetadata: false, + assetMetadata: false, + versionReporting: false, + profile: 'test-profile', + force: true, + stacks: ['stack1'], + output: 'cdk-integ.out.integ-test1', + }); + }); +}); diff --git a/packages/@aws-cdk/integ-runner/test/workers/integ-worker.test.ts b/packages/@aws-cdk/integ-runner/test/workers/integ-worker.test.ts new file mode 100644 index 0000000000000..73e4d31d25c6e --- /dev/null +++ b/packages/@aws-cdk/integ-runner/test/workers/integ-worker.test.ts @@ -0,0 +1,414 @@ +import * as child_process from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs-extra'; +import * as workerpool from 'workerpool'; +import { integTestWorker } from '../../lib/workers/extract'; +import { runIntegrationTestsInParallel, runIntegrationTests } from '../../lib/workers/integ-test-worker'; + +describe('test runner', () => { + beforeEach(() => { + jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'emptyDirSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'unlinkSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); + }); + afterEach(() => { + jest.clearAllMocks(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + + test('no snapshot', () => { + // WHEN + const test = { + fileName: 'test/test-data/integ.integ-test1.js', + }; + const spawnSyncMock = jest.spyOn(child_process, 'spawnSync').mockImplementation(); + integTestWorker({ + tests: [test], + region: 'us-east-1', + }); + + expect(spawnSyncMock).toHaveBeenCalledWith( + expect.stringMatching(/node/), + ['integ.integ-test1.js'], + expect.objectContaining({ + env: expect.objectContaining({ + CDK_INTEG_ACCOUNT: '12345678', + CDK_INTEG_REGION: 'test-region', + }), + }), + ); + }); + + test('no tests', () => { + // WHEN + const test = { + fileName: 'test/test-data/integ.integ-test2.js', + }; + jest.spyOn(child_process, 'spawnSync').mockReturnValue({ + status: 0, + stderr: Buffer.from(''), + stdout: Buffer.from(''), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const results = integTestWorker({ + tests: [test], + region: 'us-east-1', + }); + + expect(results[0]).toEqual({ fileName: expect.stringMatching(/integ.integ-test2.js$/) }); + }); + + test('has snapshot', () => { + // WHEN + const test = { + fileName: 'test/test-data/integ.test-with-snapshot.js', + }; + jest.spyOn(child_process, 'spawnSync').mockReturnValue({ + status: 0, + stderr: Buffer.from('stack1'), + stdout: Buffer.from('stack1'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const results = integTestWorker({ + tests: [test], + region: 'us-east-1', + }); + + expect(results.length).toEqual(0); + }); + + test('deploy failed', () => { + // WHEN + const test = { + fileName: 'test/test-data/integ.test-with-snapshot.js', + }; + jest.spyOn(child_process, 'spawnSync').mockReturnValue({ + status: 1, + stderr: Buffer.from('stack1'), + stdout: Buffer.from('stack1'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const results = integTestWorker({ + tests: [test], + region: 'us-east-1', + }); + + expect(results[0]).toEqual({ fileName: 'test/test-data/integ.test-with-snapshot.js' }); + }); +}); + +describe('parallel worker', () => { + let pool: workerpool.WorkerPool; + let stderrMock: jest.SpyInstance; + beforeEach(() => { + pool = workerpool.pool(path.join(__dirname, './mock-extract_worker.js')); + jest.spyOn(child_process, 'spawnSync').mockImplementation(); + stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); + jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); + }); + afterEach(async () => { + await pool.terminate(); + jest.clearAllMocks(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + test('run all integration tests', async () => { + const tests = [ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]; + await runIntegrationTests({ + tests, + pool, + regions: ['us-east-1', 'us-east-2'], + }); + + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running integration tests for failed tests...', + ); + expect(stderrMock.mock.calls[1][0]).toContain( + 'Running in parallel across regions: us-east-1, us-east-2', + ); + expect(stderrMock.mock.calls[2][0]).toContain( + 'Running test integ.another-test-with-snapshot.js in us-east-1', + ); + expect(stderrMock.mock.calls[3][0]).toContain( + 'Running test integ.test-with-snapshot.js in us-east-2', + ); + }); + + test('run tests', async () => { + const tests = [{ + fileName: 'integ.test-with-snapshot.js', + }]; + const results = await runIntegrationTestsInParallel({ + pool, + tests, + regions: ['us-east-1'], + }); + + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running test integ.test-with-snapshot.js in us-east-1', + ); + expect(results).toEqual({ + failedTests: expect.arrayContaining([ + { + fileName: 'integ.test-with-snapshot.js', + }, + ]), + metrics: expect.arrayContaining([ + { + duration: expect.anything(), + region: 'us-east-1', + tests: { + 'integ.test-with-snapshot.js': expect.anything(), + }, + }, + ]), + }); + }); + + test('run multiple tests with profiles', async () => { + const tests = [ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot2.js', + }, + { + fileName: 'integ.another-test-with-snapshot3.js', + }, + ]; + const results = await runIntegrationTestsInParallel({ + tests, + pool, + profiles: ['profile1', 'profile2'], + regions: ['us-east-1', 'us-east-2'], + }); + + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running test integ.another-test-with-snapshot3.js in profile1/us-east-1', + ); + expect(stderrMock.mock.calls[1][0]).toContain( + 'Running test integ.another-test-with-snapshot2.js in profile1/us-east-2', + ); + expect(stderrMock.mock.calls[2][0]).toContain( + 'Running test integ.another-test-with-snapshot.js in profile2/us-east-1', + ); + expect(stderrMock.mock.calls[3][0]).toContain( + 'Running test integ.test-with-snapshot.js in profile2/us-east-2', + ); + expect(results).toEqual({ + failedTests: expect.arrayContaining([ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot2.js', + }, + { + fileName: 'integ.another-test-with-snapshot3.js', + }, + ]), + metrics: expect.arrayContaining([ + { + duration: expect.any(Number), + region: 'us-east-1', + profile: 'profile1', + tests: { + 'integ.another-test-with-snapshot3.js': expect.any(Number), + }, + }, + { + duration: expect.any(Number), + region: 'us-east-2', + profile: 'profile1', + tests: { + 'integ.another-test-with-snapshot2.js': expect.any(Number), + }, + }, + { + duration: expect.any(Number), + region: 'us-east-1', + profile: 'profile2', + tests: { + 'integ.another-test-with-snapshot.js': expect.any(Number), + }, + }, + { + duration: expect.any(Number), + region: 'us-east-2', + profile: 'profile2', + tests: { + 'integ.test-with-snapshot.js': expect.any(Number), + }, + }, + ]), + }); + }); + + test('run multiple tests', async () => { + const tests = [ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]; + const results = await runIntegrationTestsInParallel({ + tests, + pool, + regions: ['us-east-1', 'us-east-2'], + }); + + expect(stderrMock.mock.calls[1][0]).toContain( + 'Running test integ.test-with-snapshot.js in us-east-2', + ); + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running test integ.another-test-with-snapshot.js in us-east-1', + ); + expect(results).toEqual({ + failedTests: expect.arrayContaining([ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]), + metrics: expect.arrayContaining([ + { + duration: expect.anything(), + region: 'us-east-2', + tests: { + 'integ.test-with-snapshot.js': expect.anything(), + }, + }, + { + duration: expect.anything(), + region: 'us-east-1', + tests: { + 'integ.another-test-with-snapshot.js': expect.anything(), + }, + }, + ]), + }); + }); + + test('more tests than regions', async () => { + const tests = [ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]; + const results = await runIntegrationTestsInParallel({ + tests, + pool, + regions: ['us-east-1'], + }); + + expect(stderrMock.mock.calls[1][0]).toContain( + 'Running test integ.test-with-snapshot.js in us-east-1', + ); + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running test integ.another-test-with-snapshot.js in us-east-1', + ); + expect(results).toEqual({ + failedTests: expect.arrayContaining([ + { + fileName: 'integ.another-test-with-snapshot.js', + }, + { + fileName: 'integ.test-with-snapshot.js', + }, + ]), + metrics: expect.arrayContaining([ + { + duration: expect.anything(), + region: 'us-east-1', + tests: { + 'integ.test-with-snapshot.js': expect.anything(), + 'integ.another-test-with-snapshot.js': expect.anything(), + }, + }, + ]), + }); + }); + + test('more regions than tests', async () => { + const tests = [ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]; + const results = await runIntegrationTestsInParallel({ + tests, + pool, + regions: ['us-east-1', 'us-east-2', 'us-west-2'], + }); + + expect(stderrMock.mock.calls[1][0]).toContain( + 'Running test integ.test-with-snapshot.js in us-east-2', + ); + expect(stderrMock.mock.calls[0][0]).toContain( + 'Running test integ.another-test-with-snapshot.js in us-east-1', + ); + expect(results).toEqual({ + failedTests: expect.arrayContaining([ + { + fileName: 'integ.test-with-snapshot.js', + }, + { + fileName: 'integ.another-test-with-snapshot.js', + }, + ]), + metrics: expect.arrayContaining([ + { + duration: expect.anything(), + region: 'us-east-2', + tests: { + 'integ.test-with-snapshot.js': expect.anything(), + }, + }, + { + duration: expect.anything(), + region: 'us-east-1', + tests: { + 'integ.another-test-with-snapshot.js': expect.anything(), + }, + }, + ]), + }); + }); +}); diff --git a/packages/@aws-cdk/integ-runner/test/workers/mock-extract_worker.ts b/packages/@aws-cdk/integ-runner/test/workers/mock-extract_worker.ts index 7479d229302b8..18f517e5f4dee 100644 --- a/packages/@aws-cdk/integ-runner/test/workers/mock-extract_worker.ts +++ b/packages/@aws-cdk/integ-runner/test/workers/mock-extract_worker.ts @@ -1,15 +1,13 @@ import * as workerpool from 'workerpool'; -import { IntegBatchResponse } from '../../lib/workers/common'; +import { IntegTestConfig } from '../../lib/runner'; import { IntegTestBatchRequest } from '../../lib/workers/integ-test-worker'; -function integTestBatch(request: IntegTestBatchRequest): IntegBatchResponse { - return { - failedTests: request.tests, - }; +function integTestWorker(request: IntegTestBatchRequest): IntegTestConfig[] { + return request.tests; } workerpool.worker({ - integTestBatch, + integTestWorker, }); diff --git a/packages/@aws-cdk/integ-runner/test/workers/snapshot-worker.test.ts b/packages/@aws-cdk/integ-runner/test/workers/snapshot-worker.test.ts new file mode 100644 index 0000000000000..d39b7e820047a --- /dev/null +++ b/packages/@aws-cdk/integ-runner/test/workers/snapshot-worker.test.ts @@ -0,0 +1,60 @@ +import * as child_process from 'child_process'; +import * as path from 'path'; +import * as fs from 'fs-extra'; +import { snapshotTestWorker } from '../../lib/workers/extract'; + +const directory = path.join(__dirname, '../test-data'); +describe('Snapshot tests', () => { + beforeEach(() => { + jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); + jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); + }); + afterEach(() => { + jest.clearAllMocks(); + jest.resetAllMocks(); + jest.restoreAllMocks(); + }); + test('no snapshot', () => { + // WHEN + const test = { + fileName: path.join(directory, 'integ.integ-test1.js'), + directory: directory, + }; + const result = snapshotTestWorker(test); + + // THEN + expect(result.length).toEqual(1); + expect(result[0]).toEqual(test); + }); + + test('has snapshot', () => { + // WHEN + jest.spyOn(child_process, 'spawnSync').mockResolvedValue; + const test = { + fileName: path.join(directory, 'integ.test-with-snapshot.js'), + directory: directory, + }; + const result = snapshotTestWorker(test); + + // THEN + expect(result.length).toEqual(0); + }); + + test('failed snapshot', () => { + // WHEN + jest.spyOn(child_process, 'spawnSync').mockRejectedValue; + const test = { + fileName: path.join(directory, 'integ.test-with-snapshot-assets.js'), + directory: directory, + }; + const result = snapshotTestWorker(test); + + // THEN + expect(result.length).toEqual(1); + expect(result[0]).toEqual(test); + }); +}); + diff --git a/packages/@aws-cdk/integ-runner/test/workers/workers.test.ts b/packages/@aws-cdk/integ-runner/test/workers/workers.test.ts deleted file mode 100644 index 040442e77b1f1..0000000000000 --- a/packages/@aws-cdk/integ-runner/test/workers/workers.test.ts +++ /dev/null @@ -1,303 +0,0 @@ -import * as child_process from 'child_process'; -import * as path from 'path'; -import * as fs from 'fs-extra'; -import * as workerpool from 'workerpool'; -import { singleThreadedSnapshotRunner } from '../../lib/workers'; -import { singleThreadedTestRunner, runIntegrationTestsInParallel, runIntegrationTests } from '../../lib/workers/integ-test-worker'; - -const directory = path.join(__dirname, '../test-data'); -describe('Snapshot tests', () => { - beforeEach(() => { - jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); - jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); - }); - afterEach(() => { - jest.clearAllMocks(); - jest.resetAllMocks(); - jest.restoreAllMocks(); - }); - test('no snapshot', () => { - // WHEN - const test = { - fileName: path.join(directory, 'integ.integ-test1.js'), - directory: directory, - }; - const result = singleThreadedSnapshotRunner([test]); - - // THEN - expect(result.failedTests.length).toEqual(1); - expect(result.failedTests[0]).toEqual(test); - }); - - test('has snapshot', () => { - // WHEN - jest.spyOn(child_process, 'spawnSync').mockResolvedValue; - const test = { - fileName: path.join(directory, 'integ.test-with-snapshot.js'), - directory: directory, - }; - const result = singleThreadedSnapshotRunner([test]); - - // THEN - expect(result.failedTests.length).toEqual(0); - }); - - test('failed snapshot', () => { - // WHEN - jest.spyOn(child_process, 'spawnSync').mockRejectedValue; - const test = { - fileName: path.join(directory, 'integ.test-with-snapshot-assets.js'), - directory: directory, - }; - const result = singleThreadedSnapshotRunner([test]); - - // THEN - expect(result.failedTests.length).toEqual(1); - expect(result.failedTests[0]).toEqual(test); - }); -}); - -describe('test runner', () => { - beforeEach(() => { - jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); - jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); - }); - afterEach(() => { - jest.clearAllMocks(); - jest.resetAllMocks(); - jest.restoreAllMocks(); - }); - - test('no snapshot', () => { - // WHEN - const test = { - fileName: path.join(directory, 'integ.integ-test1.js'), - directory: directory, - }; - const spawnSyncMock = jest.spyOn(child_process, 'spawnSync').mockImplementation(); - singleThreadedTestRunner({ - tests: [test], - region: 'us-east-1', - }); - - expect(spawnSyncMock).toHaveBeenCalledWith( - expect.stringMatching(/node/), - ['integ.integ-test1.js'], - expect.objectContaining({ - env: expect.objectContaining({ - CDK_INTEG_ACCOUNT: '12345678', - CDK_INTEG_REGION: 'test-region', - }), - }), - ); - }); -}); - -describe('parallel worker', () => { - let pool: workerpool.WorkerPool; - let stderrMock: jest.SpyInstance; - beforeEach(() => { - pool = workerpool.pool(path.join(__dirname, './mock-extract_worker.js')); - jest.spyOn(child_process, 'spawnSync').mockImplementation(); - stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; }); - jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'moveSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'removeSync').mockImplementation(() => { return true; }); - jest.spyOn(fs, 'writeFileSync').mockImplementation(() => { return true; }); - }); - afterEach(async () => { - await pool.terminate(); - jest.clearAllMocks(); - jest.resetAllMocks(); - jest.restoreAllMocks(); - }); - test('run all integration tests', async () => { - const tests = [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ]; - await runIntegrationTests({ - tests, - pool, - regions: ['us-east-1', 'us-east-2'], - }); - - expect(stderrMock.mock.calls[0][0]).toContain( - 'Running integration tests for failed tests...', - ); - expect(stderrMock.mock.calls[1][0]).toContain( - 'Running in parallel across: us-east-1, us-east-2', - ); - expect(stderrMock.mock.calls[3][0]).toContain( - 'Running test integ.test-with-snapshot.js in us-east-2', - ); - expect(stderrMock.mock.calls[2][0]).toContain( - 'Running test integ.another-test-with-snapshot.js in us-east-1', - ); - - }); - test('run tests', async () => { - const tests = [{ - fileName: 'integ.test-with-snapshot.js', - directory, - }]; - const results = await runIntegrationTestsInParallel({ - pool, - tests, - regions: ['us-east-1'], - }); - - expect(stderrMock.mock.calls[0][0]).toContain( - 'Running test integ.test-with-snapshot.js in us-east-1', - ); - expect(results).toEqual([ - { - failedTests: [{ - fileName: 'integ.test-with-snapshot.js', - directory, - }], - }, - ]); - }); - - test('run multiple tests', async () => { - const tests = [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ]; - const results = await runIntegrationTestsInParallel({ - tests, - pool, - regions: ['us-east-1', 'us-east-2'], - }); - - expect(stderrMock.mock.calls[1][0]).toContain( - 'Running test integ.test-with-snapshot.js in us-east-2', - ); - expect(stderrMock.mock.calls[0][0]).toContain( - 'Running test integ.another-test-with-snapshot.js in us-east-1', - ); - expect(results).toEqual(expect.arrayContaining([ - { - failedTests: [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - ], - }, - { - failedTests: [ - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ], - }, - ])); - }); - - test('more tests than regions', async () => { - const tests = [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ]; - const results = await runIntegrationTestsInParallel({ - tests, - pool, - regions: ['us-east-1'], - }); - - expect(stderrMock.mock.calls[1][0]).toContain( - 'Running test integ.test-with-snapshot.js in us-east-1', - ); - expect(stderrMock.mock.calls[0][0]).toContain( - 'Running test integ.another-test-with-snapshot.js in us-east-1', - ); - expect(results).toEqual([ - { - failedTests: [ - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ], - }, - { - failedTests: [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - ], - }, - ]); - }); - - test('more regions than tests', async () => { - const tests = [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ]; - const results = await runIntegrationTestsInParallel({ - tests, - pool, - regions: ['us-east-1', 'us-east-2', 'us-west-2'], - }); - - expect(stderrMock.mock.calls[1][0]).toContain( - 'Running test integ.test-with-snapshot.js in us-east-2', - ); - expect(stderrMock.mock.calls[0][0]).toContain( - 'Running test integ.another-test-with-snapshot.js in us-east-1', - ); - expect(results).toEqual(expect.arrayContaining([ - { - failedTests: [ - { - fileName: 'integ.another-test-with-snapshot.js', - directory, - }, - ], - }, - { - failedTests: [ - { - fileName: 'integ.test-with-snapshot.js', - directory, - }, - ], - }, - ])); - }); -}); From 360203f43cfdc9acc36b371863914e97ceda916f Mon Sep 17 00:00:00 2001 From: corymhall <43035978+corymhall@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:04:07 +0000 Subject: [PATCH 2/3] adding comment block for filterTests --- packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts b/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts index acb9980d3a126..39c0ea197d073 100644 --- a/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts +++ b/packages/@aws-cdk/integ-runner/lib/runner/integ-tests.ts @@ -52,6 +52,13 @@ export class IntegrationTests { return allTests; } + /** + * If the user provides a list of tests, these can either be a list of tests to include or a list of tests to exclude. + * + * - If it is a list of tests to include then we discover all available tests and check whether they have provided valid tests. + * If they have provided a test name that we don't find, then we write out that error message. + * - If it is a list of tests to exclude, then we discover all available tests and filter out the tests that were provided by the user. + */ private filterTests(discoveredTests: IntegTestConfig[], requestedTests?: string[], exclude?: boolean): IntegTestConfig[] { if (!requestedTests || requestedTests.length === 0) { return discoveredTests; From 174751acdac7827168c486d9c064a37d6ffa23cb Mon Sep 17 00:00:00 2001 From: corymhall <43035978+corymhall@users.noreply.github.com> Date: Tue, 12 Apr 2022 19:59:51 +0000 Subject: [PATCH 3/3] trigger build