feat(integ-runner): support custom --test-regex to match integ test…

… files

Co-authored-by: karakter98 <37190268+karakter98@users.noreply.github.com>

packages/@aws-cdk/integ-runner/README.md

@@ -70,6 +70,9 @@ to be a self contained CDK app. The runner will execute the following for each f
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
+- `--test-regex`
+  Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.
+
 Example:
 
 ```bash

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -13,7 +13,7 @@ import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWor
 const yargs = require('yargs');
 
 
-async function main() {
+export async function main(args: string[]) {
   const argv = yargs
     .usage('Usage: integ-runner [TEST...]')
     .option('list', { type: 'boolean', default: false, desc: 'List tests instead of running them' })
@@ -31,14 +31,16 @@ async function main() {
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
+    .option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
     .strict()
-    .argv;
+    .parse(args);
 
   const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
     maxWorkers: argv['max-workers'],
   });
 
   // list of integration tests that will be executed
+  const testRegex = arrayFromYargs(argv['test-regex']);
   const testsToRun: IntegTestWorkerConfig[] = [];
   const destructiveChanges: DestructiveChange[] = [];
   const testsFromArgs: IntegTest[] = [];
@@ -57,7 +59,7 @@ async function main() {
   let testsSucceeded = false;
   try {
     if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs();
+      const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex });
       process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
       return;
     }
@@ -69,7 +71,7 @@ async function main() {
       ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
       : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(requestedTests, exclude)));
+    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({ testRegex, tests: requestedTests, exclude })));
 
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that
@@ -184,8 +186,8 @@ function mergeTests(testFromArgs: IntegTestInfo[], failedSnapshotTests: IntegTes
   return final;
 }
 
-export function cli() {
-  main().then().catch(err => {
+export function cli(args: string[] = process.argv.slice(2)) {
+  main(args).then().catch(err => {
     logger.error(err);
     process.exitCode = 1;
   });

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -123,10 +123,9 @@ export class IntegTest {
 }
 
 /**
- * The list of tests to run can be provided in a file
- * instead of as command line arguments.
+ * Configuration options how integration test files are discovered
  */
-export interface IntegrationTestFileConfig {
+export interface IntegrationTestsDiscoveryOptions {
   /**
    * If this is set to true then the list of tests
    * provided will be excluded
@@ -135,6 +134,27 @@ export interface IntegrationTestFileConfig {
    */
   readonly exclude?: boolean;
 
+  /**
+    * List of tests to include (or exclude if `exclude=true`)
+    *
+    * @default - all matched files
+    */
+  readonly tests?: string[];
+
+  /**
+    * Detect integration test files matching any of these JavaScript regex patterns.
+    *
+    * @default
+    */
+  readonly testRegex?: string[];
+}
+
+
+/**
+ * The list of tests to run can be provided in a file
+ * instead of as command line arguments.
+ */
+export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions {
   /**
    * List of tests to include (or exclude if `exclude=true`)
    */
@@ -154,7 +174,7 @@ export class IntegrationTests {
    */
   public async fromFile(fileName: string): Promise<IntegTest[]> {
     const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    const foundTests = await this.discover();
+    const foundTests = await this.discover(file.testRegex);
 
     const allTests = this.filterTests(foundTests, file.tests, file.exclude);
 
@@ -201,17 +221,20 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(tests?: string[], exclude?: boolean): Promise<IntegTest[]> {
-    const discoveredTests = await this.discover();
+  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
+    const discoveredTests = await this.discover(options.testRegex);
 
-    const allTests = this.filterTests(discoveredTests, tests, exclude);
+    const allTests = this.filterTests(discoveredTests, options.tests, options.exclude);
 
     return allTests;
   }
 
-  private async discover(): Promise<IntegTest[]> {
+  private async discover(patterns: string[] = ['^integ\..*\.js$']): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => path.basename(fileName).startsWith('integ.') && path.basename(fileName).endsWith('.js'));
+    const integs = files.filter(fileName => patterns.some((p) => {
+      const regex = new RegExp(p);
+      return regex.test(fileName) || regex.test(path.basename(fileName));
+    }));
     return this.request(integs);
   }
 
@@ -228,7 +251,7 @@ export class IntegrationTests {
         const fullPath = path.join(dir, file);
         const statf = await fs.stat(fullPath);
         if (statf.isFile()) { ret.push(fullPath); }
-        if (statf.isDirectory()) { await recurse(path.join(fullPath)); }
+        if (statf.isDirectory()) { await recurse(fullPath); }
       }
     }
 

packages/@aws-cdk/integ-runner/test/cli.test.ts

@@ -0,0 +1,45 @@
+import * as path from 'path';
+import { main } from '../lib/cli';
+
+describe('CLI', () => {
+  const currentCwd = process.cwd();
+  beforeAll(() => {
+    process.chdir(path.join(__dirname, '..'));
+  });
+  afterAll(() => {
+    process.chdir(currentCwd);
+  });
+
+  let stdoutMock: jest.SpyInstance;
+  beforeEach(() => {
+    stdoutMock = jest.spyOn(process.stdout, 'write').mockImplementation(() => { return true; });
+  });
+  afterEach(() => {
+    stdoutMock.mockRestore();
+  });
+
+  test('find by default pattern', async () => {
+    await main(['--list', '--directory=test/test-data']);
+
+    // Expect nothing to be found since this directory doesn't contain files with the default pattern
+    expect(stdoutMock.mock.calls).toEqual([['\n']]);
+  });
+
+  test('find by custom pattern', async () => {
+    await main(['--list', '--directory=test/test-data', '--test-regex="^xxxxx\..*\.js$"']);
+
+    // Expect nothing to be found since this directory doesn't contain files with the default pattern
+    expect(stdoutMock.mock.calls).toEqual([[
+      [
+        'xxxxx.integ-test1.js',
+        'xxxxx.integ-test2.js',
+        'xxxxx.test-with-new-assets-diff.js',
+        'xxxxx.test-with-new-assets.js',
+        'xxxxx.test-with-snapshot-assets-diff.js',
+        'xxxxx.test-with-snapshot-assets.js',
+        'xxxxx.test-with-snapshot.js',
+        '',
+      ].join('\n'),
+    ]]);
+  });
+});

packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts

@@ -1,50 +1,153 @@
+import { writeFileSync } from 'fs';
 import * as mockfs from 'mock-fs';
 import { IntegrationTests } from '../../lib/runner/integration-tests';
 
 describe('IntegrationTests', () => {
   const tests = new IntegrationTests('test');
   let stderrMock: jest.SpyInstance;
   stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; });
+
   beforeEach(() => {
     mockfs({
       'test/test-data': {
         'integ.integ-test1.js': 'content',
         'integ.integ-test2.js': 'content',
         'integ.integ-test3.js': 'content',
       },
+      'other/other-data': {
+        'integ.other-test1.js': 'content',
+      },
     });
   });
 
   afterEach(() => {
     mockfs.restore();
   });
 
-  test('from cli args', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test1.js']);
+  describe('from cli args', () => {
+    test('find all', async () => {
+      const integTests = await tests.fromCliArgs();
+
+      expect(integTests.length).toEqual(3);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+      expect(integTests[2].fileName).toEqual(expect.stringMatching(/integ.integ-test3.js$/));
+    });
 
-    expect(integTests.length).toEqual(1);
-    expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
-  });
 
-  test('from cli args, test not found', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test16.js']);
+    test('find named tests', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test1.js'] });
 
-    expect(integTests.length).toEqual(0);
-    expect(stderrMock.mock.calls[0][0]).toContain(
-      'No such integ test: test-data/integ.integ-test16.js',
-    );
-    expect(stderrMock.mock.calls[1][0]).toContain(
-      'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
-    );
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+    });
+
+
+    test('test not found', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test16.js'] });
+
+      expect(integTests.length).toEqual(0);
+      expect(stderrMock.mock.calls[0][0]).toContain(
+        'No such integ test: test-data/integ.integ-test16.js',
+      );
+      expect(stderrMock.mock.calls[1][0]).toContain(
+        'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
+      );
+    });
+
+    test('exclude tests', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test1.js'], exclude: true });
+
+      const fileNames = integTests.map(test => test.fileName);
+      expect(integTests.length).toEqual(2);
+      expect(fileNames).not.toContain(
+        'test/test-data/integ.integ-test1.js',
+      );
+    });
+
+    test('match regex', async () => {
+      const integTests = await tests.fromCliArgs({ testRegex: ['1\.js$', '2\.js'] });
+
+      expect(integTests.length).toEqual(2);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+    });
+
+    test('match regex with path', async () => {
+      const otherTestDir = new IntegrationTests('.');
+      const integTests = await otherTestDir.fromCliArgs({ testRegex: ['other-data/integ\..*\.js$'] });
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.other-test1.js$/));
+    });
   });
 
-  test('from cli args, exclude', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test1.js'], true);
+  describe('from file', () => {
+    const configFile = 'integ.config.json';
+    const writeConfig = (settings: any, fileName = configFile) => {
+      writeFileSync(fileName, JSON.stringify(settings, null, 2), { encoding: 'utf-8' });
+    };
 
-    const fileNames = integTests.map(test => test.fileName);
-    expect(integTests.length).toEqual(2);
-    expect(fileNames).not.toContain(
-      'test/test-data/integ.integ-test1.js',
-    );
+    test('find all', async () => {
+      writeConfig({});
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(3);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+      expect(integTests[2].fileName).toEqual(expect.stringMatching(/integ.integ-test3.js$/));
+    });
+
+
+    test('find named tests', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test1.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+    });
+
+
+    test('test not found', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test16.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(0);
+      expect(stderrMock.mock.calls[0][0]).toContain(
+        'No such integ test: test-data/integ.integ-test16.js',
+      );
+      expect(stderrMock.mock.calls[1][0]).toContain(
+        'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
+      );
+    });
+
+    test('exclude tests', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test1.js'], exclude: true });
+      const integTests = await tests.fromFile(configFile);
+
+      const fileNames = integTests.map(test => test.fileName);
+      expect(integTests.length).toEqual(2);
+      expect(fileNames).not.toContain(
+        'test/test-data/integ.integ-test1.js',
+      );
+    });
+
+    test('match regex', async () => {
+      writeConfig({ testRegex: ['1\.js$', '2\.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(2);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+    });
+
+    test('match regex with path', async () => {
+      writeConfig({ testRegex: ['other-data/integ\..*\.js$'] });
+      const otherTestDir = new IntegrationTests('.');
+      const integTests = await otherTestDir.fromFile(configFile);
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.other-test1.js$/));
+    });
   });
 });