feat(integ-runner): support custom --test-regex to match integ test…

… files

Co-authored-by: karakter98 <37190268+karakter98@users.noreply.github.com>

packages/@aws-cdk/integ-runner/README.md

@@ -70,6 +70,9 @@ to be a self contained CDK app. The runner will execute the following for each f
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
+- `--test-regex`
+  Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.
+
 Example:
 
 ```bash

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -31,6 +31,7 @@ async function main() {
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
+    .option('test-regex', { type: 'array', default: undefined, desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.' })
     .strict()
     .argv;
 
@@ -39,6 +40,7 @@ async function main() {
   });
 
   // list of integration tests that will be executed
+  const testRegex = arrayFromYargs(argv['test-regex']);
   const testsToRun: IntegTestWorkerConfig[] = [];
   const destructiveChanges: DestructiveChange[] = [];
   const testsFromArgs: IntegTest[] = [];
@@ -57,7 +59,7 @@ async function main() {
   let testsSucceeded = false;
   try {
     if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs();
+      const tests = await new IntegrationTests(argv.directory).fromCliArgs({ testRegex });
       process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
       return;
     }
@@ -69,7 +71,7 @@ async function main() {
       ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
       : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(requestedTests, exclude)));
+    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs({ testRegex, tests: requestedTests, exclude })));
 
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -123,10 +123,9 @@ export class IntegTest {
 }
 
 /**
- * The list of tests to run can be provided in a file
- * instead of as command line arguments.
+ * Configuration options how integration test files are discovered
  */
-export interface IntegrationTestFileConfig {
+export interface IntegrationTestsDiscoveryOptions {
   /**
    * If this is set to true then the list of tests
    * provided will be excluded
@@ -135,6 +134,27 @@ export interface IntegrationTestFileConfig {
    */
   readonly exclude?: boolean;
 
+  /**
+    * List of tests to include (or exclude if `exclude=true`)
+    *
+    * @default - all matched files
+    */
+  readonly tests?: string[];
+
+  /**
+    * Detect integration test files matching any of these JavaScript regex patterns.
+    *
+    * @default
+    */
+  readonly testRegex?: string[];
+}
+
+
+/**
+ * The list of tests to run can be provided in a file
+ * instead of as command line arguments.
+ */
+export interface IntegrationTestFileConfig extends IntegrationTestsDiscoveryOptions {
   /**
    * List of tests to include (or exclude if `exclude=true`)
    */
@@ -154,7 +174,7 @@ export class IntegrationTests {
    */
   public async fromFile(fileName: string): Promise<IntegTest[]> {
     const file: IntegrationTestFileConfig = JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    const foundTests = await this.discover();
+    const foundTests = await this.discover(file.testRegex);
 
     const allTests = this.filterTests(foundTests, file.tests, file.exclude);
 
@@ -201,17 +221,20 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(tests?: string[], exclude?: boolean): Promise<IntegTest[]> {
-    const discoveredTests = await this.discover();
+  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
+    const discoveredTests = await this.discover(options.testRegex);
 
-    const allTests = this.filterTests(discoveredTests, tests, exclude);
+    const allTests = this.filterTests(discoveredTests, options.tests, options.exclude);
 
     return allTests;
   }
 
-  private async discover(): Promise<IntegTest[]> {
+  private async discover(patterns: string[] = ['^integ\..*\.js$']): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => path.basename(fileName).startsWith('integ.') && path.basename(fileName).endsWith('.js'));
+    const integs = files.filter(fileName => patterns.some((p) => {
+      const regex = new RegExp(p);
+      return regex.test(fileName) || regex.test(path.basename(fileName));
+    }));
     return this.request(integs);
   }
 
@@ -228,7 +251,7 @@ export class IntegrationTests {
         const fullPath = path.join(dir, file);
         const statf = await fs.stat(fullPath);
         if (statf.isFile()) { ret.push(fullPath); }
-        if (statf.isDirectory()) { await recurse(path.join(fullPath)); }
+        if (statf.isDirectory()) { await recurse(fullPath); }
       }
     }
 

packages/@aws-cdk/integ-runner/test/runner/integration-tests.test.ts

@@ -1,50 +1,153 @@
+import { writeFileSync } from 'fs';
 import * as mockfs from 'mock-fs';
 import { IntegrationTests } from '../../lib/runner/integration-tests';
 
 describe('IntegrationTests', () => {
   const tests = new IntegrationTests('test');
   let stderrMock: jest.SpyInstance;
   stderrMock = jest.spyOn(process.stderr, 'write').mockImplementation(() => { return true; });
+
   beforeEach(() => {
     mockfs({
       'test/test-data': {
         'integ.integ-test1.js': 'content',
         'integ.integ-test2.js': 'content',
         'integ.integ-test3.js': 'content',
       },
+      'other/other-data': {
+        'integ.other-test1.js': 'content',
+      },
     });
   });
 
   afterEach(() => {
     mockfs.restore();
   });
 
-  test('from cli args', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test1.js']);
+  describe('from cli args', () => {
+    test('find all', async () => {
+      const integTests = await tests.fromCliArgs();
+
+      expect(integTests.length).toEqual(3);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+      expect(integTests[2].fileName).toEqual(expect.stringMatching(/integ.integ-test3.js$/));
+    });
 
-    expect(integTests.length).toEqual(1);
-    expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
-  });
 
-  test('from cli args, test not found', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test16.js']);
+    test('find named tests', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test1.js'] });
 
-    expect(integTests.length).toEqual(0);
-    expect(stderrMock.mock.calls[0][0]).toContain(
-      'No such integ test: test-data/integ.integ-test16.js',
-    );
-    expect(stderrMock.mock.calls[1][0]).toContain(
-      'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
-    );
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+    });
+
+
+    test('test not found', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test16.js'] });
+
+      expect(integTests.length).toEqual(0);
+      expect(stderrMock.mock.calls[0][0]).toContain(
+        'No such integ test: test-data/integ.integ-test16.js',
+      );
+      expect(stderrMock.mock.calls[1][0]).toContain(
+        'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
+      );
+    });
+
+    test('exclude tests', async () => {
+      const integTests = await tests.fromCliArgs({ tests: ['test-data/integ.integ-test1.js'], exclude: true });
+
+      const fileNames = integTests.map(test => test.fileName);
+      expect(integTests.length).toEqual(2);
+      expect(fileNames).not.toContain(
+        'test/test-data/integ.integ-test1.js',
+      );
+    });
+
+    test('match regex', async () => {
+      const integTests = await tests.fromCliArgs({ testRegex: ['1\.js$', '2\.js'] });
+
+      expect(integTests.length).toEqual(2);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+    });
+
+    test('match regex with path', async () => {
+      const otherTestDir = new IntegrationTests('.');
+      const integTests = await otherTestDir.fromCliArgs({ testRegex: ['other-data/integ\..*\.js$'] });
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.other-test1.js$/));
+    });
   });
 
-  test('from cli args, exclude', async () => {
-    const integTests = await tests.fromCliArgs(['test-data/integ.integ-test1.js'], true);
+  describe('from file', () => {
+    const configFile = 'integ.config.json';
+    const writeConfig = (settings: any, fileName = configFile) => {
+      writeFileSync(fileName, JSON.stringify(settings, null, 2), { encoding: 'utf-8' });
+    };
 
-    const fileNames = integTests.map(test => test.fileName);
-    expect(integTests.length).toEqual(2);
-    expect(fileNames).not.toContain(
-      'test/test-data/integ.integ-test1.js',
-    );
+    test('find all', async () => {
+      writeConfig({});
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(3);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+      expect(integTests[2].fileName).toEqual(expect.stringMatching(/integ.integ-test3.js$/));
+    });
+
+
+    test('find named tests', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test1.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+    });
+
+
+    test('test not found', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test16.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(0);
+      expect(stderrMock.mock.calls[0][0]).toContain(
+        'No such integ test: test-data/integ.integ-test16.js',
+      );
+      expect(stderrMock.mock.calls[1][0]).toContain(
+        'Available tests: test-data/integ.integ-test1.js test-data/integ.integ-test2.js test-data/integ.integ-test3.js',
+      );
+    });
+
+    test('exclude tests', async () => {
+      writeConfig({ tests: ['test-data/integ.integ-test1.js'], exclude: true });
+      const integTests = await tests.fromFile(configFile);
+
+      const fileNames = integTests.map(test => test.fileName);
+      expect(integTests.length).toEqual(2);
+      expect(fileNames).not.toContain(
+        'test/test-data/integ.integ-test1.js',
+      );
+    });
+
+    test('match regex', async () => {
+      writeConfig({ testRegex: ['1\.js$', '2\.js'] });
+      const integTests = await tests.fromFile(configFile);
+
+      expect(integTests.length).toEqual(2);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.integ-test1.js$/));
+      expect(integTests[1].fileName).toEqual(expect.stringMatching(/integ.integ-test2.js$/));
+    });
+
+    test('match regex with path', async () => {
+      writeConfig({ testRegex: ['other-data/integ\..*\.js$'] });
+      const otherTestDir = new IntegrationTests('.');
+      const integTests = await otherTestDir.fromFile(configFile);
+
+      expect(integTests.length).toEqual(1);
+      expect(integTests[0].fileName).toEqual(expect.stringMatching(/integ.other-test1.js$/));
+    });
   });
 });