fix(mocha framework): Select tests based on name (#413)

Select tests in the `MochaTestFramework` based on it's name, rather than the order in which it is added to a test suite. * Change TestFramework interface to now filter based on test `id` and `name`. * Add `TestSelection` which represents a selected test using `id` and `name`. * Update `JasmineTestFramework` to now use the new `TestSelection` format (still based on ID). * Update `MochaTestFramework` to use the title to select tests. * Add integration tests to both `JasmineTestFramework` and `MochaTestFramework` to prevent this sort of thing from happening in the future. Fixes #249 BREAKING CHANGES: * Change api of `TestFramework`. It now provides an array of `TestSelection` objects, instead of an array of numbers with test ids.
stryker-mutator · Oct 20, 2017 · bb7c02f · bb7c02f
1 parent dcb4013
commit bb7c02f
Show file tree

Hide file tree

Showing 48 changed files with 896 additions and 790 deletions.
diff --git a/package.json b/package.json
@@ -27,6 +27,7 @@
     "concurrently": "^3.4.0",
     "execa": "^0.8.0",
     "glob": "^7.1.1",
+    "jasmine": "^2.8.0",
     "lerna": "^2.0.0",
     "link-parent-bin": "^0.1.1",
     "mocha": "^3.2.0",

diff --git a/packages/stryker-api/src/test_framework/TestFramework.ts b/packages/stryker-api/src/test_framework/TestFramework.ts
@@ -1,3 +1,5 @@
+import TestSelection from './TestSelection';
+
 /**
  * Represents a TestFramework which can select one or more tests to be executed.
  */
@@ -20,10 +22,10 @@ interface TestFramework {
    * will be responsible for filtering out tests with given ids.
    * The first test gets id 0, the second id 1, etc.
    * 
-   * @param indices A list of testId's to select.
+   * @param selections A list indicating the tests to select.
    * @returns A script which, if included in the test run, will filter out the correct tests.
    */
-  filter(ids: number[]): string;
+  filter(selections: TestSelection[]): string;
 }
 
 export default TestFramework;
diff --git a/packages/stryker-api/src/test_framework/TestSelection.ts b/packages/stryker-api/src/test_framework/TestSelection.ts
@@ -0,0 +1,4 @@
+export default interface TestSelection {
+  id: number;
+  name: string;
+}
diff --git a/packages/stryker-api/testResources/module/useTestFramework.ts b/packages/stryker-api/testResources/module/useTestFramework.ts
@@ -1,4 +1,4 @@
-import { TestFramework, TestFrameworkFactory, TestFrameworkSettings } from 'stryker-api/test_framework';
+import { TestFramework, TestFrameworkFactory, TestSelection, TestFrameworkSettings } from 'stryker-api/test_framework';
 
 class TestFramework1 implements TestFramework {
 
@@ -14,8 +14,8 @@ class TestFramework1 implements TestFramework {
     return '';
   }
 
-  filter(ids: number[]) {
-    return ids.toString();
+  filter(selections: TestSelection[]) {
+    return selections.map(selection => selection.id).toString();
   }
 }
 

diff --git a/packages/stryker-api/test_framework.ts b/packages/stryker-api/test_framework.ts
@@ -1,3 +1,4 @@
 export {default as TestFramework} from './src/test_framework/TestFramework';
+export {default as TestSelection} from './src/test_framework/TestSelection';
 export {default as TestFrameworkFactory} from './src/test_framework/TestFrameworkFactory';
 export {default as TestFrameworkSettings} from './src/test_framework/TestFrameworkSettings';
diff --git a/packages/stryker-jasmine/.vscode/launch.json b/packages/stryker-jasmine/.vscode/launch.json
@@ -1,123 +1,40 @@
 {
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Run unit tests",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/node_modules/grunt/bin/grunt",
-      // "preLaunchTask": "build",
-      "stopOnEntry": false,
-      "args": [
-        "mochaTest:unit"
-      ],
-      "cwd": "${workspaceRoot}/.",
-      "runtimeExecutable": null,
-      "runtimeArgs": [
-        "--nolazy"
-      ],
-      "env": {
-        "NODE_ENV": "development"
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "type": "node",
+        "request": "launch",
+        "name": "Unit tests",
+        "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha",
+        "args": [
+          "-u",
+          "tdd",
+          "--timeout",
+          "999999",
+          "--colors",
+          "${workspaceFolder}/test/helpers/**/*.js",
+          "${workspaceFolder}/test/unit/**/*.js"
+        ],
+        "internalConsoleOptions": "openOnSessionStart"
       },
-      "externalConsole": false,
-      "sourceMaps": true,
-      "outDir": "${workspaceRoot}"
-    },
-    {
-      "name": "Run integration tests",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/node_modules/grunt-cli/bin/grunt",
-      // "preLa4unchTask": "ts",
-      "stopOnEntry": false,
-      "args": [
-        "mochaTest:integration"
-      ],
-      "cwd": "${workspaceRoot}/.",
-      "runtimeExecutable": null,
-      "runtimeArgs": [
-        "--nolazy"
-      ],
-      "env": {
-        "NODE_ENV": "development"
-      },
-      "externalConsole": false,
-      "sourceMaps": true,
-      "outDir": "${workspaceRoot}"
-    },
-    {
-      "name": "Run stryker example",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/bin/stryker",
-      "preLaunchTask": "build",
-      "stopOnEntry": false,
-      "args": [
-        "--configFile",
-        "testResources/sampleProject/stryker.conf.js",
-        "--logLevel",
-        "trace",
-        "--testFramework",
-        "jasmine"
-      ],
-      "cwd": "${workspaceRoot}",
-      "runtimeExecutable": null,
-      "runtimeArgs": [
-        "--nolazy"
-      ],
-      "env": {
-        "NODE_ENV": "development"
-      },
-      "externalConsole": false,
-      "sourceMaps": true,
-      "outDir": "${workspaceRoot}"
-    },
-    {
-      "name": "Run own dog food",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/bin/stryker",
-      "preLaunchTask": "build",
-      "stopOnEntry": false,
-      "args": [
-        "--configFile",
-        "stryker.conf.js",
-        "--logLevel",
-        "info"
-      ],
-      "cwd": "${workspaceRoot}",
-      "runtimeExecutable": null,
-      "runtimeArgs": [
-        "--nolazy"
-      ],
-      "env": {
-        "NODE_ENV": "development"
-      },
-      "externalConsole": false,
-      "sourceMaps": true,
-      "outDir": "${workspaceRoot}"
-    },
-    {
-      "name": "Run stryker help",
-      "type": "node",
-      "request": "launch",
-      "program": "${workspaceRoot}/src/Stryker.js",
-      "preLaunchTask": "build",
-      "stopOnEntry": false,
-      "args": [
-        "--help"
-      ],
-      "cwd": "${workspaceRoot}/.",
-      "runtimeExecutable": null,
-      "runtimeArgs": [
-        "--nolazy"
-      ],
-      "env": {
-        "NODE_ENV": "development"
-      },
-      "externalConsole": false,
-      "sourceMaps": true,
-      "outDir": "${workspaceRoot}"
-    }
-  ]
+      {
+        "type": "node",
+        "request": "launch",
+        "name": "Integration tests",
+        "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha",
+        "args": [
+          "-u",
+          "tdd",
+          "--timeout",
+          "999999",
+          "--colors",
+          "${workspaceFolder}/test/helpers/**/*.js",
+          "${workspaceFolder}/test/integration/**/*.js"
+        ],
+        "internalConsoleOptions": "openOnSessionStart"
+      }
+    ]
 }
diff --git a/packages/stryker-jasmine/.vscode/tasks.json b/packages/stryker-jasmine/.vscode/tasks.json
@@ -0,0 +1,19 @@
+{
+    // See https://go.microsoft.com/fwlink/?LinkId=733558
+    // for the documentation about the tasks.json format
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "type": "typescript",
+            "tsconfig": "tsconfig.json",
+            "option": "watch",
+            "problemMatcher": [
+                "$tsc-watch"
+            ],
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            }
+        }
+    ]
+}
diff --git a/packages/stryker-jasmine/package.json b/packages/stryker-jasmine/package.json
@@ -9,7 +9,7 @@
     "prebuild": "npm run clean",
     "build": "tsc -p .",
     "postbuild": "tslint -p tsconfig.json",
-    "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100  mocha \"test/**/*.js\""
+    "test": "nyc --reporter=html --report-dir=reports/coverage --check-coverage --lines 100 --functions 100 --branches 100  mocha \"test/helpers/**/*.js\" \"test/unit/**/*.js\" \"test/integration/**/*.js\""
   },
   "repository": {
     "type": "git",

diff --git a/packages/stryker-jasmine/src/JasmineTestFramework.ts b/packages/stryker-jasmine/src/JasmineTestFramework.ts
@@ -1,8 +1,8 @@
-import { TestFramework, TestFrameworkSettings } from 'stryker-api/test_framework';
+import { TestFramework, TestSelection } from 'stryker-api/test_framework';
 
 export default class JasmineTestFramework implements TestFramework {
 
-  constructor(settings: TestFrameworkSettings) {
+  constructor() {
   }
 
   /**
@@ -31,24 +31,12 @@ export default class JasmineTestFramework implements TestFramework {
         });`;
   }
 
-  /**
-   * Creates a code fragment which, in included in a test run,
-   * will be responsible for filtering out tests with given ids.
-   * The first test gets id 0, the second id 1, etc.
-   *
-   * @param indices A list of testId's to select.
-   * @returns A script which, if included in the test run, will filter out the correct tests.
-   */
-  filter(ids: number[]): string {
+  filter(testSelections: TestSelection[]): string {
+    const ids = testSelections.map(selection => selection.id);
     return `    
       var currentTestId = 0;
       jasmine.getEnv().specFilter = function (spec) {
-          var filterOut = false;
-          if(${JSON.stringify(ids)}.indexOf(currentTestId) >= 0){
-            filterOut = true;
-          }
-          currentTestId++;
-          return filterOut;
+          return ${JSON.stringify(ids)}.indexOf(currentTestId++) !== -1;
       }`;
   }
 }
diff --git a/packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts b/packages/stryker-jasmine/test/integration/nestedSuiteSpec.ts
@@ -0,0 +1,96 @@
+import * as path from 'path';
+import * as execa from 'execa';
+import { expect } from 'chai';
+import { TestSelection } from 'stryker-api/test_framework';
+import JasmineTestFramework from '../../src/JasmineTestFramework';
+import * as fs from 'fs';
+import * as rimraf from 'rimraf';
+
+interface JasmineTest {
+  id: string;
+  description: string;
+  fullName: string;
+  failedExpectations: any[];
+  passedExpectations: any[];
+  status: string;
+}
+
+describe('Selecting tests with nested suites', function () {
+
+  this.timeout(10000);
+  let sut: JasmineTestFramework;
+  let testSelections: TestSelection[];
+  const jsonReporterFile = path.resolve(__dirname, '..', '..', 'testResources', 'json-reporter.js');
+  const nestedSuiteFile = path.resolve(__dirname, '..', '..', 'testResources', 'nested-suite.js');
+  const selectTestFile = path.join(__dirname, '..', '..', 'testResources', '__filterSpecs.js');
+
+  beforeEach(() => {
+    sut = new JasmineTestFramework();
+    testSelections = [
+      { id: 0, name: 'outer test 1' },
+      { id: 1, name: 'outer test 2' },
+      { id: 2, name: 'outer inner test 3' }
+    ];
+  });
+
+  afterEach(() => {
+    rimraf.sync(selectTestFile);
+  });
+
+  it('should run all tests in expected order when running all tests', () => {
+    const result = execJasmine(nestedSuiteFile);
+    expect(result.map(test => test.fullName)).deep.eq(['outer test 1', 'outer inner test 2', 'outer test 3']);
+  });
+
+  it('should only run test 1 if filtered on index 0', () => {
+    filter([0]);
+    const result = execJasmine(selectTestFile, nestedSuiteFile);
+    expect(result).lengthOf(3);
+    expect(result[0].status).eq('passed');
+    expect(result[1].status).eq('disabled');
+    expect(result[2].status).eq('disabled');
+    expect(result[0].fullName).eq('outer test 1');
+  });
+
+  it('should only run test 2 if filtered on index 1', () => {
+    filter([1]);
+    const result = execJasmine(selectTestFile, nestedSuiteFile);
+    expect(result).lengthOf(3);
+    expect(result[0].status).eq('disabled');
+    expect(result[1].status).eq('passed');
+    expect(result[2].status).eq('disabled');
+    expect(result[1].fullName).eq('outer inner test 2');
+  });
+
+  it('should only run test 3 if filtered on index 2', () => {
+    filter([2]);
+    const result = execJasmine(selectTestFile, nestedSuiteFile);
+    expect(result).lengthOf(3);
+    expect(result[0].status).eq('disabled');
+    expect(result[1].status).eq('disabled');
+    expect(result[2].status).eq('passed');
+    expect(result[2].fullName).eq('outer test 3');
+  });
+
+  it('should only run tests 1 and 3 if filtered on indices 0 and 2', () => {
+    filter([0, 2]);
+    const result = execJasmine(selectTestFile, nestedSuiteFile);
+    expect(result).lengthOf(3);
+    expect(result[0].status).eq('passed');
+    expect(result[1].status).eq('disabled');
+    expect(result[2].status).eq('passed');
+    expect(result[0].fullName).eq('outer test 1');
+    expect(result[2].fullName).eq('outer test 3');
+  });
+
+  function filter(testIds: number[]) {
+    const selections = testIds.map(id => testSelections[id]);
+    const filterFn = `(function (window) {${sut.filter(selections)}})(global);`;
+    fs.writeFileSync(selectTestFile, filterFn, 'utf8');
+  }
+
+  function execJasmine(...files: string[]): JasmineTest[] {
+    const execResult = execa.sync('jasmine', [jsonReporterFile, ...files]);
+    return JSON.parse(execResult.stdout);
+  }
+});