diff --git a/packages/api/.vscode/launch.json b/packages/api/.vscode/launch.json index 090eaac215..c7285fbf35 100644 --- a/packages/api/.vscode/launch.json +++ b/packages/api/.vscode/launch.json @@ -1,24 +1,6 @@ { "version": "0.2.0", "configurations": [ - { - "type": "node", - "request": "launch", - "name": "Integration tests", - "program": "${workspaceFolder}/../../node_modules/mocha/bin/_mocha", - "args": [ - "--timeout", - "999999", - "--colors", - "${workspaceFolder}/test/helpers/**/*.js", - "${workspaceFolder}/test/integration/**/*.js" - ], - "internalConsoleOptions": "openOnSessionStart", - "outFiles": [ - "${workspaceRoot}/test/**/*.js", - "${workspaceRoot}/src/**/*.js" - ] - }, { "type": "node", "request": "launch", @@ -35,7 +17,10 @@ "outFiles": [ "${workspaceRoot}/test/**/*.js", "${workspaceRoot}/src/**/*.js" + ], + "skipFiles": [ + "/**" ] } ] -} \ No newline at end of file +} diff --git a/packages/api/schema/stryker-core.json b/packages/api/schema/stryker-core.json index 4dd56d9a1f..dcb2672ad2 100644 --- a/packages/api/schema/stryker-core.json +++ b/packages/api/schema/stryker-core.json @@ -26,6 +26,15 @@ "Trace" ] }, + "coverageAnalysis": { + "title": "CoverageAnalysis", + "type": "string", + "enum": [ + "off", + "all", + "perTest" + ] + }, "reportType": { "title": "ReportType", "type": "string", @@ -201,13 +210,8 @@ "default": true }, "coverageAnalysis": { + "$ref": "#/definitions/coverageAnalysis", "description": "Indicates which coverage analysis strategy to use. During mutation testing, stryker will try to only run the tests that cover a particular line of code.\n\n'perTest': Analyse coverage per test.\n'all': Analyse the coverage for the entire test suite.\n'off' (default): Don't use coverage analysis", - "type": "string", - "enum": [ - "off", - "all", - "perTest" - ], "default": "off" }, "clearTextReporter": { diff --git a/packages/api/src/plugin/Contexts.ts b/packages/api/src/plugin/Contexts.ts index 03dcd8a015..dd47342817 100644 --- a/packages/api/src/plugin/Contexts.ts +++ b/packages/api/src/plugin/Contexts.ts @@ -47,6 +47,7 @@ export interface PluginContexts { [PluginKind.Reporter]: OptionsContext; [PluginKind.TestFramework]: OptionsContext; [PluginKind.TestRunner]: TestRunnerPluginContext; + [PluginKind.TestRunner2]: TestRunnerPluginContext; [PluginKind.Transpiler]: TranspilerPluginContext; [PluginKind.Checker]: OptionsContext; } diff --git a/packages/api/src/plugin/PluginKind.ts b/packages/api/src/plugin/PluginKind.ts index 31c577c238..835ad881cf 100644 --- a/packages/api/src/plugin/PluginKind.ts +++ b/packages/api/src/plugin/PluginKind.ts @@ -9,6 +9,7 @@ export enum PluginKind { OptionsEditor = 'OptionsEditor', Checker = 'Checker', TestRunner = 'TestRunner', + TestRunner2 = 'TestRunner2', TestFramework = 'TestFramework', Transpiler = 'Transpiler', Mutator = 'Mutator', diff --git a/packages/api/src/plugin/Plugins.ts b/packages/api/src/plugin/Plugins.ts index 7691a623c6..1c844fadf1 100644 --- a/packages/api/src/plugin/Plugins.ts +++ b/packages/api/src/plugin/Plugins.ts @@ -7,6 +7,7 @@ import { TestFramework } from '../../test_framework'; import { TestRunner } from '../../test_runner'; import { Transpiler } from '../../transpile'; import { OptionsEditor } from '../core/OptionsEditor'; +import { TestRunner2 } from '../../test_runner2'; import { Checker } from '../../check'; import { PluginContexts } from './Contexts'; @@ -90,6 +91,7 @@ export interface PluginInterfaces { [PluginKind.Reporter]: Reporter; [PluginKind.TestFramework]: TestFramework; [PluginKind.TestRunner]: TestRunner; + [PluginKind.TestRunner2]: TestRunner2; [PluginKind.Transpiler]: Transpiler; [PluginKind.Checker]: Checker; } diff --git a/packages/api/src/test_runner2/DryRunResult.ts b/packages/api/src/test_runner2/DryRunResult.ts new file mode 100644 index 0000000000..3750083480 --- /dev/null +++ b/packages/api/src/test_runner2/DryRunResult.ts @@ -0,0 +1,37 @@ +import { MutantCoverage } from './MutantCoverage'; +import { RunStatus } from './RunStatus'; +import { TestResult } from './TestResult'; + +export type DryRunResult = CompleteDryRunResult | TimeoutDryRunResult | ErrorDryRunResult; + +export interface CompleteDryRunResult { + /** + * The individual test results. + */ + tests: TestResult[]; + + mutantCoverage?: MutantCoverage; + + /** + * The status of the run + */ + status: RunStatus.Complete; +} +export interface TimeoutDryRunResult { + /** + * The status of the run + */ + status: RunStatus.Timeout; +} + +export interface ErrorDryRunResult { + /** + * The status of the run + */ + status: RunStatus.Error; + + /** + * If `state` is `error`, this collection should contain the error messages + */ + errorMessage: string; +} diff --git a/packages/api/src/test_runner2/MutantCoverage.ts b/packages/api/src/test_runner2/MutantCoverage.ts new file mode 100644 index 0000000000..2170c27be9 --- /dev/null +++ b/packages/api/src/test_runner2/MutantCoverage.ts @@ -0,0 +1,12 @@ +export interface MutantCoverage { + static: CoverageData; + perTest: CoveragePerTestId; +} + +export interface CoveragePerTestId { + [testId: string]: CoverageData; +} + +export interface CoverageData { + [mutantId: number]: number; +} diff --git a/packages/api/src/test_runner2/MutantRunResult.ts b/packages/api/src/test_runner2/MutantRunResult.ts new file mode 100644 index 0000000000..3713972741 --- /dev/null +++ b/packages/api/src/test_runner2/MutantRunResult.ts @@ -0,0 +1,33 @@ +export enum MutantRunStatus { + Killed = 'killed', + Survived = 'survived', + Timeout = 'timeout', + Error = 'error', +} + +export type MutantRunResult = KilledMutantRunResult | SurvivedMutantRunResult | TimeoutMutantRunResult | ErrorMutantRunResult; + +export interface TimeoutMutantRunResult { + status: MutantRunStatus.Timeout; +} + +export interface KilledMutantRunResult { + status: MutantRunStatus.Killed; + /** + * The id of the test that killed this mutant + */ + killedBy: string; + /** + * The failure message that was reported by the test + */ + failureMessage: string; +} + +export interface SurvivedMutantRunResult { + status: MutantRunStatus.Survived; +} + +export interface ErrorMutantRunResult { + status: MutantRunStatus.Error; + errorMessage: string; +} diff --git a/packages/api/src/test_runner2/RunOptions.ts b/packages/api/src/test_runner2/RunOptions.ts new file mode 100644 index 0000000000..c46b7628c0 --- /dev/null +++ b/packages/api/src/test_runner2/RunOptions.ts @@ -0,0 +1,22 @@ +import { Mutant, CoverageAnalysis } from '../../core'; + +export interface RunOptions { + /** + * The amount of time (in milliseconds) the TestRunner has to complete the test run before a timeout occurs. + */ + timeout: number; +} + +export interface DryRunOptions extends RunOptions { + /** + * Indicates whether or not mutant coverage should be collected. + */ + coverageAnalysis: CoverageAnalysis; +} + +export interface MutantRunOptions extends RunOptions { + testFilter?: string[]; + activeMutant: Mutant; +} + +export default RunOptions; diff --git a/packages/api/src/test_runner2/RunStatus.ts b/packages/api/src/test_runner2/RunStatus.ts new file mode 100644 index 0000000000..b9d8e92b4d --- /dev/null +++ b/packages/api/src/test_runner2/RunStatus.ts @@ -0,0 +1,14 @@ +export enum RunStatus { + /** + * Indicates that a test run is completed with failed or succeeded tests + */ + Complete = 'complete', + /** + * Indicates that a test run cut off early with an error + */ + Error = 'error', + /** + * Indicates that a test run timed out + */ + Timeout = 'timeout', +} diff --git a/packages/api/src/test_runner2/TestResult.ts b/packages/api/src/test_runner2/TestResult.ts new file mode 100644 index 0000000000..046fcf49bd --- /dev/null +++ b/packages/api/src/test_runner2/TestResult.ts @@ -0,0 +1,34 @@ +import { TestStatus } from './TestStatus'; + +/** + * Indicates the result of a single test + */ +interface BaseTestResult { + /** + * The id of this test. Can be the name if the test runner doesn't have an 'id' + */ + id: string; + /** + * The full human readable name of the test + */ + name: string; + /** + * The time it took to run the test + */ + timeSpentMs: number; +} + +export interface FailedTestResult extends BaseTestResult { + status: TestStatus.Failed; + failureMessage: string; +} + +export interface SkippedTestResult extends BaseTestResult { + status: TestStatus.Skipped; +} + +export interface SuccessTestResult extends BaseTestResult { + status: TestStatus.Success; +} + +export type TestResult = SuccessTestResult | FailedTestResult | SkippedTestResult; diff --git a/packages/api/src/test_runner2/TestRunner.ts b/packages/api/src/test_runner2/TestRunner.ts new file mode 100644 index 0000000000..6427bb859a --- /dev/null +++ b/packages/api/src/test_runner2/TestRunner.ts @@ -0,0 +1,10 @@ +import { DryRunOptions, MutantRunOptions } from './RunOptions'; +import { DryRunResult } from './DryRunResult'; +import { MutantRunResult } from './MutantRunResult'; + +export interface TestRunner2 { + init?(): Promise | void; + dryRun(options: DryRunOptions): Promise; + mutantRun(options: MutantRunOptions): Promise; + dispose?(): Promise | void; +} diff --git a/packages/api/src/test_runner2/TestStatus.ts b/packages/api/src/test_runner2/TestStatus.ts new file mode 100644 index 0000000000..a25850dfe2 --- /dev/null +++ b/packages/api/src/test_runner2/TestStatus.ts @@ -0,0 +1,17 @@ +/** + * Indicates what the result of a single test was. + */ +export enum TestStatus { + /** + * The test succeeded + */ + Success, + /** + * The test failed + */ + Failed, + /** + * The test was skipped (not executed) + */ + Skipped, +} diff --git a/packages/api/src/test_runner2/runResultHelpers.ts b/packages/api/src/test_runner2/runResultHelpers.ts new file mode 100644 index 0000000000..41481568a9 --- /dev/null +++ b/packages/api/src/test_runner2/runResultHelpers.ts @@ -0,0 +1,34 @@ +import { TestStatus } from './TestStatus'; +import { DryRunResult } from './DryRunResult'; +import { MutantRunResult } from './MutantRunResult'; +import { RunStatus } from './RunStatus'; +import { MutantRunStatus } from './MutantRunResult'; +import { FailedTestResult } from './TestResult'; + +export function toMutantRunResult(dryRunResult: DryRunResult): MutantRunResult { + switch (dryRunResult.status) { + case RunStatus.Complete: { + const killedBy = dryRunResult.tests.find((test): test is FailedTestResult => test.status === TestStatus.Failed); + if (killedBy) { + return { + status: MutantRunStatus.Killed, + failureMessage: killedBy.failureMessage, + killedBy: killedBy.id, + }; + } else { + return { + status: MutantRunStatus.Survived, + }; + } + } + case RunStatus.Error: + return { + status: MutantRunStatus.Error, + errorMessage: dryRunResult.errorMessage, + }; + case RunStatus.Timeout: + return { + status: MutantRunStatus.Timeout, + }; + } +} diff --git a/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts b/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts new file mode 100644 index 0000000000..9ff08cbdb6 --- /dev/null +++ b/packages/api/test/unit/test_runner2/runResultHelpers.spec.ts @@ -0,0 +1,56 @@ +import { expect } from 'chai'; + +import { toMutantRunResult, RunStatus, MutantRunResult, MutantRunStatus } from '../../../test_runner2'; +import TestStatus from '../../../src/test_runner/TestStatus'; + +describe('runResultHelpers', () => { + describe(toMutantRunResult.name, () => { + it('should convert "timeout" to "timeout"', () => { + const expected: MutantRunResult = { status: MutantRunStatus.Timeout }; + expect(toMutantRunResult({ status: RunStatus.Timeout })).deep.eq(expected); + }); + + it('should convert "error" to "error"', () => { + const expected: MutantRunResult = { status: MutantRunStatus.Error, errorMessage: 'some error' }; + expect(toMutantRunResult({ status: RunStatus.Error, errorMessage: 'some error' })).deep.eq(expected); + }); + + it('should report a failed test as "killed"', () => { + const expected: MutantRunResult = { status: MutantRunStatus.Killed, failureMessage: 'expected foo to be bar', killedBy: '42' }; + expect( + toMutantRunResult({ + status: RunStatus.Complete, + tests: [ + { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 }, + { status: TestStatus.Failed, id: '42', name: 'error', timeSpentMs: 42, failureMessage: 'expected foo to be bar' }, + { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 }, + ], + }) + ).deep.eq(expected); + }); + + it('should report only succeeded tests as "survived"', () => { + const expected: MutantRunResult = { status: MutantRunStatus.Survived }; + expect( + toMutantRunResult({ + status: RunStatus.Complete, + tests: [ + { status: TestStatus.Success, id: 'success1', name: 'success1', timeSpentMs: 42 }, + { status: TestStatus.Success, id: '42', name: 'error', timeSpentMs: 42 }, + { status: TestStatus.Success, id: 'success2', name: 'success2', timeSpentMs: 42 }, + ], + }) + ).deep.eq(expected); + }); + + it('should report an empty suite as "survived"', () => { + const expected: MutantRunResult = { status: MutantRunStatus.Survived }; + expect( + toMutantRunResult({ + status: RunStatus.Complete, + tests: [], + }) + ).deep.eq(expected); + }); + }); +}); diff --git a/packages/api/test_runner2.ts b/packages/api/test_runner2.ts new file mode 100644 index 0000000000..2ce27088c7 --- /dev/null +++ b/packages/api/test_runner2.ts @@ -0,0 +1,9 @@ +export * from './src/test_runner2/TestResult'; +export * from './src/test_runner2/TestRunner'; +export * from './src/test_runner2/TestStatus'; +export * from './src/test_runner2/DryRunResult'; +export * from './src/test_runner2/RunOptions'; +export * from './src/test_runner2/MutantCoverage'; +export * from './src/test_runner2/MutantRunResult'; +export * from './src/test_runner2/RunStatus'; +export * from './src/test_runner2/runResultHelpers'; diff --git a/packages/api/tsconfig.src.json b/packages/api/tsconfig.src.json index f83ad52656..0625b93784 100644 --- a/packages/api/tsconfig.src.json +++ b/packages/api/tsconfig.src.json @@ -15,6 +15,7 @@ "report.ts", "test_framework.ts", "test_runner.ts", + "test_runner2.ts", "transpile.ts", "plugin.ts" ], diff --git a/tsconfig.lint.json b/tsconfig.lint.json index 91aa3ba6b6..f500c4bdde 100644 --- a/tsconfig.lint.json +++ b/tsconfig.lint.json @@ -1,9 +1,19 @@ { "extends": "./tsconfig.settings.json", - // This file is a workaround for: https://github.com/palantir/tslint/issues/4137 + // This file is a workaround for: https://github.com/typescript-eslint/typescript-eslint/tree/master/packages/parser#parseroptionsproject "include": [ "packages/*/src", "packages/*/typings", - "packages/*/test" + "packages/*/test", + "packages/api/config.ts", + "packages/api/core.ts", + "packages/api/logging.ts", + "packages/api/mutant.ts", + "packages/api/report.ts", + "packages/api/test_framework.ts", + "packages/api/test_runner.ts", + "packages/api/test_runner2.ts", + "packages/api/transpile.ts", + "packages/api/plugin.ts" ] }