From 963f5cf4b89ceb7a565f3141b364a4bf3994a0d8 Mon Sep 17 00:00:00 2001 From: Mark Wubben Date: Sat, 10 Feb 2018 14:39:14 +0000 Subject: [PATCH] Refactor how tests are actually run Fixes #1684. Fixes #1416. Refs #1158. Properly support serial hooks. Hooks are divided into the following categories: * before * beforeEach * afterEach * afterEach.always * after * after.always For each category all hooks are run in the order they're declared in. This is different from tests, where serial tests are run before concurrent ones. By default hooks run concurrently. However a serial hook is not run before all preceding concurrent hooks have completed. Serial hooks are never run concurrently. Always hooks are now always run, even if --fail-fast is enabled. Internally, TestCollection, Sequence and Concurrent have been removed. This has led to a major refactor of Runner, and some smaller breaking changes and bug fixes: * Unnecessary validations have been removed * Macros can be used with hooks * A macro is recognized even if no additional arguments are given, so it can modify the test (or hook) title * --match now also matches todo tests * Skipped and todo tests are shown first in the output * --fail-fast prevents subsequent tests from running as long as the failure occurs in a serial test --- index.d.ts | 4 + index.js.flow | 4 + lib/concurrent.js | 64 ---- lib/context-ref.js | 41 +++ lib/create-chain.js | 108 ++++++ lib/main.js | 6 +- lib/runner.js | 622 ++++++++++++++++++------------- lib/sequence.js | 94 ----- lib/test-collection.js | 250 ------------- lib/test-worker.js | 79 +++- lib/test.js | 30 +- lib/validate-test.js | 48 --- profile.js | 8 +- readme.md | 32 +- test/api.js | 5 +- test/beautify-stack.js | 16 +- test/concurrent.js | 790 ---------------------------------------- test/hooks.js | 28 +- test/observable.js | 158 +++----- test/promise.js | 314 ++++++---------- test/runner.js | 560 ++++++++++++++++++---------- test/sequence.js | 780 --------------------------------------- test/test-collection.js | 381 ------------------- test/test.js | 661 +++++++++++++-------------------- 24 files changed, 1456 insertions(+), 3627 deletions(-) delete mode 100644 lib/concurrent.js create mode 100644 lib/context-ref.js create mode 100644 lib/create-chain.js delete mode 100644 lib/sequence.js delete mode 100644 lib/test-collection.js delete mode 100644 lib/validate-test.js delete mode 100644 test/concurrent.js delete mode 100644 test/sequence.js delete mode 100644 test/test-collection.js diff --git a/index.d.ts b/index.d.ts index a9580e251..72c4a9e1b 100644 --- a/index.d.ts +++ b/index.d.ts @@ -168,6 +168,10 @@ export interface SerialInterface { (title: string, macro: Macro | Macro[], ...args: Array): void; (macro: Macro | Macro[], ...args: Array): void; + after: AfterInterface; + afterEach: AfterInterface; + before: BeforeInterface; + beforeEach: BeforeInterface; cb: CbInterface; failing: FailingInterface; only: OnlyInterface; diff --git a/index.js.flow b/index.js.flow index 42a578e6e..6c460f9c6 100644 --- a/index.js.flow +++ b/index.js.flow @@ -171,6 +171,10 @@ export interface SerialInterface { (title: string, macro: Macro | Macro[], ...args: Array): void; (macro: Macro | Macro[], ...args: Array): void; + after: AfterInterface; + afterEach: AfterInterface; + before: BeforeInterface; + beforeEach: BeforeInterface; cb: CbInterface; failing: FailingInterface; only: OnlyInterface; diff --git a/lib/concurrent.js b/lib/concurrent.js deleted file mode 100644 index 3cdbb41c3..000000000 --- a/lib/concurrent.js +++ /dev/null @@ -1,64 +0,0 @@ -'use strict'; - -class Concurrent { - constructor(runnables, bail) { - if (!Array.isArray(runnables)) { - throw new TypeError('Expected an array of runnables'); - } - - this.runnables = runnables; - this.bail = bail || false; - } - - run() { - let allPassed = true; - - let pending; - let rejectPending; - let resolvePending; - const allPromises = []; - const handlePromise = promise => { - if (!pending) { - pending = new Promise((resolve, reject) => { - rejectPending = reject; - resolvePending = resolve; - }); - } - - allPromises.push(promise.then(passed => { - if (!passed) { - allPassed = false; - - if (this.bail) { - // Stop if the test failed and bail mode is on. - resolvePending(); - } - } - }, rejectPending)); - }; - - for (const runnable of this.runnables) { - const passedOrPromise = runnable.run(); - - if (!passedOrPromise) { - if (this.bail) { - // Stop if the test failed and bail mode is on. - return false; - } - - allPassed = false; - } else if (passedOrPromise !== true) { - handlePromise(passedOrPromise); - } - } - - if (pending) { - Promise.all(allPromises).then(resolvePending); - return pending.then(() => allPassed); - } - - return allPassed; - } -} - -module.exports = Concurrent; diff --git a/lib/context-ref.js b/lib/context-ref.js new file mode 100644 index 000000000..5529bfc3d --- /dev/null +++ b/lib/context-ref.js @@ -0,0 +1,41 @@ +'use strict'; +const clone = require('lodash.clone'); + +class ContextRef { + constructor() { + this.value = {}; + } + + get() { + return this.value; + } + + set(newValue) { + this.value = newValue; + } + + copy() { + return new LateBinding(this); // eslint-disable-line no-use-before-define + } +} +module.exports = ContextRef; + +class LateBinding extends ContextRef { + constructor(ref) { + super(); + this.ref = ref; + this.bound = false; + } + + get() { + if (!this.bound) { + this.set(clone(this.ref.get())); + } + return super.get(); + } + + set(newValue) { + this.bound = true; + super.set(newValue); + } +} diff --git a/lib/create-chain.js b/lib/create-chain.js new file mode 100644 index 000000000..c303b9a59 --- /dev/null +++ b/lib/create-chain.js @@ -0,0 +1,108 @@ +'use strict'; +const chainRegistry = new WeakMap(); + +function startChain(name, call, defaults) { + const fn = function () { + call(Object.assign({}, defaults), Array.from(arguments)); + }; + Object.defineProperty(fn, 'name', {value: name}); + chainRegistry.set(fn, {call, defaults, fullName: name}); + return fn; +} + +function extendChain(prev, name, flag) { + if (!flag) { + flag = name; + } + + const fn = function () { + callWithFlag(prev, flag, Array.from(arguments)); + }; + const fullName = `${chainRegistry.get(prev).fullName}.${name}`; + Object.defineProperty(fn, 'name', {value: fullName}); + prev[name] = fn; + + chainRegistry.set(fn, {flag, fullName, prev}); + return fn; +} + +function callWithFlag(prev, flag, args) { + const combinedFlags = {[flag]: true}; + do { + const step = chainRegistry.get(prev); + if (step.call) { + step.call(Object.assign({}, step.defaults, combinedFlags), args); + prev = null; + } else { + combinedFlags[step.flag] = true; + prev = step.prev; + } + } while (prev); +} + +function createHookChain(hook, isAfterHook) { + // Hook chaining rules: + // * `always` comes immediately after "after hooks" + // * `skip` must come at the end + // * no `only` + // * no repeating + extendChain(hook, 'cb', 'callback'); + extendChain(hook, 'skip', 'skipped'); + extendChain(hook.cb, 'skip', 'skipped'); + if (isAfterHook) { + extendChain(hook, 'always'); + extendChain(hook.always, 'cb', 'callback'); + extendChain(hook.always, 'skip', 'skipped'); + extendChain(hook.always.cb, 'skip', 'skipped'); + } + return hook; +} + +function createChain(fn, defaults) { + // Test chaining rules: + // * `serial` must come at the start + // * `only` and `skip` must come at the end + // * `failing` must come at the end, but can be followed by `only` and `skip` + // * `only` and `skip` cannot be chained together + // * no repeating + const root = startChain('test', fn, Object.assign({}, defaults, {type: 'test'})); + extendChain(root, 'cb', 'callback'); + extendChain(root, 'failing'); + extendChain(root, 'only', 'exclusive'); + extendChain(root, 'serial'); + extendChain(root, 'skip', 'skipped'); + extendChain(root.cb, 'failing'); + extendChain(root.cb, 'only', 'exclusive'); + extendChain(root.cb, 'skip', 'skipped'); + extendChain(root.cb.failing, 'only', 'exclusive'); + extendChain(root.cb.failing, 'skip', 'skipped'); + extendChain(root.failing, 'only', 'exclusive'); + extendChain(root.failing, 'skip', 'skipped'); + extendChain(root.serial, 'cb', 'callback'); + extendChain(root.serial, 'failing'); + extendChain(root.serial, 'only', 'exclusive'); + extendChain(root.serial, 'skip', 'skipped'); + extendChain(root.serial.cb, 'failing'); + extendChain(root.serial.cb, 'only', 'exclusive'); + extendChain(root.serial.cb, 'skip', 'skipped'); + extendChain(root.serial.cb.failing, 'only', 'exclusive'); + extendChain(root.serial.cb.failing, 'skip', 'skipped'); + + root.after = createHookChain(startChain('test.after', fn, Object.assign({}, defaults, {type: 'after'})), true); + root.afterEach = createHookChain(startChain('test.afterEach', fn, Object.assign({}, defaults, {type: 'afterEach'})), true); + root.before = createHookChain(startChain('test.before', fn, Object.assign({}, defaults, {type: 'before'})), false); + root.beforeEach = createHookChain(startChain('test.beforeEach', fn, Object.assign({}, defaults, {type: 'beforeEach'})), false); + + root.serial.after = createHookChain(startChain('test.after', fn, Object.assign({}, defaults, {serial: true, type: 'after'})), true); + root.serial.afterEach = createHookChain(startChain('test.afterEach', fn, Object.assign({}, defaults, {serial: true, type: 'afterEach'})), true); + root.serial.before = createHookChain(startChain('test.before', fn, Object.assign({}, defaults, {serial: true, type: 'before'})), false); + root.serial.beforeEach = createHookChain(startChain('test.beforeEach', fn, Object.assign({}, defaults, {serial: true, type: 'beforeEach'})), false); + + // "todo" tests cannot be chained. Allow todo tests to be flagged as needing + // to be serial. + root.todo = startChain('test.todo', fn, Object.assign({}, defaults, {type: 'test', todo: true})); + root.serial.todo = startChain('test.serial.todo', fn, Object.assign({}, defaults, {serial: true, type: 'test', todo: true})); + + return root; +} +module.exports = createChain; diff --git a/lib/main.js b/lib/main.js index 956648c58..0b7c325ab 100644 --- a/lib/main.js +++ b/lib/main.js @@ -4,15 +4,15 @@ const Runner = require('./runner'); const opts = require('./worker-options').get(); const runner = new Runner({ - bail: opts.failFast, + failFast: opts.failFast, failWithoutAssertions: opts.failWithoutAssertions, file: opts.file, match: opts.match, projectDir: opts.projectDir, + runOnlyExclusive: opts.runOnlyExclusive, serial: opts.serial, - updateSnapshots: opts.updateSnapshots, snapshotDir: opts.snapshotDir, - runOnlyExclusive: opts.runOnlyExclusive + updateSnapshots: opts.updateSnapshots }); worker.setRunner(runner); diff --git a/lib/runner.js b/lib/runner.js index bc4b6e85e..23b47eb75 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -1,275 +1,161 @@ 'use strict'; const EventEmitter = require('events'); const path = require('path'); -const Bluebird = require('bluebird'); const matcher = require('matcher'); +const ContextRef = require('./context-ref'); +const createChain = require('./create-chain'); const snapshotManager = require('./snapshot-manager'); -const TestCollection = require('./test-collection'); -const validateTest = require('./validate-test'); - -const chainRegistry = new WeakMap(); - -function startChain(name, call, defaults) { - const fn = function () { - call(Object.assign({}, defaults), Array.from(arguments)); - }; - Object.defineProperty(fn, 'name', {value: name}); - chainRegistry.set(fn, {call, defaults, fullName: name}); - return fn; -} - -function extendChain(prev, name, flag) { - if (!flag) { - flag = name; - } - - const fn = function () { - callWithFlag(prev, flag, Array.from(arguments)); - }; - const fullName = `${chainRegistry.get(prev).fullName}.${name}`; - Object.defineProperty(fn, 'name', {value: fullName}); - prev[name] = fn; - - chainRegistry.set(fn, {flag, fullName, prev}); - return fn; -} - -function callWithFlag(prev, flag, args) { - const combinedFlags = {[flag]: true}; - do { - const step = chainRegistry.get(prev); - if (step.call) { - step.call(Object.assign({}, step.defaults, combinedFlags), args); - prev = null; - } else { - combinedFlags[step.flag] = true; - prev = step.prev; - } - } while (prev); -} - -function createHookChain(hook, isAfterHook) { - // Hook chaining rules: - // * `always` comes immediately after "after hooks" - // * `skip` must come at the end - // * no `only` - // * no repeating - extendChain(hook, 'cb', 'callback'); - extendChain(hook, 'skip', 'skipped'); - extendChain(hook.cb, 'skip', 'skipped'); - if (isAfterHook) { - extendChain(hook, 'always'); - extendChain(hook.always, 'cb', 'callback'); - extendChain(hook.always, 'skip', 'skipped'); - extendChain(hook.always.cb, 'skip', 'skipped'); - } - return hook; -} - -function createChain(fn, defaults) { - // Test chaining rules: - // * `serial` must come at the start - // * `only` and `skip` must come at the end - // * `failing` must come at the end, but can be followed by `only` and `skip` - // * `only` and `skip` cannot be chained together - // * no repeating - const root = startChain('test', fn, Object.assign({}, defaults, {type: 'test'})); - extendChain(root, 'cb', 'callback'); - extendChain(root, 'failing'); - extendChain(root, 'only', 'exclusive'); - extendChain(root, 'serial'); - extendChain(root, 'skip', 'skipped'); - extendChain(root.cb, 'failing'); - extendChain(root.cb, 'only', 'exclusive'); - extendChain(root.cb, 'skip', 'skipped'); - extendChain(root.cb.failing, 'only', 'exclusive'); - extendChain(root.cb.failing, 'skip', 'skipped'); - extendChain(root.failing, 'only', 'exclusive'); - extendChain(root.failing, 'skip', 'skipped'); - extendChain(root.serial, 'cb', 'callback'); - extendChain(root.serial, 'failing'); - extendChain(root.serial, 'only', 'exclusive'); - extendChain(root.serial, 'skip', 'skipped'); - extendChain(root.serial.cb, 'failing'); - extendChain(root.serial.cb, 'only', 'exclusive'); - extendChain(root.serial.cb, 'skip', 'skipped'); - extendChain(root.serial.cb.failing, 'only', 'exclusive'); - extendChain(root.serial.cb.failing, 'skip', 'skipped'); - - root.after = createHookChain(startChain('test.after', fn, Object.assign({}, defaults, {type: 'after'})), true); - root.afterEach = createHookChain(startChain('test.afterEach', fn, Object.assign({}, defaults, {type: 'afterEach'})), true); - root.before = createHookChain(startChain('test.before', fn, Object.assign({}, defaults, {type: 'before'})), false); - root.beforeEach = createHookChain(startChain('test.beforeEach', fn, Object.assign({}, defaults, {type: 'beforeEach'})), false); - - // Todo tests cannot be chained. Allow todo tests to be flagged as needing to - // be serial. - root.todo = startChain('test.todo', fn, Object.assign({}, defaults, {type: 'test', todo: true})); - root.serial.todo = startChain('test.serial.todo', fn, Object.assign({}, defaults, {serial: true, type: 'test', todo: true})); - - return root; -} - -function wrapFunction(fn, args) { - return function (t) { - return fn.apply(this, [t].concat(args)); - }; -} +const Runnable = require('./test'); class Runner extends EventEmitter { constructor(options) { super(); options = options || {}; - + this.failFast = options.failFast === true; + this.failWithoutAssertions = options.failWithoutAssertions !== false; this.file = options.file; this.match = options.match || []; this.projectDir = options.projectDir; - this.serial = options.serial; - this.updateSnapshots = options.updateSnapshots; + this.runOnlyExclusive = options.runOnlyExclusive === true; + this.serial = options.serial === true; this.snapshotDir = options.snapshotDir; - this.runOnlyExclusive = options.runOnlyExclusive; + this.updateSnapshots = options.updateSnapshots; - this.hasStarted = false; - this.results = []; + this.activeRunnables = new Set(); + this.boundCompareTestSnapshot = this.compareTestSnapshot.bind(this); this.snapshots = null; - this.tests = new TestCollection({ - bail: options.bail, - failWithoutAssertions: options.failWithoutAssertions, - compareTestSnapshot: this.compareTestSnapshot.bind(this) - }); - - this.chain = createChain((opts, args) => { - let title; - let fn; - let macroArgIndex; - - if (this.hasStarted) { - throw new Error('All tests and hooks must be declared synchronously in your ' + - 'test file, and cannot be nested within other tests or hooks.'); - } - - if (typeof args[0] === 'string') { - title = args[0]; - fn = args[1]; - macroArgIndex = 2; - } else { - fn = args[0]; - title = null; - macroArgIndex = 1; - } - - if (this.serial) { - opts.serial = true; - } + this.stats = { + failCount: 0, + failedHookCount: 0, + hasExclusive: false, + knownFailureCount: 0, + passCount: 0, + skipCount: 0, + testCount: 0, + todoCount: 0 + }; + this.tasks = { + after: [], + afterAlways: [], + afterEach: [], + afterEachAlways: [], + before: [], + beforeEach: [], + concurrent: [], + serial: [], + todo: [] + }; - if (args.length > macroArgIndex) { - args = args.slice(macroArgIndex); - } else { - args = null; + const uniqueTestTitles = new Set(); + let hasStarted = false; + let scheduledStart = false; + this.chain = createChain((metadata, args) => { // eslint-disable-line complexity + if (hasStarted) { + throw new Error('All tests and hooks must be declared synchronously in your test file, and cannot be nested within other tests or hooks.'); } - - if (Array.isArray(fn)) { - fn.forEach(fn => { - this.addTest(title, opts, fn, args); + if (!scheduledStart) { + scheduledStart = true; + process.nextTick(() => { + hasStarted = true; + this.start(); }); - } else { - this.addTest(title, opts, fn, args); } - }, { - serial: false, - exclusive: false, - skipped: false, - todo: false, - failing: false, - callback: false, - always: false - }); - } - addTest(title, metadata, fn, args) { - if (args) { - if (fn.title) { - title = fn.title.apply(fn, [title || ''].concat(args)); - } + const specifiedTitle = typeof args[0] === 'string' ? + args.shift() : + ''; + const implementations = Array.isArray(args[0]) ? + args.shift() : + args.splice(0, 1); - fn = wrapFunction(fn, args); - } + if (metadata.todo) { + if (implementations.length > 0) { + throw new TypeError('`todo` tests are not allowed to have an implementation. Use `test.skip()` for tests with an implementation.'); + } - if (metadata.type === 'test' && this.match.length > 0) { - metadata.exclusive = matcher([title || ''], this.match).length === 1; - } + if (specifiedTitle === '') { + throw new TypeError('`todo` tests require a title'); + } - const validationError = validateTest(title, fn, metadata); - if (validationError !== null) { - throw new TypeError(validationError); - } + if (uniqueTestTitles.has(specifiedTitle)) { + throw new Error(`Duplicate test title: ${specifiedTitle}`); + } else { + uniqueTestTitles.add(specifiedTitle); + } - this.tests.add({ - metadata, - fn, - title - }); + if (this.match.length > 0) { + // --match selects TODO tests. + if (matcher([specifiedTitle], this.match).length === 1) { + metadata.exclusive = true; + this.stats.hasExclusive = true; + } + } - if (!this.scheduledStart) { - this.scheduledStart = true; - process.nextTick(() => { - this.emit('start', this._run()); - }); - } - } + this.tasks.todo.push({title: specifiedTitle, metadata}); + } else { + if (implementations.length === 0) { + throw new TypeError('Expected an implementation. Use `test.todo()` for tests without an implementation.'); + } - addTestResult(result) { - const test = result.result; - const props = { - logs: test.logs, - duration: test.duration, - title: test.title, - error: result.reason, - type: test.metadata.type, - skip: test.metadata.skipped, - todo: test.metadata.todo, - failing: test.metadata.failing - }; + for (const implementation of implementations) { + let title = implementation.title ? + implementation.title.apply(implementation, [specifiedTitle].concat(args)) : + specifiedTitle; - this.results.push(result); - this.emit('test', props); - } + if (typeof title !== 'string') { + throw new TypeError('Test & hook titles must be strings'); + } - buildStats() { - const stats = { - failCount: 0, - knownFailureCount: 0, - passCount: 0, - skipCount: 0, - testCount: 0, - todoCount: 0 - }; + if (title === '') { + if (metadata.type === 'test') { + throw new TypeError('Tests must have a title'); + } else if (metadata.always) { + title = `${metadata.type}.always hook`; + } else { + title = `${metadata.type} hook`; + } + } - for (const result of this.results) { - if (!result.passed) { - // Includes hooks - stats.failCount++; - } + if (metadata.type === 'test') { + if (uniqueTestTitles.has(title)) { + throw new Error(`Duplicate test title: ${title}`); + } else { + uniqueTestTitles.add(title); + } + } - const metadata = result.result.metadata; - if (metadata.type === 'test') { - stats.testCount++; - - if (metadata.skipped) { - stats.skipCount++; - } else if (metadata.todo) { - stats.todoCount++; - } else if (result.passed) { - if (metadata.failing) { - stats.knownFailureCount++; - } else { - stats.passCount++; + const task = { + title, + implementation, + args, + metadata: Object.assign({}, metadata) + }; + + if (metadata.type === 'test') { + if (this.match.length > 0) { + // --match overrides .only() + task.metadata.exclusive = matcher([title], this.match).length === 1; + } + if (task.metadata.exclusive) { + this.stats.hasExclusive = true; + } + + this.tasks[metadata.serial ? 'serial' : 'concurrent'].push(task); + } else if (!metadata.skipped) { + this.tasks[metadata.type + (metadata.always ? 'Always' : '')].push(task); } } } - } - - return stats; + }, { + serial: false, + exclusive: false, + skipped: false, + todo: false, + failing: false, + callback: false, + always: false + }); } compareTestSnapshot(options) { @@ -302,21 +188,265 @@ class Runner extends EventEmitter { } } - _run() { - this.hasStarted = true; + onRun(runnable) { + this.activeRunnables.add(runnable); + } + + onRunComplete(runnable) { + this.activeRunnables.delete(runnable); + } + + attributeLeakedError(err) { + for (const runnable of this.activeRunnables) { + if (runnable.attributeLeakedError(err)) { + return true; + } + } + return false; + } - if (this.runOnlyExclusive && !this.tests.hasExclusive) { - return Promise.resolve(null); + beforeExitHandler() { + for (const runnable of this.activeRunnables) { + runnable.finishDueToInactivity(); } + } + + runMultiple(runnables) { + let allPassed = true; + const storedResults = []; + const runAndStoreResult = runnable => { + return this.runSingle(runnable).then(result => { + if (!result.passed) { + allPassed = false; + } + storedResults.push(result); + }); + }; + + let waitForSerial = Promise.resolve(); + return runnables.reduce((prev, runnable) => { + if (runnable.metadata.serial || this.serial) { + waitForSerial = prev.then(() => { + // Serial runnables run as long as there was no previous failure, unless + // the runnable should always be run. + return (allPassed || runnable.metadata.always) && runAndStoreResult(runnable); + }); + return waitForSerial; + } + + return Promise.all([ + prev, + waitForSerial.then(() => { + // Concurrent runnables are kicked off after the previous serial + // runnables have completed, as long as there was no previous failure + // (or if the runnable should always be run). One concurrent runnable's + // failure does not prevent the next runnable from running. + return (allPassed || runnable.metadata.always) && runAndStoreResult(runnable); + }) + ]); + }, waitForSerial).then(() => ({allPassed, storedResults})); + } - this.tests.on('test', result => { - this.addTestResult(result); + runSingle(runnable) { + this.onRun(runnable); + return runnable.run().then(result => { + // If run() throws or rejects then the entire test run crashes, so + // onRunComplete() doesn't *have* to be inside a finally(). + this.onRunComplete(runnable); + return result; }); - return Bluebird.try(() => this.tests.build().run()); } - attributeLeakedError(err) { - return this.tests.attributeLeakedError(err); + runHooks(tasks, contextRef, titleSuffix) { + const hooks = tasks.map(task => new Runnable({ + contextRef, + failWithoutAssertions: false, + fn: task.args.length === 0 ? + task.implementation : + t => task.implementation.apply(null, [t].concat(task.args)), + compareTestSnapshot: this.boundCompareTestSnapshot, + metadata: task.metadata, + title: `${task.title}${titleSuffix || ''}` + })); + return this.runMultiple(hooks, this.serial).then(outcome => { + if (outcome.allPassed) { + return true; + } + + // Only emit results for failed hooks. + for (const result of outcome.storedResults) { + if (!result.passed) { + this.stats.failedHookCount++; + this.emit('hook-failed', result); + } + } + return false; + }); + } + + runTest(task, contextRef) { + const hookSuffix = ` for ${task.title}`; + return this.runHooks(this.tasks.beforeEach, contextRef, hookSuffix).then(hooksOk => { + // Don't run the test if a `beforeEach` hook failed. + if (!hooksOk) { + return false; + } + + const test = new Runnable({ + contextRef, + failWithoutAssertions: this.failWithoutAssertions, + fn: task.args.length === 0 ? + task.implementation : + t => task.implementation.apply(null, [t].concat(task.args)), + compareTestSnapshot: this.boundCompareTestSnapshot, + metadata: task.metadata, + title: task.title + }); + return this.runSingle(test).then(result => { + if (!result.passed) { + this.stats.failCount++; + this.emit('test', result); + // Don't run `afterEach` hooks if the test failed. + return false; + } + + if (result.metadata.failing) { + this.stats.knownFailureCount++; + } else { + this.stats.passCount++; + } + this.emit('test', result); + return this.runHooks(this.tasks.afterEach, contextRef, hookSuffix); + }); + }).then(hooksAndTestOk => { + return this.runHooks(this.tasks.afterEachAlways, contextRef, hookSuffix).then(alwaysOk => { + return hooksAndTestOk && alwaysOk; + }); + }); + } + + start() { + const runOnlyExclusive = this.stats.hasExclusive || this.runOnlyExclusive; + + const todoTitles = []; + for (const task of this.tasks.todo) { + if (runOnlyExclusive && !task.metadata.exclusive) { + continue; + } + + this.stats.testCount++; + this.stats.todoCount++; + todoTitles.push(task.title); + } + + const concurrentTests = []; + const serialTests = []; + const skippedTests = []; + for (const task of this.tasks.serial) { + if (runOnlyExclusive && !task.metadata.exclusive) { + continue; + } + + this.stats.testCount++; + if (task.metadata.skipped) { + this.stats.skipCount++; + skippedTests.push({ + failing: task.metadata.failing, + title: task.title + }); + } else { + serialTests.push(task); + } + } + for (const task of this.tasks.concurrent) { + if (runOnlyExclusive && !task.metadata.exclusive) { + continue; + } + + this.stats.testCount++; + if (task.metadata.skipped) { + this.stats.skipCount++; + skippedTests.push({ + failing: task.metadata.failing, + title: task.title + }); + } else if (this.serial) { + serialTests.push(task); + } else { + concurrentTests.push(task); + } + } + + if (concurrentTests.length === 0 && serialTests.length === 0) { + this.emit('start', { + // `ended` is always resolved with `undefined`. + ended: Promise.resolve(undefined), + skippedTests, + stats: this.stats, + todoTitles + }); + // Don't run any hooks if there are no tests to run. + return; + } + + const contextRef = new ContextRef(); + + // Note that the hooks and tests always begin running asynchronously. + const beforePromise = this.runHooks(this.tasks.before, contextRef); + const serialPromise = beforePromise.then(beforeHooksOk => { + // Don't run tests if a `before` hook failed. + if (!beforeHooksOk) { + return false; + } + + return serialTests.reduce((prev, task) => { + return prev.then(prevOk => { + // Prevent subsequent tests from running if `failFast` is enabled and + // the previous test failed. + if (!prevOk && this.failFast) { + return false; + } + + return this.runTest(task, contextRef.copy()); + }); + }, Promise.resolve(true)); + }); + const concurrentPromise = Promise.all([beforePromise, serialPromise]).then(prevOkays => { + const beforeHooksOk = prevOkays[0]; + const serialOk = prevOkays[1]; + // Don't run tests if a `before` hook failed, or if `failFast` is enabled + // and a previous serial test failed. + if (!beforeHooksOk || (!serialOk && this.failFast)) { + return false; + } + + // If a concurrent test fails, even if `failFast` is enabled it won't + // stop other concurrent tests from running. + return Promise.all(concurrentTests.map(task => { + return this.runTest(task, contextRef.copy()); + })).then(allOkays => allOkays.every(ok => ok)); + }); + + const beforeExitHandler = this.beforeExitHandler.bind(this); + process.on('beforeExit', beforeExitHandler); + + const ended = concurrentPromise + // Only run `after` hooks if all hooks and tests passed. + .then(ok => ok && this.runHooks(this.tasks.after, contextRef)) + // Always run `after.always` hooks. + .then(() => this.runHooks(this.tasks.afterAlways, contextRef)) + .then(() => { + process.removeListener('beforeExit', beforeExitHandler); + // `ended` is always resolved with `undefined`. + return undefined; + }); + + this.emit('start', { + ended, + skippedTests, + stats: this.stats, + todoTitles + }); } } diff --git a/lib/sequence.js b/lib/sequence.js deleted file mode 100644 index 1e5960a98..000000000 --- a/lib/sequence.js +++ /dev/null @@ -1,94 +0,0 @@ -'use strict'; - -const beforeExitSubscribers = new Set(); -const beforeExitHandler = () => { - for (const subscriber of beforeExitSubscribers) { - subscriber(); - } -}; -const onBeforeExit = subscriber => { - if (beforeExitSubscribers.size === 0) { - // Only listen for the event once, no matter how many Sequences are run - // concurrently. - process.on('beforeExit', beforeExitHandler); - } - - beforeExitSubscribers.add(subscriber); - return { - dispose() { - beforeExitSubscribers.delete(subscriber); - if (beforeExitSubscribers.size === 0) { - process.removeListener('beforeExit', beforeExitHandler); - } - } - }; -}; - -class Sequence { - constructor(runnables, bail) { - if (!Array.isArray(runnables)) { - throw new TypeError('Expected an array of runnables'); - } - - this.runnables = runnables; - this.bail = bail || false; - } - - run() { - const iterator = this.runnables[Symbol.iterator](); - - let activeRunnable; - const beforeExit = onBeforeExit(() => { - if (activeRunnable.finishDueToInactivity) { - activeRunnable.finishDueToInactivity(); - } - }); - - let allPassed = true; - const finish = () => { - beforeExit.dispose(); - return allPassed; - }; - - const runNext = () => { - let promise; - - for (let next = iterator.next(); !next.done; next = iterator.next()) { - activeRunnable = next.value; - const passedOrPromise = activeRunnable.run(); - if (!passedOrPromise) { - allPassed = false; - - if (this.bail) { - // Stop if the test failed and bail mode is on. - break; - } - } else if (passedOrPromise !== true) { - promise = passedOrPromise; - break; - } - } - - if (!promise) { - return finish(); - } - - return promise.then(passed => { - if (!passed) { - allPassed = false; - - if (this.bail) { - // Stop if the test failed and bail mode is on. - return finish(); - } - } - - return runNext(); - }); - }; - - return runNext(); - } -} - -module.exports = Sequence; diff --git a/lib/test-collection.js b/lib/test-collection.js deleted file mode 100644 index 0569e88e6..000000000 --- a/lib/test-collection.js +++ /dev/null @@ -1,250 +0,0 @@ -'use strict'; -const EventEmitter = require('events'); -const clone = require('lodash.clone'); -const Concurrent = require('./concurrent'); -const Sequence = require('./sequence'); -const Test = require('./test'); - -class ContextRef { - constructor() { - this.value = {}; - } - - get() { - return this.value; - } - - set(newValue) { - this.value = newValue; - } - - copy() { - return new LateBinding(this); // eslint-disable-line no-use-before-define - } -} - -class LateBinding extends ContextRef { - constructor(ref) { - super(); - this.ref = ref; - this.bound = false; - } - - get() { - if (!this.bound) { - this.set(clone(this.ref.get())); - } - return super.get(); - } - - set(newValue) { - this.bound = true; - super.set(newValue); - } -} - -class TestCollection extends EventEmitter { - constructor(options) { - super(); - - this.bail = options.bail; - this.failWithoutAssertions = options.failWithoutAssertions; - this.compareTestSnapshot = options.compareTestSnapshot; - this.hasExclusive = false; - this.testCount = 0; - - this.tests = { - concurrent: [], - serial: [] - }; - - this.hooks = { - before: [], - beforeEach: [], - after: [], - afterAlways: [], - afterEach: [], - afterEachAlways: [] - }; - - this.pendingTestInstances = new Set(); - this.uniqueTestTitles = new Set(); - - this._emitTestResult = this._emitTestResult.bind(this); - } - - add(test) { - const metadata = test.metadata; - const type = metadata.type; - - if (test.title === '' || typeof test.title !== 'string') { - if (type === 'test') { - throw new TypeError('Tests must have a title'); - } else { - test.title = `${type} hook`; - } - } - - if (type === 'test') { - if (this.uniqueTestTitles.has(test.title)) { - throw new Error(`Duplicate test title: ${test.title}`); - } else { - this.uniqueTestTitles.add(test.title); - } - } - - // Add a hook - if (type !== 'test') { - this.hooks[type + (metadata.always ? 'Always' : '')].push(test); - return; - } - - this.testCount++; - - // Add `.only()` tests if `.only()` was used previously - if (this.hasExclusive && !metadata.exclusive) { - return; - } - - if (metadata.exclusive && !this.hasExclusive) { - this.tests.concurrent = []; - this.tests.serial = []; - this.hasExclusive = true; - } - - if (metadata.serial) { - this.tests.serial.push(test); - } else { - this.tests.concurrent.push(test); - } - } - - _skippedTest(test) { - return { - run: () => { - this._emitTestResult({ - passed: true, - result: test - }); - - return true; - } - }; - } - - _emitTestResult(result) { - this.pendingTestInstances.delete(result.result); - this.emit('test', result); - } - - _buildHooks(hooks, testTitle, contextRef) { - return hooks.map(hook => { - const test = this._buildHook(hook, testTitle, contextRef); - - if (hook.metadata.skipped || hook.metadata.todo) { - return this._skippedTest(test); - } - - return test; - }); - } - - _buildHook(hook, testTitle, contextRef) { - let title = hook.title; - - if (testTitle) { - title += ` for ${testTitle}`; - } - - const test = new Test({ - contextRef, - failWithoutAssertions: false, - fn: hook.fn, - compareTestSnapshot: this.compareTestSnapshot, - metadata: hook.metadata, - onResult: this._emitTestResult, - title - }); - this.pendingTestInstances.add(test); - return test; - } - - _buildTest(test, contextRef) { - test = new Test({ - contextRef, - failWithoutAssertions: this.failWithoutAssertions, - fn: test.fn, - compareTestSnapshot: this.compareTestSnapshot, - metadata: test.metadata, - onResult: this._emitTestResult, - title: test.title - }); - this.pendingTestInstances.add(test); - return test; - } - - _buildTestWithHooks(test, contextRef) { - if (test.metadata.skipped || test.metadata.todo) { - return new Sequence([this._skippedTest(this._buildTest(test))], true); - } - - const copiedRef = contextRef.copy(); - - const beforeHooks = this._buildHooks(this.hooks.beforeEach, test.title, copiedRef); - const afterHooks = this._buildHooks(this.hooks.afterEach, test.title, copiedRef); - - let sequence = new Sequence([].concat(beforeHooks, this._buildTest(test, copiedRef), afterHooks), true); - if (this.hooks.afterEachAlways.length > 0) { - const afterAlwaysHooks = new Sequence(this._buildHooks(this.hooks.afterEachAlways, test.title, copiedRef)); - sequence = new Sequence([sequence, afterAlwaysHooks], false); - } - return sequence; - } - - _buildTests(tests, contextRef) { - return tests.map(test => this._buildTestWithHooks(test, contextRef)); - } - - _hasUnskippedTests() { - return this.tests.serial.concat(this.tests.concurrent) - .some(test => { - return !(test.metadata && test.metadata.skipped === true); - }); - } - - build() { - const contextRef = new ContextRef(); - - const serialTests = new Sequence(this._buildTests(this.tests.serial, contextRef), this.bail); - const concurrentTests = new Concurrent(this._buildTests(this.tests.concurrent, contextRef), this.bail); - const allTests = new Sequence([serialTests, concurrentTests]); - - let finalTests; - // Only run before and after hooks when there are unskipped tests - if (this._hasUnskippedTests()) { - const beforeHooks = new Sequence(this._buildHooks(this.hooks.before, null, contextRef)); - const afterHooks = new Sequence(this._buildHooks(this.hooks.after, null, contextRef)); - finalTests = new Sequence([beforeHooks, allTests, afterHooks], true); - } else { - finalTests = new Sequence([allTests], true); - } - - if (this.hooks.afterAlways.length > 0) { - const afterAlwaysHooks = new Sequence(this._buildHooks(this.hooks.afterAlways, null, contextRef)); - finalTests = new Sequence([finalTests, afterAlwaysHooks], false); - } - - return finalTests; - } - - attributeLeakedError(err) { - for (const test of this.pendingTestInstances) { - if (test.attributeLeakedError(err)) { - return true; - } - } - return false; - } -} - -module.exports = TestCollection; diff --git a/lib/test-worker.js b/lib/test-worker.js index 93ea6aace..c4b1e8ac3 100644 --- a/lib/test-worker.js +++ b/lib/test-worker.js @@ -43,7 +43,14 @@ function exit() { // Reference the IPC channel so the exit sequence can be completed. adapter.forceRefChannel(); - const stats = runner.buildStats(); + const stats = { + failCount: runner.stats.failCount + runner.stats.failedHookCount, + knownFailureCount: runner.stats.knownFailureCount, + passCount: runner.stats.passCount, + skipCount: runner.stats.skipCount, + testCount: runner.stats.testCount, + todoCount: runner.stats.todoCount + }; adapter.send('results', {stats}); } @@ -57,33 +64,67 @@ exports.setRunner = newRunner => { touchedFiles.add(file); } }); - runner.on('start', promise => { + runner.on('start', started => { adapter.send('stats', { - testCount: runner.tests.testCount, - hasExclusive: runner.tests.hasExclusive + testCount: started.stats.testCount, + hasExclusive: started.stats.hasExclusive }); - promise.then(() => { + for (const partial of started.skippedTests) { + adapter.send('test', { + duration: null, + error: null, + failing: partial.failing, + logs: [], + skip: true, + title: partial.title, + todo: false, + type: 'test' + }); + } + for (const title of started.todoTitles) { + adapter.send('test', { + duration: null, + error: null, + failing: false, + logs: [], + skip: true, + title, + todo: true, + type: 'test' + }); + } + + started.ended.then(() => { runner.saveSnapshotState(); return exit(); }).catch(err => { handleUncaughtException(err); }); }); - runner.on('test', props => { - const hasError = typeof props.error !== 'undefined'; - // Don't send anything if it's a passed hook - if (!hasError && props.type !== 'test') { - return; - } - - if (hasError) { - props.error = serializeError(props.error); - } else { - props.error = null; - } - - adapter.send('test', props); + runner.on('hook-failed', result => { + adapter.send('test', { + duration: result.duration, + error: serializeError(result.error), + failing: result.metadata.failing, + logs: result.logs, + skip: result.metadata.skip, + title: result.title, + todo: result.metadata.todo, + type: result.metadata.type + }); + }); + runner.on('test', result => { + adapter.send('test', { + duration: result.duration, + error: result.passed ? null : serializeError(result.error), + failing: result.metadata.failing, + logs: result.logs, + skip: result.metadata.skip, + title: result.title, + todo: result.metadata.todo, + type: result.metadata.type + }); }); }; diff --git a/lib/test.js b/lib/test.js index 654c90a3a..115b02106 100644 --- a/lib/test.js +++ b/lib/test.js @@ -103,7 +103,6 @@ class Test { this.failWithoutAssertions = options.failWithoutAssertions; this.fn = isGeneratorFn(options.fn) ? co.wrap(options.fn) : options.fn; this.metadata = options.metadata; - this.onResult = options.onResult; this.title = options.title; this.logs = []; @@ -317,7 +316,7 @@ class Test { values: [formatErrorValue('Error thrown in test:', result.error)] })); } - return this.finish(); + return this.finishPromised(); } const returnedObservable = isObservable(result.retval); @@ -335,11 +334,11 @@ class Test { if (returnedObservable || returnedPromise) { const asyncType = returnedObservable ? 'observables' : 'promises'; this.saveFirstError(new Error(`Do not return ${asyncType} from tests declared via \`test.cb(...)\`, if you want to return a promise simply declare the test via \`test(...)\``)); - return this.finish(); + return this.finishPromised(); } if (this.calledEnd) { - return this.finish(); + return this.finishPromised(); } return new Promise(resolve => { @@ -386,7 +385,7 @@ class Test { }); } - return this.finish(); + return this.finishPromised(); } finish() { @@ -401,26 +400,27 @@ class Test { this.duration = nowAndTimers.now() - this.startedAt; - let reason = this.assertError; - let passed = !reason; + let error = this.assertError; + let passed = !error; if (this.metadata.failing) { passed = !passed; if (passed) { - reason = undefined; + error = null; } else { - reason = new Error('Test was expected to fail, but succeeded, you should stop marking the test as failing'); + error = new Error('Test was expected to fail, but succeeded, you should stop marking the test as failing'); } } - this.onResult({ + return { + duration: this.duration, + error, + logs: this.logs, + metadata: this.metadata, passed, - result: this, - reason - }); - - return passed; + title: this.title + }; } finishPromised() { diff --git a/lib/validate-test.js b/lib/validate-test.js deleted file mode 100644 index 8258a5990..000000000 --- a/lib/validate-test.js +++ /dev/null @@ -1,48 +0,0 @@ -'use strict'; - -function validate(title, fn, metadata) { - if (metadata.type !== 'test') { - if (metadata.exclusive) { - return '`only` is only for tests and cannot be used with hooks'; - } - - if (metadata.failing) { - return '`failing` is only for tests and cannot be used with hooks'; - } - - if (metadata.todo) { - return '`todo` is only for documentation of future tests and cannot be used with hooks'; - } - } - - if (metadata.todo) { - if (typeof fn === 'function') { - return '`todo` tests are not allowed to have an implementation. Use ' + - '`test.skip()` for tests with an implementation.'; - } - - if (typeof title !== 'string') { - return '`todo` tests require a title'; - } - - if (metadata.skipped || metadata.failing || metadata.exclusive) { - return '`todo` tests are just for documentation and cannot be used with `skip`, `only`, or `failing`'; - } - } else if (typeof fn !== 'function') { - return 'Expected an implementation. Use `test.todo()` for tests without an implementation.'; - } - - if (metadata.always) { - if (!(metadata.type === 'after' || metadata.type === 'afterEach')) { - return '`always` can only be used with `after` and `afterEach`'; - } - } - - if (metadata.skipped && metadata.exclusive) { - return '`only` tests cannot be skipped'; - } - - return null; -} - -module.exports = validate; diff --git a/profile.js b/profile.js index b7b1897e0..35acdce80 100644 --- a/profile.js +++ b/profile.js @@ -99,6 +99,7 @@ babelConfigHelper.build(process.cwd(), cacheDir, babelConfigHelper.validate(conf const events = new EventEmitter(); events.on('loaded-file', () => {}); + let failCount = 0; let uncaughtExceptionCount = 0; // Mock the behavior of a parent process @@ -130,8 +131,13 @@ babelConfigHelper.build(process.cwd(), cacheDir, babelConfigHelper.validate(conf console.log('RESULTS:', data.stats); + failCount = data.stats.failCount; + setImmediate(() => process.emit('ava-teardown')); + }); + + events.on('teardown', () => { if (process.exit) { - process.exit(data.stats.failCount + uncaughtExceptionCount); // eslint-disable-line unicorn/no-process-exit + process.exit(failCount + uncaughtExceptionCount); // eslint-disable-line unicorn/no-process-exit } }); diff --git a/readme.md b/readme.md index fbf48fbff..5004a2b21 100644 --- a/readme.md +++ b/readme.md @@ -524,17 +524,15 @@ test.failing('demonstrate some bug', t => { AVA lets you register hooks that are run before and after your tests. This allows you to run setup and/or teardown code. -`test.before()` registers a hook to be run before the first test in your test file. Similarly `test.after()` registers a hook to be run after the last test. Use `test.after.always()` to register a hook that will **always** run once your tests and other hooks complete. `.always()` hooks run regardless of whether there were earlier failures or if all tests were skipped, so they are ideal for cleanup tasks. There are two exceptions to this however. If you use `--fail-fast` AVA will stop testing as soon as a failure occurs, and it won't run any hooks including the `.always()` hooks. Uncaught exceptions will crash your tests, possibly preventing `.always()` hooks from running. +`test.before()` registers a hook to be run before the first test in your test file. Similarly `test.after()` registers a hook to be run after the last test. Use `test.after.always()` to register a hook that will **always** run once your tests and other hooks complete. `.always()` hooks run regardless of whether there were earlier failures, so they are ideal for cleanup tasks. Note however that uncaught exceptions, unhandled rejections or timeouts will crash your tests, possibly preventing `.always()` hooks from running. -`test.beforeEach()` registers a hook to be run before each test in your test file. Similarly `test.afterEach()` a hook to be run after each test. Use `test.afterEach.always()` to register an after hook that is called even if other test hooks, or the test itself, fail. `.always()` hooks are ideal for cleanup tasks. +`test.beforeEach()` registers a hook to be run before each test in your test file. Similarly `test.afterEach()` a hook to be run after each test. Use `test.afterEach.always()` to register an after hook that is called even if other test hooks, or the test itself, fail. -If a test is skipped with the `.skip` modifier, the respective `.beforeEach()` and `.afterEach()` hooks are not run. Likewise, if all tests in a test file are skipped `.before()` and `.after()` hooks for the file are not run. Hooks modified with `.always()` will always run, even if all tests are skipped. +If a test is skipped with the `.skip` modifier, the respective `.beforeEach()`, `.afterEach()` and `.afterEach.always()` hooks are not run. Likewise, if all tests in a test file are skipped `.before()`, `.after()` and `.after.always()` hooks for the file are not run. -**Note**: If the `--fail-fast` flag is specified, AVA will stop after the first test failure and the `.always` hook will **not** run. +Like `test()` these methods take an optional title and an implementation function. The title is shown if your hook fails to execute. The implementation is called with an [execution object](#t). You can use assertions in your hooks. You can also pass a [macro function](#test-macros) and additional arguments. -Like `test()` these methods take an optional title and a callback function. The title is shown if your hook fails to execute. The callback is called with an [execution object](#t). - -`before` hooks execute before `beforeEach` hooks. `afterEach` hooks execute before `after` hooks. Within their category the hooks execute in the order they were defined. +`.before()` hooks execute before `.beforeEach()` hooks. `.afterEach()` hooks execute before `.after()` hooks. Within their category the hooks execute in the order they were defined. By default hooks execute concurrently, but you can use `test.serial` to ensure only that single hook is run at a time. Unlike with tests, serial hooks are *not* run before other hooks: ```js test.before(t => { @@ -542,7 +540,15 @@ test.before(t => { }); test.before(t => { - // This runs after the above, but before tests + // This runs concurrently with the above +}); + +test.serial.before(t => { + // This runs after the above +}); + +test.serial.before(t => { + // This too runs after the above, and before tests }); test.after('cleanup', t => { @@ -590,13 +596,13 @@ test.afterEach.cb(t => { }); ``` -Keep in mind that the `beforeEach` and `afterEach` hooks run just before and after a test is run, and that by default tests run concurrently. If you need to set up global state for each test (like spying on `console.log` [for example](https://github.com/avajs/ava/issues/560)), you'll need to make sure the tests are [run serially](#running-tests-serially). +Keep in mind that the `.beforeEach()` and `.afterEach()` hooks run just before and after a test is run, and that by default tests run concurrently. This means each multiple `.beforeEach()` hooks may run concurrently. Using `test.serial.beforeEach()` does not change this. If you need to set up global state for each test (like spying on `console.log` [for example](https://github.com/avajs/ava/issues/560)), you'll need to make sure the tests themselves are [run serially](#running-tests-serially). -Remember that AVA runs each test file in its own process. You may not have to clean up global state in a `after`-hook since that's only called right before the process exits. +Remember that AVA runs each test file in its own process. You may not have to clean up global state in a `.after()`-hook since that's only called right before the process exits. #### Test context -The `beforeEach` & `afterEach` hooks can share context with the test: +The `.beforeEach()` & `.afterEach()` hooks can share context with the test: ```js test.beforeEach(t => { @@ -620,7 +626,7 @@ test('context is unicorn', t => { }); ``` -Context sharing is *not* available to `before` and `after` hooks. +Context sharing is *not* available to `.before()` and `.after()` hooks. ### Test macros @@ -821,7 +827,7 @@ Should contain the actual test. Type: `object` -The execution object of a particular test. Each test implementation receives a different object. Contains the [assertions](#assertions) as well as `.plan(count)` and `.end()` methods. `t.context` can contain shared state from `beforeEach` hooks. `t.title` returns the test's title. +The execution object of a particular test. Each test implementation receives a different object. Contains the [assertions](#assertions) as well as `.plan(count)` and `.end()` methods. `t.context` can contain shared state from `.beforeEach()` hooks. `t.title` returns the test's title. ###### `t.plan(count)` diff --git a/test/api.js b/test/api.js index 3658954cb..077ca2d3b 100644 --- a/test/api.js +++ b/test/api.js @@ -227,8 +227,11 @@ test('fail-fast mode - single file', t => { }, { ok: false, title: 'second fail' + }, { + ok: true, + title: 'third pass' }]); - t.is(result.passCount, 1); + t.is(result.passCount, 2); t.is(result.failCount, 1); }); }); diff --git a/test/beautify-stack.js b/test/beautify-stack.js index b88e0ba5d..d23ac0186 100644 --- a/test/beautify-stack.js +++ b/test/beautify-stack.js @@ -1,7 +1,9 @@ 'use strict'; +require('../lib/worker-options').set({}); + const proxyquire = require('proxyquire').noPreserveCache(); const test = require('tap').test; -const Sequence = require('../lib/sequence'); +const Runner = require('../lib/runner'); const beautifyStack = proxyquire('../lib/beautify-stack', { debug() { @@ -56,20 +58,20 @@ test('returns empty string without any arguments', t => { test('beautify stack - removes uninteresting lines', t => { try { - const seq = new Sequence([{ + const runner = new Runner(); + runner.runSingle({ run() { fooFunc(); } - }]); - seq.run(); + }); } catch (err) { const stack = beautifyStack(err.stack); t.match(stack, /fooFunc/); t.match(stack, /barFunc/); - // The runNext line is introduced by Sequence. It's internal so it should + // The runSingle line is introduced by Runner. It's internal so it should // be stripped. - t.match(err.stack, /runNext/); - t.notMatch(stack, /runNext/); + t.match(err.stack, /runSingle/); + t.notMatch(stack, /runSingle/); t.end(); } }); diff --git a/test/concurrent.js b/test/concurrent.js deleted file mode 100644 index 78edd3178..000000000 --- a/test/concurrent.js +++ /dev/null @@ -1,790 +0,0 @@ -'use strict'; -const tap = require('tap'); -const isPromise = require('is-promise'); -const Concurrent = require('../lib/concurrent'); - -let results = []; -const test = (name, fn) => { - tap.test(name, t => { - results = []; - return fn(t); - }); -}; -function collect(result) { - if (isPromise(result)) { - return result.then(collect); - } - - results.push(result); - return result.passed; -} - -function pass(val) { - return { - run() { - return collect({ - passed: true, - result: val - }); - } - }; -} - -function fail(val) { - return { - run() { - return collect({ - passed: false, - reason: val - }); - } - }; -} - -function failWithTypeError() { - return { - run() { - throw new TypeError('Unexpected Error'); - } - }; -} - -function passAsync(val) { - return { - run() { - return collect(Promise.resolve({ - passed: true, - result: val - })); - } - }; -} - -function failAsync(err) { - return { - run() { - return collect(Promise.resolve({ - passed: false, - reason: err - })); - } - }; -} - -function reject(err) { - return { - run() { - return Promise.reject(err); - } - }; -} - -test('all sync - all pass - no bail', t => { - const passed = new Concurrent( - [ - pass('a'), - pass('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - no failure - bail', t => { - const passed = new Concurrent( - [ - pass('a'), - pass('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - begin failure - no bail', t => { - const passed = new Concurrent( - [ - fail('a'), - pass('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - mid failure - no bail', t => { - const passed = new Concurrent( - [ - pass('a'), - fail('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - end failure - no bail', t => { - const passed = new Concurrent( - [ - pass('a'), - pass('b'), - fail('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all sync - multiple failure - no bail', t => { - const passed = new Concurrent( - [ - fail('a'), - pass('b'), - fail('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all sync - begin failure - bail', t => { - const passed = new Concurrent( - [ - fail('a'), - pass('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - } - ]); - t.end(); -}); - -test('all sync - mid failure - bail', t => { - const passed = new Concurrent( - [ - pass('a'), - fail('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - } - ]); - t.end(); -}); - -test('all sync - end failure - bail', t => { - const passed = new Concurrent( - [ - pass('a'), - pass('b'), - fail('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all async - no failure - no bail', t => { - return new Concurrent( - [ - passAsync('a'), - passAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('all async - no failure - bail', t => { - return new Concurrent( - [ - passAsync('a'), - passAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('last async - no failure - no bail', t => { - return new Concurrent( - [ - pass('a'), - pass('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('mid async - no failure - no bail', t => { - return new Concurrent( - [ - pass('a'), - passAsync('b'), - pass('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'c' - }, - { - passed: true, - result: 'b' - } - ]); - }); -}); - -test('first async - no failure - no bail', t => { - return new Concurrent( - [ - passAsync('a'), - pass('b'), - pass('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - }, - { - passed: true, - result: 'a' - } - ]); - }); -}); - -test('last async - no failure - bail', t => { - return new Concurrent( - [ - pass('a'), - pass('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('mid async - no failure - bail', t => { - return new Concurrent( - [ - pass('a'), - passAsync('b'), - pass('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'c' - }, - { - passed: true, - result: 'b' - } - ]); - }); -}); - -test('first async - no failure - bail', t => { - return new Concurrent( - [ - passAsync('a'), - pass('b'), - pass('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - }, - { - passed: true, - result: 'a' - } - ]); - }); -}); - -test('all async - begin failure - bail', t => { - return new Concurrent( - [ - failAsync('a'), - passAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('all async - mid failure - bail', t => { - return new Concurrent( - [ - passAsync('a'), - failAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('all async - end failure - bail', t => { - return new Concurrent( - [ - passAsync('a'), - passAsync('b'), - failAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - }); -}); - -test('all async - begin failure - no bail', t => { - return new Concurrent( - [ - failAsync('a'), - passAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('all async - mid failure - no bail', t => { - return new Concurrent( - [ - passAsync('a'), - failAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - }); -}); - -test('all async - end failure - no bail', t => { - return new Concurrent( - [ - passAsync('a'), - passAsync('b'), - failAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - }); -}); - -test('all async - multiple failure - no bail', t => { - return new Concurrent( - [ - failAsync('a'), - passAsync('b'), - failAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - }); -}); - -test('rejections are just passed through - no bail', t => { - return new Concurrent( - [ - pass('a'), - pass('b'), - reject('foo') - ], - false - ).run().catch(err => { - t.is(err, 'foo'); - }); -}); - -test('rejections are just passed through - bail', t => { - return new Concurrent( - [ - pass('a'), - pass('b'), - reject('foo') - ], - true - ).run().catch(err => { - t.is(err, 'foo'); - }); -}); - -test('sequences of sequences', t => { - const passed = new Concurrent([ - new Concurrent([pass('a'), pass('b')]), - new Concurrent([pass('c')]) - ]).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - - t.end(); -}); - -test('must be called with array of runnables', t => { - t.throws(() => { - new Concurrent(pass('a')).run(); - }, {message: 'Expected an array of runnables'}); - t.end(); -}); - -test('should throw an error then test.run() fails with not AvaError', t => { - t.throws(() => { - new Concurrent([failWithTypeError()]).run(); - }, {message: 'Unexpected Error'}); - t.end(); -}); diff --git a/test/hooks.js b/test/hooks.js index 9c957ca1e..78229f77c 100644 --- a/test/hooks.js +++ b/test/hooks.js @@ -32,7 +32,7 @@ function fork(testPath) { const promiseEnd = (runner, next) => { return new Promise(resolve => { - runner.on('start', resolve); + runner.on('start', data => resolve(data.ended)); next(runner); }).then(() => runner); }; @@ -69,9 +69,8 @@ test('after', t => { arr.push('a'); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 1); - t.is(stats.failCount, 0); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.failCount, 0); t.strictDeepEqual(arr, ['a', 'b']); }); }); @@ -89,9 +88,8 @@ test('after not run if test failed', t => { throw new Error('something went wrong'); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 0); - t.is(stats.failCount, 1); + t.is(runner.stats.passCount, 0); + t.is(runner.stats.failCount, 1); t.strictDeepEqual(arr, []); }); }); @@ -109,9 +107,8 @@ test('after.always run even if test failed', t => { throw new Error('something went wrong'); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 0); - t.is(stats.failCount, 1); + t.is(runner.stats.passCount, 0); + t.is(runner.stats.failCount, 1); t.strictDeepEqual(arr, ['a']); }); }); @@ -125,6 +122,8 @@ test('after.always run even if before failed', t => { throw new Error('something went wrong'); }); + runner.chain('test', a => a.pass()); + runner.chain.after.always(() => { arr.push('a'); }); @@ -228,8 +227,7 @@ test('fail if beforeEach hook fails', t => { a.pass(); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.failCount, 1); + t.is(runner.stats.failedHookCount, 1); t.strictDeepEqual(arr, ['a']); }); }); @@ -448,8 +446,7 @@ test('shared context', t => { a.context.prop = 'afterEach'; }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.failCount, 0); + t.is(runner.stats.failCount, 0); }); }); @@ -466,8 +463,7 @@ test('shared context of any type', t => { a.is(a.context, 'foo'); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.failCount, 0); + t.is(runner.stats.failCount, 0); }); }); diff --git a/test/observable.js b/test/observable.js index 6cfd25c71..819fd4f59 100644 --- a/test/observable.js +++ b/test/observable.js @@ -5,31 +5,28 @@ const test = require('tap').test; const Test = require('../lib/test'); const Observable = require('zen-observable'); // eslint-disable-line import/order -function ava(fn, onResult) { +function ava(fn) { return new Test({ contextRef: null, failWithoutAssertions: true, fn, metadata: {type: 'test', callback: false}, - onResult, title: '[anonymous]' }); } -ava.cb = function (fn, onResult) { +ava.cb = function (fn) { return new Test({ contextRef: null, failWithoutAssertions: true, fn, metadata: {type: 'test', callback: true}, - onResult, title: '[anonymous]' }); }; test('returning an observable from a legacy async fn is an error', t => { - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.plan(2); const observable = Observable.of(); @@ -41,18 +38,14 @@ test('returning an observable from a legacy async fn is an error', t => { }); return observable; - }, r => { - result = r; - }).run(); - - t.is(passed, false); - t.match(result.reason.message, /Do not return observables/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.match(result.error.message, /Do not return observables/); + }); }); test('handle throws with erroring observable', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -60,18 +53,15 @@ test('handle throws with erroring observable', t => { }); return a.throws(observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with erroring observable returned by function', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -79,18 +69,15 @@ test('handle throws with erroring observable returned by function', t => { }); return a.throws(() => observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with long running erroring observable', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -100,50 +87,39 @@ test('handle throws with long running erroring observable', t => { }); return a.throws(observable, /abc/); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with completed observable', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const observable = Observable.of(); return a.throws(observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with completed observable returned by function', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const observable = Observable.of(); return a.throws(() => observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with regex', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -151,18 +127,15 @@ test('handle throws with regex', t => { }); return a.throws(observable, /abc/); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with string', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -170,18 +143,15 @@ test('handle throws with string', t => { }); return a.throws(observable, 'abc'); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with false-positive observable', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -190,34 +160,27 @@ test('handle throws with false-positive observable', t => { }); return a.throws(observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle notThrows with completed observable', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const observable = Observable.of(); return a.notThrows(observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle notThrows with thrown observable', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -225,18 +188,14 @@ test('handle notThrows with thrown observable', t => { }); return a.notThrows(observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle notThrows with erroring observable returned by function', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const observable = new Observable(observer => { @@ -244,11 +203,8 @@ test('handle notThrows with erroring observable returned by function', t => { }); return a.notThrows(() => observable); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); diff --git a/test/promise.js b/test/promise.js index 8b1cef09c..7ad06bc59 100644 --- a/test/promise.js +++ b/test/promise.js @@ -5,24 +5,22 @@ const Promise = require('bluebird'); const test = require('tap').test; const Test = require('../lib/test'); -function ava(fn, onResult) { +function ava(fn) { return new Test({ contextRef: null, failWithoutAssertions: true, fn, metadata: {type: 'test', callback: false}, - onResult, title: '[anonymous]' }); } -ava.cb = function (fn, onResult) { +ava.cb = function (fn) { return new Test({ contextRef: null, failWithoutAssertions: true, fn, metadata: {type: 'test', callback: true}, - onResult, title: '[anonymous]' }); }; @@ -42,27 +40,22 @@ function fail() { } test('returning a promise from a legacy async fn is an error', t => { - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.plan(1); return Promise.resolve(true).then(() => { a.pass(); a.end(); }); - }, r => { - result = r; - }).run(); - - t.is(passed, false); - t.match(result.reason.message, /Do not return promises/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.match(result.error.message, /Do not return promises/); + }); }); test('assertion plan is tested after returned promise resolves', t => { - let result; const start = Date.now(); - ava(a => { + const instance = ava(a => { a.plan(2); const defer = Promise.defer(); @@ -75,20 +68,17 @@ test('assertion plan is tested after returned promise resolves', t => { a.pass(); return defer.promise; - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.planCount, 2); - t.is(result.result.assertCount, 2); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 2); + t.is(instance.assertCount, 2); t.true(Date.now() - start >= 500); - t.end(); }); }); test('missing assertion will fail the test', t => { - let result; - ava(a => { + return ava(a => { a.plan(2); const defer = Promise.defer(); @@ -99,18 +89,14 @@ test('missing assertion will fail the test', t => { }, 200); return defer.promise; - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.assertion, 'plan'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.assertion, 'plan'); }); }); test('extra assertion will fail the test', t => { - let result; - ava(a => { + return ava(a => { a.plan(2); const defer = Promise.defer(); @@ -126,51 +112,41 @@ test('extra assertion will fail the test', t => { }, 500); return defer.promise; - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.assertion, 'plan'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.assertion, 'plan'); }); }); test('handle throws with rejected promise', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.reject(new Error()); return a.throws(promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with rejected promise returned by function', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.reject(new Error()); return a.throws(() => promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); // TODO(team): This is a very slow test, and I can't figure out why we need it - James test('handle throws with long running rejected promise', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = new Promise((resolve, reject) => { @@ -180,270 +156,208 @@ test('handle throws with long running rejected promise', t => { }); return a.throws(promise, /abc/); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle throws with resolved promise', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.resolve(); return a.throws(promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with resolved promise returned by function', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.resolve(); return a.throws(() => promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with regex', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.reject(new Error('abc')); return a.throws(promise, /abc/); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('throws with regex will fail if error message does not match', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.reject(new Error('abc')); return a.throws(promise, /def/); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with string', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.reject(new Error('abc')); return a.throws(promise, 'abc'); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('throws with string argument will reject if message does not match', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.reject(new Error('abc')); return a.throws(promise, 'def'); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('does not handle throws with string reject', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.reject('abc'); // eslint-disable-line prefer-promise-reject-errors return a.throws(promise, 'abc'); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle throws with false-positive promise', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.resolve(new Error()); return a.throws(promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle notThrows with resolved promise', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.resolve(); return a.notThrows(promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle notThrows with rejected promise', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.reject(new Error()); return a.notThrows(promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('handle notThrows with resolved promise returned by function', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(1); const promise = Promise.resolve(); return a.notThrows(() => promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('handle notThrows with rejected promise returned by function', t => { - let result; - ava(a => { + return ava(a => { a.plan(1); const promise = Promise.reject(new Error()); return a.notThrows(() => promise); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('assert pass', t => { - let result; - ava(a => { + const instance = ava(a => { return pass().then(() => { a.pass(); }); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('assert fail', t => { - let result; - ava(a => { + return ava(a => { return pass().then(() => { a.fail(); }); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); }); }); test('reject', t => { - let result; - ava(a => { + return ava(a => { return fail().then(() => { a.pass(); }); - }, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.message, 'Rejected promise returned by test'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Rejected promise returned by test. Reason:'); - t.match(result.reason.values[0].formatted, /.*Error.*\n.*message: 'unicorn'/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.message, 'Rejected promise returned by test'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Rejected promise returned by test. Reason:'); + t.match(result.error.values[0].formatted, /.*Error.*\n.*message: 'unicorn'/); }); }); test('reject with non-Error', t => { - let result; - ava( - () => Promise.reject('failure'), // eslint-disable-line prefer-promise-reject-errors - r => { - result = r; - } - ).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.message, 'Rejected promise returned by test'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Rejected promise returned by test. Reason:'); - t.match(result.reason.values[0].formatted, /failure/); - t.end(); + return ava(() => { + return Promise.reject('failure'); // eslint-disable-line prefer-promise-reject-errors + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.message, 'Rejected promise returned by test'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Rejected promise returned by test. Reason:'); + t.match(result.error.values[0].formatted, /failure/); }); }); diff --git a/test/runner.js b/test/runner.js index e58700e2e..4ed269dd0 100644 --- a/test/runner.js +++ b/test/runner.js @@ -9,7 +9,7 @@ const noop = () => {}; const promiseEnd = (runner, next) => { return new Promise(resolve => { - runner.on('start', resolve); + runner.on('start', data => resolve(data.ended)); next(runner); }).then(() => runner); }; @@ -121,12 +121,15 @@ test('anything can be skipped', t => { }); }); -test('include skipped tests in results', t => { - const titles = []; +test('emit skipped tests at start', t => { + t.plan(1); const runner = new Runner(); - runner.on('test', test => { - titles.push(test.title); + runner.on('start', data => { + t.strictDeepEqual(data.skippedTests, [ + {failing: false, title: 'test.serial.skip'}, + {failing: true, title: 'test.failing.skip'} + ]); }); return promiseEnd(runner, () => { @@ -136,81 +139,84 @@ test('include skipped tests in results', t => { runner.chain.beforeEach('beforeEach', noop); runner.chain.beforeEach.skip('beforeEach.skip', noop); - runner.chain.serial('test', a => a.pass()); - runner.chain.serial.skip('test.skip', noop); + runner.chain.serial('test.serial', a => a.pass()); + runner.chain.serial.skip('test.serial.skip', noop); + + runner.chain.failing('test.failing', a => a.fail()); + runner.chain.failing.skip('test.failing.skip', noop); runner.chain.after('after', noop); runner.chain.after.skip('after.skip', noop); runner.chain.afterEach('afterEach', noop); runner.chain.afterEach.skip('afterEach.skip', noop); - }).then(() => { - t.strictDeepEqual(titles, [ - 'before', - 'before.skip', - 'beforeEach for test', - 'beforeEach.skip for test', - 'test', - 'afterEach for test', - 'afterEach.skip for test', - 'test.skip', - 'after', - 'after.skip' - ]); }); }); test('test types and titles', t => { - t.plan(10); + t.plan(20); - const fn = a => { - a.pass(); - }; - - function named(a) { - a.pass(); - } + const fail = a => a.fail(); + const pass = a => a.pass(); - return promiseEnd(new Runner(), runner => { - runner.chain.before(named); - runner.chain.beforeEach(fn); - runner.chain.after(fn); - runner.chain.afterEach(named); - runner.chain('test', fn); - - const tests = [ - { - type: 'before', - title: 'before hook' - }, - { - type: 'beforeEach', - title: 'beforeEach hook for test' - }, - { - type: 'test', - title: 'test' - }, - { - type: 'afterEach', - title: 'afterEach hook for test' - }, - { - type: 'after', - title: 'after hook' - } - ]; + const check = (setup, expect) => { + const runner = new Runner(); + const assert = data => { + const expected = expect.shift(); + t.is(data.title, expected.title); + t.is(data.metadata.type, expected.type); + }; + runner.on('hook-failed', assert); + runner.on('test', assert); + return promiseEnd(runner, () => setup(runner.chain)); + }; - runner.on('test', props => { - const test = tests.shift(); - t.is(props.title, test.title); - t.is(props.type, test.type); - }); - }); + return Promise.all([ + check(chain => { + chain.before(fail); + chain('test', pass); + }, [ + {type: 'before', title: 'before hook'} + ]), + check(chain => { + chain('test', pass); + chain.after(fail); + }, [ + {type: 'test', title: 'test'}, + {type: 'after', title: 'after hook'} + ]), + check(chain => { + chain('test', pass); + chain.after.always(fail); + }, [ + {type: 'test', title: 'test'}, + {type: 'after', title: 'after.always hook'} + ]), + check(chain => { + chain.beforeEach(fail); + chain('test', fail); + }, [ + {type: 'beforeEach', title: 'beforeEach hook for test'} + ]), + check(chain => { + chain('test', pass); + chain.afterEach(fail); + }, [ + {type: 'test', title: 'test'}, + {type: 'afterEach', title: 'afterEach hook for test'} + ]), + check(chain => { + chain('test', pass); + chain.afterEach.always(fail); + }, [ + {type: 'test', title: 'test'}, + {type: 'afterEach', title: 'afterEach.always hook for test'} + ]) + ]); }); test('skip test', t => { - t.plan(5); + t.plan(4); const arr = []; return promiseEnd(new Runner(), runner => { @@ -222,31 +228,48 @@ test('skip test', t => { runner.chain.skip('skip', () => { arr.push('b'); }); - - t.throws(() => { - runner.chain.skip('should be a todo'); - }, new TypeError('Expected an implementation. Use `test.todo()` for tests without an implementation.')); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.testCount, 2); - t.is(stats.passCount, 1); - t.is(stats.skipCount, 1); + t.is(runner.stats.testCount, 2); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.skipCount, 1); t.strictDeepEqual(arr, ['a']); }); }); -test('test throws when given no function', t => { +test('tests must have a non-empty title)', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { + t.throws(() => { + runner.chain('', t => t.pass()); + }, new TypeError('Tests must have a title')); + }); +}); + +test('test titles must be unique', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { + runner.chain('title', t => t.pass()); + + t.throws(() => { + runner.chain('title', t => t.pass()); + }, new Error('Duplicate test title: title')); + }); +}); + +test('tests must have an implementation', t => { t.plan(1); const runner = new Runner(); t.throws(() => { - runner.chain(); + runner.chain('title'); }, new TypeError('Expected an implementation. Use `test.todo()` for tests without an implementation.')); }); test('todo test', t => { - t.plan(6); + t.plan(4); const arr = []; return promiseEnd(new Runner(), runner => { @@ -256,20 +279,43 @@ test('todo test', t => { }); runner.chain.todo('todo'); + }).then(runner => { + t.is(runner.stats.testCount, 2); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.todoCount, 1); + t.strictDeepEqual(arr, ['a']); + }); +}); + +test('todo tests must not have an implementation', t => { + t.plan(1); + return promiseEnd(new Runner(), runner => { t.throws(() => { runner.chain.todo('todo', () => {}); }, new TypeError('`todo` tests are not allowed to have an implementation. Use `test.skip()` for tests with an implementation.')); + }); +}); +test('todo tests must have a title', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { t.throws(() => { runner.chain.todo(); }, new TypeError('`todo` tests require a title')); - }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.testCount, 2); - t.is(stats.passCount, 1); - t.is(stats.todoCount, 1); - t.strictDeepEqual(arr, ['a']); + }); +}); + +test('todo test titles must be unique', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { + runner.chain('title', t => t.pass()); + + t.throws(() => { + runner.chain.todo('title'); + }, new Error('Duplicate test title: title')); }); }); @@ -288,56 +334,23 @@ test('only test', t => { a.pass(); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.testCount, 1); - t.is(stats.passCount, 1); + t.is(runner.stats.testCount, 1); + t.is(runner.stats.passCount, 1); t.strictDeepEqual(arr, ['b']); }); }); -test('throws if you give a function to todo', t => { - const runner = new Runner(); - - t.throws(() => { - runner.chain.todo('todo with function', noop); - }, new TypeError('`todo` tests are not allowed to have an implementation. Use ' + - '`test.skip()` for tests with an implementation.')); - - t.end(); -}); - -test('throws if todo has no title', t => { - const runner = new Runner(); - - t.throws(() => { - runner.chain.todo(); - }, new TypeError('`todo` tests require a title')); - - t.end(); -}); - -test('validate accepts skipping failing tests', t => { - t.plan(2); - - return promiseEnd(new Runner(), runner => { - runner.chain.failing.skip('skip failing', noop); - }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.testCount, 1); - t.is(stats.skipCount, 1); - }); -}); - -test('runOnlyExclusive option test', t => { +test('options.runOnlyExclusive means only exclusive tests are run', t => { t.plan(1); - const arr = []; return promiseEnd(new Runner({runOnlyExclusive: true}), runner => { runner.chain('test', () => { - arr.push('a'); + t.fail(); + }); + + runner.chain.only('test 2', () => { + t.pass(); }); - }).then(() => { - t.strictDeepEqual(arr, []); }); }); @@ -369,72 +382,103 @@ test('options.serial forces all tests to be serial', t => { }); }); -test('options.bail will bail out', t => { - t.plan(1); +test('options.failFast does not stop concurrent tests from running', t => { + const expected = ['first', 'second']; + t.plan(expected.length); - return promiseEnd(new Runner({bail: true}), runner => { - runner.chain('test', a => { - t.pass(); + promiseEnd(new Runner({failFast: true}), runner => { + let block; + let resume; + runner.chain.beforeEach(() => { + if (block) { + return block; + } + + block = new Promise(resolve => { + resume = resolve; + }); + }); + + runner.chain('first', a => { + resume(); a.fail(); }); - runner.chain('test 2', () => { - t.fail(); + runner.chain('second', a => { + a.pass(); + }); + + runner.on('test', data => { + t.is(data.title, expected.shift()); }); }); }); -test('options.bail will bail out (async)', t => { - t.plan(1); +test('options.failFast && options.serial stops subsequent tests from running ', t => { + const expected = ['first']; + t.plan(expected.length); - let bailed = false; - promiseEnd(new Runner({bail: true}), runner => { - runner.chain.cb('cb', a => { - setTimeout(() => { - a.fail(); - a.end(); - }, 100); - a.pass(); + promiseEnd(new Runner({failFast: true, serial: true}), runner => { + let block; + let resume; + runner.chain.beforeEach(() => { + if (block) { + return block; + } + + block = new Promise(resolve => { + resume = resolve; + }); }); - // Note that because the first test is asynchronous, the second test is - // run and the `setTimeout` call still occurs. The runner should end though - // as soon as the first test fails. - // See the `bail + serial` test below for comparison - runner.chain.cb('cb 2', a => { - setTimeout(() => { - t.true(bailed); - t.end(); - a.end(); - }, 300); + runner.chain('first', a => { + resume(); + a.fail(); + }); + + runner.chain('second', a => { a.pass(); }); - }).then(() => { - bailed = true; + + runner.on('test', data => { + t.is(data.title, expected.shift()); + }); }); }); -test('options.bail + serial - tests will never happen (async)', t => { - t.plan(1); +test('options.failFast & failing serial test stops subsequent tests from running ', t => { + const expected = ['first']; + t.plan(expected.length); - const tests = []; - return promiseEnd(new Runner({bail: true, serial: true}), runner => { - runner.chain.cb('cb', a => { - setTimeout(() => { - tests.push(1); - a.fail(); - a.end(); - }, 100); + promiseEnd(new Runner({failFast: true, serial: true}), runner => { + let block; + let resume; + runner.chain.beforeEach(() => { + if (block) { + return block; + } + + block = new Promise(resolve => { + resume = resolve; + }); }); - runner.chain.cb('cb 2', a => { - setTimeout(() => { - a.end(); - t.fail(); - }, 300); + runner.chain.serial('first', a => { + resume(); + a.fail(); + }); + + runner.chain.serial('second', a => { + a.pass(); + }); + + runner.chain('third', a => { + a.pass(); + }); + + runner.on('test', data => { + t.is(data.title, expected.shift()); }); - }).then(() => { - t.strictDeepEqual(tests, [1]); }); }); @@ -462,10 +506,9 @@ test('options.match will not run tests with non-matching titles', t => { a.pass(); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.skipCount, 0); - t.is(stats.passCount, 2); - t.is(stats.testCount, 2); + t.is(runner.stats.skipCount, 0); + t.is(runner.stats.passCount, 2); + t.is(runner.stats.testCount, 2); }); }); @@ -484,10 +527,9 @@ test('options.match hold no effect on hooks with titles', t => { a.pass(); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.skipCount, 0); - t.is(stats.passCount, 1); - t.is(stats.testCount, 1); + t.is(runner.stats.skipCount, 0); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.testCount, 1); }); }); @@ -505,25 +547,40 @@ test('options.match overrides .only', t => { a.pass(); }); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.skipCount, 0); - t.is(stats.passCount, 2); - t.is(stats.testCount, 2); + t.is(runner.stats.skipCount, 0); + t.is(runner.stats.passCount, 2); + t.is(runner.stats.testCount, 2); + }); +}); + +test('options.match matches todo tests', t => { + t.plan(2); + + return promiseEnd(new Runner({match: ['*oo']}), runner => { + runner.chain.todo('moo'); + runner.chain.todo('oom'); + }).then(runner => { + t.is(runner.stats.testCount, 1); + t.is(runner.stats.todoCount, 1); }); }); test('macros: Additional args will be spread as additional args on implementation function', t => { - t.plan(3); + t.plan(4); return promiseEnd(new Runner(), runner => { + runner.chain.before(function (a) { + t.deepEqual(slice.call(arguments, 1), ['foo', 'bar']); + a.pass(); + }, 'foo', 'bar'); + runner.chain('test1', function (a) { t.deepEqual(slice.call(arguments, 1), ['foo', 'bar']); a.pass(); }, 'foo', 'bar'); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 1); - t.is(stats.testCount, 1); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.testCount, 1); }); }); @@ -558,9 +615,32 @@ test('macros: Customize test names attaching a `title` function', t => { runner.chain('supplied', macroFn, 'B'); runner.chain(macroFn, 'C'); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 3); - t.is(stats.testCount, 3); + t.is(runner.stats.passCount, 3); + t.is(runner.stats.testCount, 3); + }); +}); + +test('macros: test titles must be strings', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { + t.throws(() => { + const macro = t => t.pass(); + macro.title = () => []; + runner.chain(macro); + }, new TypeError('Test & hook titles must be strings')); + }); +}); + +test('macros: hook titles must be strings', t => { + t.plan(1); + + return promiseEnd(new Runner(), runner => { + t.throws(() => { + const macro = t => t.pass(); + macro.title = () => []; + runner.chain.before(macro); + }, new TypeError('Test & hook titles must be strings')); }); }); @@ -581,9 +661,8 @@ test('match applies to macros', t => { runner.chain(macroFn, 'foo'); runner.chain(macroFn, 'bar'); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 1); - t.is(stats.testCount, 1); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.testCount, 1); }); }); @@ -618,9 +697,8 @@ test('arrays of macros', t => { runner.chain('C', macroFnA, 'C'); runner.chain('D', macroFnB, 'D'); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 6); - t.is(stats.testCount, 6); + t.is(runner.stats.passCount, 6); + t.is(runner.stats.testCount, 6); t.is(expectedArgsA.length, 0); t.is(expectedArgsB.length, 0); }); @@ -655,8 +733,98 @@ test('match applies to arrays of macros', t => { runner.chain([fooMacro, barMacro, bazMacro], 'foo'); runner.chain([fooMacro, barMacro, bazMacro], 'bar'); }).then(runner => { - const stats = runner.buildStats(); - t.is(stats.passCount, 1); - t.is(stats.testCount, 1); + t.is(runner.stats.passCount, 1); + t.is(runner.stats.testCount, 1); + }); +}); + +test('silently skips other tests when .only is used', t => { + return promiseEnd(new Runner(), runner => { + runner.chain('skip me', a => a.pass()); + runner.chain.serial('skip me too', a => a.pass()); + runner.chain.only('only me', a => a.pass()); + }).then(runner => { + t.is(runner.stats.passCount, 1); + t.is(runner.stats.skipCount, 0); + }); +}); + +test('subsequent always hooks are run even if earlier always hooks fail', t => { + t.plan(3); + return promiseEnd(new Runner(), runner => { + runner.chain('test', a => a.pass()); + runner.chain.serial.after.always(a => { + t.pass(); + a.fail(); + }); + runner.chain.serial.after.always(a => { + t.pass(); + a.fail(); + }); + runner.chain.after.always(a => { + t.pass(); + a.fail(); + }); + }); +}); + +test('hooks run concurrently, but can be serialized', t => { + t.plan(7); + + let activeCount = 0; + return promiseEnd(new Runner(), runner => { + runner.chain('test', a => a.pass()); + + runner.chain.before(() => { + t.is(activeCount, 0); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 20)).then(() => { + activeCount--; + }); + }); + + runner.chain.before(() => { + t.is(activeCount, 1); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 10)).then(() => { + activeCount--; + }); + }); + + runner.chain.serial.before(() => { + t.is(activeCount, 0); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 10)).then(() => { + activeCount--; + }); + }); + + runner.chain.before(() => { + t.is(activeCount, 0); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 20)).then(() => { + activeCount--; + }); + }); + + runner.chain.before(() => { + t.is(activeCount, 1); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 10)).then(() => { + activeCount--; + }); + }); + + runner.chain.serial.before(() => { + t.is(activeCount, 0); + activeCount++; + return new Promise(resolve => setTimeout(resolve, 10)).then(() => { + activeCount--; + }); + }); + + runner.chain.serial.before(() => { + t.is(activeCount, 0); + }); }); }); diff --git a/test/sequence.js b/test/sequence.js deleted file mode 100644 index 555fba97e..000000000 --- a/test/sequence.js +++ /dev/null @@ -1,780 +0,0 @@ -'use strict'; -const tap = require('tap'); -const Promise = require('bluebird'); -const isPromise = require('is-promise'); -const Sequence = require('../lib/sequence'); - -let results = []; -const test = (name, fn) => { - tap.test(name, t => { - results = []; - return fn(t); - }); -}; -function collect(result) { - if (isPromise(result)) { - return result.then(collect); - } - - results.push(result); - return result.passed; -} - -function pass(val) { - return { - run() { - return collect({ - passed: true, - result: val - }); - } - }; -} - -function fail(val) { - return { - run() { - return collect({ - passed: false, - reason: val - }); - } - }; -} - -function passAsync(val) { - return { - run() { - return collect(Promise.resolve({ - passed: true, - result: val - })); - } - }; -} - -function failAsync(err) { - return { - run() { - return collect(Promise.resolve({ - passed: false, - reason: err - })); - } - }; -} - -function reject(err) { - return { - run() { - return Promise.reject(err); - } - }; -} - -test('all sync - no failure - no bail', t => { - const passed = new Sequence( - [ - pass('a'), - pass('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - no failure - bail', t => { - const passed = new Sequence( - [ - pass('a'), - pass('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - begin failure - no bail', t => { - const passed = new Sequence( - [ - fail('a'), - pass('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - mid failure - no bail', t => { - const passed = new Sequence( - [ - pass('a'), - fail('b'), - pass('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a'}, - { - passed: false, - reason: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); -}); - -test('all sync - end failure - no bail', t => { - const passed = new Sequence( - [ - pass('a'), - pass('b'), - fail('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all sync - multiple failure - no bail', t => { - const passed = new Sequence( - [ - fail('a'), - pass('b'), - fail('c') - ], - false - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all sync - begin failure - bail', t => { - const passed = new Sequence( - [ - fail('a'), - pass('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - } - ]); - t.end(); -}); - -test('all sync - mid failure - bail', t => { - const passed = new Sequence( - [ - pass('a'), - fail('b'), - pass('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - } - ]); - t.end(); -}); - -test('all sync - end failure - bail', t => { - const passed = new Sequence( - [ - pass('a'), - pass('b'), - fail('c') - ], - true - ).run(); - - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); -}); - -test('all async - no failure - no bail', t => { - new Sequence( - [ - passAsync('a'), - passAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - no failure - bail', t => { - new Sequence( - [ - passAsync('a'), - passAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('last async - no failure - no bail', t => { - new Sequence( - [ - pass('a'), - pass('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('mid async - no failure - no bail', t => { - new Sequence( - [ - pass('a'), - passAsync('b'), - pass('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('first async - no failure - no bail', t => { - new Sequence( - [ - passAsync('a'), - pass('b'), - pass('c') - ], - false - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('last async - no failure - bail', t => { - new Sequence( - [ - pass('a'), - pass('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('mid async - no failure - bail', t => { - new Sequence( - [ - pass('a'), - passAsync('b'), - pass('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('first async - no failure - bail', t => { - new Sequence( - [ - passAsync('a'), - pass('b'), - pass('c') - ], - true - ).run().then(passed => { - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - begin failure - bail', t => { - new Sequence( - [ - failAsync('a'), - passAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - } - ]); - t.end(); - }); -}); - -test('all async - mid failure - bail', t => { - new Sequence( - [ - passAsync('a'), - failAsync('b'), - passAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - } - ]); - t.end(); - }); -}); - -test('all async - end failure - bail', t => { - new Sequence( - [ - passAsync('a'), - passAsync('b'), - failAsync('c') - ], - true - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - begin failure - no bail', t => { - new Sequence( - [ - failAsync('a'), - passAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - mid failure - no bail', t => { - new Sequence( - [ - passAsync('a'), - failAsync('b'), - passAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: false, - reason: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - end failure - no bail', t => { - new Sequence( - [ - passAsync('a'), - passAsync('b'), - failAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); - }); -}); - -test('all async - multiple failure - no bail', t => { - new Sequence( - [ - failAsync('a'), - passAsync('b'), - failAsync('c') - ], - false - ).run().then(passed => { - t.equal(passed, false); - t.strictDeepEqual(results, [ - { - passed: false, - reason: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: false, - reason: 'c' - } - ]); - t.end(); - }); -}); - -test('rejections are just passed through - no bail', t => { - new Sequence( - [ - pass('a'), - pass('b'), - reject('foo') - ], - false - ).run().catch(err => { - t.is(err, 'foo'); - t.end(); - }); -}); - -test('rejections are just passed through - bail', t => { - new Sequence( - [ - pass('a'), - pass('b'), - reject('foo') - ], - true - ).run().catch(err => { - t.is(err, 'foo'); - t.end(); - }); -}); - -test('needs at least one sequence runnable', t => { - t.throws(() => { - new Sequence().run(); - }, {message: 'Expected an array of runnables'}); - t.end(); -}); - -test('sequences of sequences', t => { - const passed = new Sequence([ - new Sequence([pass('a'), pass('b')]), - new Sequence([pass('c')]) - ]).run(); - - t.equal(passed, true); - t.strictDeepEqual(results, [ - { - passed: true, - result: 'a' - }, - { - passed: true, - result: 'b' - }, - { - passed: true, - result: 'c' - } - ]); - - t.end(); -}); diff --git a/test/test-collection.js b/test/test-collection.js deleted file mode 100644 index c90106681..000000000 --- a/test/test-collection.js +++ /dev/null @@ -1,381 +0,0 @@ -'use strict'; -require('../lib/worker-options').set({}); - -const test = require('tap').test; -const TestCollection = require('../lib/test-collection'); - -function defaults() { - return { - type: 'test', - serial: false, - exclusive: false, - skipped: false, - callback: false, - always: false - }; -} - -function metadata(opts) { - return Object.assign(defaults(), opts); -} - -function mockTest(opts, title) { - return { - title, - metadata: metadata(opts) - }; -} - -function titles(tests) { - if (!tests) { - tests = []; - } - - return tests.map(test => test.title); -} - -function removeEmptyProps(obj) { - if (Array.isArray(obj) && obj.length === 0) { - return null; - } - - if (obj.constructor !== Object) { - return obj; - } - - let cleanObj = null; - - Object.keys(obj).forEach(key => { - const value = removeEmptyProps(obj[key]); - - if (value) { - if (!cleanObj) { - cleanObj = {}; - } - - cleanObj[key] = value; - } - }); - - return cleanObj; -} - -function serialize(collection) { - const serialized = { - tests: { - concurrent: titles(collection.tests.concurrent), - serial: titles(collection.tests.serial) - }, - hooks: { - before: titles(collection.hooks.before), - beforeEach: titles(collection.hooks.beforeEach), - after: titles(collection.hooks.after), - afterAlways: titles(collection.hooks.afterAlways), - afterEach: titles(collection.hooks.afterEach), - afterEachAlways: titles(collection.hooks.afterEachAlways) - } - }; - - return removeEmptyProps(serialized); -} - -test('hasExclusive is set when an exclusive test is added', t => { - const collection = new TestCollection({}); - t.false(collection.hasExclusive); - collection.add(mockTest({exclusive: true}, 'foo')); - t.true(collection.hasExclusive); - t.end(); -}); - -test('adding a concurrent test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({}, 'foo')); - t.strictDeepEqual(serialize(collection), { - tests: { - concurrent: ['foo'] - } - }); - t.end(); -}); - -test('adding a serial test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({serial: true}, 'bar')); - t.strictDeepEqual(serialize(collection), { - tests: { - serial: ['bar'] - } - }); - t.end(); -}); - -test('adding a before test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({type: 'before'}, 'baz')); - t.strictDeepEqual(serialize(collection), { - hooks: { - before: ['baz'] - } - }); - t.end(); -}); - -test('adding a beforeEach test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({type: 'beforeEach'}, 'foo')); - t.strictDeepEqual(serialize(collection), { - hooks: { - beforeEach: ['foo'] - } - }); - t.end(); -}); - -test('adding a after test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({type: 'after'}, 'bar')); - t.strictDeepEqual(serialize(collection), { - hooks: { - after: ['bar'] - } - }); - t.end(); -}); - -test('adding a after.always test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({ - type: 'after', - always: true - }, 'bar')); - t.strictDeepEqual(serialize(collection), { - hooks: { - afterAlways: ['bar'] - } - }); - t.end(); -}); - -test('adding a afterEach test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({type: 'afterEach'}, 'baz')); - t.strictDeepEqual(serialize(collection), { - hooks: { - afterEach: ['baz'] - } - }); - t.end(); -}); - -test('adding a afterEach.always test', t => { - const collection = new TestCollection({}); - collection.add(mockTest({ - type: 'afterEach', - always: true - }, 'baz')); - t.strictDeepEqual(serialize(collection), { - hooks: { - afterEachAlways: ['baz'] - } - }); - t.end(); -}); - -test('adding a bunch of different types', t => { - const collection = new TestCollection({}); - collection.add(mockTest({}, 'a')); - collection.add(mockTest({}, 'b')); - collection.add(mockTest({serial: true}, 'c')); - collection.add(mockTest({serial: true}, 'd')); - collection.add(mockTest({type: 'before'}, 'e')); - t.strictDeepEqual(serialize(collection), { - tests: { - concurrent: ['a', 'b'], - serial: ['c', 'd'] - }, - hooks: { - before: ['e'] - } - }); - t.end(); -}); - -test('skips before and after hooks when all tests are skipped', t => { - t.plan(5); - - const collection = new TestCollection({}); - collection.add({ - metadata: metadata({type: 'before'}), - fn: a => a.fail() - }); - collection.add({ - metadata: metadata({type: 'after'}), - fn: a => a.fail() - }); - collection.add({ - title: 'some serial test', - metadata: metadata({skipped: true, serial: true}), - fn: a => a.fail() - }); - collection.add({ - title: 'some concurrent test', - metadata: metadata({skipped: true}), - fn: a => a.fail() - }); - - const log = []; - collection.on('test', result => { - t.is(result.result.metadata.skipped, true); - t.is(result.result.metadata.type, 'test'); - log.push(result.result.title); - }); - - collection.build().run(); - - t.strictDeepEqual(log, [ - 'some serial test', - 'some concurrent test' - ]); - - t.end(); -}); - -test('runs after.always hook, even if all tests are skipped', t => { - t.plan(6); - - const collection = new TestCollection({}); - collection.add({ - title: 'some serial test', - metadata: metadata({skipped: true, serial: true}), - fn: a => a.fail() - }); - collection.add({ - title: 'some concurrent test', - metadata: metadata({skipped: true}), - fn: a => a.fail() - }); - collection.add({ - title: 'after always', - metadata: metadata({type: 'after', always: true}), - fn: a => a.pass() - }); - - const log = []; - collection.on('test', result => { - if (result.result.metadata.type === 'after') { - t.is(result.result.metadata.skipped, false); - } else { - t.is(result.result.metadata.skipped, true); - t.is(result.result.metadata.type, 'test'); - } - log.push(result.result.title); - }); - - collection.build().run(); - - t.strictDeepEqual(log, [ - 'some serial test', - 'some concurrent test', - 'after always' - ]); - - t.end(); -}); - -test('skips beforeEach and afterEach hooks when test is skipped', t => { - t.plan(3); - - const collection = new TestCollection({}); - collection.add({ - metadata: metadata({type: 'beforeEach'}), - fn: a => a.fail() - }); - collection.add({ - metadata: metadata({type: 'afterEach'}), - fn: a => a.fail() - }); - collection.add({ - title: 'some test', - metadata: metadata({skipped: true}), - fn: a => a.fail() - }); - - const log = []; - collection.on('test', result => { - t.is(result.result.metadata.skipped, true); - t.is(result.result.metadata.type, 'test'); - log.push(result.result.title); - }); - - collection.build().run(); - - t.strictDeepEqual(log, [ - 'some test' - ]); - - t.end(); -}); - -test('foo', t => { - const collection = new TestCollection({}); - const log = []; - - function logger(result) { - t.is(result.passed, true); - log.push(result.result.title); - } - - function add(title, opts) { - collection.add({ - title, - metadata: metadata(opts), - fn: a => a.pass() - }); - } - - add('after1', {type: 'after'}); - add('after.always', { - type: 'after', - always: true - }); - add('beforeEach1', {type: 'beforeEach'}); - add('before1', {type: 'before'}); - add('beforeEach2', {type: 'beforeEach'}); - add('afterEach1', {type: 'afterEach'}); - add('afterEach.always', { - type: 'afterEach', - always: true - }); - add('test1', {}); - add('afterEach2', {type: 'afterEach'}); - add('test2', {}); - add('after2', {type: 'after'}); - add('before2', {type: 'before'}); - - collection.on('test', logger); - - const passed = collection.build().run(); - t.is(passed, true); - - t.strictDeepEqual(log, [ - 'before1', - 'before2', - 'beforeEach1 for test1', - 'beforeEach2 for test1', - 'test1', - 'afterEach1 for test1', - 'afterEach2 for test1', - 'afterEach.always for test1', - 'beforeEach1 for test2', - 'beforeEach2 for test2', - 'test2', - 'afterEach1 for test2', - 'afterEach2 for test2', - 'afterEach.always for test2', - 'after1', - 'after2', - 'after.always' - ]); - - t.end(); -}); diff --git a/test/test.js b/test/test.js index 7cbfa7ec8..0bd63d4d7 100644 --- a/test/test.js +++ b/test/test.js @@ -6,7 +6,6 @@ const delay = require('delay'); const Test = require('../lib/test'); const failingTestHint = 'Test was expected to fail, but succeeded, you should stop marking the test as failing'; -const noop = () => {}; class ContextRef { constructor() { @@ -20,255 +19,212 @@ class ContextRef { } } -function ava(fn, contextRef, onResult) { +function ava(fn, contextRef) { return new Test({ contextRef: contextRef || new ContextRef(), failWithoutAssertions: true, fn, metadata: {type: 'test', callback: false}, - onResult: onResult || noop, - title: '[anonymous]' + title: 'test' }); } -ava.failing = (fn, contextRef, onResult) => { +ava.failing = (fn, contextRef) => { return new Test({ contextRef: contextRef || new ContextRef(), failWithoutAssertions: true, fn, metadata: {type: 'test', callback: false, failing: true}, - onResult: onResult || noop, - title: '[anonymous]' + title: 'test.failing' }); }; -ava.cb = (fn, contextRef, onResult) => { +ava.cb = (fn, contextRef) => { return new Test({ contextRef: contextRef || new ContextRef(), failWithoutAssertions: true, fn, metadata: {type: 'test', callback: true}, - onResult: onResult || noop, - title: '[anonymous]' + title: 'test.cb' }); }; -ava.cb.failing = (fn, contextRef, onResult) => { +ava.cb.failing = (fn, contextRef) => { return new Test({ contextRef: contextRef || new ContextRef(), failWithoutAssertions: true, fn, metadata: {type: 'test', callback: true, failing: true}, - onResult: onResult || noop, - title: '[anonymous]' + title: 'test.cb.failing' }); }; test('run test', t => { - const passed = ava(a => { + return ava(a => { a.fail(); - }).run(); - - t.is(passed, false); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + }); }); test('multiple asserts', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.pass(); a.pass(); a.pass(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.assertCount, 3); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 3); + }); }); test('plan assertions', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.plan(2); a.pass(); a.pass(); - }, null, r => { - result = r; - }).run(); - t.is(passed, true); - t.is(result.result.planCount, 2); - t.is(result.result.assertCount, 2); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 2); + t.is(instance.assertCount, 2); + }); }); test('run more assertions than planned', t => { - let result; - const passed = ava(a => { + return ava(a => { a.plan(2); a.pass(); a.pass(); a.pass(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.ok(result.reason); - t.match(result.reason.message, /Planned for 2 assertions, but got 3\./); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.ok(result.error); + t.match(result.error.message, /Planned for 2 assertions, but got 3\./); + t.is(result.error.name, 'AssertionError'); + }); }); test('fails if no assertions are run', t => { - let result; - const passed = ava(() => {}, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.ok(result.reason); - t.is(result.reason.name, 'Error'); - t.match(result.reason.message, /Test finished without running any assertions/); - t.end(); + return ava(() => {}).run().then(result => { + t.is(result.passed, false); + t.ok(result.error); + t.is(result.error.name, 'Error'); + t.match(result.error.message, /Test finished without running any assertions/); + }); }); test('fails if no assertions are run, unless so planned', t => { - const passed = ava(a => a.plan(0)).run(); - t.is(passed, true); - t.end(); + return ava(a => a.plan(0)).run().then(result => { + t.is(result.passed, true); + }); }); test('fails if no assertions are run, unless an ended callback test', t => { - const passed = ava.cb(a => a.end()).run(); - t.is(passed, true); - t.end(); + return ava.cb(a => a.end()).run().then(result => { + t.is(result.passed, true); + }); }); test('wrap non-assertion errors', t => { const err = new Error(); - let result; - const passed = ava(() => { + return ava(() => { throw err; - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, 'Error thrown in test'); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Error thrown in test:'); - t.match(result.reason.values[0].formatted, /Error/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, 'Error thrown in test'); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Error thrown in test:'); + t.match(result.error.values[0].formatted, /Error/); + }); }); test('end can be used as callback without maintaining thisArg', t => { - ava.cb(a => { + return ava.cb(a => { a.pass(); setTimeout(a.end); - }).run().then(passed => { - t.is(passed, true); - t.end(); + }).run().then(result => { + t.is(result.passed, true); }); }); test('end can be used as callback with error', t => { const err = new Error('failed'); - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.end(err); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, 'Callback called with an error'); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Callback called with an error:'); - t.match(result.reason.values[0].formatted, /.*Error.*\n.*message: 'failed'/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, 'Callback called with an error'); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Callback called with an error:'); + t.match(result.error.values[0].formatted, /.*Error.*\n.*message: 'failed'/); + }); }); test('end can be used as callback with a non-error as its error argument', t => { const nonError = {foo: 'bar'}; - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.end(nonError); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.ok(result.reason); - t.is(result.reason.message, 'Callback called with an error'); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Callback called with an error:'); - t.match(result.reason.values[0].formatted, /.*\{.*\n.*foo: 'bar'/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.ok(result.error); + t.is(result.error.message, 'Callback called with an error'); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Callback called with an error:'); + t.match(result.error.values[0].formatted, /.*\{.*\n.*foo: 'bar'/); + }); }); test('title returns the test title', t => { t.plan(1); - new Test({ + return new Test({ fn(a) { t.is(a.title, 'foo'); a.pass(); }, metadata: {type: 'test', callback: false}, - onResult: noop, title: 'foo' }).run(); }); test('handle non-assertion errors even when planned', t => { const err = new Error('bar'); - let result; - const passed = ava(a => { + return ava(a => { a.plan(1); throw err; - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.message, 'Error thrown in test'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.message, 'Error thrown in test'); + }); }); test('handle testing of arrays', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.deepEqual(['foo', 'bar'], ['foo', 'bar']); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); + }); }); test('handle falsy testing of arrays', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.notDeepEqual(['foo', 'bar'], ['foo', 'bar', 'cat']); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); + }); }); test('handle testing of objects', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.deepEqual({ foo: 'foo', bar: 'bar' @@ -276,18 +232,15 @@ test('handle testing of objects', t => { foo: 'foo', bar: 'bar' }); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); + }); }); test('handle falsy testing of objects', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.notDeepEqual({ foo: 'foo', bar: 'bar' @@ -296,194 +249,150 @@ test('handle falsy testing of objects', t => { bar: 'bar', cat: 'cake' }); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); + }); }); test('planned async assertion', t => { - let result; - ava.cb(a => { + const instance = ava.cb(a => { a.plan(1); setTimeout(() => { a.pass(); a.end(); }, 100); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('async assertion with `.end()`', t => { - let result; - ava.cb(a => { + const instance = ava.cb(a => { setTimeout(() => { a.pass(); a.end(); }, 100); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.assertCount, 1); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.assertCount, 1); }); }); test('more assertions than planned should emit an assertion error', t => { - let result; - const passed = ava(a => { + return ava(a => { a.plan(1); a.pass(); a.pass(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); + }); }); test('record test duration', t => { - let result; - ava.cb(a => { + return ava.cb(a => { a.plan(1); setTimeout(() => { a.true(true); a.end(); }, 1234); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.true(result.result.duration >= 1000); - t.end(); + }).run().then(result => { + t.is(result.passed, true); + t.true(result.duration >= 1000); }); }); test('wait for test to end', t => { - let avaTest; - - let result; - ava.cb(a => { + const instance = ava.cb(a => { a.plan(1); - - avaTest = a; - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.planCount, 1); - t.is(result.result.assertCount, 1); - t.true(result.result.duration >= 1000); - t.end(); + setTimeout(() => { + a.pass(); + a.end(); + }, 1234); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 1); + t.is(instance.assertCount, 1); + t.true(result.duration >= 1000); }); - - setTimeout(() => { - avaTest.pass(); - avaTest.end(); - }, 1234); }); test('fails with the first assertError', t => { - let result; - const passed = ava(a => { + return ava(a => { a.plan(2); a.is(1, 2); a.is(3, 4); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Difference:'); - t.match(result.reason.values[0].formatted, /- 1\n\+ 2/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Difference:'); + t.match(result.error.values[0].formatted, /- 1\n\+ 2/); + }); }); test('failing pending assertion causes test to fail, not promise rejection', t => { - let result; return ava(a => { - return a.throws(Promise.resolve()) - .then(() => { - throw new Error('Should be ignored'); - }); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.notMatch(result.reason.message, /Rejected promise returned by test/); + return a.throws(Promise.resolve()).then(() => { + throw new Error('Should be ignored'); + }); + }).run().then(result => { + t.is(result.passed, false); + t.notMatch(result.error.message, /Rejected promise returned by test/); }); }); test('fails with thrown falsy value', t => { - let result; - const passed = ava(() => { + return ava(() => { throw 0; // eslint-disable-line no-throw-literal - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, 'Error thrown in test'); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Error thrown in test:'); - t.match(result.reason.values[0].formatted, /0/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, 'Error thrown in test'); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Error thrown in test:'); + t.match(result.error.values[0].formatted, /0/); + }); }); test('fails with thrown non-error object', t => { const obj = {foo: 'bar'}; - let result; - const passed = ava(() => { + return ava(() => { throw obj; - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, 'Error thrown in test'); - t.is(result.reason.name, 'AssertionError'); - t.is(result.reason.values.length, 1); - t.is(result.reason.values[0].label, 'Error thrown in test:'); - t.match(result.reason.values[0].formatted, /.*\{.*\n.*foo: 'bar'/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, 'Error thrown in test'); + t.is(result.error.name, 'AssertionError'); + t.is(result.error.values.length, 1); + t.is(result.error.values[0].label, 'Error thrown in test:'); + t.match(result.error.values[0].formatted, /.*\{.*\n.*foo: 'bar'/); + }); }); test('skipped assertions count towards the plan', t => { - let result; - const passed = ava(a => { + const instance = ava(a => { a.plan(2); a.pass(); a.skip.fail(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, true); - t.is(result.result.planCount, 2); - t.is(result.result.assertCount, 2); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 2); + t.is(instance.assertCount, 2); + }); }); test('throws and notThrows work with promises', t => { let asyncCalled = false; - let result; - ava(a => { + const instance = ava(a => { a.plan(2); return Promise.all([ a.throws(delay.reject(10, new Error('foo')), 'foo'), @@ -491,50 +400,39 @@ test('throws and notThrows work with promises', t => { asyncCalled = true; })) ]); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.planCount, 2); - t.is(result.result.assertCount, 2); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 2); + t.is(instance.assertCount, 2); t.is(asyncCalled, true); - t.end(); }); }); test('end should not be called multiple times', t => { - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.pass(); a.end(); a.end(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, '`t.end()` called more than once'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, '`t.end()` called more than once'); + }); }); test('cb test that throws sync', t => { - let result; const err = new Error('foo'); - const passed = ava.cb(() => { + return ava.cb(() => { throw err; - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, 'Error thrown in test'); - t.is(result.reason.name, 'AssertionError'); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, 'Error thrown in test'); + t.is(result.error.name, 'AssertionError'); + }); }); test('multiple resolving and rejecting promises passed to t.throws/t.notThrows', t => { - let result; - ava(a => { + const instance = ava(a => { a.plan(6); const promises = []; for (let i = 0; i < 3; i++) { @@ -544,57 +442,43 @@ test('multiple resolving and rejecting promises passed to t.throws/t.notThrows', ); } return Promise.all(promises); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, true); - t.is(result.result.planCount, 6); - t.is(result.result.assertCount, 6); - t.end(); + }); + return instance.run().then(result => { + t.is(result.passed, true); + t.is(instance.planCount, 6); + t.is(instance.assertCount, 6); }); }); test('fails if test ends while there are pending assertions', t => { - let result; - const passed = ava(a => { + return ava(a => { a.throws(Promise.reject(new Error())); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.name, 'Error'); - t.match(result.reason.message, /Test finished, but an assertion is still pending/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'Error'); + t.match(result.error.message, /Test finished, but an assertion is still pending/); + }); }); test('fails if callback test ends while there are pending assertions', t => { - let result; - const passed = ava.cb(a => { + return ava.cb(a => { a.throws(Promise.reject(new Error())); a.end(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.name, 'Error'); - t.match(result.reason.message, /Test finished, but an assertion is still pending/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'Error'); + t.match(result.error.message, /Test finished, but an assertion is still pending/); + }); }); test('fails if async test ends while there are pending assertions', t => { - let result; - ava(a => { + return ava(a => { a.throws(Promise.reject(new Error())); return Promise.resolve(); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.name, 'Error'); - t.match(result.reason.message, /Test finished, but an assertion is still pending/); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.name, 'Error'); + t.match(result.error.message, /Test finished, but an assertion is still pending/); }); }); @@ -645,117 +529,90 @@ test('contextRef', t => { }); test('failing tests should fail', t => { - const passed = ava.failing('foo', a => { + return ava.failing('foo', a => { a.fail(); - }).run(); - - t.is(passed, true); - t.end(); + }).run().then(result => { + t.is(result.passed, true); + }); }); test('failing callback tests should end without error', t => { const err = new Error('failed'); - const passed = ava.cb.failing(a => { + return ava.cb.failing(a => { a.end(err); - }).run(); - - t.is(passed, true); - t.end(); + }).run().then(result => { + t.is(result.passed, true); + }); }); test('failing tests must not pass', t => { - let result; - const passed = ava.failing(a => { + return ava.failing(a => { a.pass(); - }, null, r => { - result = r; - }).run(); - - t.is(passed, false); - t.is(result.reason.message, failingTestHint); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, failingTestHint); + }); }); test('failing callback tests must not pass', t => { - const passed = ava.cb.failing(a => { + return ava.cb.failing(a => { a.pass(); a.end(); - }).run(); - - t.is(passed, false); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + }); }); test('failing tests must not return a fulfilled promise', t => { - let result; - ava.failing(a => { - return Promise.resolve() - .then(() => { - a.pass(); - }); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.message, failingTestHint); - t.end(); + return ava.failing(a => { + return Promise.resolve().then(() => a.pass()); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, failingTestHint); }); }); test('failing tests pass when returning a rejected promise', t => { - ava.failing(a => { + return ava.failing(a => { a.plan(1); - return a.notThrows(delay(10), 'foo') - .then(() => Promise.reject()); - }).run().then(passed => { - t.is(passed, true); - t.end(); + return a.notThrows(delay(10), 'foo').then(() => Promise.reject()); + }).run().then(result => { + t.is(result.passed, true); }); }); test('failing tests pass with `t.throws(nonThrowingPromise)`', t => { - ava.failing(a => { + return ava.failing(a => { return a.throws(Promise.resolve(10)); - }).run().then(passed => { - t.is(passed, true); - t.end(); + }).run().then(result => { + t.is(result.passed, true); }); }); test('failing tests fail with `t.notThrows(throws)`', t => { - let result; - ava.failing(a => { + return ava.failing(a => { return a.notThrows(Promise.resolve('foo')); - }, null, r => { - result = r; - }).run().then(passed => { - t.is(passed, false); - t.is(result.reason.message, failingTestHint); - t.end(); + }).run().then(result => { + t.is(result.passed, false); + t.is(result.error.message, failingTestHint); }); }); test('log from tests', t => { - let result; - - ava(a => { + return ava(a => { a.log('a log message from a test'); t.true(true); a.log('another log message from a test'); a.log({b: 1, c: {d: 2}}, 'complex log', 5, 5.1); a.log(); - }, null, r => { - result = r; - }).run(); - - t.deepEqual( - result.result.logs, - [ - 'a log message from a test', - 'another log message from a test', - '{\n b: 1,\n c: {\n d: 2,\n },\n} complex log 5 5.1' - ] - ); - - t.end(); + }).run().then(result => { + t.deepEqual( + result.logs, + [ + 'a log message from a test', + 'another log message from a test', + '{\n b: 1,\n c: {\n d: 2,\n },\n} complex log 5 5.1' + ] + ); + }); });