diff --git a/README.md b/README.md index 3a76561a..a4b0b189 100755 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ global manipulation. Our goal with **lab** is to keep the execution engine as si - [Multiple Reporters](#multiple-reporters) - See Below - [Custom Reporters](#custom-reporters) - See Below - `--shuffle` - randomize the order that test scripts are executed. Will not work with `--id`. +- `--seed` - use this seed to randomize the order with `--shuffle`. This is useful to debug order dependent test failures. - `-s`, `--silence` - silence test output, defaults to false. - `-S`, `--sourcemaps` - enables sourcemap support for stack traces and code coverage, disabled by default. - `-t`, `--threshold` - sets the minimum code test coverage percentage to 100%. diff --git a/lib/cli.js b/lib/cli.js index cca2896b..3732cdbb 100755 --- a/lib/cli.js +++ b/lib/cli.js @@ -293,6 +293,11 @@ internals.options = function () { multiple: true, default: null }, + seed: { + type: 'string', + description: 'use this seed to randomize the order with `--shuffle`. This is useful to debug order dependent test failures', + default: null + }, shuffle: { type: 'boolean', description: 'shuffle script execution order', @@ -396,7 +401,7 @@ internals.options = function () { const keys = ['assert', 'colors', 'context-timeout', 'coverage', 'coverage-exclude', 'coverage-path', 'debug', 'dry', 'environment', 'flat', 'globals', 'grep', 'lint', 'lint-errors-threshold', 'lint-fix', 'lint-options', 'lint-warnings-threshold', - 'linter', 'output', 'parallel', 'pattern', 'rejections', 'reporter', 'shuffle', 'silence', + 'linter', 'output', 'parallel', 'pattern', 'rejections', 'reporter', 'seed', 'shuffle', 'silence', 'silent-skips', 'sourcemaps', 'threshold', 'timeout', 'transform', 'verbose']; for (let i = 0; i < keys.length; ++i) { if (argv.hasOwnProperty(keys[i]) && argv[keys[i]] !== undefined && argv[keys[i]] !== null) { diff --git a/lib/reporters/console.js b/lib/reporters/console.js index 2be0c961..a8bb7302 100755 --- a/lib/reporters/console.js +++ b/lib/reporters/console.js @@ -262,6 +262,10 @@ internals.Reporter.prototype.end = function (notebook) { } } + if (notebook.seed) { + output += 'Randomized with seed: ' + notebook.seed + '. Use --shuffle --seed ' + notebook.seed + ' to run tests in same order again.\n'; + } + // Coverage const coverage = notebook.coverage; diff --git a/lib/runner.js b/lib/runner.js index 25523b3f..32f5a3ca 100755 --- a/lib/runner.js +++ b/lib/runner.js @@ -5,6 +5,7 @@ const Domain = require('domain'); const Items = require('items'); const Hoek = require('hoek'); +const Seedrandom = require('seedrandom'); const Reporters = require('./reporters'); const Coverage = require('./coverage'); const Linters = require('./lint'); @@ -51,6 +52,7 @@ internals.defaults = { rejections: false, reporter: 'console', shuffle: false, + seed: Math.random(), // schedule: true, threshold: 0, @@ -93,6 +95,10 @@ exports.report = function (scripts, options, callback) { result.coverage = Coverage.analyze(settings); } + if (settings.shuffle) { + result.seed = settings.seed; + } + return next(null, result); }); }; @@ -135,7 +141,7 @@ exports.execute = function (scripts, options, reporter, callback) { scripts = [].concat(scripts); if (settings.shuffle) { - internals.shuffle(scripts); + internals.shuffle(scripts, settings.seed); } const experiments = scripts.map((script) => { @@ -241,12 +247,13 @@ internals.enableSkip = (element) => { element.options.skip = true; }; +internals.shuffle = function (scripts, seed) { -internals.shuffle = function (scripts) { + const random = Seedrandom(seed); const last = scripts.length - 1; for (let i = 0; i < scripts.length; ++i) { - const rand = i + Math.floor(Math.random() * (last - i + 1)); + const rand = i + Math.floor(random() * (last - i + 1)); const temp = scripts[i]; scripts[i] = scripts[rand]; scripts[rand] = temp; diff --git a/package.json b/package.json index b42c1807..276b536d 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "json-stable-stringify": "1.x.x", "json-stringify-safe": "5.x.x", "mkdirp": "0.5.x", + "seedrandom": "^2.4.2", "source-map-support": "0.4.x" }, "devDependencies": { diff --git a/test/cli.js b/test/cli.js index cfbf4c51..bf4b1564 100755 --- a/test/cli.js +++ b/test/cli.js @@ -333,6 +333,21 @@ describe('CLI', () => { }); }); + it('reports the used seed for randomization', (done) => { + + RunCli(['test/cli', '--shuffle'], (error, result) => { + + if (error) { + done(error); + } + + expect(result.errorOutput).to.equal(''); + expect(result.code).to.equal(0); + expect(result.output).to.contain('seed'); + done(); + }); + }); + it('runs a range of tests (-i 3-4)', (done) => { // The range may need to adjust as new tests are added (if they are skipped for example) diff --git a/test/reporters.js b/test/reporters.js index 2f78b9da..8d40a0cf 100755 --- a/test/reporters.js +++ b/test/reporters.js @@ -325,6 +325,23 @@ describe('Reporter', () => { }); }); + it('includes the used seed for shuffle in the output', (done) => { + + const reporter = Reporters.generate({ reporter: 'console' }); + const notebook = { + tests: [], + seed: 1234 + }; + + reporter.finalize(notebook, (err, code, output) => { + + expect(output).to.contain('1234'); + expect(output).to.contain('seed'); + expect(err).not.to.exist(); + done(); + }); + }); + describe('console', () => { it('generates a report', (done) => { diff --git a/test/runner.js b/test/runner.js index 5f655729..e0068dbd 100755 --- a/test/runner.js +++ b/test/runner.js @@ -892,27 +892,74 @@ describe('Runner', () => { }); }); - const random = Math.random; - let first = true; - Math.random = function () { + const scripts = [script1, script2, script3, script4, script5]; + Lab.execute(scripts, { dry: true, shuffle: true, seed: 0.3 }, null, (err, notebook1) => { - if (first) { - first = false; - return 0.3; - } + expect(err).not.to.exist(); + Lab.execute(scripts, { dry: true, shuffle: true, seed: 0.7 }, null, (err, notebook2) => { - return 0.7; - }; + expect(err).not.to.exist(); + expect(notebook1.tests).to.not.equal(notebook2.tests); + done(); + }); + }); + }); + + it('shuffle allows to set a seed to use to re-use order of a previous test run', (done) => { + + const script1 = Lab.script(); + script1.experiment('test1', () => { + + script1.test('1', (testDone) => { + + testDone(); + }); + }); + + const script2 = Lab.script(); + script2.experiment('test2', () => { + + script2.test('2', (testDone) => { + + testDone(); + }); + }); + + const script3 = Lab.script(); + script3.experiment('test3', () => { + + script3.test('3', (testDone) => { + + testDone(); + }); + }); + + const script4 = Lab.script(); + script4.experiment('test4', () => { + + script4.test('4', (testDone) => { + + testDone(); + }); + }); + + const script5 = Lab.script(); + script5.experiment('test5', () => { + + script5.test('5', (testDone) => { + + testDone(); + }); + }); const scripts = [script1, script2, script3, script4, script5]; - Lab.execute(scripts, { dry: true, shuffle: true }, null, (err, notebook1) => { + Lab.execute(scripts, { dry: true, shuffle: true, seed: 1234 }, null, (err, notebook1) => { expect(err).not.to.exist(); - Lab.execute(scripts, { dry: true, shuffle: true }, null, (err, notebook2) => { + Lab.execute(scripts, { dry: true, shuffle: true, seed: 1234 }, null, (err, notebook2) => { expect(err).not.to.exist(); - expect(notebook1.tests).to.not.equal(notebook2.tests); - Math.random = random; + expect(notebook1.tests).to.equal(notebook2.tests); done(); }); }); @@ -1209,6 +1256,26 @@ describe('Runner', () => { }); }); + it('reports the used seed', (done) => { + + const script = Lab.script(); + script.experiment('test', () => { + + script.test('1', (testDone) => { + + testDone(); + }); + }); + + Lab.report(script, { output: false, seed: 1234, shuffle: true }, (err, code, output) => { + + expect(err).not.to.exist(); + expect(code).to.equal(0); + expect(output).to.contain('1234'); + done(); + }); + }); + it('uses provided linter', (done) => { const script = Lab.script();