Allow setting a seed for --shuffle with --seed. (#657)

This is useful, when you run into an order dependent test failure. The
runner reports the used seed after the test run. If you want to maintain
the same test run order for the next run, you can re-use the seed and
you get the same order.

For example: `lab test/unit --shuffle --seed 1234`

Fixes #656

README.md

@@ -53,6 +53,7 @@ global manipulation. Our goal with **lab** is to keep the execution engine as si
     - [Multiple Reporters](#multiple-reporters) - See Below
     - [Custom Reporters](#custom-reporters) - See Below
 - `--shuffle` - randomize the order that test scripts are executed.  Will not work with `--id`.
+- `--seed` - use this seed to randomize the order with `--shuffle`. This is useful to debug order dependent test failures.
 - `-s`, `--silence` - silence test output, defaults to false.
 - `-S`, `--sourcemaps` - enables sourcemap support for stack traces and code coverage, disabled by default.
 - `-t`, `--threshold` - sets the minimum code test coverage percentage to 100%.

lib/cli.js

@@ -293,6 +293,11 @@ internals.options = function () {
             multiple: true,
             default: null
         },
+        seed: {
+            type: 'string',
+            description: 'use this seed to randomize the order with `--shuffle`. This is useful to debug order dependent test failures',
+            default: null
+        },
         shuffle: {
             type: 'boolean',
             description: 'shuffle script execution order',
@@ -396,7 +401,7 @@ internals.options = function () {
     const keys = ['assert', 'colors', 'context-timeout', 'coverage', 'coverage-exclude',
         'coverage-path', 'debug', 'dry', 'environment', 'flat', 'globals', 'grep',
         'lint', 'lint-errors-threshold', 'lint-fix', 'lint-options', 'lint-warnings-threshold',
-        'linter', 'output', 'parallel', 'pattern', 'rejections', 'reporter', 'shuffle', 'silence',
+        'linter', 'output', 'parallel', 'pattern', 'rejections', 'reporter', 'seed', 'shuffle', 'silence',
         'silent-skips', 'sourcemaps', 'threshold', 'timeout', 'transform', 'verbose'];
     for (let i = 0; i < keys.length; ++i) {
         if (argv.hasOwnProperty(keys[i]) && argv[keys[i]] !== undefined && argv[keys[i]] !== null) {

lib/reporters/console.js

@@ -262,6 +262,10 @@ internals.Reporter.prototype.end = function (notebook) {
         }
     }
 
+    if (notebook.seed) {
+        output += 'Randomized with seed: ' + notebook.seed + '. Use --shuffle --seed ' + notebook.seed + ' to run tests in same order again.\n';
+    }
+
     // Coverage
 
     const coverage = notebook.coverage;

lib/runner.js

@@ -5,6 +5,7 @@
 const Domain = require('domain');
 const Items = require('items');
 const Hoek = require('hoek');
+const Seedrandom = require('seedrandom');
 const Reporters = require('./reporters');
 const Coverage = require('./coverage');
 const Linters = require('./lint');
@@ -51,6 +52,7 @@ internals.defaults = {
     rejections: false,
     reporter: 'console',
     shuffle: false,
+    seed: Math.random(),
 
     // schedule: true,
     threshold: 0,
@@ -93,6 +95,10 @@ exports.report = function (scripts, options, callback) {
                 result.coverage = Coverage.analyze(settings);
             }
 
+            if (settings.shuffle) {
+                result.seed = settings.seed;
+            }
+
             return next(null, result);
         });
     };
@@ -135,7 +141,7 @@ exports.execute = function (scripts, options, reporter, callback) {
     scripts = [].concat(scripts);
 
     if (settings.shuffle) {
-        internals.shuffle(scripts);
+        internals.shuffle(scripts, settings.seed);
     }
 
     const experiments = scripts.map((script) => {
@@ -241,12 +247,13 @@ internals.enableSkip = (element) => {
     element.options.skip = true;
 };
 
+internals.shuffle = function (scripts, seed) {
 
-internals.shuffle = function (scripts) {
+    const random = Seedrandom(seed);
 
     const last = scripts.length - 1;
     for (let i = 0; i < scripts.length; ++i) {
-        const rand = i + Math.floor(Math.random() * (last - i + 1));
+        const rand = i + Math.floor(random() * (last - i + 1));
         const temp = scripts[i];
         scripts[i] = scripts[rand];
         scripts[rand] = temp;

package.json

@@ -24,6 +24,7 @@
     "json-stable-stringify": "1.x.x",
     "json-stringify-safe": "5.x.x",
     "mkdirp": "0.5.x",
+    "seedrandom": "^2.4.2",
     "source-map-support": "0.4.x"
   },
   "devDependencies": {

test/cli.js

@@ -333,6 +333,21 @@ describe('CLI', () => {
         });
     });
 
+    it('reports the used seed for randomization', (done) => {
+
+        RunCli(['test/cli', '--shuffle'], (error, result) => {
+
+            if (error) {
+                done(error);
+            }
+
+            expect(result.errorOutput).to.equal('');
+            expect(result.code).to.equal(0);
+            expect(result.output).to.contain('seed');
+            done();
+        });
+    });
+
     it('runs a range of tests (-i 3-4)', (done) => {
 
         // The range may need to adjust as new tests are added (if they are skipped for example)

test/reporters.js

@@ -325,6 +325,23 @@ describe('Reporter', () => {
         });
     });
 
+    it('includes the used seed for shuffle in the output', (done) => {
+
+        const reporter = Reporters.generate({ reporter: 'console' });
+        const notebook = {
+            tests: [],
+            seed: 1234
+        };
+
+        reporter.finalize(notebook, (err, code, output) => {
+
+            expect(output).to.contain('1234');
+            expect(output).to.contain('seed');
+            expect(err).not.to.exist();
+            done();
+        });
+    });
+
     describe('console', () => {
 
         it('generates a report', (done) => {

test/runner.js

@@ -892,27 +892,74 @@ describe('Runner', () => {
             });
         });
 
-        const random = Math.random;
-        let first = true;
-        Math.random = function () {
+        const scripts = [script1, script2, script3, script4, script5];
+        Lab.execute(scripts, { dry: true, shuffle: true, seed: 0.3 }, null, (err, notebook1) => {
 
-            if (first) {
-                first = false;
-                return 0.3;
-            }
+            expect(err).not.to.exist();
+            Lab.execute(scripts, { dry: true, shuffle: true, seed: 0.7 }, null, (err, notebook2) => {
 
-            return 0.7;
-        };
+                expect(err).not.to.exist();
+                expect(notebook1.tests).to.not.equal(notebook2.tests);
+                done();
+            });
+        });
+    });
+
+    it('shuffle allows to set a seed to use to re-use order of a previous test run', (done) => {
+
+        const script1 = Lab.script();
+        script1.experiment('test1', () => {
+
+            script1.test('1', (testDone) => {
+
+                testDone();
+            });
+        });
+
+        const script2 = Lab.script();
+        script2.experiment('test2', () => {
+
+            script2.test('2', (testDone) => {
+
+                testDone();
+            });
+        });
+
+        const script3 = Lab.script();
+        script3.experiment('test3', () => {
+
+            script3.test('3', (testDone) => {
+
+                testDone();
+            });
+        });
+
+        const script4 = Lab.script();
+        script4.experiment('test4', () => {
+
+            script4.test('4', (testDone) => {
+
+                testDone();
+            });
+        });
+
+        const script5 = Lab.script();
+        script5.experiment('test5', () => {
+
+            script5.test('5', (testDone) => {
+
+                testDone();
+            });
+        });
 
         const scripts = [script1, script2, script3, script4, script5];
-        Lab.execute(scripts, { dry: true, shuffle: true }, null, (err, notebook1) => {
+        Lab.execute(scripts, { dry: true, shuffle: true, seed: 1234 }, null, (err, notebook1) => {
 
             expect(err).not.to.exist();
-            Lab.execute(scripts, { dry: true, shuffle: true }, null, (err, notebook2) => {
+            Lab.execute(scripts, { dry: true, shuffle: true, seed: 1234 }, null, (err, notebook2) => {
 
                 expect(err).not.to.exist();
-                expect(notebook1.tests).to.not.equal(notebook2.tests);
-                Math.random = random;
+                expect(notebook1.tests).to.equal(notebook2.tests);
                 done();
             });
         });
@@ -1209,6 +1256,26 @@ describe('Runner', () => {
         });
     });
 
+    it('reports the used seed', (done) => {
+
+        const script = Lab.script();
+        script.experiment('test', () => {
+
+            script.test('1', (testDone) => {
+
+                testDone();
+            });
+        });
+
+        Lab.report(script, { output: false, seed: 1234, shuffle: true }, (err, code, output) => {
+
+            expect(err).not.to.exist();
+            expect(code).to.equal(0);
+            expect(output).to.contain('1234');
+            done();
+        });
+    });
+
     it('uses provided linter', (done) => {
 
         const script = Lab.script();