Benchmark: Inline code from benchmark.js (#2019)

Author: IvanGoncharov

package.json

@@ -49,7 +49,6 @@
     "@babel/preset-env": "7.4.5",
     "@babel/register": "7.4.4",
     "babel-eslint": "10.0.2",
-    "benchmark": "2.1.4",
     "chai": "4.2.0",
     "eslint": "5.16.0",
     "eslint-plugin-flowtype": "3.11.1",

resources/benchmark.js

@@ -5,8 +5,9 @@
 const os = require('os');
 const fs = require('fs');
 const path = require('path');
-const { Benchmark } = require('benchmark');
+const assert = require('assert');
 
+const { red, green, yellow, cyan, grey } = require('./colors');
 const {
   exec,
   copyFile,
@@ -16,8 +17,17 @@ const {
   readdirRecursive,
 } = require('./utils');
 
+const NS_PER_SEC = 1e9;
 const LOCAL = 'local';
 
+const minTime = 0.05 * NS_PER_SEC;
+// The maximum time a benchmark is allowed to run before finishing.
+const maxTime = 5 * NS_PER_SEC;
+// The minimum sample size required to perform statistical analysis.
+const minSamples = 15;
+// The default number of times to execute a test on a benchmark's first cycle.
+const initCount = 10;
+
 function LOCAL_DIR(...paths) {
   return path.join(__dirname, '..', ...paths);
 }
@@ -93,43 +103,132 @@ function runBenchmark(benchmark, environments) {
   const benches = environments.map(environment => {
     const module = require(path.join(environment.distPath, benchmark));
     benchmarkName = module.name;
-    return new Benchmark(environment.revision, module.measure);
+    return {
+      name: environment.revision,
+      fn: module.measure,
+    };
   });
 
   console.log('⏱️   ' + benchmarkName);
+  const results = [];
   for (let i = 0; i < benches.length; ++i) {
-    benches[i].run({ async: false });
-    process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
+    const { name, fn } = benches[i];
+    try {
+      const samples = collectSamples(fn);
+      results.push({ name, samples, ...computeStats(samples) });
+      process.stdout.write('  ' + cyan(i + 1) + ' tests completed.\u000D');
+    } catch (error) {
+      console.log('  ' + name + ': ' + red(String(error)));
+    }
   }
   console.log('\n');
 
-  beautifyBenchmark(benches);
+  beautifyBenchmark(results);
   console.log('');
 }
 
-function beautifyBenchmark(results) {
-  const benches = results.map(result => ({
-    name: result.name,
-    error: result.error,
-    ops: result.hz,
-    deviation: result.stats.rme,
-    numRuns: result.stats.sample.length,
-  }));
+function collectSamples(fn) {
+  clock(initCount, fn); // initial warm up
+
+  // Cycles a benchmark until a run `count` can be established.
+  // Resolve time span required to achieve a percent uncertainty of at most 1%.
+  // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html.
+  let count = initCount;
+  let clocked = 0;
+  while ((clocked = clock(count, fn)) < minTime) {
+    // Calculate how many more iterations it will take to achieve the `minTime`.
+    count += Math.ceil(((minTime - clocked) * count) / clocked);
+  }
 
-  const nameMaxLen = maxBy(benches, ({ name }) => name.length);
-  const opsTop = maxBy(benches, ({ ops }) => ops);
-  const opsMaxLen = maxBy(benches, ({ ops }) => beautifyNumber(ops).length);
+  let elapsed = 0;
+  const samples = [];
 
-  for (const bench of benches) {
-    if (bench.error) {
-      console.log('  ' + bench.name + ': ' + red(String(bench.error)));
-      continue;
-    }
-    printBench(bench);
+  // If time permits, increase sample size to reduce the margin of error.
+  while (samples.length < minSamples || elapsed < maxTime) {
+    clocked = clock(count, fn);
+    assert(clocked > 0);
+
+    elapsed += clocked;
+    // Compute the seconds per operation.
+    samples.push(clocked / count);
+  }
+
+  return samples;
+}
+
+// Clocks the time taken to execute a test per cycle (secs).
+function clock(count, fn) {
+  const start = process.hrtime.bigint();
+  for (let i = 0; i < count; ++i) {
+    fn();
+  }
+  return Number(process.hrtime.bigint() - start);
+}
+
+// T-Distribution two-tailed critical values for 95% confidence.
+// See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm.
+const tTable = /* prettier-ignore */ {
+  '1':  12.706, '2':  4.303, '3':  3.182, '4':  2.776, '5':  2.571, '6':  2.447,
+  '7':  2.365,  '8':  2.306, '9':  2.262, '10': 2.228, '11': 2.201, '12': 2.179,
+  '13': 2.16,   '14': 2.145, '15': 2.131, '16': 2.12,  '17': 2.11,  '18': 2.101,
+  '19': 2.093,  '20': 2.086, '21': 2.08,  '22': 2.074, '23': 2.069, '24': 2.064,
+  '25': 2.06,   '26': 2.056, '27': 2.052, '28': 2.048, '29': 2.045, '30': 2.042,
+  infinity: 1.96,
+};
+
+// Computes stats on benchmark results.
+function computeStats(samples) {
+  assert(samples.length > 1);
+
+  // Compute the sample mean (estimate of the population mean).
+  let mean = 0;
+  for (const x of samples) {
+    mean += x;
+  }
+  mean /= samples.length;
+
+  // Compute the sample variance (estimate of the population variance).
+  let variance = 0;
+  for (const x of samples) {
+    variance += Math.pow(x - mean, 2);
+  }
+  variance /= samples.length - 1;
+
+  // Compute the sample standard deviation (estimate of the population standard deviation).
+  const sd = Math.sqrt(variance);
+
+  // Compute the standard error of the mean (a.k.a. the standard deviation of the sampling distribution of the sample mean).
+  const sem = sd / Math.sqrt(samples.length);
+
+  // Compute the degrees of freedom.
+  const df = samples.length - 1;
+
+  // Compute the critical value.
+  const critical = tTable[df] || tTable.infinity;
+
+  // Compute the margin of error.
+  const moe = sem * critical;
+
+  // The relative margin of error (expressed as a percentage of the mean).
+  const rme = (moe / mean) * 100 || 0;
+
+  return {
+    ops: NS_PER_SEC / mean,
+    deviation: rme,
+  };
+}
+
+function beautifyBenchmark(results) {
+  const nameMaxLen = maxBy(results, ({ name }) => name.length);
+  const opsTop = maxBy(results, ({ ops }) => ops);
+  const opsMaxLen = maxBy(results, ({ ops }) => beautifyNumber(ops).length);
+
+  for (const result of results) {
+    printBench(result);
   }
 
   function printBench(bench) {
-    const { name, ops, deviation, numRuns } = bench;
+    const { name, ops, deviation, samples } = bench;
     console.log(
       '  ' +
         nameStr() +
@@ -139,7 +238,7 @@ function beautifyBenchmark(results) {
         grey('\xb1') +
         deviationStr() +
         cyan('%') +
-        grey(' (' + numRuns + ' runs sampled)'),
+        grey(' (' + samples.length + ' runs sampled)'),
     );
 
     function nameStr() {
@@ -160,22 +259,6 @@ function beautifyBenchmark(results) {
   }
 }
 
-function red(str) {
-  return '\u001b[31m' + str + '\u001b[0m';
-}
-function green(str) {
-  return '\u001b[32m' + str + '\u001b[0m';
-}
-function yellow(str) {
-  return '\u001b[33m' + str + '\u001b[0m';
-}
-function cyan(str) {
-  return '\u001b[36m' + str + '\u001b[0m';
-}
-function grey(str) {
-  return '\u001b[90m' + str + '\u001b[0m';
-}
-
 function beautifyNumber(num) {
   return Number(num.toFixed(num > 100 ? 0 : 2)).toLocaleString();
 }

resources/colors.js

@@ -0,0 +1,31 @@
+// @noflow
+
+'use strict';
+
+function red(str) {
+  return '\u001b[31m' + str + '\u001b[0m';
+}
+
+function green(str) {
+  return '\u001b[32m' + str + '\u001b[0m';
+}
+
+function yellow(str) {
+  return '\u001b[33m' + str + '\u001b[0m';
+}
+
+function cyan(str) {
+  return '\u001b[36m' + str + '\u001b[0m';
+}
+
+function grey(str) {
+  return '\u001b[90m' + str + '\u001b[0m';
+}
+
+module.exports = {
+  red,
+  green,
+  yellow,
+  cyan,
+  grey,
+};

yarn.lock

@@ -751,14 +751,6 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
   integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
 
-benchmark@2.1.4:
-  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
-  integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
-  dependencies:
-    lodash "^4.17.4"
-    platform "^1.3.3"
-
 brace-expansion@^1.1.7:
   version "1.1.11"
   resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@@ -1699,7 +1691,7 @@ lodash.flattendeep@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2"
   integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=
 
-lodash@^4.17.11, lodash@^4.17.4:
+lodash@^4.17.11:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
   integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
@@ -2129,11 +2121,6 @@ pkg-dir@^3.0.0:
   dependencies:
     find-up "^3.0.0"
 
-platform@^1.3.3:
-  version "1.3.5"
-  resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
-  integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==
-
 prelude-ls@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"