benchmark: support for multiple http benchmarkers

Makefile

@@ -620,13 +620,6 @@ ifeq ($(XZ), 0)
  ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.xz.done"
 endif
 
-haswrk=$(shell which wrk > /dev/null 2>&1; echo $$?)
-wrk:
-ifneq ($(haswrk), 0)
- @echo "please install wrk before proceeding. More information can be found in benchmark/README.md." >&2
- @exit 1
-endif
-
 bench-net: all
  @$(NODE) benchmark/run.js net
 
@@ -636,7 +629,7 @@ bench-crypto: all
 bench-tls: all
  @$(NODE) benchmark/run.js tls
 
-bench-http: wrk all
+bench-http: all
  @$(NODE) benchmark/run.js http
 
 bench-fs: all

benchmark/README.md

@@ -14,9 +14,28 @@ This folder contains benchmarks to measure the performance of the Node.js APIs.
 
 ## Prerequisites
 
-Most of the http benchmarks require [`wrk`][wrk] to be installed. It may be
-available through your preferred package manager. If not, `wrk` can be built
-[from source][wrk] via `make`.
+Most of the HTTP benchmarks require a benchmarker to be installed, this can be
+either [`wrk`][wrk] or [`autocannon`][autocannon].
+
+`Autocannon` is a Node script that can be installed using
+`npm install -g autocannon`. It will use the Node executable that is in the
+path, hence if you want to compare two HTTP benchmark runs make sure that the
+Node version in the path is not altered.
+
+`wrk` may be available through your preferred package manger. If not, you can
+easily build it [from source][wrk] via `make`.
+
+To select which tool will be used to run your HTTP benchmark you can:
+* When running the benchmakrs, set `NODE_HTTP_BENCHMARKER` environment variable
+to desired benchmarker.
+* To select the default benchmarker for a particular benchmark, specify it as
+`benchmarker` key (e.g. `benchmarker: 'wrk'`) in configuration passed to
+`createBenchmark`. This can be overridden by `NODE_HTTP_BENCHMARKER` in run
+time.
+
+If you do not specify which benchmarker to use, all of the installed tools will
+be used to run the benchmarks. This will also happen if you pass `all` as the
+desired benchmark tool.
 
 To analyze the results `R` should be installed. Check you package manager or
 download it from https://www.r-project.org/.
@@ -287,5 +306,6 @@ function main(conf) {
 }
 ```
 
+[autocannon]: https://github.com/mcollina/autocannon
 [wrk]: https://github.com/wg/wrk
 [t-test]: https://en.wikipedia.org/wiki/Student%27s_t-test#Equal_or_unequal_sample_sizes.2C_unequal_variances

benchmark/common.js

@@ -88,52 +88,157 @@ Benchmark.prototype._queue = function(options) {
  return queue;
 };
 
-function hasWrk() {
- const result = child_process.spawnSync('wrk', ['-h']);
- if (result.error && result.error.code === 'ENOENT') {
- console.error('Couldn\'t locate `wrk` which is needed for running ' +
- 'benchmarks. Check benchmark/README.md for further instructions.');
- process.exit(1);
- }
+function AutocannonBenchmarker() {
+ const autocannon_exe = process.platform === 'win32'
+ ? 'autocannon.cmd'
+ : 'autocannon';
+ this.present = function() {
+ var result = child_process.spawnSync(autocannon_exe, ['-h']);
+ if (result.error && result.error.code === 'ENOENT')
+ return false;
+ else
+ return true;
+ };
+ this.create = function(path, duration, connections) {
+ const args = ['-d', duration, '-c', connections, '-j', '-n',
+ 'http://127.0.0.1:' + exports.PORT + path ];
+ var child = child_process.spawn(autocannon_exe, args);
+ child.stdout.setEncoding('utf8');
+ return child;
+ };
+ this.processResults = function(output) {
+ let result;
+ try {
+ result = JSON.parse(output);
+ } catch (err) {
+ // Do nothing, let next line handle this
+ }
+ if (!result || !result.requests || !result.requests.average) {
+ return undefined;
+ } else {
+ return result.requests.average;
+ }
+ };
 }
 
-// benchmark an http server.
-const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/;
-Benchmark.prototype.http = function(urlPath, args, cb) {
- hasWrk();
- const self = this;
-
- const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath;
- args = args.concat(urlFull);
-
- const childStart = process.hrtime();
- const child = child_process.spawn('wrk', args);
- child.stderr.pipe(process.stderr);
+function WrkBenchmarker() {
+ this.present = function() {
+ var result = child_process.spawnSync('wrk', ['-h']);
+ if (result.error && result.error.code === 'ENOENT')
+ return false;
+ else
+ return true;
+ };
+ this.create = function(path, duration, connections) {
+ const args = ['-d', duration, '-c', connections, '-t', 8,
+ 'http://127.0.0.1:' + exports.PORT + path ];
+ var child = child_process.spawn('wrk', args);
+ child.stdout.setEncoding('utf8');
+ child.stderr.pipe(process.stderr);
+ return child;
+ };
+ const regexp = /Requests\/sec:[ \t]+([0-9\.]+)/;
+ this.processResults = function(output) {
+ const match = output.match(regexp);
+ const result = match && +match[1];
+ if (!result)
+ return undefined;
+ else
+ return result;
+ };
+}
 
- // Collect stdout
- let stdout = '';
- child.stdout.on('data', (chunk) => stdout += chunk.toString());
+const HTTPBenchmarkers = {
+ autocannon: new AutocannonBenchmarker(),
+ wrk: new WrkBenchmarker()
+};
 
- child.once('close', function(code) {
- const elapsed = process.hrtime(childStart);
- if (cb) cb(code);
+// Benchmark an http server.
+Benchmark.prototype.http = function(urlPath, duration, connections, cb) {
+ const self = this;
+ duration = 1;
+
+ const picked_benchmarker = process.env.NODE_HTTP_BENCHMARKER ||
+ this.config.benchmarker || 'all';
+ const benchmarkers = picked_benchmarker === 'all'
+ ? Object.keys(HTTPBenchmarkers)
+ : [picked_benchmarker];
+
+ // See if any benchmarker is available. Also test if all used benchmarkers
+ // are defined
+ var any_available = false;
+ for (var i = 0; i < benchmarkers.length; ++i) {
+ const benchmarker = benchmarkers[i];
+ const http_benchmarker = HTTPBenchmarkers[benchmarker];
+ if (http_benchmarker === undefined) {
+ console.error('Unknown http benchmarker: ', benchmarker);
+ process.exit(1);
+ }
+ if (http_benchmarker.present()) {
+ any_available = true;
+ }
+ }
+ if (!any_available) {
+ console.error('Couldn\'t locate any of the required http benchmarkers ' +
+ '(' + benchmarkers.join(', ') + '). Check ' +
+ 'benchmark/README.md for further instructions.');
+ process.exit(1);
+ }
 
- if (code) {
- console.error('wrk failed with ' + code);
- process.exit(code);
+ function runHttpBenchmarker(index, collected_code) {
+ // All benchmarkers executed
+ if (index === benchmarkers.length) {
+ if (cb)
+ cb(collected_code);
+ if (collected_code !== 0)
+ process.exit(1);
+ return;
  }
 
- // Extract requests pr second and check for odd results
- const match = stdout.match(WRK_REGEXP);
- if (!match || match.length <= 1) {
- console.error('wrk produced strange output:');
- console.error(stdout);
- process.exit(1);
+ // Run next benchmarker
+ const benchmarker = benchmarkers[index];
+ self.config.benchmarker = benchmarker;
+
+ const http_benchmarker = HTTPBenchmarkers[benchmarker];
+ if (http_benchmarker.present()) {
+ const child_start = process.hrtime();
+ var child = http_benchmarker.create(urlPath, duration, connections);
+
+ // Collect stdout
+ let stdout = '';
+ child.stdout.on('data', (chunk) => stdout += chunk.toString());
+
+ child.once('close', function(code) {
+ const elapsed = process.hrtime(child_start);
+ if (code) {
+ if (stdout === '') {
+ console.error(benchmarker + ' failed with ' + code);
+ } else {
+ console.error(benchmarker + ' failed with ' + code + '. Output: ');
+ console.error(stdout);
+ }
+ runHttpBenchmarker(index + 1, code);
+ return;
+ }
+
+ var result = http_benchmarker.processResults(stdout);
+ if (!result) {
+ console.error(benchmarker + ' produced strange output');
+ console.error(stdout);
+ runHttpBenchmarker(index + 1, 1);
+ return;
+ }
+
+ self.report(result, elapsed);
+ runHttpBenchmarker(index + 1, collected_code);
+ });
+ } else {
+ runHttpBenchmarker(index + 1, collected_code);
  }
+ }
 
- // Report rate
- self.report(+match[1], elapsed);
- });
+ // Run with all benchmarkers
+ runHttpBenchmarker(0, 0);
 };
 
 Benchmark.prototype._run = function() {

benchmark/http/chunked.js

@@ -20,8 +20,6 @@ function main(conf) {
  const http = require('http');
  var chunk = Buffer.alloc(conf.size, '8');
 
- var args = ['-d', '10s', '-t', 8, '-c', conf.c];
-
  var server = http.createServer(function(req, res) {
  function send(left) {
  if (left === 0) return res.end();
@@ -34,7 +32,7 @@ function main(conf) {
  });
 
  server.listen(common.PORT, function() {
- bench.http('/', args, function() {
+ bench.http('/', 10, conf.c, function() {
  server.close();
  });
  });

benchmark/http/cluster.js

@@ -27,9 +27,8 @@ function main(conf) {
 
  setTimeout(function() {
  var path = '/' + conf.type + '/' + conf.length;
- var args = ['-d', '10s', '-t', 8, '-c', conf.c];
 
- bench.http(path, args, function() {
+ bench.http(path, 10, conf.c, function() {
  w1.destroy();
  w2.destroy();
  });

benchmark/http/end-vs-write-end.js

@@ -43,14 +43,13 @@ function main(conf) {
  }
 
  var method = conf.method === 'write' ? write : end;
- var args = ['-d', '10s', '-t', 8, '-c', conf.c];
 
  var server = http.createServer(function(req, res) {
  method(res);
  });
 
  server.listen(common.PORT, function() {
- bench.http('/', args, function() {
+ bench.http('/', 10, conf.c, function() {
  server.close();
  });
  });

benchmark/http/simple.js

@@ -15,9 +15,8 @@ function main(conf) {
  var server = require('./_http_simple.js');
  setTimeout(function() {
  var path = '/' + conf.type + '/' + conf.length + '/' + conf.chunks;
- var args = ['-d', '10s', '-t', 8, '-c', conf.c];
 
- bench.http(path, args, function() {
+ bench.http(path, 10, conf.c, function() {
  server.close();
  });
  }, 2000);