Benchmark task

This adds a benchmark task inspired by #1167 and #1163. Invoke with `yarn benchmark`.

Allows filtering down which benchmark to run, ala jest, and supplying which revisions to run against.

.npmignore

@@ -16,4 +16,5 @@ coverage
 resources
 src
 dist
+__tests__
 npm

package.json

@@ -25,6 +25,7 @@
  "testonly:cover": "babel-node ./node_modules/.bin/isparta cover --root src --report html _mocha -- $npm_package_options_mocha",
  "testonly:coveralls": "babel-node ./node_modules/.bin/isparta cover --root src --report lcovonly _mocha -- $npm_package_options_mocha && cat ./coverage/lcov.info | coveralls",
  "lint": "eslint --rulesdir ./resources/lint src || (printf '\\033[33mTry: \\033[7m npm run lint -- --fix \\033[0m\\n' && exit 1)",
+ "benchmark": "node ./resources/benchmark.js",
  "prettier": "prettier --write 'src/**/*.js'",
  "check": "flow check",
  "check-cover": "for file in {src/*.js,src/**/*.js}; do echo $file; flow coverage $file; done",
@@ -52,6 +53,8 @@
  "babel-plugin-transform-flow-strip-types": "6.22.0",
  "babel-plugin-transform-object-rest-spread": "6.26.0",
  "babel-preset-env": "^1.5.2",
+ "beautify-benchmark": "0.2.4",
+ "benchmark": "2.1.4",
  "chai": "4.1.2",
  "chai-json-equal": "0.0.1",
  "chai-spies-next": "0.9.3",

resources/benchmark.js

@@ -0,0 +1,148 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const { Suite } = require('benchmark');
+const beautifyBenchmark = require('beautify-benchmark');
+const { execSync } = require('child_process');
+const os = require('os');
+const path = require('path');
+
+// Like build:cjs, but includes __tests__ and copies other files.
+const BUILD_CMD = 'babel src --optional runtime --copy-files --out-dir dist/';
+const LOCAL = 'local';
+const LOCAL_DIR = path.join(__dirname, '../');
+const TEMP_DIR = os.tmpdir();
+
+// Returns the complete git hash for a given git revision reference.
+function hashForRevision(revision) {
+ if (revision === LOCAL) {
+ return revision;
+ }
+ const out = execSync(`git rev-parse "${revision}"`, { encoding: 'utf8' });
+ const match = /[0-9a-f]{8,40}/.exec(out);
+ if (!match) {
+ throw new Error(`Bad results for revision ${revision}: ${out}`);
+ }
+ return match[0];
+}
+
+// Returns the temporary directory which hosts the files for this git hash.
+function dirForHash(hash) {
+ if (hash === LOCAL) {
+ return path.join(__dirname, '../');
+ }
+ return path.join(TEMP_DIR, 'graphql-js-benchmark', hash);
+}
+
+// Build a benchmarkable environment for the given revision.
+function prepareRevision(revision) {
+ console.log(`🍳 Preparing ${revision}...`);
+ const hash = hashForRevision(revision);
+ const dir = dirForHash(hash);
+ if (hash === LOCAL) {
+ execSync(`(cd "${dir}" && yarn run ${BUILD_CMD})`);
+ } else {
+ execSync(`
+ if [ ! -d "${dir}" ]; then
+ mkdir -p "${dir}" &&
+ git archive "${hash}" | tar -xC "${dir}" &&
+ (cd "${dir}" && yarn install);
+ fi &&
+ # Copy in local tests so the same logic applies to each revision.
+ for file in $(cd "${LOCAL_DIR}src"; find . -path '*/__tests__/*.js');
+ do cp "${LOCAL_DIR}src/$file" "${dir}/src/$file";
+ done &&
+ (cd "${dir}" && yarn run ${BUILD_CMD})
+ `);
+ }
+}
+
+// Find all benchmark tests to be run.
+function findBenchmarks() {
+ const out = execSync(
+ `(cd ${LOCAL_DIR}src; find . -path '*/__tests__/*-benchmark.js')`,
+ { encoding: 'utf8' },
+ );
+ return out.split('\n').filter(Boolean);
+}
+
+// Run a given benchmark test with the provided revisions.
+function runBenchmark(benchmark, revisions) {
+ const modules = revisions.map(revision =>
+ require(path.join(
+ dirForHash(hashForRevision(revision)),
+ 'dist',
+ benchmark,
+ )),
+ );
+ const suite = new Suite(modules[0].name, {
+ onStart(event) {
+ console.log('⏱️ ' + event.currentTarget.name);
+ },
+ onCycle(event) {
+ beautifyBenchmark.add(event.target);
+ },
+ onComplete() {
+ beautifyBenchmark.log();
+ },
+ });
+ for (let i = 0; i < revisions.length; i++) {
+ suite.add(revisions[i], modules[i].measure);
+ }
+ suite.run();
+}
+
+// Prepare all revisions and run benchmarks matching a pattern against them.
+function prepareAndRunBenchmarks(benchmarkPatterns, revisions) {
+ const benchmarks = findBenchmarks().filter(
+ benchmark =>
+ benchmarkPatterns.length === 0 ||
+ benchmarkPatterns.some(pattern => benchmark.indexOf(pattern) !== -1),
+ );
+ if (benchmarks.length === 0) {
+ console.warn(
+ 'No benchmarks matching: ' +
+ `\u001b[1m${benchmarkPatterns.join('\u001b[0m or \u001b[1m')}\u001b[0m`,
+ );
+ return;
+ }
+ revisions.forEach(revision => prepareRevision(revision));
+ benchmarks.forEach(benchmark => runBenchmark(benchmark, revisions));
+}
+
+function getArguments(argv) {
+ const revsIdx = argv.indexOf('--revs');
+ const revsArgs = revsIdx === -1 ? [] : argv.slice(revsIdx + 1);
+ const benchmarkPatterns = revsIdx === -1 ? argv : argv.slice(0, revsIdx);
+ let assumeArgs;
+ let revisions;
+ switch (revsArgs.length) {
+ case 0:
+ assumeArgs = [...benchmarkPatterns, '--revs', 'local', 'HEAD'];
+ revisions = [LOCAL, 'HEAD'];
+ break;
+ case 1:
+ assumeArgs = [...benchmarkPatterns, '--revs', 'local', revsArgs[0]];
+ revisions = [LOCAL, revsArgs[0]];
+ break;
+ default:
+ revisions = revsArgs;
+ break;
+ }
+ if (assumeArgs) {
+ console.warn(
+ `Assuming you meant: \u001b[1mbenchmark ${assumeArgs.join(' ')}\u001b[0m`,
+ );
+ }
+ return { benchmarkPatterns, revisions };
+}
+
+// Get the revisions and make things happen!
+if (require.main === module) {
+ const { benchmarkPatterns, revisions } = getArguments(process.argv.slice(2));
+ prepareAndRunBenchmarks(benchmarkPatterns, revisions);
+}

src/language/__tests__/parser-benchmark.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) 2015-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { join } from 'path';
+import { readFileSync } from 'fs';
+import { parse } from '../parser';
+
+const kitchenSink = readFileSync(join(__dirname, '/kitchen-sink.graphql'), {
+ encoding: 'utf8',
+});
+
+export const name = 'Parse kitchen sink';
+export function measure() {
+ parse(kitchenSink);
+}

yarn.lock

@@ -786,6 +786,17 @@ bcrypt-pbkdf@^1.0.0:
  dependencies:
  tweetnacl "^0.14.3"
 
+beautify-benchmark@0.2.4:
+ version "0.2.4"
+ resolved "https://registry.yarnpkg.com/beautify-benchmark/-/beautify-benchmark-0.2.4.tgz#3151def14c1a2e0d07ff2e476861c7ed0e1ae39b"
+
+benchmark@2.1.4:
+ version "2.1.4"
+ resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
+ dependencies:
+ lodash "^4.17.4"
+ platform "^1.3.3"
+
 binary-extensions@^1.0.0:
  version "1.11.0"
  resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205"
@@ -2240,6 +2251,10 @@ pinkie@^2.0.0:
  version "2.0.4"
  resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
 
+platform@^1.3.3:
+ version "1.3.5"
+ resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
+
 pluralize@^7.0.0:
  version "7.0.0"
  resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777"