Add benchmark README and latest numbers (#689)

* add benchmark README and latest numbers * chore: update readme chore: update readme * chore: update readme chore: update readme * chore: update benchmarks * generate latest benchmark numbers Co-authored-by: Daniel Dyla <dyladan@users.noreply.github.com> Co-authored-by: Olivier Albertini <olivier.albertini@montreal.ca>
open-telemetry · Jan 24, 2020 · 69947da · 69947da
1 parent d3af8c4
commit 69947da
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 31 deletions.
diff --git a/benchmark/README.md b/benchmark/README.md
@@ -0,0 +1,67 @@
+# Benchmarks
+
+## How to run
+
+To run your benchmark, just:
+```sh
+$ npm run bench
+```
+
+The minimum sample size is set to 10 to perform statistical analysis on benchmark, you can re-configure that in `benchmark.js`.
+
+> NOTE: If you're interested in writing benchmark for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples.
+
+## Results
+
+### `v0.3.3` release
+
+```
+Beginning NoopTracerRegistry Benchmark...
+  5 tests completed.
+
+  #startSpan                     x 731,516,636 ops/sec ±2.57% (20 runs sampled)
+  #startSpan:parent              x 744,353,590 ops/sec ±3.03% (20 runs sampled)
+  #startSpan with attribute      x 737,451,332 ops/sec ±3.75% (20 runs sampled)
+  #startSpan with 30 attributes  x   1,658,688 ops/sec ±1.23% (20 runs sampled)
+  #startSpan with 100 attributes x     535,082 ops/sec ±1.55% (20 runs sampled)
+
+Beginning BasicTracerRegistry Benchmark...
+  5 tests completed.
+
+  #startSpan                     x 80,633 ops/sec ±3.57% (20 runs sampled)
+  #startSpan:parent              x 56,228 ops/sec ±2.18% (20 runs sampled)
+  #startSpan with attribute      x 86,710 ops/sec ±1.80% (20 runs sampled)
+  #startSpan with 30 attributes  x 36,331 ops/sec ±1.29% (20 runs sampled)
+  #startSpan with 100 attributes x  3,549 ops/sec ±3.59% (20 runs sampled)
+
+Beginning BasicTracerRegistry with SimpleSpanProcessor Benchmark...
+  5 tests completed.
+
+  #startSpan                     x 74,539 ops/sec ±4.49% (20 runs sampled)
+  #startSpan:parent              x 48,953 ops/sec ±4.98% (20 runs sampled)
+  #startSpan with attribute      x 79,686 ops/sec ±2.54% (20 runs sampled)
+  #startSpan with 30 attributes  x 26,491 ops/sec ±13.68% (20 runs sampled)
+  #startSpan with 100 attributes x  2,464 ops/sec ±19.64% (20 runs sampled)
+
+Beginning BasicTracerRegistry with BatchSpanProcessor Benchmark...
+  5 tests completed.
+
+  #startSpan                     x 74,974 ops/sec ±3.57% (20 runs sampled)
+  #startSpan:parent              x 42,390 ops/sec ±20.68% (20 runs sampled)
+  #startSpan with attribute      x 76,497 ops/sec ±2.93% (20 runs sampled)
+  #startSpan with 30 attributes  x 33,042 ops/sec ±2.03% (20 runs sampled)
+  #startSpan with 100 attributes x  3,459 ops/sec ±4.56% (20 runs sampled)
+
+
+Beginning B3Format Benchmark...
+  2 tests completed.
+
+  #Inject  x 5,086,366 ops/sec ±3.18% (100 runs sampled)
+  #Extract x 4,859,557 ops/sec ±3.80% (100 runs sampled)
+
+Beginning HttpTraceContext Benchmark...
+  2 tests completed.
+
+  #Inject  x 13,660,710 ops/sec ±1.84% (100 runs sampled)
+  #Extract x  1,692,010 ops/sec ±0.83% (100 runs sampled)
+```
diff --git a/benchmark/benchmark.js b/benchmark/benchmark.js
@@ -4,10 +4,9 @@ const Benchmark = require('benchmark');
 const benchmarks = require('beautify-benchmark');
 
 Benchmark.options.maxTime = 0;
-// @todo : Change it to between 50-100 or keep it random.
-Benchmark.options.minSamples = 10;
 
-module.exports = () => {
+module.exports = (minSamples) => {
+  Benchmark.options.minSamples = minSamples;
   const suite = new Benchmark.Suite();
 
   return suite

diff --git a/benchmark/index.js b/benchmark/index.js
@@ -3,5 +3,5 @@
 const execSync = require('child_process').execSync;
 const exec = cmd => execSync(cmd, { stdio: [0, 1, 2] });
 
-exec('node benchmark/tracer');
-exec('node benchmark/propagator');
+exec('node benchmark/tracer.js');
+exec('node benchmark/propagator.js');
diff --git a/benchmark/propagator.js b/benchmark/propagator.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const benchmark = require('./benchmark');
-const opentelemetry = require('@opentelemetry/core');
+const opentelemetry = require('../packages/opentelemetry-core');
 
 const setups = [
   {
@@ -26,7 +26,7 @@ const setups = [
 for (const setup of setups) {
   console.log(`Beginning ${setup.name} Benchmark...`);
   const propagator = setup.propagator;
-  const suite = benchmark()
+  const suite = benchmark(100)
     .add('#Inject', function () {
       propagator.inject({
         traceId: 'd4cda95b652f4a1592b449d5929fda1b',

diff --git a/benchmark/tracer.js b/benchmark/tracer.js
@@ -1,28 +1,34 @@
 'use strict';
 
 const benchmark = require('./benchmark');
-const opentelemetry = require('@opentelemetry/core');
-const { BasicTracerRegistry, BatchSpanProcessor, InMemorySpanExporter, SimpleSpanProcessor } = require('@opentelemetry/tracing');
-const { NodeTracerRegistry } = require('@opentelemetry/node');
+const opentelemetry = require('../packages/opentelemetry-core');
+const { BasicTracerRegistry, BatchSpanProcessor, InMemorySpanExporter, SimpleSpanProcessor } = require('../packages/opentelemetry-tracing');
 
-const exporter = new InMemorySpanExporter();
 const logger = new opentelemetry.NoopLogger();
 
 const setups = [
+  {
+    name: 'NoopTracerRegistry',
+    registry: opentelemetry.getTracerRegistry()
+  },
   {
     name: 'BasicTracerRegistry',
     registry: new BasicTracerRegistry({ logger })
   },
   {
-    name: 'NodeTracerRegistry',
-    registry: new NodeTracerRegistry({ logger })
+    name: 'BasicTracerRegistry with SimpleSpanProcessor',
+    registry: getRegistry(new SimpleSpanProcessor(new InMemorySpanExporter()))
+  },
+  {
+    name: 'BasicTracerRegistry with BatchSpanProcessor',
+    registry: getRegistry(new BatchSpanProcessor(new InMemorySpanExporter()))
   }
 ];
 
 for (const setup of setups) {
   console.log(`Beginning ${setup.name} Benchmark...`);
   const tracer = setup.registry.getTracer("benchmark");
-  const suite = benchmark()
+  const suite = benchmark(20)
     .add('#startSpan', function () {
       const span = tracer.startSpan('op');
       span.end();
@@ -51,25 +57,14 @@ for (const setup of setups) {
         span.setAttribute('attr-key-' + j, 'attr-value-' + j);
       }
       span.end();
-    })
-    .add('#startSpan with SimpleSpanProcessor', function () {
-      const simpleSpanProcessor = new SimpleSpanProcessor(exporter);
-
-      registry.addSpanProcessor(simpleSpanProcessor);
-      const span = tracer.startSpan('op');
-      span.end();
-
-      simpleSpanProcessor.shutdown();
-    })
-    .add('#startSpan with BatchSpanProcessor', function () {
-      const batchSpanProcessor = new BatchSpanProcessor(exporter);
-
-      registry.addSpanProcessor(batchSpanProcessor);
-      const span = tracer.startSpan('op');
-      span.end();
-      batchSpanProcessor.shutdown();
     });
 
   // run async
   suite.run({ async: false });
 }
+function getRegistry(processor) {
+  const registry = new BasicTracerRegistry({ logger });
+  registry.addSpanProcessor(processor);
+  return registry;
+}
+