1.0.23

- concurrency option for async benchmarks
evanwashere · Dec 25, 2024 · 77990ba · 77990ba
1 parent e13bf4c
commit 77990ba
Show file tree

Hide file tree

Showing 5 changed files with 157 additions and 23 deletions.
diff --git a/package.json b/package.json
@@ -2,7 +2,7 @@
   "name": "mitata",
   "type": "module",
   "license": "MIT",
-  "version": "1.0.22",
+  "version": "1.0.23",
   "main": "src/main.mjs",
   "types": "src/main.d.mts",
   "files": ["src", "license", "readme.md"],

diff --git a/readme.md b/readme.md
@@ -18,8 +18,10 @@ try mitata in browser with ai assistant at [https://bolt.new/~/mitata](https://b
 
 ## Recommendations
 
+- read [writing good benchmarks](#writing-good-benchmarks)
 - use dedicated hardware for running benchmarks
 - run with garbage collection enabled (e.g. `node --expose-gc ...`)
+- install optional [hardware counters](#hardware-counters) extension to see cpu stats like IPC (instructions per cycle)
 - make sure your runtime has high-resolution timers and other relevant options/permissions enabled
 
 ## Quick Start
@@ -114,7 +116,7 @@ bench('lots of allocations', () => {
 
 ## universal compatibility
 
-Out of box mitata can detect engine/runtime it's running on and fall back to using [alternative](https://github.com/evanwashere/mitata/blob/master/src/lib.mjs#L43) non-standard I/O functions. If your engine or runtime is missing support, open an issue or pr requesting for support.
+Out of box mitata can detect engine/runtime it's running on and fall back to using [alternative](https://github.com/evanwashere/mitata/blob/master/src/lib.mjs#L45) non-standard I/O functions. If your engine or runtime is missing support, open an issue or pr requesting for support.
 
 ### how to use mitata with engine CLIs like d8, jsc, graaljs, spidermonkey
 
@@ -179,6 +181,33 @@ bench('deleting $keys from object', function* (state) {
 }).args('keys', [1, 10, 100]);
 ```
 
+### concurrency
+
+`concurrency` option enables transparent concurrent execution of asynchronous benchmark, providing insights into:
+- scalability of async functions
+- potential bottlenecks in parallel code
+- performance under different levels of concurrency
+
+*(note: concurrent benchmarks may have higher variance due to scheduling, contention, event loop and async overhead)*
+
+```js
+bench('sleepAsync(1000) x $concurrency', function* () {
+  // concurrency inherited from arguments
+  yield async () => await sleepAsync(1000);
+}).args('concurrency', [1, 5, 10]);
+
+bench('sleepAsync(1000) x 5', function* () {
+  yield {
+    // concurrency is set manually
+    concurrency: 5,
+
+    async bench() {
+      await sleepAsync(1000);
+    },
+  };
+});
+```
+
 ## hardware counters
 
 `bun add @mitata/counters`
@@ -188,6 +217,7 @@ bench('deleting $keys from object', function* (state) {
 supported on: `macos (apple silicon) | linux (amd64, aarch64)`
 
 macos:
+- [Apple Silicon CPU optimization guide/handbook](https://developer.apple.com/documentation/apple-silicon/cpu-optimization-guide)
 - Xcode must be installed for complete cpu counters support
 - Instruments.app (CPU Counters) has to be closed during benchmarking
 
@@ -429,7 +459,87 @@ a / b x 10,311,999 ops/sec (11 runs sampled) v8-never-optimize=true min..max=(95
 ```
 </details>
 
+## writing good benchmarks
+
+Creating accurate and meaningful benchmarks requires careful attention to how modern JavaScript engines optimize code. This covers essential concepts and best practices to ensure your benchmarks measure actual performance characteristics rather than optimization artifacts.
+
+### dead code elimination
+
+JIT can detect and eliminate code that has no observable effects. To ensure your benchmark code executes as intended, you must create observable side effects.
+
+```js
+import { do_not_optimize } from 'mitata';
+
+bench(function* () {
+  // ❌ Bad: jit can see that function has zero side-effects
+  yield () => new Array(0);
+  // will get optimized to:
+  /*
+    yield () => {};
+  */
+
+  // ✅ Good: do_not_optimize(value) emits code that causes side-effects
+  yield () => do_not_optimize(new Array(0));
+});
+```
+
+### garbage collection pressure
+
+For benchmarks involving significant memory allocations, controlling garbage collection frequency can improve results consistency.
+
+```js
+// ❌ Bad: unpredictable gc pauses
+bench(() => {
+  const bigArray = new Array(1000000);
+});
+
+// ✅ Good: gc before each (batch-)iteration
+bench(() => {
+  const bigArray = new Array(1000000);
+}).gc('inner'); // run gc before each iteration
+```
+
+### loop invariant code motion optimization
+
+JavaScript engines can optimize away repeated computations by hoisting them out of loops or caching results. Use computed parameters to prevent loop invariant code motion optimization.
+
+```js
+bench(function* (ctx) {
+  const str = 'abc';
+
+  // ❌ Bad: JIT sees that both str and 'c' search value are constants/comptime-known
+  yield () => str.includes('c');
+  // will get optimized to:
+  /*
+    yield () => true;
+  */
+
+  // ❌ Bad: JIT sees that computation doesn't depend on anything inside loop
+  const substr = ctx.get('substr');
+  yield () => str.includes(substr);
+  // will get optimized to:
+  /*
+    const $0 = str.includes(substr);
+    yield () => $0;
+  */
+
+  // ✅ Good: using computed parameters prevents jit from performing any loop optimizations
+  yield {
+    [0]() {
+      return str;
+    },
+
+    [1]() {
+      return substr;
+    },
+
+    bench(str, substr) {
+      return do_not_optimize(str.includes(substr));
+    },
+  };
+}).args('substr', ['c']);
+```
 
-## License
+## license
 
-MIT © [Evan](https://github.com/evanwashere)
+MIT © [evanwashere](https://github.com/evanwashere)
diff --git a/src/lib.d.mts b/src/lib.d.mts
@@ -46,6 +46,7 @@ interface stats {
 interface k_options {
   now?: () => number;
   inner_gc?: boolean;
+  concurrency?: number;
   min_samples?: number;
   max_samples?: number;
   min_cpu_time?: number;
@@ -58,6 +59,7 @@ interface k_options {
   gc?: boolean | (() => void);
 }
 
+export const k_concurrency: number;
 export const k_min_samples: number;
 export const k_max_samples: number;
 export const k_min_cpu_time: number;

diff --git a/src/lib.mjs b/src/lib.mjs
@@ -19,6 +19,7 @@ export async function generator(gen, opts = {}) {
 
   const g = gen(ctx);
   const n = await g.next();
+
   if (n.done || 'fn' !== kind(n.value)) {
     if ('fn' !== kind(n.value?.bench, true)) throw new TypeError('expected benchmarkable yield from generator');
 
@@ -31,6 +32,7 @@ export async function generator(gen, opts = {}) {
     }
   }
 
+  opts.concurrency ??= n.value?.concurrency ?? opts.args?.concurrency;
   const stats = await fn('fn' === kind(n.value) ? n.value : n.value.bench, opts);
   if (!(await g.next()).done) throw new TypeError('expected generator to yield once');
 
@@ -111,6 +113,7 @@ export function kind(fn, _ = false) {
   ) return 'iter';
 }
 
+export const k_concurrency = 1;
 export const k_min_samples = 12;
 export const k_batch_unroll = 4;
 export const k_max_samples = 1e9;
@@ -127,6 +130,7 @@ function defaults(opts) {
   opts.params ??= {};
   opts.inner_gc ??= false;
   opts.$counters ??= false;
+  opts.concurrency ??= k_concurrency;
   opts.min_samples ??= k_min_samples;
   opts.max_samples ??= k_max_samples;
   opts.min_cpu_time ??= k_min_cpu_time;
@@ -175,8 +179,10 @@ export async function fn(fn, opts = {}) {
     let samples = new Array(2 ** 20);
 
     ${!params.length ? '' : Array.from({ length: params.length }, (_, o) => `
-      let param_${o} = ${!batch ? 'null' : `new Array(${opts.batch_samples})`};
-    `.trim()).join(' ')}
+      ${Array.from({ length: opts.concurrency }, (_, c) => `
+        let param_${o}_${c} = ${!batch ? 'null' : `new Array(${opts.batch_samples})`};
+      `.trim()).join(' ')}
+    `.trim()).join('\n')}
 
     ${!opts.gc ? '' : `$gc();`}
 
@@ -185,10 +191,18 @@ export async function fn(fn, opts = {}) {
 
       ${!params.length ? '' : `
         ${!batch ? `
-          ${Array.from({ length: params.length }, (_, o) => `if ((param_${o} = $params[${o}]()) instanceof Promise) param_${o} = await param_${o};`).join(' ')}
+          ${Array.from({ length: params.length }, (_, o) => `
+            ${Array.from({ length: opts.concurrency }, (_, c) => `
+              if ((param_${o}_${c} = $params[${o}]()) instanceof Promise) param_${o}_${c} = await param_${o}_${c};
+            `.trim()).join(' ')}
+          `.trim()).join('\n')}
         ` : `
           for (let o = 0; o < ${opts.batch_samples}; o++) {
-            ${Array.from({ length: params.length }, (_, o) => `if ((param_${o}[o] = $params[${o}]()) instanceof Promise) param_${o}[o] = await param_${o}[o];`).join(' ')}
+            ${Array.from({ length: params.length }, (_, o) => `
+              ${Array.from({ length: opts.concurrency }, (_, c) => `
+                if ((param_${o}_${c}[o] = $params[${o}]()) instanceof Promise) param_${o}_${c}[o] = await param_${o}_${c}[o];
+              `.trim()).join(' ')}
+            `.trim()).join('\n')}
           }
         `}
       `}
@@ -205,22 +219,30 @@ export async function fn(fn, opts = {}) {
       ${!opts.$counters ? '' : '$counters.before();'} const t0 = $now();
 
       ${!batch ? `
-        ${!async ? '' : 'await '} ${!params.length ? `
-          $fn();
-        ` : `
-          $fn(${Array.from({ length: params.length }, (_, o) => `param_${o}`).join(', ')});
-        `}
+        ${!async ? '' : (1 >= opts.concurrency ? '' : 'await Promise.all([')}
+          ${Array.from({ length: opts.concurrency }, (_, c) => `
+            ${!async ? '' : (1 < opts.concurrency ? '' : 'await ')} ${(!params.length ? `
+              $fn()
+            ` : `
+              $fn(${Array.from({ length: params.length }, (_, o) => `param_${o}_${c}`).join(', ')})
+            `).trim()}${!async ? ';' : (1 < opts.concurrency ? ',' : ';')}
+          `.trim()).join('\n')}
+        ${!async ? '' : (1 >= opts.concurrency ? '' : ']);')}
       ` : `
         for (let o = 0; o < ${(opts.batch_samples / opts.batch_unroll) | 0}; o++) {
-          ${!params.length ? `
-            ${new Array(opts.batch_unroll).fill(`${!async ? '' : 'await'} $fn();`).join(' ')}
-          ` : `
-            const param_offset = o * ${opts.batch_unroll};
-
-            ${Array.from({ length: opts.batch_unroll }, (_, u) => `
-              ${!async ? '' : 'await'} $fn(${Array.from({ length: params.length }, (_, o) => `param_${o}[${u === 0 ? '' : `${u} + `}param_offset]`).join(', ')});
-            `.trim()).join('\n' + ' '.repeat(12))}
-          `}
+          ${!params.length ? '' : `const param_offset = o * ${opts.batch_unroll};`}
+
+          ${Array.from({ length: opts.batch_unroll }, (_, u) => `
+            ${!async ? '' : (1 >= opts.concurrency ? '' : 'await Promise.all([')}
+              ${Array.from({ length: opts.concurrency }, (_, c) => `
+                ${!async ? '' : (1 < opts.concurrency ? '' : 'await ')} ${(!params.length ? `
+                  $fn()
+                ` : `
+                  $fn(${Array.from({ length: params.length }, (_, o) => `param_${o}_${c}[${u === 0 ? '' : `${u} + `}param_offset]`).join(', ')})
+                `).trim()}${!async ? ';' : (1 < opts.concurrency ? ',' : ';')}
+              `.trim()).join(' ')}
+            ${!async ? '' : (1 >= opts.concurrency ? '' : ']);')}
+          `.trim()).join('\n')}
         }
       `}
 

diff --git a/tools/package.json b/tools/package.json
@@ -5,6 +5,6 @@
     "tinybench": "^3.0.7",
     "bench-node": "^0.4.1",
     "cronometro": "^4.0.0",
-    "@mitata/counters": "^0.0.3"
+    "@mitata/counters": "latest"
   }
 }