Fix harness to not call timer repeatedly in the measured loop. (#38)

Co-authored-by: Ömer Sinan Ağacan <omersa@google.com>
Co-authored-by: Vyacheslav Egorov <vegorov@google.com>

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.2.0
+
+- Change measuring algorithm in `BenchmarkBase` to avoid calling stopwatch
+methods repeatedly in the measuring loop. This makes measurement work better
+for `run` methods which are small themselves.
+
 ## 2.1.0
 
 - Add AsyncBenchmarkBase.

lib/benchmark_harness.dart

@@ -5,6 +5,7 @@
 library benchmark_harness;
 
 import 'dart:async';
+import 'dart:math' as math;
 
 part 'src/benchmark_base.dart';
 part 'src/async_benchmark_base.dart';

lib/src/benchmark_base.dart

@@ -4,63 +4,97 @@
 
 part of benchmark_harness;
 
+const int _minimumMeasureDurationMillis = 2000;
+
 class BenchmarkBase {
   final String name;
   final ScoreEmitter emitter;
 
-  // Empty constructor.
   const BenchmarkBase(this.name, {this.emitter = const PrintEmitter()});
 
-  // The benchmark code.
-  // This function is not used, if both [warmup] and [exercise] are overwritten.
+  /// The benchmark code.
+  ///
+  /// This function is not used, if both [warmup] and [exercise] are overwritten.
   void run() {}
 
-  // Runs a short version of the benchmark. By default invokes [run] once.
+  /// Runs a short version of the benchmark. By default invokes [run] once.
   void warmup() {
     run();
   }
 
-  // Exercises the benchmark. By default invokes [run] 10 times.
+  /// Exercises the benchmark. By default invokes [run] 10 times.
   void exercise() {
     for (var i = 0; i < 10; i++) {
       run();
     }
   }
 
-  // Not measured setup code executed prior to the benchmark runs.
+  /// Not measured setup code executed prior to the benchmark runs.
   void setup() {}
 
-  // Not measures teardown code executed after the benchmark runs.
+  /// Not measured teardown code executed after the benchmark runs.
   void teardown() {}
 
-  // Measures the score for this benchmark by executing it repeatedly until
-  // time minimum has been reached.
-  static double measureFor(void Function() f, int minimumMillis) {
-    var minimumMicros = minimumMillis * 1000;
-    var iter = 0;
-    var watch = Stopwatch();
-    watch.start();
-    var elapsed = 0;
-    while (elapsed < minimumMicros) {
-      f();
-      elapsed = watch.elapsedMicroseconds;
-      iter++;
+  /// Measures the score for this benchmark by executing it enough times
+  /// to reach [minimumMillis].
+  static _Measurement _measureForImpl(void Function() f, int minimumMillis) {
+    final minimumMicros = minimumMillis * 1000;
+    var iter = 2;
+    final watch = Stopwatch()..start();
+    while (true) {
+      watch.reset();
+      for (var i = 0; i < iter; i++) {
+        f();
+      }
+      final elapsed = watch.elapsedMicroseconds;
+      final measurement = _Measurement(elapsed, iter);
+      if (measurement.elapsedMicros >= minimumMicros) {
+        return measurement;
+      }
+
+      iter = measurement.estimateIterationsNeededToReach(
+          minimumMicros: minimumMicros);
     }
-    return elapsed / iter;
   }
 
-  // Measures the score for the benchmark and returns it.
+  /// Measures the score for this benchmark by executing it repeatedly until
+  /// time minimum has been reached.
+  static double measureFor(void Function() f, int minimumMillis) =>
+      _measureForImpl(f, minimumMillis).score;
+
+  /// Measures the score for the benchmark and returns it.
   double measure() {
     setup();
     // Warmup for at least 100ms. Discard result.
-    measureFor(warmup, 100);
+    _measureForImpl(warmup, 100);
     // Run the benchmark for at least 2000ms.
-    var result = measureFor(exercise, 2000);
+    var result = _measureForImpl(exercise, _minimumMeasureDurationMillis);
     teardown();
-    return result;
+    return result.score;
   }
 
   void report() {
     emitter.emit(name, measure());
   }
 }
+
+class _Measurement {
+  final int elapsedMicros;
+  final int iterations;
+
+  _Measurement(this.elapsedMicros, this.iterations);
+
+  double get score => elapsedMicros / iterations;
+
+  int estimateIterationsNeededToReach({required int minimumMicros}) {
+    final elapsed = roundDownToMillisecond(elapsedMicros);
+    return elapsed == 0
+        ? iterations * 1000
+        : (iterations * math.max(minimumMicros / elapsed, 1.5)).ceil();
+  }
+
+  static int roundDownToMillisecond(int micros) => (micros ~/ 1000) * 1000;
+
+  @override
+  String toString() => '$elapsedMicros in $iterations iterations';
+}

pubspec.yaml

@@ -1,5 +1,5 @@
 name: benchmark_harness
-version: 2.1.0
+version: 2.2.0
 description: The official Dart project benchmark harness.
 repository: https://github.com/dart-lang/benchmark_harness