Benchmark run time baseline estimation

build.zig

@@ -59,6 +59,7 @@ fn setupTesting(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
 fn setupExamples(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) void {
     const example_step = b.step("test_examples", "Build examples");
     const example_names = [_][]const u8{
+        "baseline",
         "basic",
         "bubble_sort",
         "hooks",

examples/baseline.zig

@@ -0,0 +1,43 @@
+const std = @import("std");
+const zbench = @import("zbench");
+
+fn noop(_: std.mem.Allocator) void {
+    // does nothing
+}
+
+test "benchmark timer baseline" {
+    const stdout = std.io.getStdOut().writer();
+    var bench = zbench.Benchmark.init(std.testing.allocator, .{});
+    defer bench.deinit();
+
+    try bench.add("Bench baseline, n=10 ", noop, .{
+        .iterations = 10,
+        .baseline_correction = false,
+    });
+    try bench.add("Bench baseline, n=10 ", noop, .{
+        .iterations = 10,
+        .baseline_correction = true,
+    });
+
+    try bench.add("Bench baseline, n=1k ", noop, .{
+        .iterations = 1_000,
+        .baseline_correction = false,
+    });
+    try bench.add("Bench baseline, n=1k ", noop, .{
+        .iterations = 1_000,
+        .baseline_correction = true,
+    });
+
+    try bench.add("Bench baseline, n=10k", noop, .{
+        .iterations = 10_000,
+        .baseline_correction = false,
+    });
+
+    try bench.add("Bench baseline, n=10k", noop, .{
+        .iterations = 10_000,
+        .baseline_correction = true,
+    });
+
+    try stdout.writeAll("\n");
+    try bench.run(stdout);
+}

util/optional.zig

@@ -37,4 +37,10 @@ test "Optional" {
     const b: Foo = optional(Foo, .{ .abc = 6 }, a);
     try std.testing.expectEqual(@as(u8, 6), b.abc);
     try std.testing.expectEqual(@as(u8, 10), b.xyz);
+
+    const Bar = struct { flag_a: bool, flag_b: bool };
+    const c: Bar = Bar{ .flag_a = false, .flag_b = true };
+    const d: Bar = optional(Bar, .{ .flag_a = true, .flag_b = true }, c);
+    try std.testing.expect(d.flag_a);
+    try std.testing.expect(d.flag_b);
 }

util/runner.zig

@@ -21,7 +21,7 @@ const State = union(enum) {
 
         /// Maximum number of iterations the benchmark can run. This limit helps
         /// to avoid excessively long benchmark runs.
-        max_iterations: u16,
+        max_iterations: u32,
 
         /// Time budget for the benchmark in nanoseconds. This value is used to
         /// determine how long a single benchmark should be allowed to run
@@ -58,8 +58,8 @@ state: State,
 
 pub fn init(
     allocator: std.mem.Allocator,
-    iterations: u16,
-    max_iterations: u16,
+    iterations: u32,
+    max_iterations: u32,
     time_budget_ns: u64,
     track_allocations: bool,
 ) Error!Runner {
@@ -139,6 +139,7 @@ pub fn finish(self: *Runner) Error!Readings {
             .allocator = self.allocator,
             .iterations = 0,
             .timings_ns = &.{},
+            .baseline_ns = &.{},
             .allocations = null,
         },
         .running => |st| st.readings,
@@ -175,26 +176,26 @@ test "Runner" {
     var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, false);
     {
         errdefer r.abort();
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(400_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
-        try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null)));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(400_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
+        try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, 0, null)));
     }
     const result = try r.finish();
     defer result.deinit();
@@ -208,26 +209,26 @@ test "Runner - memory tracking" {
     var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, true);
     {
         errdefer r.abort();
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 256, .count = 1 })));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 256, .count = 1 })));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 512, .count = 2 })));
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 512, .count = 2 })));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 1024, .count = 4 })));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 1024, .count = 4 })));
-        try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 2048, .count = 8 })));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 2045, .count = 8 })));
-        try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 4096, .count = 16 })));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 4096, .count = 16 })));
-        try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 8192, .count = 32 })));
-
-        try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 1, .count = 2 })));
-        try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 2, .count = 4 })));
-        try expectEq(Step.more, try r.next(Reading.init(300, .{ .max = 4, .count = 8 })));
-        try expectEq(Step.more, try r.next(Reading.init(400, .{ .max = 8, .count = 16 })));
-        try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 16, .count = 32 })));
-        try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 32, .count = 64 })));
-        try expectEq(Step.more, try r.next(Reading.init(300, .{ .max = 64, .count = 128 })));
-        try expectEq(@as(?Step, null), try r.next(Reading.init(400, .{ .max = 128, .count = 256 })));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 256, .count = 1 })));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 256, .count = 1 })));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 512, .count = 2 })));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 512, .count = 2 })));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 1024, .count = 4 })));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 1024, .count = 4 })));
+        try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 2048, .count = 8 })));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 2045, .count = 8 })));
+        try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 4096, .count = 16 })));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 4096, .count = 16 })));
+        try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 8192, .count = 32 })));
+
+        try expectEq(Step.more, try r.next(Reading.init(100, 0, .{ .max = 1, .count = 2 })));
+        try expectEq(Step.more, try r.next(Reading.init(200, 0, .{ .max = 2, .count = 4 })));
+        try expectEq(Step.more, try r.next(Reading.init(300, 0, .{ .max = 4, .count = 8 })));
+        try expectEq(Step.more, try r.next(Reading.init(400, 0, .{ .max = 8, .count = 16 })));
+        try expectEq(Step.more, try r.next(Reading.init(100, 0, .{ .max = 16, .count = 32 })));
+        try expectEq(Step.more, try r.next(Reading.init(200, 0, .{ .max = 32, .count = 64 })));
+        try expectEq(Step.more, try r.next(Reading.init(300, 0, .{ .max = 64, .count = 128 })));
+        try expectEq(@as(?Step, null), try r.next(Reading.init(400, 0, .{ .max = 128, .count = 256 })));
     }
     const result = try r.finish();
     defer result.deinit();

util/runner/types.zig

@@ -6,14 +6,17 @@ pub const Step = enum { more };
 
 pub const Reading = struct {
     timing_ns: u64,
+    baseline_ns: u64,
     allocation: ?AllocationReading,
 
     pub fn init(
         timing_ns: u64,
+        baseline_ns: u64,
         allocation: ?AllocationReading,
     ) Reading {
         return .{
             .timing_ns = timing_ns,
+            .baseline_ns = baseline_ns,
             .allocation = allocation,
         };
     }
@@ -23,6 +26,7 @@ pub const Readings = struct {
     allocator: std.mem.Allocator,
     iterations: usize,
     timings_ns: []u64,
+    baseline_ns: []u64,
     allocations: ?AllocationReadings,
 
     pub fn init(
@@ -34,6 +38,7 @@ pub const Readings = struct {
             .allocator = allocator,
             .iterations = n,
             .timings_ns = try allocator.alloc(u64, n),
+            .baseline_ns = try allocator.alloc(u64, n),
             .allocations = if (track_allocations)
                 try AllocationReadings.init(allocator, n)
             else
@@ -43,11 +48,13 @@ pub const Readings = struct {
 
     pub fn deinit(self: Readings) void {
         self.allocator.free(self.timings_ns);
+        self.allocator.free(self.baseline_ns);
         if (self.allocations) |allocs| allocs.deinit(self.allocator);
     }
 
     pub fn set(self: *Readings, i: usize, reading: Reading) void {
         self.timings_ns[i] = reading.timing_ns;
+        self.baseline_ns[i] = reading.baseline_ns;
         if (self.allocations) |allocs| {
             if (reading.allocation) |x| {
                 allocs.maxes[i] = x.max;

zbench.zig

@@ -31,11 +31,11 @@ pub const Hooks = struct {
 pub const Config = struct {
     /// Number of iterations the benchmark has been run. Initialized to 0.
     /// If 0 then zBench will calculate an value.
-    iterations: u16 = 0,
+    iterations: u32 = 0,
 
     /// Maximum number of iterations the benchmark can run. Default is 16384.
     /// This limit helps to avoid excessively long benchmark runs.
-    max_iterations: u16 = 16384,
+    max_iterations: u32 = 16384,
 
     /// Time budget for the benchmark in nanoseconds. Default is 2e9 (2 seconds).
     /// This value is used to determine how long a single benchmark should be allowed to run
@@ -55,6 +55,9 @@ pub const Config = struct {
     /// Track memory allocations made using the Allocator provided to
     /// benchmarks.
     track_allocations: bool = false,
+
+    /// experimental: subtract average runtime baseline from timing results
+    baseline_correction: bool = false,
 };
 
 /// A benchmark definition.
@@ -89,12 +92,22 @@ const Definition = struct {
         defer if (self.config.hooks.after_each) |hook| hook();
 
         var t = try std.time.Timer.start();
+        const baseline: u64 = t.read();
+        t.reset();
+
         switch (self.defn) {
             .simple => |func| func(allocator),
             .parameterised => |x| x.func(@ptrCast(x.context), allocator),
         }
+
+        var lap = t.read();
+        if (self.config.baseline_correction) {
+            lap -|= baseline;
+        }
+
         return Runner.Reading{
-            .timing_ns = t.read(),
+            .timing_ns = lap,
+            .baseline_ns = baseline,
             .allocation = if (tracking) |trk| AllocationReading{
                 .max = trk.maxAllocated(),
                 .count = trk.allocationCount(),
@@ -286,6 +299,7 @@ pub const Benchmark = struct {
                 progress_node.setEstimatedTotalItems(0);
                 progress_node.setCompletedItems(0);
                 progress.refresh();
+
                 try x.prettyPrint(arena.allocator(), writer, true);
                 _ = arena.reset(.retain_capacity);
             },
@@ -296,7 +310,7 @@ pub const Benchmark = struct {
 /// Write the prettyPrint() header to a writer.
 pub fn prettyPrintHeader(writer: anytype) !void {
     try writer.print(
-        "{s:<22} {s:<8} {s:<14} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10}\n",
+        "{s:<22} {s:<8} {s:<14} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10} {s:<22}\n",
         .{
             "benchmark",
             "runs",
@@ -306,10 +320,11 @@ pub fn prettyPrintHeader(writer: anytype) !void {
             "p75",
             "p99",
             "p995",
+            "baseline",
         },
     );
     const dashes = "-------------------------";
-    try writer.print(dashes ++ dashes ++ dashes ++ dashes ++ dashes ++ "\n", .{});
+    try writer.print(dashes ++ dashes ++ dashes ++ dashes ++ dashes ++ dashes ++ "\n", .{});
 }
 
 /// Get a copy of the system information, cpu type, cores, memory, etc.
@@ -343,37 +358,46 @@ pub const Result = struct {
     ) !void {
         var buf: [128]u8 = undefined;
 
-        const timings_ns = self.readings.timings_ns;
-        const s = try Statistics(u64).init(allocator, timings_ns);
+        const stats_timings = try Statistics(u64).init(allocator, self.readings.timings_ns);
+        const stats_baseline = try Statistics(u64).init(allocator, self.readings.baseline_ns);
+
         // Benchmark name, number of iterations, and total time
         try writer.print("{s:<22} ", .{self.name});
         try setColor(colors, writer, Color.cyan);
         try writer.print("{d:<8} {s:<15}", .{
             self.readings.iterations,
-            std.fmt.fmtDuration(s.total),
+            std.fmt.fmtDuration(stats_timings.total),
         });
         // Mean + standard deviation
         try setColor(colors, writer, Color.green);
         try writer.print("{s:<23}", .{
             try std.fmt.bufPrint(&buf, "{:.3} ± {:.3}", .{
-                std.fmt.fmtDuration(s.mean),
-                std.fmt.fmtDuration(s.stddev),
+                std.fmt.fmtDuration(stats_timings.mean),
+                std.fmt.fmtDuration(stats_timings.stddev),
             }),
         });
         // Minimum and maximum
         try setColor(colors, writer, Color.blue);
         try writer.print("{s:<29}", .{
             try std.fmt.bufPrint(&buf, "({:.3} ... {:.3})", .{
-                std.fmt.fmtDuration(s.min),
-                std.fmt.fmtDuration(s.max),
+                std.fmt.fmtDuration(stats_timings.min),
+                std.fmt.fmtDuration(stats_timings.max),
             }),
         });
         // Percentiles
         try setColor(colors, writer, Color.cyan);
         try writer.print("{:<10} {:<10} {:<10}", .{
-            std.fmt.fmtDuration(s.percentiles.p75),
-            std.fmt.fmtDuration(s.percentiles.p99),
-            std.fmt.fmtDuration(s.percentiles.p995),
+            std.fmt.fmtDuration(stats_timings.percentiles.p75),
+            std.fmt.fmtDuration(stats_timings.percentiles.p99),
+            std.fmt.fmtDuration(stats_timings.percentiles.p995),
+        });
+        // Baseline
+        try setColor(colors, writer, Color.red);
+        try writer.print("{s:<23}", .{
+            try std.fmt.bufPrint(&buf, "{:.3} ± {:.3}", .{
+                std.fmt.fmtDuration(stats_baseline.mean),
+                std.fmt.fmtDuration(stats_baseline.stddev),
+            }),
         });
         // End of line
         try setColor(colors, writer, Color.reset);