Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Benchmark run time baseline estimation #78

Closed
wants to merge 18 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ pub const Config = struct {
max_iterations: u16 = 16384,
time_budget_ns: u64 = 2e9, // 2 seconds
hooks: Hooks = .{},
track_allocations: bool = false,
track_allocations: bool = false,
};
```

Expand Down
1 change: 1 addition & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ fn setupTesting(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.b
fn setupExamples(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) void {
const example_step = b.step("test_examples", "Build examples");
const example_names = [_][]const u8{
"baseline",
"basic",
"bubble_sort",
"hooks",
Expand Down
53 changes: 53 additions & 0 deletions examples/baseline.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
const std = @import("std");
const zbench = @import("zbench");

fn noop(_: std.mem.Allocator) void {
// does nothing
}

test "benchmark timer baseline" {
const stdout = std.io.getStdOut().writer();
var bench = zbench.Benchmark.init(std.testing.allocator, .{});
defer bench.deinit();

try bench.add("Correction off, n=10 ", noop, .{
.iterations = 10,
.baseline_correction = false,
});
try bench.add("Correction on, n=10 ", noop, .{
.iterations = 10,
.baseline_correction = true,
});

try bench.add("Correction off, n=1k ", noop, .{
.iterations = 1_000,
.baseline_correction = false,
});
try bench.add("Correction on, n=1k ", noop, .{
.iterations = 1_000,
.baseline_correction = true,
});

try bench.add("Correction off, n=10k", noop, .{
.iterations = 10_000,
.baseline_correction = false,
});

try bench.add("Correction on, n=10k", noop, .{
.iterations = 10_000,
.baseline_correction = true,
});

try bench.add("Correction off, n=1M ", noop, .{
.iterations = 1_000_000,
.baseline_correction = false,
});

try bench.add("Correction on, n=1M ", noop, .{
.iterations = 1_000_000,
.baseline_correction = true,
});

try stdout.writeAll("\n");
try bench.run(stdout);
}
20 changes: 18 additions & 2 deletions util/color.zig
Original file line number Diff line number Diff line change
@@ -1,26 +1,42 @@
const std = @import("std");

pub const Color = enum {
black,
grey,
red,
red_bright,
green,
green_bright,
yellow,
yellow_bright,
blue,
blue_bright,
magenta,
magenta_bright,
cyan,
cyan_bright,
reset,
none,
default,

// Return the ANSI escape code for this color.
pub fn code(self: Color) []const u8 {
return switch (self) {
.black => "\x1b[30m",
.grey => "\x1b[90m", // this is actually 'bright black'
.red => "\x1b[31m",
.red_bright => "\x1b[91m",
.green => "\x1b[32m",
.green_bright => "\x1b[92m",
.yellow => "\x1b[33m",
.yellow_bright => "\x1b[93m",
.blue => "\x1b[34m",
.blue_bright => "\x1b[94m",
.magenta => "\x1b[35m",
.magenta_bright => "\x1b[95m",
.cyan => "\x1b[36m",
.cyan_bright => "\x1b[96m",
.reset => "\x1b[0m",
.none => "",
.default => "",
};
}
};
6 changes: 6 additions & 0 deletions util/optional.zig
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,10 @@ test "Optional" {
const b: Foo = optional(Foo, .{ .abc = 6 }, a);
try std.testing.expectEqual(@as(u8, 6), b.abc);
try std.testing.expectEqual(@as(u8, 10), b.xyz);

const Bar = struct { flag_a: bool, flag_b: bool };
const c: Bar = Bar{ .flag_a = false, .flag_b = true };
const d: Bar = optional(Bar, .{ .flag_a = true, .flag_b = true }, c);
try std.testing.expect(d.flag_a);
try std.testing.expect(d.flag_b);
}
87 changes: 44 additions & 43 deletions util/runner.zig
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ const State = union(enum) {

/// Maximum number of iterations the benchmark can run. This limit helps
/// to avoid excessively long benchmark runs.
max_iterations: u16,
max_iterations: u32,

/// Time budget for the benchmark in nanoseconds. This value is used to
/// determine how long a single benchmark should be allowed to run
Expand Down Expand Up @@ -58,8 +58,8 @@ state: State,

pub fn init(
allocator: std.mem.Allocator,
iterations: u16,
max_iterations: u16,
iterations: u32,
max_iterations: u32,
time_budget_ns: u64,
track_allocations: bool,
) Error!Runner {
Expand Down Expand Up @@ -139,6 +139,7 @@ pub fn finish(self: *Runner) Error!Readings {
.allocator = self.allocator,
.iterations = 0,
.timings_ns = &.{},
.baseline_ns = &.{},
.allocations = null,
},
.running => |st| st.readings,
Expand Down Expand Up @@ -175,26 +176,26 @@ test "Runner" {
var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, false);
{
errdefer r.abort();
try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));

try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(400_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, null)));
try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));

try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(400_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, null)));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, null)));
try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, 0, null)));
}
const result = try r.finish();
defer result.deinit();
Expand All @@ -208,26 +209,26 @@ test "Runner - memory tracking" {
var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, true);
{
errdefer r.abort();
try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 256, .count = 1 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 256, .count = 1 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 512, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 512, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 1024, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 1024, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 2048, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 2045, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 4096, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 4096, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 8192, .count = 32 })));

try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 1, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 2, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(300, .{ .max = 4, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(400, .{ .max = 8, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 16, .count = 32 })));
try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 32, .count = 64 })));
try expectEq(Step.more, try r.next(Reading.init(300, .{ .max = 64, .count = 128 })));
try expectEq(@as(?Step, null), try r.next(Reading.init(400, .{ .max = 128, .count = 256 })));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 256, .count = 1 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 256, .count = 1 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 512, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 512, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 1024, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 1024, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(100_000_000, 0, .{ .max = 2048, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 2045, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(300_000_000, 0, .{ .max = 4096, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 4096, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(200_000_000, 0, .{ .max = 8192, .count = 32 })));

try expectEq(Step.more, try r.next(Reading.init(100, 0, .{ .max = 1, .count = 2 })));
try expectEq(Step.more, try r.next(Reading.init(200, 0, .{ .max = 2, .count = 4 })));
try expectEq(Step.more, try r.next(Reading.init(300, 0, .{ .max = 4, .count = 8 })));
try expectEq(Step.more, try r.next(Reading.init(400, 0, .{ .max = 8, .count = 16 })));
try expectEq(Step.more, try r.next(Reading.init(100, 0, .{ .max = 16, .count = 32 })));
try expectEq(Step.more, try r.next(Reading.init(200, 0, .{ .max = 32, .count = 64 })));
try expectEq(Step.more, try r.next(Reading.init(300, 0, .{ .max = 64, .count = 128 })));
try expectEq(@as(?Step, null), try r.next(Reading.init(400, 0, .{ .max = 128, .count = 256 })));
}
const result = try r.finish();
defer result.deinit();
Expand Down
7 changes: 7 additions & 0 deletions util/runner/types.zig
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,17 @@ pub const Step = enum { more };

pub const Reading = struct {
timing_ns: u64,
baseline_ns: u64,
allocation: ?AllocationReading,

pub fn init(
timing_ns: u64,
baseline_ns: u64,
allocation: ?AllocationReading,
) Reading {
return .{
.timing_ns = timing_ns,
.baseline_ns = baseline_ns,
.allocation = allocation,
};
}
Expand All @@ -23,6 +26,7 @@ pub const Readings = struct {
allocator: std.mem.Allocator,
iterations: usize,
timings_ns: []u64,
baseline_ns: []u64,
allocations: ?AllocationReadings,

pub fn init(
Expand All @@ -34,6 +38,7 @@ pub const Readings = struct {
.allocator = allocator,
.iterations = n,
.timings_ns = try allocator.alloc(u64, n),
.baseline_ns = try allocator.alloc(u64, n),
.allocations = if (track_allocations)
try AllocationReadings.init(allocator, n)
else
Expand All @@ -43,11 +48,13 @@ pub const Readings = struct {

pub fn deinit(self: Readings) void {
self.allocator.free(self.timings_ns);
self.allocator.free(self.baseline_ns);
if (self.allocations) |allocs| allocs.deinit(self.allocator);
}

pub fn set(self: *Readings, i: usize, reading: Reading) void {
self.timings_ns[i] = reading.timing_ns;
self.baseline_ns[i] = reading.baseline_ns;
if (self.allocations) |allocs| {
if (reading.allocation) |x| {
allocs.maxes[i] = x.max;
Expand Down
Loading
Loading