fuzzing: implement limited fuzzing

Adds the limit option to `--fuzz=[limit]`. the limit expresses a number
of iterations that *each fuzz test* will perform at maximum before
exiting. The limit argument supports also 'K', 'M', and 'G' suffixeds
(e.g. '10K').

Does not imply `--web-ui` (like unlimited fuzzing does) and prints a
fuzzing report at the end.

Closes #22900 but does not implement the time based limit, as after
internal discussions we concluded to be problematic to both implement
and use correctly.

Author: kristoff-it

lib/compiler/build_runner.zig

@@ -112,7 +112,7 @@ pub fn main() !void {
     var steps_menu = false;
     var output_tmp_nonce: ?[16]u8 = null;
     var watch = false;
-    var fuzz = false;
+    var fuzz: ?std.Build.Fuzz.Mode = null;
     var debounce_interval_ms: u16 = 50;
     var webui_listen: ?std.net.Address = null;
 
@@ -274,10 +274,44 @@ pub fn main() !void {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
             } else if (mem.eql(u8, arg, "--fuzz")) {
-                fuzz = true;
+                fuzz = .{ .forever = undefined };
                 if (webui_listen == null) {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
+            } else if (mem.startsWith(u8, arg, "--fuzz=")) {
+                const value = arg["--fuzz=".len..];
+                if (value.len == 0) fatal("missing argument to --fuzz\n", .{});
+
+                const unit: u8 = value[value.len - 1];
+                const digits = switch (value[value.len - 1]) {
+                    '0'...'9' => value,
+                    'K', 'M', 'G' => value[0 .. value.len - 1],
+                    else => fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]\n",
+                        .{},
+                    ),
+                };
+
+                const amount = std.fmt.parseInt(u64, digits, 10) catch {
+                    fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]\n",
+                        .{},
+                    );
+                };
+
+                const normalized_amount = std.math.mul(u64, amount, switch (unit) {
+                    else => unreachable,
+                    '0'...'9' => 1,
+                    'K' => 1000,
+                    'M' => 1_000_000,
+                    'G' => 1_000_000_000,
+                }) catch fatal("fuzzing limit amount overflows u64\n", .{});
+
+                fuzz = .{
+                    .limit = .{
+                        .amount = normalized_amount,
+                    },
+                };
             } else if (mem.eql(u8, arg, "-fincremental")) {
                 graph.incremental = true;
             } else if (mem.eql(u8, arg, "-fno-incremental")) {
@@ -476,6 +510,7 @@ pub fn main() !void {
             targets.items,
             main_progress_node,
             &run,
+            fuzz,
         ) catch |err| switch (err) {
             error.UncleanExit => {
                 assert(!run.watch and run.web_server == null);
@@ -485,7 +520,8 @@ pub fn main() !void {
         };
 
         if (run.web_server) |*web_server| {
-            web_server.finishBuild(.{ .fuzz = fuzz });
+            if (fuzz) |mode| assert(mode == .forever);
+            web_server.finishBuild(.{ .fuzz = fuzz != null });
         }
 
         if (!watch and run.web_server == null) {
@@ -651,6 +687,7 @@ fn runStepNames(
     step_names: []const []const u8,
     parent_prog_node: std.Progress.Node,
     run: *Run,
+    fuzz: ?std.Build.Fuzz.Mode,
 ) !void {
     const gpa = run.gpa;
     const step_stack = &run.step_stack;
@@ -676,6 +713,7 @@ fn runStepNames(
             });
         }
     }
+
     assert(run.memory_blocked_steps.items.len == 0);
 
     var test_skip_count: usize = 0;
@@ -724,6 +762,45 @@ fn runStepNames(
         }
     }
 
+    const ttyconf = run.ttyconf;
+
+    if (fuzz) |mode| blk: {
+        switch (builtin.os.tag) {
+            // Current implementation depends on two things that need to be ported to Windows:
+            // * Memory-mapping to share data between the fuzzer and build runner.
+            // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
+            //   many addresses to source locations).
+            .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
+            else => {},
+        }
+        if (@bitSizeOf(usize) != 64) {
+            // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
+            // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
+            // on 32-bit platforms.
+            // Affects or affected by issues #5185, #22523, and #22464.
+            fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
+        }
+
+        switch (mode) {
+            .forever => break :blk,
+            .limit => {},
+        }
+
+        assert(mode == .limit);
+        var f = std.Build.Fuzz.init(
+            gpa,
+            thread_pool,
+            step_stack.keys(),
+            parent_prog_node,
+            ttyconf,
+            mode,
+        ) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
+        defer f.deinit();
+
+        f.start();
+        f.waitAndPrintReport();
+    }
+
     // A proper command line application defaults to silently succeeding.
     // The user may request verbose mode if they have a different preference.
     const failures_only = switch (run.summary) {
@@ -737,8 +814,6 @@ fn runStepNames(
         std.Progress.setStatus(.failure);
     }
 
-    const ttyconf = run.ttyconf;
-
     if (run.summary != .none) {
         const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
         defer std.debug.unlockStderrWriter();
@@ -1366,7 +1441,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
         \\  --watch                      Continuously rebuild when source files are modified
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
         \\  --webui[=ip]                 Enable the web interface on the given IP address
-        \\  --fuzz                       Continuously search for unit test failures (implies '--webui')
+        \\  --fuzz[=limit]               Continuously search for unit test failures with an optional 
+        \\                               limit to the max number of iterations. The argument supports
+        \\                               an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
+        \\                               '--webui' when no limit is specified.
         \\  --time-report                Force full rebuild and provide detailed information on
         \\                               compilation time of Zig source code (implies '--webui')
         \\     -fincremental             Enable incremental compilation

lib/compiler/test_runner.zig

@@ -2,6 +2,7 @@
 const builtin = @import("builtin");
 
 const std = @import("std");
+const fatal = std.process.fatal;
 const testing = std.testing;
 const assert = std.debug.assert;
 const fuzz_abi = std.Build.abi.fuzz;
@@ -62,13 +63,13 @@ pub fn main() void {
     }
 
     if (listen) {
-        return mainServer() catch @panic("internal test runner failure");
+        return mainServer(opt_cache_dir) catch @panic("internal test runner failure");
     } else {
         return mainTerminal();
     }
 }
 
-fn mainServer() !void {
+fn mainServer(opt_cache_dir: ?[]const u8) !void {
     @disableInstrumentation();
     var stdin_reader = std.fs.File.stdin().readerStreaming(&stdin_buffer);
     var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer);
@@ -78,9 +79,66 @@ fn mainServer() !void {
         .zig_version = builtin.zig_version_string,
     });
 
-    if (builtin.fuzz) {
+    if (builtin.fuzz) blk: {
+        const cache_dir = opt_cache_dir.?;
         const coverage_id = fuzz_abi.fuzzer_coverage_id();
-        try server.serveU64Message(.coverage_id, coverage_id);
+        const coverage_file_path: std.Build.Cache.Path = .{
+            .root_dir = .{
+                .path = cache_dir,
+                .handle = std.fs.cwd().openDir(cache_dir, .{}) catch |err| {
+                    if (err == error.FileNotFound) {
+                        try server.serveCoverageIdMessage(coverage_id, 0, 0, 0);
+                        break :blk;
+                    }
+
+                    fatal("failed to access cache dir '{s}': {s}", .{
+                        cache_dir, @errorName(err),
+                    });
+                },
+            },
+            .sub_path = "v/" ++ std.fmt.hex(coverage_id),
+        };
+
+        var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
+            if (err == error.FileNotFound) {
+                try server.serveCoverageIdMessage(coverage_id, 0, 0, 0);
+                break :blk;
+            }
+
+            fatal("failed to load coverage file '{f}': {s}", .{
+                coverage_file_path, @errorName(err),
+            });
+        };
+        defer coverage_file.close();
+
+        var rbuf: [0x1000]u8 = undefined;
+        var r = coverage_file.reader(&rbuf);
+
+        var header: fuzz_abi.SeenPcsHeader = undefined;
+        r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
+            fatal("failed to read from coverage file '{f}': {s}", .{
+                coverage_file_path, @errorName(err),
+            });
+        };
+
+        if (header.pcs_len == 0) {
+            fatal("corrupted coverage file '{f}': pcs_len was zero", .{
+                coverage_file_path,
+            });
+        }
+
+        var seen_count: usize = 0;
+        const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
+        for (0..chunk_count) |_| {
+            const seen = r.interface.takeInt(usize, .little) catch |err| {
+                fatal("failed to read from coverage file '{f}': {s}", .{
+                    coverage_file_path, @errorName(err),
+                });
+            };
+            seen_count += @popCount(seen);
+        }
+
+        try server.serveCoverageIdMessage(coverage_id, header.n_runs, header.unique_runs, seen_count);
     }
 
     while (true) {
@@ -158,13 +216,18 @@ fn mainServer() !void {
                 if (!builtin.fuzz) unreachable;
 
                 const index = try server.receiveBody_u32();
+                const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
+                const amount_or_instance = try server.receiveBody_u64();
+
                 const test_fn = builtin.test_functions[index];
                 const entry_addr = @intFromPtr(test_fn.func);
 
                 try server.serveU64Message(.fuzz_start_addr, entry_addr);
                 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
                 is_fuzz_test = false;
                 fuzz_test_index = index;
+                fuzz_mode = mode;
+                fuzz_amount_or_instance = amount_or_instance;
 
                 test_fn.func() catch |err| switch (err) {
                     error.SkipZigTest => return,
@@ -178,6 +241,8 @@ fn mainServer() !void {
                 };
                 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
                 if (log_err_count != 0) @panic("error logs detected");
+                assert(mode != .forever);
+                std.process.exit(0);
             },
 
             else => {
@@ -343,6 +408,8 @@ pub fn mainSimple() anyerror!void {
 
 var is_fuzz_test: bool = undefined;
 var fuzz_test_index: u32 = undefined;
+var fuzz_mode: fuzz_abi.LimitKind = undefined;
+var fuzz_amount_or_instance: u64 = undefined;
 
 pub fn fuzz(
     context: anytype,
@@ -401,9 +468,11 @@ pub fn fuzz(
 
         global.ctx = context;
         fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
+
         for (options.corpus) |elem|
             fuzz_abi.fuzzer_new_input(.fromSlice(elem));
-        fuzz_abi.fuzzer_main();
+
+        fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
         return;
     }
 

lib/fuzzer.zig

@@ -600,9 +600,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {
 }
 
 /// fuzzer_init_test must be called first
-export fn fuzzer_main() void {
-    while (true) {
-        fuzzer.cycle();
+export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
+    switch (limit_kind) {
+        .forever => while (true) fuzzer.cycle(),
+        .iterations => for (0..amount -| 1) |_| fuzzer.cycle(),
     }
 }