Merge pull request #25342 from ziglang/fuzz-limit

fuzzing: implement limited fuzzing

Author: andrewrk

lib/compiler/build_runner.zig

@@ -112,7 +112,7 @@ pub fn main() !void {
     var steps_menu = false;
     var output_tmp_nonce: ?[16]u8 = null;
     var watch = false;
-    var fuzz = false;
+    var fuzz: ?std.Build.Fuzz.Mode = null;
     var debounce_interval_ms: u16 = 50;
     var webui_listen: ?std.net.Address = null;
 
@@ -274,10 +274,44 @@ pub fn main() !void {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
             } else if (mem.eql(u8, arg, "--fuzz")) {
-                fuzz = true;
+                fuzz = .{ .forever = undefined };
                 if (webui_listen == null) {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
+            } else if (mem.startsWith(u8, arg, "--fuzz=")) {
+                const value = arg["--fuzz=".len..];
+                if (value.len == 0) fatal("missing argument to --fuzz", .{});
+
+                const unit: u8 = value[value.len - 1];
+                const digits = switch (unit) {
+                    '0'...'9' => value,
+                    'K', 'M', 'G' => value[0 .. value.len - 1],
+                    else => fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
+                        .{},
+                    ),
+                };
+
+                const amount = std.fmt.parseInt(u64, digits, 10) catch {
+                    fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
+                        .{},
+                    );
+                };
+
+                const normalized_amount = std.math.mul(u64, amount, switch (unit) {
+                    else => unreachable,
+                    '0'...'9' => 1,
+                    'K' => 1000,
+                    'M' => 1_000_000,
+                    'G' => 1_000_000_000,
+                }) catch fatal("fuzzing limit amount overflows u64", .{});
+
+                fuzz = .{
+                    .limit = .{
+                        .amount = normalized_amount,
+                    },
+                };
             } else if (mem.eql(u8, arg, "-fincremental")) {
                 graph.incremental = true;
             } else if (mem.eql(u8, arg, "-fno-incremental")) {
@@ -476,6 +510,7 @@ pub fn main() !void {
             targets.items,
             main_progress_node,
             &run,
+            fuzz,
         ) catch |err| switch (err) {
             error.UncleanExit => {
                 assert(!run.watch and run.web_server == null);
@@ -485,7 +520,12 @@ pub fn main() !void {
         };
 
         if (run.web_server) |*web_server| {
-            web_server.finishBuild(.{ .fuzz = fuzz });
+            if (fuzz) |mode| if (mode != .forever) fatal(
+                "error: limited fuzzing is not implemented yet for --webui",
+                .{},
+            );
+
+            web_server.finishBuild(.{ .fuzz = fuzz != null });
         }
 
         if (!watch and run.web_server == null) {
@@ -651,6 +691,7 @@ fn runStepNames(
     step_names: []const []const u8,
     parent_prog_node: std.Progress.Node,
     run: *Run,
+    fuzz: ?std.Build.Fuzz.Mode,
 ) !void {
     const gpa = run.gpa;
     const step_stack = &run.step_stack;
@@ -676,6 +717,7 @@ fn runStepNames(
             });
         }
     }
+
     assert(run.memory_blocked_steps.items.len == 0);
 
     var test_skip_count: usize = 0;
@@ -724,6 +766,45 @@ fn runStepNames(
         }
     }
 
+    const ttyconf = run.ttyconf;
+
+    if (fuzz) |mode| blk: {
+        switch (builtin.os.tag) {
+            // Current implementation depends on two things that need to be ported to Windows:
+            // * Memory-mapping to share data between the fuzzer and build runner.
+            // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
+            //   many addresses to source locations).
+            .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
+            else => {},
+        }
+        if (@bitSizeOf(usize) != 64) {
+            // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
+            // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
+            // on 32-bit platforms.
+            // Affects or affected by issues #5185, #22523, and #22464.
+            fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
+        }
+
+        switch (mode) {
+            .forever => break :blk,
+            .limit => {},
+        }
+
+        assert(mode == .limit);
+        var f = std.Build.Fuzz.init(
+            gpa,
+            thread_pool,
+            step_stack.keys(),
+            parent_prog_node,
+            ttyconf,
+            mode,
+        ) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
+        defer f.deinit();
+
+        f.start();
+        f.waitAndPrintReport();
+    }
+
     // A proper command line application defaults to silently succeeding.
     // The user may request verbose mode if they have a different preference.
     const failures_only = switch (run.summary) {
@@ -737,8 +818,6 @@ fn runStepNames(
         std.Progress.setStatus(.failure);
     }
 
-    const ttyconf = run.ttyconf;
-
     if (run.summary != .none) {
         const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
         defer std.debug.unlockStderrWriter();
@@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
         \\  --watch                      Continuously rebuild when source files are modified
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
         \\  --webui[=ip]                 Enable the web interface on the given IP address
-        \\  --fuzz                       Continuously search for unit test failures (implies '--webui')
+        \\  --fuzz[=limit]               Continuously search for unit test failures with an optional 
+        \\                               limit to the max number of iterations. The argument supports
+        \\                               an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
+        \\                               '--webui' when no limit is specified.
         \\  --time-report                Force full rebuild and provide detailed information on
         \\                               compilation time of Zig source code (implies '--webui')
         \\     -fincremental             Enable incremental compilation

lib/compiler/test_runner.zig

@@ -2,6 +2,7 @@
 const builtin = @import("builtin");
 
 const std = @import("std");
+const fatal = std.process.fatal;
 const testing = std.testing;
 const assert = std.debug.assert;
 const fuzz_abi = std.Build.abi.fuzz;
@@ -55,12 +56,13 @@ pub fn main() void {
         }
     }
 
-    fba.reset();
     if (builtin.fuzz) {
         const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
         fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
     }
 
+    fba.reset();
+
     if (listen) {
         return mainServer() catch @panic("internal test runner failure");
     } else {
@@ -79,8 +81,13 @@ fn mainServer() !void {
     });
 
     if (builtin.fuzz) {
-        const coverage_id = fuzz_abi.fuzzer_coverage_id();
-        try server.serveU64Message(.coverage_id, coverage_id);
+        const coverage = fuzz_abi.fuzzer_coverage();
+        try server.serveCoverageIdMessage(
+            coverage.id,
+            coverage.runs,
+            coverage.unique,
+            coverage.seen,
+        );
     }
 
     while (true) {
@@ -158,26 +165,33 @@ fn mainServer() !void {
                 if (!builtin.fuzz) unreachable;
 
                 const index = try server.receiveBody_u32();
+                const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
+                const amount_or_instance = try server.receiveBody_u64();
+
                 const test_fn = builtin.test_functions[index];
                 const entry_addr = @intFromPtr(test_fn.func);
 
                 try server.serveU64Message(.fuzz_start_addr, entry_addr);
                 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
                 is_fuzz_test = false;
                 fuzz_test_index = index;
+                fuzz_mode = mode;
+                fuzz_amount_or_instance = amount_or_instance;
 
                 test_fn.func() catch |err| switch (err) {
                     error.SkipZigTest => return,
                     else => {
                         if (@errorReturnTrace()) |trace| {
                             std.debug.dumpStackTrace(trace.*);
                         }
-                        std.debug.print("failed with error.{s}\n", .{@errorName(err)});
+                        std.debug.print("failed with error.{t}\n", .{err});
                         std.process.exit(1);
                     },
                 };
                 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
                 if (log_err_count != 0) @panic("error logs detected");
+                assert(mode != .forever);
+                std.process.exit(0);
             },
 
             else => {
@@ -240,11 +254,11 @@ fn mainTerminal() void {
             else => {
                 fail_count += 1;
                 if (have_tty) {
-                    std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{
-                        i + 1, test_fn_list.len, test_fn.name, @errorName(err),
+                    std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
+                        i + 1, test_fn_list.len, test_fn.name, err,
                     });
                 } else {
-                    std.debug.print("FAIL ({s})\n", .{@errorName(err)});
+                    std.debug.print("FAIL ({t})\n", .{err});
                 }
                 if (@errorReturnTrace()) |trace| {
                     std.debug.dumpStackTrace(trace.*);
@@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {
 
 var is_fuzz_test: bool = undefined;
 var fuzz_test_index: u32 = undefined;
+var fuzz_mode: fuzz_abi.LimitKind = undefined;
+var fuzz_amount_or_instance: u64 = undefined;
 
 pub fn fuzz(
     context: anytype,
@@ -383,7 +399,7 @@ pub fn fuzz(
                 else => {
                     std.debug.lockStdErr();
                     if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
-                    std.debug.print("failed with error.{s}\n", .{@errorName(err)});
+                    std.debug.print("failed with error.{t}\n", .{err});
                     std.process.exit(1);
                 },
             };
@@ -401,9 +417,11 @@ pub fn fuzz(
 
         global.ctx = context;
         fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
+
         for (options.corpus) |elem|
             fuzz_abi.fuzzer_new_input(.fromSlice(elem));
-        fuzz_abi.fuzzer_main();
+
+        fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
         return;
     }