commit 5ecef2934af9349507061b2ff0498a089e7f5de0 (tree)
parent bca057afd28bdcc38b123c05915ca7d775a2a17d
Author: Kendall Condon <goon.pri.low@gmail.com>
Date: Wed, 11 Mar 2026 21:19:22 -0400
rerun fuzz tests from name instead of index
The indexes can change between recompilation due to conditional
compilation and compiler quirks. While unit test names are still not a
perfect solution, they are better than indexes.
Diffstat:
4 files changed, 44 insertions(+), 24 deletions(-)
diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig
@@ -180,7 +180,23 @@ fn mainServer(init: std.process.Init.Minimal) !void {
// since they are not present.
if (!builtin.fuzz) unreachable;
- const index = try server.receiveBody_u32();
+ const index: u32 = @intCast(index: {
+ testing.allocator_instance = .{};
+ defer if (testing.allocator_instance.deinit() == .leak) {
+ @panic("internal test runner memory leak");
+ };
+
+ const name_len = try server.receiveBody_u32();
+ const name = try server.in.readAlloc(testing.allocator, @intCast(name_len));
+ defer testing.allocator.free(name);
+ for (0.., builtin.test_functions) |i, test_fn| {
+ if (std.mem.eql(u8, name, test_fn.name)) {
+ break :index i;
+ }
+ } else {
+ std.debug.panic("fuzz test {s} no longer exists", .{name});
+ }
+ });
const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
const amount_or_instance = try server.receiveBody_u64();
diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig
@@ -145,9 +145,9 @@ pub fn start(fuzz: *Fuzz) void {
}
for (fuzz.run_steps) |run| {
- for (run.fuzz_tests.items) |unit_test_index| {
+ for (run.fuzz_tests.items) |unit_test_name| {
assert(run.rebuilt_executable != null);
- fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
+ fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_name });
}
}
}
@@ -193,17 +193,16 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod
run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
}
-fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
+fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_name: []const u8) void {
const owner = run.step.owner;
const gpa = owner.allocator;
const graph = owner.graph;
const io = graph.io;
- const test_name = run.cached_test_metadata.?.testName(unit_test_index);
- const prog_node = fuzz.prog_node.start(test_name, 0);
+ const prog_node = fuzz.prog_node.start(unit_test_name, 0);
defer prog_node.end();
- run.rerunInFuzzMode(fuzz, unit_test_index, prog_node) catch |err| switch (err) {
+ run.rerunInFuzzMode(fuzz, unit_test_name, prog_node) catch |err| switch (err) {
error.MakeFailed => {
var buf: [256]u8 = undefined;
const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {
@@ -214,7 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
return;
},
else => {
- log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, test_name, err });
+ log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, unit_test_name, err });
return;
},
};
@@ -588,7 +587,7 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
\\
, .{
cov.run.step.name,
- cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
+ cov.run.fuzz_tests.items[0],
cov.id,
cov.cumulative.runs,
header.n_runs,
diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig
@@ -88,9 +88,10 @@ dep_output_file: ?*Output,
has_side_effects: bool,
-/// If this is a Zig unit test binary, this tracks the indexes of the unit
-/// tests that are also fuzz tests.
-fuzz_tests: std.ArrayList(u32),
+/// If this is a Zig unit test binary, this tracks the names of the unit
+/// tests that are also fuzz tests. Indexes cannot be used as they may
+/// change between reruns.
+fuzz_tests: std.ArrayList([]const u8),
cached_test_metadata: ?CachedTestMetadata = null,
/// Populated during the fuzz phase if this run step corresponds to a unit test
@@ -1067,7 +1068,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
pub fn rerunInFuzzMode(
run: *Run,
fuzz: *std.Build.Fuzz,
- unit_test_index: u32,
+ unit_test_name: []const u8,
prog_node: std.Progress.Node,
) !void {
const step = &run.step;
@@ -1138,7 +1139,7 @@ pub fn rerunInFuzzMode(
.unit_test_timeout_ns = null, // don't time out fuzz tests for now
.gpa = fuzz.gpa,
}, .{
- .unit_test_index = unit_test_index,
+ .unit_test_name = unit_test_name,
.fuzz = fuzz,
});
}
@@ -1210,7 +1211,7 @@ fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {
const FuzzContext = struct {
fuzz: *std.Build.Fuzz,
- unit_test_index: u32,
+ unit_test_name: []const u8,
};
fn runCommand(
@@ -1843,7 +1844,7 @@ fn waitZigTest(
sendRunFuzzTestMessage(
io,
child.stdin.?,
- ctx.unit_test_index,
+ ctx.unit_test_name,
.forever,
0, // instance ID; will be used by multiprocess forever fuzzing in the future
) catch |err| return .{ .write_failed = err };
@@ -1852,7 +1853,7 @@ fn waitZigTest(
sendRunFuzzTestMessage(
io,
child.stdin.?,
- ctx.unit_test_index,
+ ctx.unit_test_name,
.iterations,
limit.amount,
) catch |err| return .{ .write_failed = err };
@@ -2001,10 +2002,10 @@ fn waitZigTest(
results.leak_count +|= leak_count;
results.log_err_count +|= log_err_count;
- if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
+ if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index));
if (tr_hdr.flags.status == .fail) {
- const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
+ const name = md.testName(tr_hdr.index);
const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
stderr.tossBuffered();
if (stderr_bytes.len == 0) {
@@ -2013,12 +2014,12 @@ fn waitZigTest(
try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
}
} else if (leak_count > 0) {
- const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
+ const name = md.testName(tr_hdr.index);
const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
stderr.tossBuffered();
try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
} else if (log_err_count > 0) {
- const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
+ const name = md.testName(tr_hdr.index);
const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
stderr.tossBuffered();
try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
@@ -2148,7 +2149,7 @@ fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, in
fn sendRunFuzzTestMessage(
io: Io,
file: Io.File,
- index: u32,
+ test_name: []const u8,
kind: std.Build.abi.fuzz.LimitKind,
amount_or_instance: u64,
) !void {
@@ -2160,7 +2161,10 @@ fn sendRunFuzzTestMessage(
w.interface.writeStruct(header, .little) catch |err| switch (err) {
error.WriteFailed => return w.err.?,
};
- w.interface.writeInt(u32, index, .little) catch |err| switch (err) {
+ w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) {
+ error.WriteFailed => return w.err.?,
+ };
+ w.interface.writeAll(test_name) catch |err| switch (err) {
error.WriteFailed => return w.err.?,
};
w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {
diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig
@@ -35,7 +35,8 @@ pub const Message = struct {
run_test,
/// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
/// The message body is:
- /// - a u32 test index.
+ /// - a u32 test name len.
+ /// - a test name with the above length
/// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
/// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
start_fuzzing,
