motiejus/zig
1
Fork 0
Code Releases Activity

add --debug-rt CLI arg to the compiler + bonus edits

Browse Source 
The flag makes compiler_rt and libfuzzer be in debug mode.

Also:
* fuzzer: override debug logs and disable debug logs for frequently
  called functions
* std.Build.Fuzz: fix bug of rerunning the old unit test binary
* report errors from rebuilding the unit tests better
* link.Elf: additionally add tsan lib and fuzzer lib to the hash
This commit is contained in:
Andrew Kelley
2024-07-24 17:41:44 -07:00
parent 90dfd86ebe
commit a3c74aca99
9 changed files with 116 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/compiler/build_runner.zig
View File

@@ -208,6 +208,8 @@ pub fn main() !void {
try debug_log_scopes.append(next_arg);
} else if (mem.eql(u8, arg, "--debug-pkg-config")) {
builder.debug_pkg_config = true;
} else if (mem.eql(u8, arg, "--debug-rt")) {
graph.debug_compiler_runtime_libs = true;
} else if (mem.eql(u8, arg, "--debug-compile-errors")) {
builder.debug_compile_errors = true;
} else if (mem.eql(u8, arg, "--system")) {
@@ -1072,7 +1074,8 @@ fn workerMakeOneStep(
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
const gpa = b.allocator;
printErrorMessages(gpa, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {};
}
handle_result: {
@@ -1126,14 +1129,12 @@ fn workerMakeOneStep(
}
pub fn printErrorMessages(
b: *std.Build,
gpa: Allocator,
failing_step: *Step,
ttyconf: std.io.tty.Config,
stderr: File,
prominent_compile_errors: bool,
) !void {
const gpa = b.allocator;
// Provide context for where these error messages are coming from by
// printing the corresponding Step subtree.
@@ -1313,6 +1314,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void {
\\ --seed [integer] For shuffling dependency traversal order (default: random)
\\ --debug-log [scope] Enable debugging the compiler
\\ --debug-pkg-config Fail if unknown pkg-config flags encountered
\\ --debug-rt Debug compiler runtime libraries
\\ --verbose-link Enable compiler debug output for linking
\\ --verbose-air Enable compiler debug output for Zig AIR
\\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR

72
lib/fuzzer.zig
View File

@@ -1,14 +1,40 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
pub const std_options = .{
.logFn = logOverride,
};
var log_file: ?std.fs.File = null;
fn logOverride(
comptime level: std.log.Level,
comptime scope: @TypeOf(.EnumLiteral),
comptime format: []const u8,
args: anytype,
) void {
const f = if (log_file) |f| f else f: {
const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file");
log_file = f;
break :f f;
};
const prefix1 = comptime level.asText();
const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): ";
f.writer().print(prefix1 ++ prefix2 ++ format ++ "\n", args) catch @panic("failed to write to fuzzer log");
}
export threadlocal var __sancov_lowest_stack: usize = 0;
export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void {
std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop });
}
export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void {
std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end });
export fn __sanitizer_cov_pcs_init(pc_start: [*]const usize, pc_end: [*]const usize) void {
std.log.debug("__sanitizer_cov_pcs_init pc_start={*}, pc_end={*}", .{ pc_start, pc_end });
fuzzer.pc_range = .{
.start = @intFromPtr(pc_start),
.end = @intFromPtr(pc_start),
};
}
export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void {
@@ -48,34 +74,45 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void {
const len = cases_ptr[0];
const val_size_in_bits = cases_ptr[1];
const cases = cases_ptr[2..][0..len];
std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
pc, val, val_size_in_bits, cases.len,
});
_ = val;
_ = pc;
_ = val_size_in_bits;
_ = cases;
//std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{
// pc, val, val_size_in_bits, cases.len,
//});
}
export fn __sanitizer_cov_trace_pc_indir(callee: usize) void {
const pc = @returnAddress();
std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
_ = callee;
_ = pc;
//std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee });
}
fn handleCmp(pc: usize, arg1: u64, arg2: u64) void {
std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
_ = pc;
_ = arg1;
_ = arg2;
//std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 });
}
const Fuzzer = struct {
gpa: Allocator,
rng: std.Random.DefaultPrng,
input: std.ArrayListUnmanaged(u8),
pc_range: PcRange,
count: usize,
const Slice = extern struct {
ptr: [*]const u8,
len: usize,
fn toSlice(s: Slice) []const u8 {
fn toZig(s: Slice) []const u8 {
return s.ptr[0..s.len];
}
fn fromSlice(s: []const u8) Slice {
fn fromZig(s: []const u8) Slice {
return .{
.ptr = s.ptr,
.len = s.len,
@@ -83,14 +120,27 @@ const Fuzzer = struct {
}
};
const PcRange = struct {
start: usize,
end: usize,
};
fn next(f: *Fuzzer) ![]const u8 {
const gpa = f.gpa;
// Prepare next input.
const rng = fuzzer.rng.random();
const len = rng.uintLessThan(usize, 64);
try f.input.resize(gpa, len);
rng.bytes(f.input.items);
f.resetCoverage();
f.count += 1;
return f.input.items;
}
fn resetCoverage(f: *Fuzzer) void {
_ = f;
}
};
var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{};
@@ -99,10 +149,12 @@ var fuzzer: Fuzzer = .{
.gpa = general_purpose_allocator.allocator(),
.rng = std.Random.DefaultPrng.init(0),
.input = .{},
.pc_range = .{ .start = 0, .end = 0 },
.count = 0,
};
export fn fuzzer_next() Fuzzer.Slice {
return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) {
return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) {
error.OutOfMemory => @panic("out of memory"),
});
}

1
lib/std/Build.zig
View File

@@ -113,6 +113,7 @@ pub const Graph = struct {
arena: Allocator,
system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .{},
system_package_mode: bool = false,
debug_compiler_runtime_libs: bool = false,
cache: Cache,
zig_exe: [:0]const u8,
env_map: EnvMap,

36
lib/std/Build/Fuzz.zig
View File

@@ -55,22 +55,32 @@ pub fn start(
}
fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
const compile_step = run.producer.?;
const prog_node = parent_prog_node.start(compile_step.step.name, 0);
const gpa = run.step.owner.allocator;
const stderr = std.io.getStdErr();
const compile = run.producer.?;
const prog_node = parent_prog_node.start(compile.step.name, 0);
defer prog_node.end();
if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| {
const result = compile.rebuildInFuzzMode(prog_node);
const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0;
const show_error_msgs = compile.step.result_error_msgs.items.len > 0;
const show_stderr = compile.step.result_stderr.len > 0;
if (show_error_msgs or show_compile_errors or show_stderr) {
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {};
}
if (result) |rebuilt_bin_path| {
run.rebuilt_executable = rebuilt_bin_path;
} else |err| switch (err) {
error.MakeFailed => {
const b = run.step.owner;
const stderr = std.io.getStdErr();
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {};
},
error.MakeFailed => {},
else => {
std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{
compile_step.step.name, @errorName(err),
compile.step.name, @errorName(err),
});
},
}
@@ -82,6 +92,7 @@ fn fuzzWorkerRun(
ttyconf: std.io.tty.Config,
parent_prog_node: std.Progress.Node,
) void {
const gpa = run.step.owner.allocator;
const test_name = run.cached_test_metadata.?.testName(unit_test_index);
const prog_node = parent_prog_node.start(test_name, 0);
@@ -89,11 +100,10 @@ fn fuzzWorkerRun(
run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) {
error.MakeFailed => {
const b = run.step.owner;
const stderr = std.io.getStdErr();
std.debug.lockStdErr();
defer std.debug.unlockStdErr();
build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {};
build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {};
},
else => {
std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{

10
lib/std/Build/Step/Compile.zig
View File

@@ -1483,6 +1483,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 {
try zig_args.append("--global-cache-dir");
try zig_args.append(b.graph.global_cache_root.path orelse ".");
if (b.graph.debug_compiler_runtime_libs) try zig_args.append("--debug-rt");
try zig_args.append("--name");
try zig_args.append(compile.name);
@@ -1840,6 +1842,14 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
}
pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {
const gpa = c.step.owner.allocator;
c.step.result_error_msgs.clearRetainingCapacity();
c.step.result_stderr = "";
c.step.result_error_bundle.deinit(gpa);
c.step.result_error_bundle = std.zig.ErrorBundle.empty;
const zig_args = try getZigArgs(c, true);
const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false);
return maybe_output_bin_path.?;

5
lib/std/Build/Step/Run.zig
View File

@@ -865,7 +865,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress.
},
.artifact => |pa| {
const artifact = pa.artifact;
const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?;
const file_path = if (artifact == run.producer.?)
run.rebuilt_executable.?
else
(artifact.installed_path orelse artifact.generated_bin.?.path.?);
try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
},
.output_file, .output_directory => unreachable,

4
src/Compilation.zig
View File

@@ -2180,7 +2180,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
comp.bin_file = try link.File.createEmpty(arena, comp, emit, whole.lf_open_opts);
}
},
.incremental => {},
.incremental => {
log.debug("Compilation.update for {s}, CacheMode.incremental", .{comp.root_name});
},
}
// From this point we add a preliminary set of file system inputs that

2
src/link/Elf.zig
View File

@@ -2286,6 +2286,8 @@ fn linkWithLLD(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: s
}
try man.addOptionalFile(module_obj_path);
try man.addOptionalFile(compiler_rt_path);
try man.addOptionalFile(if (comp.tsan_lib) |l| l.full_object_path else null);
try man.addOptionalFile(if (comp.fuzzer_lib) |l| l.full_object_path else null);
// We can skip hashing libc and libc++ components that we are in charge of building from Zig
// installation sources because they are always a product of the compiler version + target information.

5
src/main.zig
View File

@@ -655,6 +655,7 @@ const usage_build_generic =
\\ --debug-log [scope] Enable printing debug/info log messages for scope
\\ --debug-compile-errors Crash with helpful diagnostics at the first compile error
\\ --debug-link-snapshot Enable dumping of the linker's state in JSON format
\\ --debug-rt Debug compiler runtime libraries
\\
;
@@ -912,6 +913,7 @@ fn buildOutputType(
var minor_subsystem_version: ?u16 = null;
var mingw_unicode_entry_point: bool = false;
var enable_link_snapshots: bool = false;
var debug_compiler_runtime_libs = false;
var opt_incremental: ?bool = null;
var install_name: ?[]const u8 = null;
var hash_style: link.File.Elf.HashStyle = .both;
@@ -1367,6 +1369,8 @@ fn buildOutputType(
} else {
enable_link_snapshots = true;
}
} else if (mem.eql(u8, arg, "--debug-rt")) {
debug_compiler_runtime_libs = true;
} else if (mem.eql(u8, arg, "-fincremental")) {
dev.check(.incremental);
opt_incremental = true;
@@ -3408,6 +3412,7 @@ fn buildOutputType(
// noise when --search-prefix and --mod are combined.
.global_cc_argv = try cc_argv.toOwnedSlice(arena),
.file_system_inputs = &file_system_inputs,
.debug_compiler_runtime_libs = debug_compiler_runtime_libs,
}) catch |err| switch (err) {
error.LibCUnavailable => {
const triple_name = try target.zigTriple(arena);